--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:17:09.218043Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-542f9-767cbb57f5-vn4jq namespace: kserve-ci-e2e-test resourceVersion: "20127" uid: 85fa4d2c-a3ca-4413-8fbe-252a0777e49f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-542f9-767cbb57f5-vn4jq to ip-10-0-130-158.ec2.internal metadata: creationTimestamp: "2026-07-07T15:17:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:17:09Z" name: ensemble-graph-542f9-767cbb57f5-vn4jq.18c00ae48fad82d5 namespace: kserve-ci-e2e-test resourceVersion: "20132" uid: 01f58964-ece1-4e8b-8a5f-4aa2271c7fa9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:09Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-542f9-767cbb57f5-vn4jq namespace: kserve-ci-e2e-test resourceVersion: "20129" uid: 85fa4d2c-a3ca-4413-8fbe-252a0777e49f kind: Event lastTimestamp: "2026-07-07T15:17:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-542f9-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T15:17:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:09Z" name: ensemble-graph-542f9-767cbb57f5-vn4jq.18c00ae4996dd56a namespace: kserve-ci-e2e-test resourceVersion: "20145" uid: 42190001-7082-4019-b80c-d6f866b350c4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:10Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-542f9-767cbb57f5-vn4jq namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: 85fa4d2c-a3ca-4413-8fbe-252a0777e49f kind: Event lastTimestamp: "2026-07-07T15:17:10Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:17:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:17:10Z" name: ensemble-graph-542f9-767cbb57f5-vn4jq.18c00ae4ccd4a89a namespace: kserve-ci-e2e-test resourceVersion: "20156" uid: 323ea563-46f3-436d-b7c2-1a5d900efe77 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-542f9} kind: Pod name: ensemble-graph-542f9-767cbb57f5-vn4jq namespace: kserve-ci-e2e-test resourceVersion: "20129" uid: 85fa4d2c-a3ca-4413-8fbe-252a0777e49f kind: Event lastTimestamp: "2026-07-07T15:17:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b3cb5f47c4ea9d876e5490bc83b15040508a0aac484b7e9d11de2faf2dbcdfe4" already present on machine metadata: creationTimestamp: "2026-07-07T15:17:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:10Z" name: ensemble-graph-542f9-767cbb57f5-vn4jq.18c00ae4ce71ad8c namespace: kserve-ci-e2e-test resourceVersion: "20158" uid: beb1f607-1565-4af9-942e-88709bb8cad3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-542f9} kind: Pod name: ensemble-graph-542f9-767cbb57f5-vn4jq namespace: kserve-ci-e2e-test resourceVersion: "20129" uid: 85fa4d2c-a3ca-4413-8fbe-252a0777e49f kind: Event lastTimestamp: "2026-07-07T15:17:10Z" message: 'Created container: ensemble-graph-542f9' metadata: creationTimestamp: "2026-07-07T15:17:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:10Z" name: ensemble-graph-542f9-767cbb57f5-vn4jq.18c00ae4d19e99fd namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: 3fad6e8a-0ed1-4f75-b562-5005cc195194 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-542f9} kind: Pod name: ensemble-graph-542f9-767cbb57f5-vn4jq namespace: kserve-ci-e2e-test resourceVersion: "20129" uid: 85fa4d2c-a3ca-4413-8fbe-252a0777e49f kind: Event lastTimestamp: "2026-07-07T15:17:10Z" message: Started container ensemble-graph-542f9 metadata: creationTimestamp: "2026-07-07T15:17:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:10Z" name: ensemble-graph-542f9-767cbb57f5-vn4jq.18c00ae4d212ff42 namespace: kserve-ci-e2e-test resourceVersion: "20160" uid: 5b878c33-df5f-4f43-8beb-53c7a6c3466c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-542f9} kind: Pod name: ensemble-graph-542f9-767cbb57f5-vn4jq namespace: kserve-ci-e2e-test resourceVersion: "20129" uid: 85fa4d2c-a3ca-4413-8fbe-252a0777e49f kind: Event lastTimestamp: "2026-07-07T15:17:19Z" message: Stopping container ensemble-graph-542f9 metadata: creationTimestamp: "2026-07-07T15:17:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:19Z" name: ensemble-graph-542f9-767cbb57f5-vn4jq.18c00ae6e9859e5c namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: c5c155df-26f1-43c5-947b-24057009c711 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:17:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-542f9} kind: Pod name: ensemble-graph-542f9-767cbb57f5-vn4jq namespace: kserve-ci-e2e-test resourceVersion: "20129" uid: 85fa4d2c-a3ca-4413-8fbe-252a0777e49f kind: Event lastTimestamp: "2026-07-07T15:17:47Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:47Z" name: ensemble-graph-542f9-767cbb57f5-vn4jq.18c00ae79cdbe85a namespace: kserve-ci-e2e-test resourceVersion: "20663" uid: 3a19a0fd-d117-4f22-8596-b3c748111f6d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-542f9-767cbb57f5 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: e25eb446-0998-49c9-967c-d2afff22cc81 kind: Event lastTimestamp: "2026-07-07T15:17:09Z" message: 'Created pod: ensemble-graph-542f9-767cbb57f5-vn4jq' metadata: creationTimestamp: "2026-07-07T15:17:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:09Z" name: ensemble-graph-542f9-767cbb57f5.18c00ae48f27127b namespace: kserve-ci-e2e-test resourceVersion: "20128" uid: c49e916b-344d-49aa-b301-e0ea12bf830e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-542f9 namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: a79007eb-051c-4bfc-a81b-2a3b9ee30c2c kind: Event lastTimestamp: "2026-07-07T15:17:09Z" message: Scaled up replica set ensemble-graph-542f9-767cbb57f5 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:17:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:09Z" name: ensemble-graph-542f9.18c00ae48e5385c7 namespace: kserve-ci-e2e-test resourceVersion: "20124" uid: 8815864a-2cde-4f6a-9c13-cc8edd4e4476 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-542f9 namespace: kserve-ci-e2e-test resourceVersion: "20139" uid: 14fc0b86-ad26-446c-87cc-4d4229cd3645 kind: Event lastTimestamp: "2026-07-07T15:17:09Z" message: InferenceGraph [ensemble-graph-542f9] is Ready metadata: creationTimestamp: "2026-07-07T15:17:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:17:09Z" name: ensemble-graph-542f9.18c00ae494d345a8 namespace: kserve-ci-e2e-test resourceVersion: "20142" uid: a20579ba-4f1d-4c79-890e-bd09837db204 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:18:19.545486Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-5aeac-8558df8fd8-mz2x2 namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: 1c8ac236-3402-4144-b4ec-902df600ec17 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-5aeac-8558df8fd8-mz2x2 to ip-10-0-128-215.ec2.internal metadata: creationTimestamp: "2026-07-07T15:18:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:18:19Z" name: ensemble-graph-5aeac-8558df8fd8-mz2x2.18c00af4ef853218 namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: 1921a5ad-3e10-415c-81e8-b48fc3f8935d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:19Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-5aeac-8558df8fd8-mz2x2 namespace: kserve-ci-e2e-test resourceVersion: "21107" uid: 1c8ac236-3402-4144-b4ec-902df600ec17 kind: Event lastTimestamp: "2026-07-07T15:18:19Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:18:19Z" name: ensemble-graph-5aeac-8558df8fd8-mz2x2.18c00af5094926b1 namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: 59413e37-e6f7-4128-a2ec-3ace4a49aa8d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5aeac} kind: Pod name: ensemble-graph-5aeac-8558df8fd8-mz2x2 namespace: kserve-ci-e2e-test resourceVersion: "21098" uid: 1c8ac236-3402-4144-b4ec-902df600ec17 kind: Event lastTimestamp: "2026-07-07T15:18:20Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b3cb5f47c4ea9d876e5490bc83b15040508a0aac484b7e9d11de2faf2dbcdfe4" already present on machine metadata: creationTimestamp: "2026-07-07T15:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:20Z" name: ensemble-graph-5aeac-8558df8fd8-mz2x2.18c00af50acfebdd namespace: kserve-ci-e2e-test resourceVersion: "21124" uid: a6a7e31e-6d41-4fc7-9ee2-437f61cadb78 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5aeac} kind: Pod name: ensemble-graph-5aeac-8558df8fd8-mz2x2 namespace: kserve-ci-e2e-test resourceVersion: "21098" uid: 1c8ac236-3402-4144-b4ec-902df600ec17 kind: Event lastTimestamp: "2026-07-07T15:18:20Z" message: 'Created container: ensemble-graph-5aeac' metadata: creationTimestamp: "2026-07-07T15:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:20Z" name: ensemble-graph-5aeac-8558df8fd8-mz2x2.18c00af50da578d9 namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 0765f42b-6a7a-4cf0-a01c-9ed533acdefc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5aeac} kind: Pod name: ensemble-graph-5aeac-8558df8fd8-mz2x2 namespace: kserve-ci-e2e-test resourceVersion: "21098" uid: 1c8ac236-3402-4144-b4ec-902df600ec17 kind: Event lastTimestamp: "2026-07-07T15:18:20Z" message: Started container ensemble-graph-5aeac metadata: creationTimestamp: "2026-07-07T15:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:20Z" name: ensemble-graph-5aeac-8558df8fd8-mz2x2.18c00af50e23f51b namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: ad383283-575d-44af-ab73-ac3ce967949e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5aeac} kind: Pod name: ensemble-graph-5aeac-8558df8fd8-mz2x2 namespace: kserve-ci-e2e-test resourceVersion: "21098" uid: 1c8ac236-3402-4144-b4ec-902df600ec17 kind: Event lastTimestamp: "2026-07-07T15:26:34Z" message: Stopping container ensemble-graph-5aeac metadata: creationTimestamp: "2026-07-07T15:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:34Z" name: ensemble-graph-5aeac-8558df8fd8-mz2x2.18c00b681df88ccf namespace: kserve-ci-e2e-test resourceVersion: "24601" uid: 6aeab6cd-d0b7-4d60-8778-dded3b43a583 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:26:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5aeac} kind: Pod name: ensemble-graph-5aeac-8558df8fd8-mz2x2 namespace: kserve-ci-e2e-test resourceVersion: "21098" uid: 1c8ac236-3402-4144-b4ec-902df600ec17 kind: Event lastTimestamp: "2026-07-07T15:27:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:01Z" name: ensemble-graph-5aeac-8558df8fd8-mz2x2.18c00b68b035226b namespace: kserve-ci-e2e-test resourceVersion: "24930" uid: 34c81b22-27ba-439c-8959-0d5e1e1f3a17 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-5aeac-8558df8fd8 namespace: kserve-ci-e2e-test resourceVersion: "21093" uid: a6991a2c-6456-4834-9dc6-074e2e94ba3f kind: Event lastTimestamp: "2026-07-07T15:18:19Z" message: 'Created pod: ensemble-graph-5aeac-8558df8fd8-mz2x2' metadata: creationTimestamp: "2026-07-07T15:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:19Z" name: ensemble-graph-5aeac-8558df8fd8.18c00af4ef118558 namespace: kserve-ci-e2e-test resourceVersion: "21100" uid: 8fdb6162-0534-42a4-b618-4bc651e93eeb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-5aeac namespace: kserve-ci-e2e-test resourceVersion: "21091" uid: 3f50a01f-1f05-42e9-acc3-7e6c06d6e6ad kind: Event lastTimestamp: "2026-07-07T15:18:19Z" message: Scaled up replica set ensemble-graph-5aeac-8558df8fd8 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:19Z" name: ensemble-graph-5aeac.18c00af4ee39ce8f namespace: kserve-ci-e2e-test resourceVersion: "21094" uid: c9710699-395e-4408-b046-99514a412fe6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-5aeac namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: 4b166617-1e03-4915-b5da-ebf7f127c530 kind: Event lastTimestamp: "2026-07-07T15:18:19Z" message: InferenceGraph [ensemble-graph-5aeac] is Ready metadata: creationTimestamp: "2026-07-07T15:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:18:19Z" name: ensemble-graph-5aeac.18c00af4f5307c19 namespace: kserve-ci-e2e-test resourceVersion: "21111" uid: 86be481a-a95f-4aa3-b6f5-ee14b2813415 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:18:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-5aeac namespace: kserve-ci-e2e-test resourceVersion: "21092" uid: 91a3af1a-617d-4e20-b3ee-3a3866cb4a12 kind: Event lastTimestamp: "2026-07-07T15:18:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:49Z" name: ensemble-graph-5aeac.18c00af86d2fde06 namespace: kserve-ci-e2e-test resourceVersion: "21374" uid: 7f598462-5b78-4792-8a0c-6c38f39f10a1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:18:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-5aeac namespace: kserve-ci-e2e-test resourceVersion: "21092" uid: 91a3af1a-617d-4e20-b3ee-3a3866cb4a12 kind: Event lastTimestamp: "2026-07-07T15:18:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:49Z" name: ensemble-graph-5aeac.18c00af86d930f3b namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: e30282f6-13ac-4608-84f8-cca29413f668 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:06:54.265218Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: 31c08cbd-5315-4979-8937-4559fee41cf0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3c9c9-predictor-7d847f679-zghqk to ip-10-0-128-215.ec2.internal metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:06:54Z" name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk.18c00a5561a21546 namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: 2aff6b4a-6d4c-4937-855b-e66d2a0be365 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:54Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 31c08cbd-5315-4979-8937-4559fee41cf0 kind: Event lastTimestamp: "2026-07-07T15:06:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-3c9c9-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:54Z" name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk.18c00a556e545b4b namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: d06c6812-83a8-49ca-9d37-fa9a0980670b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk namespace: kserve-ci-e2e-test resourceVersion: "15027" uid: 31c08cbd-5315-4979-8937-4559fee41cf0 kind: Event lastTimestamp: "2026-07-07T15:06:55Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:06:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:06:55Z" name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk.18c00a559f164ba4 namespace: kserve-ci-e2e-test resourceVersion: "15084" uid: 62bf8eb9-ec77-450e-b1ad-59d913c81312 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 31c08cbd-5315-4979-8937-4559fee41cf0 kind: Event lastTimestamp: "2026-07-07T15:06:55Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:06:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:55Z" name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk.18c00a55a110faa6 namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: 4816ec39-cd53-4969-9d80-6c51093c2ce3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 31c08cbd-5315-4979-8937-4559fee41cf0 kind: Event lastTimestamp: "2026-07-07T15:06:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:06:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:55Z" name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk.18c00a55a3903359 namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: 3a76413c-2089-442e-bcb2-d5cbc3148391 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 31c08cbd-5315-4979-8937-4559fee41cf0 kind: Event lastTimestamp: "2026-07-07T15:06:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:06:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:55Z" name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk.18c00a55a401662b namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 3ef50c95-3ce2-407d-b836-0fcc4c8889c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 31c08cbd-5315-4979-8937-4559fee41cf0 kind: Event lastTimestamp: "2026-07-07T15:06:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:06:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:55Z" name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk.18c00a55a40c8f10 namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: 844bf99c-fd27-488d-924e-4ec78e2bcadf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 31c08cbd-5315-4979-8937-4559fee41cf0 kind: Event lastTimestamp: "2026-07-07T15:06:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:06:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:55Z" name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk.18c00a55a68b24a5 namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: f9ad3067-c1e3-41ab-98bb-9c9f34125045 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 31c08cbd-5315-4979-8937-4559fee41cf0 kind: Event lastTimestamp: "2026-07-07T15:06:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:06:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:55Z" name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk.18c00a55a749846e namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: f51f30f8-55a0-4a42-b3a4-f80d832dc558 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:06:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 31c08cbd-5315-4979-8937-4559fee41cf0 kind: Event lastTimestamp: "2026-07-07T15:07:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:06:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:33Z" name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk.18c00a560e573772 namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 0690e2d8-5373-4393-b70f-841707814154 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 31c08cbd-5315-4979-8937-4559fee41cf0 kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:09Z" name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk.18c00ad69f6fe1ea namespace: kserve-ci-e2e-test resourceVersion: "19460" uid: 2faec87e-8fbf-4dc6-9657-8fc30b31873a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 31c08cbd-5315-4979-8937-4559fee41cf0 kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:09Z" name: error-404-isvc-3c9c9-predictor-7d847f679-zghqk.18c00ad69f70a76c namespace: kserve-ci-e2e-test resourceVersion: "19461" uid: 546c2882-b926-49df-954d-ef2a050b0929 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3c9c9-predictor-7d847f679 namespace: kserve-ci-e2e-test resourceVersion: "15004" uid: 7e4ee0d1-5671-49a0-9a2e-40cf08681cce kind: Event lastTimestamp: "2026-07-07T15:06:54Z" message: 'Created pod: error-404-isvc-3c9c9-predictor-7d847f679-zghqk' metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:54Z" name: error-404-isvc-3c9c9-predictor-7d847f679.18c00a5560eb359e namespace: kserve-ci-e2e-test resourceVersion: "15013" uid: 4389505c-37f3-4beb-af6e-027cacea74c9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3c9c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "15001" uid: 26b84c7f-0d48-4847-9885-f2b50912fca1 kind: Event lastTimestamp: "2026-07-07T15:06:54Z" message: Scaled up replica set error-404-isvc-3c9c9-predictor-7d847f679 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:54Z" name: error-404-isvc-3c9c9-predictor.18c00a555e320df6 namespace: kserve-ci-e2e-test resourceVersion: "15006" uid: 1263d8d5-775b-4b46-9ec3-f3a9d09dbc38 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3c9c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 6d1688a2-8ddc-4bd3-80f0-16e665711f02 kind: Event lastTimestamp: "2026-07-07T15:07:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:09Z" name: error-404-isvc-3c9c9-predictor.18c00a58dd05f5c8 namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: 6d996362-33be-4b02-b594-a1de3d34f837 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3c9c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 6d1688a2-8ddc-4bd3-80f0-16e665711f02 kind: Event lastTimestamp: "2026-07-07T15:07:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:09Z" name: error-404-isvc-3c9c9-predictor.18c00a58dd8aada4 namespace: kserve-ci-e2e-test resourceVersion: "15248" uid: 8c37036c-0ef4-4c5d-82cb-329579d780ef reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:07:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3c9c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "15246" uid: 6d1688a2-8ddc-4bd3-80f0-16e665711f02 kind: Event lastTimestamp: "2026-07-07T15:08:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:07:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:09Z" name: error-404-isvc-3c9c9-predictor.18c00a5c5c61dd86 namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: aec9b3f7-6c9a-4526-9359-70b9d521dd9e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:07:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3c9c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "15246" uid: 6d1688a2-8ddc-4bd3-80f0-16e665711f02 kind: Event lastTimestamp: "2026-07-07T15:08:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:07:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:09Z" name: error-404-isvc-3c9c9-predictor.18c00a5c5cd6b34c namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 49a16319-ec6a-4188-8b11-062a3d0941d4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:06:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3c9c9 namespace: kserve-ci-e2e-test resourceVersion: "14984" uid: 44bc15aa-a971-4649-9d15-dcab7d435810 kind: Event lastTimestamp: "2026-07-07T15:07:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:07:43Z" name: error-404-isvc-3c9c9.18c00a555e77bf26 namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: 60b1ebdf-b224-4e2a-a9d7-0ce54436dcb3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3c9c9 namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 44bc15aa-a971-4649-9d15-dcab7d435810 kind: Event lastTimestamp: "2026-07-07T15:07:43Z" message: InferenceService [error-404-isvc-3c9c9] is Ready metadata: creationTimestamp: "2026-07-07T15:07:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:07:43Z" name: error-404-isvc-3c9c9.18c00a60ca5569ec namespace: kserve-ci-e2e-test resourceVersion: "15738" uid: 63243e3f-c486-42fd-930a-d6f8fa64ff57 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:16:09.294565Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv namespace: kserve-ci-e2e-test resourceVersion: "19433" uid: cd396cf3-880b-4780-8cfa-c7589254de62 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv to ip-10-0-128-215.ec2.internal metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:16:09Z" name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv.18c00ad69bf5edc4 namespace: kserve-ci-e2e-test resourceVersion: "19438" uid: 07737c2d-0eef-4480-8e4c-ed2b75840fc3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: cd396cf3-880b-4780-8cfa-c7589254de62 kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:16:09Z" name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv.18c00ad6b60ee065 namespace: kserve-ci-e2e-test resourceVersion: "19471" uid: 115edfaa-7089-4072-b274-39175ed08c62 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: cd396cf3-880b-4780-8cfa-c7589254de62 kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:09Z" name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv.18c00ad6b7794774 namespace: kserve-ci-e2e-test resourceVersion: "19473" uid: 7f1bf89e-3f23-4382-bdaf-1e178e7310aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: cd396cf3-880b-4780-8cfa-c7589254de62 kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:09Z" name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv.18c00ad6ba933867 namespace: kserve-ci-e2e-test resourceVersion: "19474" uid: 7615a9c6-5b81-421f-951c-a18e8805a13c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: cd396cf3-880b-4780-8cfa-c7589254de62 kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:09Z" name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv.18c00ad6bb024f09 namespace: kserve-ci-e2e-test resourceVersion: "19475" uid: 5d7f14dd-3bd6-454a-b054-77c2310bfd75 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: cd396cf3-880b-4780-8cfa-c7589254de62 kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:09Z" name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv.18c00ad6bb0edcc8 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: 63da66e3-a473-401d-859d-4b92bfdf9574 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: cd396cf3-880b-4780-8cfa-c7589254de62 kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:09Z" name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv.18c00ad6bfa50e18 namespace: kserve-ci-e2e-test resourceVersion: "19477" uid: 3be6e1f4-1c75-4e7d-880a-b8123423f851 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: cd396cf3-880b-4780-8cfa-c7589254de62 kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:09Z" name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv.18c00ad6c019ec10 namespace: kserve-ci-e2e-test resourceVersion: "19479" uid: 36044e72-a2f0-436f-bb09-3fc6b0d588ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:16:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: cd396cf3-880b-4780-8cfa-c7589254de62 kind: Event lastTimestamp: "2026-07-07T15:16:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:16:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:47Z" name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv.18c00ad70c17e252 namespace: kserve-ci-e2e-test resourceVersion: "19910" uid: 58ff15d8-8a00-48d8-a8b5-4c69c74a28bb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: cd396cf3-880b-4780-8cfa-c7589254de62 kind: Event lastTimestamp: "2026-07-07T15:17:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:17:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:19Z" name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv.18c00ae6f5575dd0 namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: f69f63b7-c994-4da3-8898-77a604e409cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: cd396cf3-880b-4780-8cfa-c7589254de62 kind: Event lastTimestamp: "2026-07-07T15:17:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:17:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:19Z" name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv.18c00ae6f557de52 namespace: kserve-ci-e2e-test resourceVersion: "20289" uid: 198b56be-25e0-4f68-a74c-872704a25f1a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: cd396cf3-880b-4780-8cfa-c7589254de62 kind: Event lastTimestamp: "2026-07-07T15:17:22Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:22Z" name: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv.18c00ae7943cca6d namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: baf4dd29-bead-4982-87a7-a8b05e4ee545 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-542f9-predictor-5df9bc96cc namespace: kserve-ci-e2e-test resourceVersion: "19424" uid: dc86f34c-9045-4f3e-aae9-8e5ce79b366c kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: 'Created pod: error-404-isvc-542f9-predictor-5df9bc96cc-xnvdv' metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:16:09Z" name: error-404-isvc-542f9-predictor-5df9bc96cc.18c00ad69b6f1a8b namespace: kserve-ci-e2e-test resourceVersion: "19434" uid: cda0695f-c4dd-42d6-a07e-7ddc5e05c198 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-542f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19418" uid: 2768183c-cc5a-44c4-bcf5-dac52f674ab9 kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: Scaled up replica set error-404-isvc-542f9-predictor-5df9bc96cc from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:16:09Z" name: error-404-isvc-542f9-predictor.18c00ad6990c89fd namespace: kserve-ci-e2e-test resourceVersion: "19427" uid: 5da42ac1-db58-452e-8453-59ff98fb3ad0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:16:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-542f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19422" uid: a63c4c08-c05b-47f7-a695-cce5e33d2704 kind: Event lastTimestamp: "2026-07-07T15:16:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:16:39Z" name: error-404-isvc-542f9-predictor.18c00ada175bc354 namespace: kserve-ci-e2e-test resourceVersion: "19721" uid: f0082678-cee4-4600-a84b-0db1a851ae5b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:16:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-542f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19422" uid: a63c4c08-c05b-47f7-a695-cce5e33d2704 kind: Event lastTimestamp: "2026-07-07T15:16:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:16:39Z" name: error-404-isvc-542f9-predictor.18c00ada17ccec25 namespace: kserve-ci-e2e-test resourceVersion: "19723" uid: c5efe8da-80ba-4720-b984-682417045148 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:16:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-542f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19609" uid: a63c4c08-c05b-47f7-a695-cce5e33d2704 kind: Event lastTimestamp: "2026-07-07T15:17:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:09Z" name: error-404-isvc-542f9-predictor.18c00ae1156dbeff namespace: kserve-ci-e2e-test resourceVersion: "20141" uid: cfc9e8da-e156-462d-a9b5-88fa3376180c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:16:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-542f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19609" uid: a63c4c08-c05b-47f7-a695-cce5e33d2704 kind: Event lastTimestamp: "2026-07-07T15:17:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:09Z" name: error-404-isvc-542f9-predictor.18c00ae115e0ff5e namespace: kserve-ci-e2e-test resourceVersion: "20143" uid: 2a5acbf0-6d3d-40a5-b9b9-8d16f8b8fc06 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-542f9 namespace: kserve-ci-e2e-test resourceVersion: "19390" uid: 67f09d02-4311-4d34-9955-a8db8f3fc012 kind: Event lastTimestamp: "2026-07-07T15:16:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:16:57Z" name: error-404-isvc-542f9.18c00ad69919f5f4 namespace: kserve-ci-e2e-test resourceVersion: "20013" uid: d6f7ece3-6253-46dc-a020-ec1e0aa1e06a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-542f9 namespace: kserve-ci-e2e-test resourceVersion: "19390" uid: 67f09d02-4311-4d34-9955-a8db8f3fc012 kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: 'Failed to update status for InferenceService "error-404-isvc-542f9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-542f9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:16:09Z" name: error-404-isvc-542f9.18c00ad699b9ddd8 namespace: kserve-ci-e2e-test resourceVersion: "19429" uid: 6c180a47-c7fc-4fe9-b2b2-5cb04c75ff04 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-542f9 namespace: kserve-ci-e2e-test resourceVersion: "19390" uid: 67f09d02-4311-4d34-9955-a8db8f3fc012 kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-542f9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:16:09Z" name: error-404-isvc-542f9.18c00ad699ba3725 namespace: kserve-ci-e2e-test resourceVersion: "19430" uid: 571c1fba-fa4b-4b1e-be38-1e739bf2490f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-542f9 namespace: kserve-ci-e2e-test resourceVersion: "20009" uid: 67f09d02-4311-4d34-9955-a8db8f3fc012 kind: Event lastTimestamp: "2026-07-07T15:16:57Z" message: InferenceService [error-404-isvc-542f9] is Ready metadata: creationTimestamp: "2026-07-07T15:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:16:57Z" name: error-404-isvc-542f9.18c00ae1c96c0b49 namespace: kserve-ci-e2e-test resourceVersion: "20011" uid: 08345a91-faea-4ed9-9e8e-6ec4b5ca24f8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:17:19.701061Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: 0ec4559f-825f-473b-a2b6-9ecf8bbe6429 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx to ip-10-0-128-215.ec2.internal metadata: creationTimestamp: "2026-07-07T15:17:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:17:19Z" name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx.18c00ae70083b5da namespace: kserve-ci-e2e-test resourceVersion: "20343" uid: a77f1270-3505-4c15-85a2-6d688f4ef4c8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: 0ec4559f-825f-473b-a2b6-9ecf8bbe6429 kind: Event lastTimestamp: "2026-07-07T15:17:20Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:17:20Z" name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx.18c00ae71a1a4878 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: fcdec5d7-af3f-4a2e-92f4-ee3bf13aca13 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 0ec4559f-825f-473b-a2b6-9ecf8bbe6429 kind: Event lastTimestamp: "2026-07-07T15:17:20Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:20Z" name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx.18c00ae71baf8ce4 namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: 2880d6e0-767a-4233-ab3b-4b943c72e873 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 0ec4559f-825f-473b-a2b6-9ecf8bbe6429 kind: Event lastTimestamp: "2026-07-07T15:17:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:20Z" name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx.18c00ae71ea3d917 namespace: kserve-ci-e2e-test resourceVersion: "20371" uid: 0a42fee7-e95d-4ceb-ad3c-434d41bfa3c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 0ec4559f-825f-473b-a2b6-9ecf8bbe6429 kind: Event lastTimestamp: "2026-07-07T15:17:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:20Z" name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx.18c00ae71f205d40 namespace: kserve-ci-e2e-test resourceVersion: "20372" uid: 83528343-84fa-4787-abc2-be4815cde109 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 0ec4559f-825f-473b-a2b6-9ecf8bbe6429 kind: Event lastTimestamp: "2026-07-07T15:17:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:20Z" name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx.18c00ae71f2b3de5 namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: 7f517b21-8d0d-4a20-ad2e-f7721d1822a9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 0ec4559f-825f-473b-a2b6-9ecf8bbe6429 kind: Event lastTimestamp: "2026-07-07T15:17:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:20Z" name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx.18c00ae721fad1d0 namespace: kserve-ci-e2e-test resourceVersion: "20377" uid: 4a76ee55-a49c-4504-80b8-60d859d013d0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 0ec4559f-825f-473b-a2b6-9ecf8bbe6429 kind: Event lastTimestamp: "2026-07-07T15:17:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:20Z" name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx.18c00ae72275eb39 namespace: kserve-ci-e2e-test resourceVersion: "20378" uid: 5545efa9-4d0a-41b0-a60b-0064d7d52466 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T15:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 0ec4559f-825f-473b-a2b6-9ecf8bbe6429 kind: Event lastTimestamp: "2026-07-07T15:26:36Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:36Z" name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx.18c00ae72d7fb843 namespace: kserve-ci-e2e-test resourceVersion: "24734" uid: e67bf34c-c7ce-491b-a6e0-e78c86cbe405 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 0ec4559f-825f-473b-a2b6-9ecf8bbe6429 kind: Event lastTimestamp: "2026-07-07T15:26:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:34Z" name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx.18c00b68430321d9 namespace: kserve-ci-e2e-test resourceVersion: "24690" uid: 6efc8d08-b534-40ae-8d5e-a0267d5db1d5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 0ec4559f-825f-473b-a2b6-9ecf8bbe6429 kind: Event lastTimestamp: "2026-07-07T15:26:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:34Z" name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx.18c00b684303830e namespace: kserve-ci-e2e-test resourceVersion: "24692" uid: d57a92c8-2370-4759-a15d-d57b5363e376 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 0ec4559f-825f-473b-a2b6-9ecf8bbe6429 kind: Event lastTimestamp: "2026-07-07T15:26:36Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:36Z" name: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx.18c00b68a22d076d namespace: kserve-ci-e2e-test resourceVersion: "24732" uid: 9c15e545-6a9a-4d25-82db-120186613faa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5aeac-predictor-59dfb478c9 namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: 829deeb4-1aad-48f9-9977-5d1f8253a268 kind: Event lastTimestamp: "2026-07-07T15:17:19Z" message: 'Created pod: error-404-isvc-5aeac-predictor-59dfb478c9-8pbmx' metadata: creationTimestamp: "2026-07-07T15:17:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:19Z" name: error-404-isvc-5aeac-predictor-59dfb478c9.18c00ae6fff5cbc8 namespace: kserve-ci-e2e-test resourceVersion: "20340" uid: 72333ddc-3d04-4db0-842b-029a52cb77fe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5aeac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: d5dd4cc5-1887-4406-9c72-49088c7eb846 kind: Event lastTimestamp: "2026-07-07T15:17:19Z" message: Scaled up replica set error-404-isvc-5aeac-predictor-59dfb478c9 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:17:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:19Z" name: error-404-isvc-5aeac-predictor.18c00ae6fe059d7c namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: f80245a9-19af-422a-b5b5-40900e0e30c2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:17:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5aeac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 51fc8fbf-1799-49e9-9f38-310db2d4eabd kind: Event lastTimestamp: "2026-07-07T15:17:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:17:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:49Z" name: error-404-isvc-5aeac-predictor.18c00aea7c9f01e4 namespace: kserve-ci-e2e-test resourceVersion: "20682" uid: e2bf2eab-5816-4112-8faf-50aa9d0e3b96 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:17:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5aeac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 51fc8fbf-1799-49e9-9f38-310db2d4eabd kind: Event lastTimestamp: "2026-07-07T15:17:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:17:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:49Z" name: error-404-isvc-5aeac-predictor.18c00aea7d2568a8 namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 53fcaa4c-86ee-4b75-a80d-5832b9c91b94 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:18:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5aeac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 51fc8fbf-1799-49e9-9f38-310db2d4eabd kind: Event lastTimestamp: "2026-07-07T15:18:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:18:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:19Z" name: error-404-isvc-5aeac-predictor.18c00af17aec2293 namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: a435fdc6-848e-49ae-a3a8-e29779ec3b83 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:18:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5aeac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 51fc8fbf-1799-49e9-9f38-310db2d4eabd kind: Event lastTimestamp: "2026-07-07T15:18:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:18:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:19Z" name: error-404-isvc-5aeac-predictor.18c00af17b7c0092 namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: ef354046-1528-4839-bdae-1b95edbd02ac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:17:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5aeac namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: dac38812-8d8f-42a5-825a-0ad77358d4a9 kind: Event lastTimestamp: "2026-07-07T15:18:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:17:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:18:06Z" name: error-404-isvc-5aeac.18c00ae6fe39dfc0 namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: 079e9d52-ee0b-40db-8078-9432d189a69d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5aeac namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: dac38812-8d8f-42a5-825a-0ad77358d4a9 kind: Event lastTimestamp: "2026-07-07T15:17:19Z" message: 'Failed to update status for InferenceService "error-404-isvc-5aeac": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5aeac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:17:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:17:19Z" name: error-404-isvc-5aeac.18c00ae6feb1c61e namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: f4a1eb6a-7c17-46a2-a867-adefa4d4daf3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5aeac namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: dac38812-8d8f-42a5-825a-0ad77358d4a9 kind: Event lastTimestamp: "2026-07-07T15:17:19Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5aeac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:17:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:17:19Z" name: error-404-isvc-5aeac.18c00ae6feb211c6 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: 233dfcf6-02c0-4842-ad7b-efe803d1aed4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5aeac namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: dac38812-8d8f-42a5-825a-0ad77358d4a9 kind: Event lastTimestamp: "2026-07-07T15:18:06Z" message: InferenceService [error-404-isvc-5aeac] is Ready metadata: creationTimestamp: "2026-07-07T15:18:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:18:06Z" name: error-404-isvc-5aeac.18c00af1ea516f4f namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 5980787c-f5e2-43d7-9e3f-8460a55446f9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:27:45.019442Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8 namespace: kserve-ci-e2e-test resourceVersion: "25572" uid: c2eb94fe-213a-402e-942d-5ce48933167b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8 to ip-10-0-128-215.ec2.internal metadata: creationTimestamp: "2026-07-07T15:27:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:27:45Z" name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8.18c00b78986504f8 namespace: kserve-ci-e2e-test resourceVersion: "25577" uid: 3fccee97-7af9-4a20-8374-7aa539900a7e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8 namespace: kserve-ci-e2e-test resourceVersion: "25573" uid: c2eb94fe-213a-402e-942d-5ce48933167b kind: Event lastTimestamp: "2026-07-07T15:27:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-73e96-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T15:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:45Z" name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8.18c00b78a4fce48c namespace: kserve-ci-e2e-test resourceVersion: "25588" uid: fb60a2fc-1e7b-4b90-8a0f-c90612eea066 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8 namespace: kserve-ci-e2e-test resourceVersion: "25582" uid: c2eb94fe-213a-402e-942d-5ce48933167b kind: Event lastTimestamp: "2026-07-07T15:27:46Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:27:46Z" name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8.18c00b78d62e320a namespace: kserve-ci-e2e-test resourceVersion: "25611" uid: ca9fa588-e332-4a7d-be4f-488e8b56ebc0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8 namespace: kserve-ci-e2e-test resourceVersion: "25573" uid: c2eb94fe-213a-402e-942d-5ce48933167b kind: Event lastTimestamp: "2026-07-07T15:27:46Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:46Z" name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8.18c00b78d7c40ed6 namespace: kserve-ci-e2e-test resourceVersion: "25614" uid: 7c500dc8-7145-46af-9b27-a6a0fb1e6573 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8 namespace: kserve-ci-e2e-test resourceVersion: "25573" uid: c2eb94fe-213a-402e-942d-5ce48933167b kind: Event lastTimestamp: "2026-07-07T15:27:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:46Z" name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8.18c00b78db445a92 namespace: kserve-ci-e2e-test resourceVersion: "25616" uid: f9db333a-d9ad-42c7-9948-ac0f83eaf181 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8 namespace: kserve-ci-e2e-test resourceVersion: "25573" uid: c2eb94fe-213a-402e-942d-5ce48933167b kind: Event lastTimestamp: "2026-07-07T15:27:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:46Z" name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8.18c00b78dbbfab39 namespace: kserve-ci-e2e-test resourceVersion: "25617" uid: 3da70d54-4665-4f8d-b7cf-524a6ac144de reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8 namespace: kserve-ci-e2e-test resourceVersion: "25573" uid: c2eb94fe-213a-402e-942d-5ce48933167b kind: Event lastTimestamp: "2026-07-07T15:27:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:46Z" name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8.18c00b78dbca2ca3 namespace: kserve-ci-e2e-test resourceVersion: "25618" uid: d883f76b-47c0-443e-a937-f34dc2623398 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8 namespace: kserve-ci-e2e-test resourceVersion: "25573" uid: c2eb94fe-213a-402e-942d-5ce48933167b kind: Event lastTimestamp: "2026-07-07T15:27:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:46Z" name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8.18c00b78df9682a8 namespace: kserve-ci-e2e-test resourceVersion: "25619" uid: 379bb6f4-a9fc-4e3c-a392-23ec142a5a3a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8 namespace: kserve-ci-e2e-test resourceVersion: "25573" uid: c2eb94fe-213a-402e-942d-5ce48933167b kind: Event lastTimestamp: "2026-07-07T15:27:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:46Z" name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8.18c00b78e01379b2 namespace: kserve-ci-e2e-test resourceVersion: "25620" uid: 964fc08e-094f-41e8-8ea6-6f83883e2aaa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T15:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8 namespace: kserve-ci-e2e-test resourceVersion: "25573" uid: c2eb94fe-213a-402e-942d-5ce48933167b kind: Event lastTimestamp: "2026-07-07T15:37:02Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:37:02Z" name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8.18c00b790160201e namespace: kserve-ci-e2e-test resourceVersion: "30010" uid: ae3f9ed3-623d-4c8f-9dd3-1db871a79794 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:36:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8 namespace: kserve-ci-e2e-test resourceVersion: "25573" uid: c2eb94fe-213a-402e-942d-5ce48933167b kind: Event lastTimestamp: "2026-07-07T15:36:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:36:59Z" name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8.18c00bf9c0560914 namespace: kserve-ci-e2e-test resourceVersion: "29983" uid: f9474ec7-f23a-4cd0-9dc6-1427c34941ef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:36:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8 namespace: kserve-ci-e2e-test resourceVersion: "25573" uid: c2eb94fe-213a-402e-942d-5ce48933167b kind: Event lastTimestamp: "2026-07-07T15:36:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:36:59Z" name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8.18c00bf9c056d8ae namespace: kserve-ci-e2e-test resourceVersion: "29984" uid: ab2a5c77-0f5a-41d2-9820-878825eb6494 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:37:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8 namespace: kserve-ci-e2e-test resourceVersion: "25573" uid: c2eb94fe-213a-402e-942d-5ce48933167b kind: Event lastTimestamp: "2026-07-07T15:37:02Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:37:02Z" name: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8.18c00bfa75c0e3fd namespace: kserve-ci-e2e-test resourceVersion: "30009" uid: 1d362d10-ab09-4d9d-873e-01e9403f4303 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-73e96-predictor-78d9dc85b9 namespace: kserve-ci-e2e-test resourceVersion: "25561" uid: 6f2e66b0-9628-4184-a449-0cd4109fd69b kind: Event lastTimestamp: "2026-07-07T15:27:45Z" message: 'Created pod: error-404-isvc-73e96-predictor-78d9dc85b9-bh7l8' metadata: creationTimestamp: "2026-07-07T15:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:45Z" name: error-404-isvc-73e96-predictor-78d9dc85b9.18c00b78977daa71 namespace: kserve-ci-e2e-test resourceVersion: "25574" uid: e1e38e6b-c023-4a52-b3f3-05802074d6e6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-73e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "25560" uid: 0b305e08-b9d8-4789-886b-efd2aee95b4e kind: Event lastTimestamp: "2026-07-07T15:27:44Z" message: Scaled up replica set error-404-isvc-73e96-predictor-78d9dc85b9 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:44Z" name: error-404-isvc-73e96-predictor.18c00b78947ffd7f namespace: kserve-ci-e2e-test resourceVersion: "25564" uid: 9204b518-9fe4-4b83-91b4-2e76350df24e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-73e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "25562" uid: 9a7241d2-859c-4d13-971a-d295e4238c82 kind: Event lastTimestamp: "2026-07-07T15:27:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:27:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:59Z" name: error-404-isvc-73e96-predictor.18c00b7c137baa22 namespace: kserve-ci-e2e-test resourceVersion: "25771" uid: cbe5632f-b267-4e2a-9534-523feeac9c3b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-73e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "25562" uid: 9a7241d2-859c-4d13-971a-d295e4238c82 kind: Event lastTimestamp: "2026-07-07T15:27:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:27:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:59Z" name: error-404-isvc-73e96-predictor.18c00b7c13fec319 namespace: kserve-ci-e2e-test resourceVersion: "25772" uid: a718206c-acf5-4b85-bed3-a1fcf4d7e56f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:28:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-73e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "25770" uid: 9a7241d2-859c-4d13-971a-d295e4238c82 kind: Event lastTimestamp: "2026-07-07T15:29:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:29:00Z" name: error-404-isvc-73e96-predictor.18c00b7f92c768aa namespace: kserve-ci-e2e-test resourceVersion: "26627" uid: 747c6973-89cf-42ef-bc07-ff839c8a5ba9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:28:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-73e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "25770" uid: 9a7241d2-859c-4d13-971a-d295e4238c82 kind: Event lastTimestamp: "2026-07-07T15:29:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:29:00Z" name: error-404-isvc-73e96-predictor.18c00b7f9343202a namespace: kserve-ci-e2e-test resourceVersion: "26628" uid: a38602a1-19d8-481c-aac7-bf97ab346c29 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:27:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-73e96 namespace: kserve-ci-e2e-test resourceVersion: "25532" uid: ce6c5a59-bdae-4d99-9f86-e297d73ba90f kind: Event lastTimestamp: "2026-07-07T15:28:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:28:32Z" name: error-404-isvc-73e96.18c00b78954db309 namespace: kserve-ci-e2e-test resourceVersion: "26379" uid: a8220b7b-2cf4-41ec-b67a-e9e465a0a9ba reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-73e96 namespace: kserve-ci-e2e-test resourceVersion: "25532" uid: ce6c5a59-bdae-4d99-9f86-e297d73ba90f kind: Event lastTimestamp: "2026-07-07T15:27:44Z" message: 'Failed to update status for InferenceService "error-404-isvc-73e96": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-73e96": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:27:44Z" name: error-404-isvc-73e96.18c00b789650ba19 namespace: kserve-ci-e2e-test resourceVersion: "25571" uid: cd6c2120-ba86-4d3f-b91d-626ff7747637 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-73e96 namespace: kserve-ci-e2e-test resourceVersion: "25532" uid: ce6c5a59-bdae-4d99-9f86-e297d73ba90f kind: Event lastTimestamp: "2026-07-07T15:27:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-73e96": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:27:45Z" name: error-404-isvc-73e96.18c00b78965102fa namespace: kserve-ci-e2e-test resourceVersion: "25575" uid: f67de7d5-6016-4c4f-b255-29a353b7f4d3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-73e96 namespace: kserve-ci-e2e-test resourceVersion: "26376" uid: ce6c5a59-bdae-4d99-9f86-e297d73ba90f kind: Event lastTimestamp: "2026-07-07T15:28:32Z" message: InferenceService [error-404-isvc-73e96] is Ready metadata: creationTimestamp: "2026-07-07T15:28:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:28:32Z" name: error-404-isvc-73e96.18c00b83bcf7bde8 namespace: kserve-ci-e2e-test resourceVersion: "26378" uid: 73fc0573-3896-43ae-b30b-226cc847c027 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:26:34.779979Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf namespace: kserve-ci-e2e-test resourceVersion: "24664" uid: b1ff71cc-091f-473a-bac2-cc9977ab8fc5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-761e8-predictor-7869d59c55-4qcxf to ip-10-0-128-215.ec2.internal metadata: creationTimestamp: "2026-07-07T15:26:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:26:34Z" name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf.18c00b683dcbe423 namespace: kserve-ci-e2e-test resourceVersion: "24669" uid: c8aed937-840c-4055-9c0f-7b74c3e70606 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf namespace: kserve-ci-e2e-test resourceVersion: "24680" uid: b1ff71cc-091f-473a-bac2-cc9977ab8fc5 kind: Event lastTimestamp: "2026-07-07T15:26:35Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:26:35Z" name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf.18c00b6857d6c413 namespace: kserve-ci-e2e-test resourceVersion: "24702" uid: e108a1af-a0e8-4f8e-9905-dcffb7d1a6de reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf namespace: kserve-ci-e2e-test resourceVersion: "24665" uid: b1ff71cc-091f-473a-bac2-cc9977ab8fc5 kind: Event lastTimestamp: "2026-07-07T15:26:35Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:35Z" name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf.18c00b68599859d2 namespace: kserve-ci-e2e-test resourceVersion: "24706" uid: eba37730-4652-47d0-9a4d-9a637dfa628b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf namespace: kserve-ci-e2e-test resourceVersion: "24665" uid: b1ff71cc-091f-473a-bac2-cc9977ab8fc5 kind: Event lastTimestamp: "2026-07-07T15:26:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:35Z" name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf.18c00b685ca1b5c1 namespace: kserve-ci-e2e-test resourceVersion: "24710" uid: 915507ea-242f-49fe-a305-08ff51759c98 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf namespace: kserve-ci-e2e-test resourceVersion: "24665" uid: b1ff71cc-091f-473a-bac2-cc9977ab8fc5 kind: Event lastTimestamp: "2026-07-07T15:26:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:35Z" name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf.18c00b685d2207dc namespace: kserve-ci-e2e-test resourceVersion: "24711" uid: 7caadf7a-f604-4f28-ab2a-d1984795c7d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf namespace: kserve-ci-e2e-test resourceVersion: "24665" uid: b1ff71cc-091f-473a-bac2-cc9977ab8fc5 kind: Event lastTimestamp: "2026-07-07T15:26:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:35Z" name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf.18c00b685d2d3131 namespace: kserve-ci-e2e-test resourceVersion: "24712" uid: e0c4ccc8-d643-4059-9410-bf9c9ea20290 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf namespace: kserve-ci-e2e-test resourceVersion: "24665" uid: b1ff71cc-091f-473a-bac2-cc9977ab8fc5 kind: Event lastTimestamp: "2026-07-07T15:26:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:35Z" name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf.18c00b685ff124e1 namespace: kserve-ci-e2e-test resourceVersion: "24713" uid: 03eec3f5-3a6a-4e2c-84a1-104197a1359d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf namespace: kserve-ci-e2e-test resourceVersion: "24665" uid: b1ff71cc-091f-473a-bac2-cc9977ab8fc5 kind: Event lastTimestamp: "2026-07-07T15:26:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:35Z" name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf.18c00b6860708c32 namespace: kserve-ci-e2e-test resourceVersion: "24714" uid: 3f4bef1d-3729-4096-a236-129cdf98af5b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:26:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf namespace: kserve-ci-e2e-test resourceVersion: "24665" uid: b1ff71cc-091f-473a-bac2-cc9977ab8fc5 kind: Event lastTimestamp: "2026-07-07T15:27:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:12Z" name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf.18c00b68a466fe9e namespace: kserve-ci-e2e-test resourceVersion: "25153" uid: 7c9ae44d-34d6-40a3-82c5-87492c33c655 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf namespace: kserve-ci-e2e-test resourceVersion: "24665" uid: b1ff71cc-091f-473a-bac2-cc9977ab8fc5 kind: Event lastTimestamp: "2026-07-07T15:27:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:44Z" name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf.18c00b7886ff214f namespace: kserve-ci-e2e-test resourceVersion: "25517" uid: ee2de803-9248-4885-b6c5-94a2218c1c15 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf namespace: kserve-ci-e2e-test resourceVersion: "24665" uid: b1ff71cc-091f-473a-bac2-cc9977ab8fc5 kind: Event lastTimestamp: "2026-07-07T15:27:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:44Z" name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf.18c00b7886ff8de2 namespace: kserve-ci-e2e-test resourceVersion: "25521" uid: 1f25ad04-8a09-4c80-ae29-6d015efd504f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf namespace: kserve-ci-e2e-test resourceVersion: "24665" uid: b1ff71cc-091f-473a-bac2-cc9977ab8fc5 kind: Event lastTimestamp: "2026-07-07T15:27:47Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:47Z" name: error-404-isvc-761e8-predictor-7869d59c55-4qcxf.18c00b792c81300c namespace: kserve-ci-e2e-test resourceVersion: "25638" uid: 2eb7094e-5c62-4b2c-bbe2-a584bb672f0b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-761e8-predictor-7869d59c55 namespace: kserve-ci-e2e-test resourceVersion: "24652" uid: d77e1382-fce6-47dd-a38f-3ce1c8d37d42 kind: Event lastTimestamp: "2026-07-07T15:26:34Z" message: 'Created pod: error-404-isvc-761e8-predictor-7869d59c55-4qcxf' metadata: creationTimestamp: "2026-07-07T15:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:26:34Z" name: error-404-isvc-761e8-predictor-7869d59c55.18c00b683cfe00b7 namespace: kserve-ci-e2e-test resourceVersion: "24666" uid: 21b4f441-e311-4df2-b627-01b537259275 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-761e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24645" uid: d593ca2e-b7f1-4d12-87d6-13fbcb01e30a kind: Event lastTimestamp: "2026-07-07T15:26:34Z" message: Scaled up replica set error-404-isvc-761e8-predictor-7869d59c55 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:26:34Z" name: error-404-isvc-761e8-predictor.18c00b6839c71fbe namespace: kserve-ci-e2e-test resourceVersion: "24655" uid: 59ea553d-3b22-4aa0-ae82-5d0b05993667 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:26:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-761e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24651" uid: 18bf626b-9e73-4493-8ac9-c3aafe82e9a5 kind: Event lastTimestamp: "2026-07-07T15:27:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:04Z" name: error-404-isvc-761e8-predictor.18c00b6bb87b2dc4 namespace: kserve-ci-e2e-test resourceVersion: "24957" uid: 1e750469-a174-480e-b059-65143b2c97cc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:26:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-761e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24651" uid: 18bf626b-9e73-4493-8ac9-c3aafe82e9a5 kind: Event lastTimestamp: "2026-07-07T15:27:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:04Z" name: error-404-isvc-761e8-predictor.18c00b6bb920f913 namespace: kserve-ci-e2e-test resourceVersion: "24958" uid: 9b061064-094f-49e0-a524-cd706387e6e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:27:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-761e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24842" uid: 18bf626b-9e73-4493-8ac9-c3aafe82e9a5 kind: Event lastTimestamp: "2026-07-07T15:27:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:34Z" name: error-404-isvc-761e8-predictor.18c00b72b704aadb namespace: kserve-ci-e2e-test resourceVersion: "25384" uid: a4d9891d-e2d1-44f2-930e-3fe8ba9ddaab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:27:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-761e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24842" uid: 18bf626b-9e73-4493-8ac9-c3aafe82e9a5 kind: Event lastTimestamp: "2026-07-07T15:27:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:34Z" name: error-404-isvc-761e8-predictor.18c00b72b77f89c0 namespace: kserve-ci-e2e-test resourceVersion: "25385" uid: fd7b1934-00df-4941-80c6-8685ad257496 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:26:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-761e8 namespace: kserve-ci-e2e-test resourceVersion: "24621" uid: bd95641b-a8f4-4df2-ac9b-442dceaf37cf kind: Event lastTimestamp: "2026-07-07T15:27:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:27:22Z" name: error-404-isvc-761e8.18c00b683a4ecb20 namespace: kserve-ci-e2e-test resourceVersion: "25265" uid: 7087bb28-7711-4e92-97a5-429937f4aac3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-761e8 namespace: kserve-ci-e2e-test resourceVersion: "24621" uid: bd95641b-a8f4-4df2-ac9b-442dceaf37cf kind: Event lastTimestamp: "2026-07-07T15:26:34Z" message: 'Failed to update status for InferenceService "error-404-isvc-761e8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-761e8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:26:34Z" name: error-404-isvc-761e8.18c00b683abf929f namespace: kserve-ci-e2e-test resourceVersion: "24660" uid: a2a498ac-4a75-4800-824a-9755ede69a23 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-761e8 namespace: kserve-ci-e2e-test resourceVersion: "24621" uid: bd95641b-a8f4-4df2-ac9b-442dceaf37cf kind: Event lastTimestamp: "2026-07-07T15:26:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-761e8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:26:34Z" name: error-404-isvc-761e8.18c00b683abfd8d8 namespace: kserve-ci-e2e-test resourceVersion: "24661" uid: ec44db31-1cf8-42e0-96e1-f59c5507630e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-761e8 namespace: kserve-ci-e2e-test resourceVersion: "25262" uid: bd95641b-a8f4-4df2-ac9b-442dceaf37cf kind: Event lastTimestamp: "2026-07-07T15:27:22Z" message: InferenceService [error-404-isvc-761e8] is Ready metadata: creationTimestamp: "2026-07-07T15:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:27:22Z" name: error-404-isvc-761e8.18c00b736168473b namespace: kserve-ci-e2e-test resourceVersion: "25264" uid: 50eea5b4-7033-4fa7-b050-1f0ad33ab4b2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:07:30.143570Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv namespace: kserve-ci-e2e-test resourceVersion: "15552" uid: 196ffcbf-7496-4f27-8599-d1b6d11fe543 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv to ip-10-0-128-215.ec2.internal metadata: creationTimestamp: "2026-07-07T15:07:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:07:30Z" name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv.18c00a5dbc263cf0 namespace: kserve-ci-e2e-test resourceVersion: "15560" uid: ab41c5cf-3dc9-4194-9359-c09bbc770616 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: 196ffcbf-7496-4f27-8599-d1b6d11fe543 kind: Event lastTimestamp: "2026-07-07T15:07:30Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:07:30Z" name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv.18c00a5dd645c967 namespace: kserve-ci-e2e-test resourceVersion: "15592" uid: b79382c4-22c5-4ea5-868d-993fac5edcac reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: 196ffcbf-7496-4f27-8599-d1b6d11fe543 kind: Event lastTimestamp: "2026-07-07T15:07:30Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:30Z" name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv.18c00a5dd7eda35e namespace: kserve-ci-e2e-test resourceVersion: "15594" uid: 50671703-20fa-460d-865f-035cef15ecdc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: 196ffcbf-7496-4f27-8599-d1b6d11fe543 kind: Event lastTimestamp: "2026-07-07T15:07:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:30Z" name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv.18c00a5dda832add namespace: kserve-ci-e2e-test resourceVersion: "15595" uid: 3d6bae5d-7674-4874-b576-e6b00a54ccdf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: 196ffcbf-7496-4f27-8599-d1b6d11fe543 kind: Event lastTimestamp: "2026-07-07T15:07:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:30Z" name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv.18c00a5ddaeda2e5 namespace: kserve-ci-e2e-test resourceVersion: "15596" uid: b7a3718c-7342-4c39-b318-b666f24608c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: 196ffcbf-7496-4f27-8599-d1b6d11fe543 kind: Event lastTimestamp: "2026-07-07T15:07:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:30Z" name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv.18c00a5ddaf79d87 namespace: kserve-ci-e2e-test resourceVersion: "15597" uid: a21602c4-b0e4-4701-93d6-581ae96a158e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: 196ffcbf-7496-4f27-8599-d1b6d11fe543 kind: Event lastTimestamp: "2026-07-07T15:07:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:30Z" name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv.18c00a5dde2a40a5 namespace: kserve-ci-e2e-test resourceVersion: "15598" uid: 954c0311-7a07-483e-aa23-568eafba298f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: 196ffcbf-7496-4f27-8599-d1b6d11fe543 kind: Event lastTimestamp: "2026-07-07T15:07:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:30Z" name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv.18c00a5ddea090ca namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: cabd5b40-2366-442a-bc3d-fa6589e830ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:07:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: 196ffcbf-7496-4f27-8599-d1b6d11fe543 kind: Event lastTimestamp: "2026-07-07T15:08:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:07:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:08Z" name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv.18c00a5e3c635de9 namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: 25c2bd65-46d1-4390-b6ba-ffc668214154 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: 196ffcbf-7496-4f27-8599-d1b6d11fe543 kind: Event lastTimestamp: "2026-07-07T15:16:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:44Z" name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv.18c00adee163e0ef namespace: kserve-ci-e2e-test resourceVersion: "19798" uid: 49d34693-a4c8-44f7-8d1e-b522d2b96e2d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: 196ffcbf-7496-4f27-8599-d1b6d11fe543 kind: Event lastTimestamp: "2026-07-07T15:16:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:44Z" name: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv.18c00adee1649a59 namespace: kserve-ci-e2e-test resourceVersion: "19800" uid: f34abc98-6ce2-432b-9292-b9c3ad7e7a20 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8996c-predictor-85d8b95c6c namespace: kserve-ci-e2e-test resourceVersion: "15545" uid: 1b7dfb52-9186-4c23-9426-a76f8932a309 kind: Event lastTimestamp: "2026-07-07T15:07:30Z" message: 'Created pod: error-404-isvc-8996c-predictor-85d8b95c6c-hwmlv' metadata: creationTimestamp: "2026-07-07T15:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:30Z" name: error-404-isvc-8996c-predictor-85d8b95c6c.18c00a5dbb919990 namespace: kserve-ci-e2e-test resourceVersion: "15555" uid: a017c197-43ac-49d1-ae11-04244822182b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8996c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15543" uid: 382f2984-5e3a-4d31-8398-62b31d077931 kind: Event lastTimestamp: "2026-07-07T15:07:30Z" message: Scaled up replica set error-404-isvc-8996c-predictor-85d8b95c6c from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:30Z" name: error-404-isvc-8996c-predictor.18c00a5db907dd16 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 65104a81-0d7d-47db-8e6e-d6142b36d14a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8996c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15544" uid: a918a608-66e6-4b07-a7e3-aae893ca08af kind: Event lastTimestamp: "2026-07-07T15:07:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:07:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:45Z" name: error-404-isvc-8996c-predictor.18c00a6137b8fa55 namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: 658993dd-a25c-485d-aa05-f4ffe578dba6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8996c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15544" uid: a918a608-66e6-4b07-a7e3-aae893ca08af kind: Event lastTimestamp: "2026-07-07T15:07:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:07:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:45Z" name: error-404-isvc-8996c-predictor.18c00a6138368c2e namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: 42a33a20-e4fe-4597-bd71-3e5cb2b4f6e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:08:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8996c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: a918a608-66e6-4b07-a7e3-aae893ca08af kind: Event lastTimestamp: "2026-07-07T15:09:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:09:00Z" name: error-404-isvc-8996c-predictor.18c00a64b7060fef namespace: kserve-ci-e2e-test resourceVersion: "16435" uid: 36d19bea-bc71-48e1-8e25-04cecbd536dc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:08:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8996c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: a918a608-66e6-4b07-a7e3-aae893ca08af kind: Event lastTimestamp: "2026-07-07T15:09:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:09:00Z" name: error-404-isvc-8996c-predictor.18c00a64b76abe49 namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: d0094df4-4507-464a-8f9b-1526dcc3c5f2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:07:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8996c namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: 2c7186a8-e1a1-4106-bf6c-381443ed1fc7 kind: Event lastTimestamp: "2026-07-07T15:08:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:08:18Z" name: error-404-isvc-8996c.18c00a5db91ab8ac namespace: kserve-ci-e2e-test resourceVersion: "16094" uid: 118d1b8f-af5b-469d-8318-b75dbc792e1a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8996c namespace: kserve-ci-e2e-test resourceVersion: "16089" uid: 2c7186a8-e1a1-4106-bf6c-381443ed1fc7 kind: Event lastTimestamp: "2026-07-07T15:08:18Z" message: InferenceService [error-404-isvc-8996c] is Ready metadata: creationTimestamp: "2026-07-07T15:08:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:08:18Z" name: error-404-isvc-8996c.18c00a68fa9aef45 namespace: kserve-ci-e2e-test resourceVersion: "16091" uid: 0a1e4a5a-53d5-447c-a384-80088d98c052 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:17:55.265556Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk namespace: kserve-ci-e2e-test resourceVersion: "20811" uid: 9da5da07-7a4e-4229-9fbf-e91d5e9def8b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk to ip-10-0-128-215.ec2.internal metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:17:55Z" name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk.18c00aef4852d78b namespace: kserve-ci-e2e-test resourceVersion: "20815" uid: 427213b4-0009-49a2-ad6f-d5712b9e73be reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 9da5da07-7a4e-4229-9fbf-e91d5e9def8b kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:17:55Z" name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk.18c00aef624bbac7 namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 102c621d-acd6-4cbf-979c-c91616d765c7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: 9da5da07-7a4e-4229-9fbf-e91d5e9def8b kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:55Z" name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk.18c00aef63d92377 namespace: kserve-ci-e2e-test resourceVersion: "20858" uid: 816fecef-4e0c-47bd-a384-9fe9ce316fd3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: 9da5da07-7a4e-4229-9fbf-e91d5e9def8b kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:55Z" name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk.18c00aef6698d9ca namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: 836ecaa4-28ae-4b9a-a4be-8014358afef4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: 9da5da07-7a4e-4229-9fbf-e91d5e9def8b kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:55Z" name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk.18c00aef6714fba8 namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: 4e07e6c3-3540-4db9-ae6a-5adf46a5e772 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: 9da5da07-7a4e-4229-9fbf-e91d5e9def8b kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:55Z" name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk.18c00aef671ffb81 namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 97895add-7f83-4db1-88cc-2435061afb17 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: 9da5da07-7a4e-4229-9fbf-e91d5e9def8b kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:55Z" name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk.18c00aef69e7e476 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: bfa06182-119d-4257-8ee2-eb339038db29 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: 9da5da07-7a4e-4229-9fbf-e91d5e9def8b kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:55Z" name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk.18c00aef6a63e603 namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: e8b19736-22d6-4192-a988-a9a20c06299f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:17:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: 9da5da07-7a4e-4229-9fbf-e91d5e9def8b kind: Event lastTimestamp: "2026-07-07T15:18:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:17:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:33Z" name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk.18c00aefd381eded namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: eccca763-95f3-4e80-9970-ccc9b69fa0e0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: 9da5da07-7a4e-4229-9fbf-e91d5e9def8b kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:10Z" name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk.18c00b7071d8ee09 namespace: kserve-ci-e2e-test resourceVersion: "25032" uid: e2b4ffa0-b337-447b-9d26-5d91c0e44d2e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: 9da5da07-7a4e-4229-9fbf-e91d5e9def8b kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:10Z" name: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk.18c00b7071d96d86 namespace: kserve-ci-e2e-test resourceVersion: "25034" uid: 0c85e5ad-bb86-4751-86d6-8237c5d292f4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a9c06-predictor-77c555c7dc namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: 800c4d2c-f113-44e6-9be2-aaf98d5c6aaa kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: 'Created pod: error-404-isvc-a9c06-predictor-77c555c7dc-n9tbk' metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:55Z" name: error-404-isvc-a9c06-predictor-77c555c7dc.18c00aef47db36bf namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: e14975ec-c53a-4580-b80e-ecd734ad948f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a9c06-predictor namespace: kserve-ci-e2e-test resourceVersion: "20801" uid: 38cd60dc-300c-45b6-a82a-91745c9594d0 kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: Scaled up replica set error-404-isvc-a9c06-predictor-77c555c7dc from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:55Z" name: error-404-isvc-a9c06-predictor.18c00aef46415d09 namespace: kserve-ci-e2e-test resourceVersion: "20805" uid: 327573c1-5b50-4ba6-aaae-feb558480583 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:18:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9c06-predictor namespace: kserve-ci-e2e-test resourceVersion: "20802" uid: a91d1941-fd3d-4b98-9088-05dc16a5a54c kind: Event lastTimestamp: "2026-07-07T15:18:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:18:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:25Z" name: error-404-isvc-a9c06-predictor.18c00af2c4c29a5d namespace: kserve-ci-e2e-test resourceVersion: "21169" uid: 4351790d-a726-45be-983f-db0d7393ba97 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:18:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9c06-predictor namespace: kserve-ci-e2e-test resourceVersion: "20802" uid: a91d1941-fd3d-4b98-9088-05dc16a5a54c kind: Event lastTimestamp: "2026-07-07T15:18:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:18:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:25Z" name: error-404-isvc-a9c06-predictor.18c00af2c54ce180 namespace: kserve-ci-e2e-test resourceVersion: "21170" uid: a57800c8-7c7f-4af6-ba10-a9e1a9b80425 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:18:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9c06-predictor namespace: kserve-ci-e2e-test resourceVersion: "21018" uid: a91d1941-fd3d-4b98-9088-05dc16a5a54c kind: Event lastTimestamp: "2026-07-07T15:19:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:18:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:19:25Z" name: error-404-isvc-a9c06-predictor.18c00af9c319613f namespace: kserve-ci-e2e-test resourceVersion: "21674" uid: 4646040b-31ed-4e38-8f76-1ec7bcfae67a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:18:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9c06-predictor namespace: kserve-ci-e2e-test resourceVersion: "21018" uid: a91d1941-fd3d-4b98-9088-05dc16a5a54c kind: Event lastTimestamp: "2026-07-07T15:19:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:18:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:19:25Z" name: error-404-isvc-a9c06-predictor.18c00af9c36b6d3d namespace: kserve-ci-e2e-test resourceVersion: "21675" uid: facedaf5-f372-46e6-a55c-0c4ac8cbb488 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9c06 namespace: kserve-ci-e2e-test resourceVersion: "20779" uid: 3507bd2c-e820-4f8d-8ee0-cd0f216a96e3 kind: Event lastTimestamp: "2026-07-07T15:18:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:18:43Z" name: error-404-isvc-a9c06.18c00aef464cd99b namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: 3da164f7-905d-4400-8cdb-63f5d7caf258 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9c06 namespace: kserve-ci-e2e-test resourceVersion: "20779" uid: 3507bd2c-e820-4f8d-8ee0-cd0f216a96e3 kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: 'Failed to update status for InferenceService "error-404-isvc-a9c06": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a9c06": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:17:55Z" name: error-404-isvc-a9c06.18c00aef46a60585 namespace: kserve-ci-e2e-test resourceVersion: "20808" uid: 926f9391-5ba9-4125-9899-f7a5f7befb3f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9c06 namespace: kserve-ci-e2e-test resourceVersion: "20779" uid: 3507bd2c-e820-4f8d-8ee0-cd0f216a96e3 kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a9c06": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:17:55Z" name: error-404-isvc-a9c06.18c00aef46a65a90 namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: bafcb19b-4e48-4e1b-bb96-afae8f9bebe6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9c06 namespace: kserve-ci-e2e-test resourceVersion: "21332" uid: 3507bd2c-e820-4f8d-8ee0-cd0f216a96e3 kind: Event lastTimestamp: "2026-07-07T15:18:43Z" message: InferenceService [error-404-isvc-a9c06] is Ready metadata: creationTimestamp: "2026-07-07T15:18:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:18:43Z" name: error-404-isvc-a9c06.18c00afa90b8a118 namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: 71416aa8-614b-439a-983e-cdc083ba97c4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:27:10.359870Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x namespace: kserve-ci-e2e-test resourceVersion: "25093" uid: 49efee85-7499-48bf-b795-523fc057f59d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x to ip-10-0-128-215.ec2.internal metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:27:10Z" name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x.18c00b708686fe24 namespace: kserve-ci-e2e-test resourceVersion: "25099" uid: 161147ac-4290-4408-a38e-49c49b9f5d57 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x namespace: kserve-ci-e2e-test resourceVersion: "25101" uid: 49efee85-7499-48bf-b795-523fc057f59d kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:27:10Z" name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x.18c00b70a081894a namespace: kserve-ci-e2e-test resourceVersion: "25120" uid: df1a2b31-53bc-46d3-be17-66fcda186e9c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x namespace: kserve-ci-e2e-test resourceVersion: "25098" uid: 49efee85-7499-48bf-b795-523fc057f59d kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:10Z" name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x.18c00b70a35b4802 namespace: kserve-ci-e2e-test resourceVersion: "25122" uid: 73cf431e-424c-43ff-accb-fea911e5efec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x namespace: kserve-ci-e2e-test resourceVersion: "25098" uid: 49efee85-7499-48bf-b795-523fc057f59d kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:10Z" name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x.18c00b70a6a15d52 namespace: kserve-ci-e2e-test resourceVersion: "25124" uid: e103bef6-e459-4dbd-9870-e959223bb520 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x namespace: kserve-ci-e2e-test resourceVersion: "25098" uid: 49efee85-7499-48bf-b795-523fc057f59d kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:10Z" name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x.18c00b70a71d6037 namespace: kserve-ci-e2e-test resourceVersion: "25125" uid: b85cfd28-35ed-4511-acaf-80b5af981b23 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x namespace: kserve-ci-e2e-test resourceVersion: "25098" uid: 49efee85-7499-48bf-b795-523fc057f59d kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:10Z" name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x.18c00b70a728a03d namespace: kserve-ci-e2e-test resourceVersion: "25126" uid: ca78ba37-23db-46bb-ae95-6bc62312b009 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x namespace: kserve-ci-e2e-test resourceVersion: "25098" uid: 49efee85-7499-48bf-b795-523fc057f59d kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:10Z" name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x.18c00b70aa4c7c8b namespace: kserve-ci-e2e-test resourceVersion: "25127" uid: 153b386c-4ae1-4f12-999c-3e72889db577 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x namespace: kserve-ci-e2e-test resourceVersion: "25098" uid: 49efee85-7499-48bf-b795-523fc057f59d kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:10Z" name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x.18c00b70aac920a6 namespace: kserve-ci-e2e-test resourceVersion: "25128" uid: ee72c06c-2fd2-47e6-aa9a-43ab8c522689 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x namespace: kserve-ci-e2e-test resourceVersion: "25098" uid: 49efee85-7499-48bf-b795-523fc057f59d kind: Event lastTimestamp: "2026-07-07T15:27:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:48Z" name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x.18c00b710ed45619 namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: 5a5a6099-03b3-4751-b86d-89e751985154 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:44:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x namespace: kserve-ci-e2e-test resourceVersion: "25098" uid: 49efee85-7499-48bf-b795-523fc057f59d kind: Event lastTimestamp: "2026-07-07T15:44:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:44:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:44:29Z" name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x.18c00c627fb773b0 namespace: kserve-ci-e2e-test resourceVersion: "33085" uid: f5afb9be-2704-46cc-a09a-5e33cc5cb98d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:44:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x namespace: kserve-ci-e2e-test resourceVersion: "25098" uid: 49efee85-7499-48bf-b795-523fc057f59d kind: Event lastTimestamp: "2026-07-07T15:44:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:44:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:44:29Z" name: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x.18c00c627fb7ef2d namespace: kserve-ci-e2e-test resourceVersion: "33086" uid: b21685e0-1026-4846-a3a7-ec6053ae2b84 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c3e9b-predictor-7578b96d6f namespace: kserve-ci-e2e-test resourceVersion: "25082" uid: 828499b6-695e-4578-8783-742e65941612 kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: 'Created pod: error-404-isvc-c3e9b-predictor-7578b96d6f-9zh9x' metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:10Z" name: error-404-isvc-c3e9b-predictor-7578b96d6f.18c00b70855c4041 namespace: kserve-ci-e2e-test resourceVersion: "25095" uid: 1b090f5f-492f-4402-b7a7-3a3ae81c1d88 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c3e9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25079" uid: 12637100-487d-44f3-9aa8-1a49c42ff989 kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: Scaled up replica set error-404-isvc-c3e9b-predictor-7578b96d6f from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:10Z" name: error-404-isvc-c3e9b-predictor.18c00b70817b041a namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: 3d8ffb49-2473-407d-82a0-f4079d2f36fd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:27:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3e9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25081" uid: 0d3fb34a-dd2a-49ec-b7a5-160f73218cb7 kind: Event lastTimestamp: "2026-07-07T15:27:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:27:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:40Z" name: error-404-isvc-c3e9b-predictor.18c00b74005ca5ba namespace: kserve-ci-e2e-test resourceVersion: "25437" uid: c99ab74a-fc82-47ad-85c3-5d3fd4c00810 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:27:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3e9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25081" uid: 0d3fb34a-dd2a-49ec-b7a5-160f73218cb7 kind: Event lastTimestamp: "2026-07-07T15:27:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:27:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:40Z" name: error-404-isvc-c3e9b-predictor.18c00b7400eaefc4 namespace: kserve-ci-e2e-test resourceVersion: "25438" uid: 67f88061-5ca2-43ce-82b6-752be63e0311 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:27:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3e9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25289" uid: 0d3fb34a-dd2a-49ec-b7a5-160f73218cb7 kind: Event lastTimestamp: "2026-07-07T15:28:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:28:25Z" name: error-404-isvc-c3e9b-predictor.18c00b7afece3380 namespace: kserve-ci-e2e-test resourceVersion: "26308" uid: dbd2ca15-5682-4006-a4f3-de9ba5d42533 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:27:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3e9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25289" uid: 0d3fb34a-dd2a-49ec-b7a5-160f73218cb7 kind: Event lastTimestamp: "2026-07-07T15:28:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:28:25Z" name: error-404-isvc-c3e9b-predictor.18c00b7aff5e7881 namespace: kserve-ci-e2e-test resourceVersion: "26309" uid: 9593b0fa-0113-4e83-8d3b-470eab23db09 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3e9b namespace: kserve-ci-e2e-test resourceVersion: "25050" uid: 75ca1d2d-c77e-490d-9326-b5e7b49ce7de kind: Event lastTimestamp: "2026-07-07T15:27:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:27:58Z" name: error-404-isvc-c3e9b.18c00b7081c2ab5b namespace: kserve-ci-e2e-test resourceVersion: "25763" uid: e9beff6e-03b9-419b-8560-79880772005d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3e9b namespace: kserve-ci-e2e-test resourceVersion: "25050" uid: 75ca1d2d-c77e-490d-9326-b5e7b49ce7de kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: 'Failed to update status for InferenceService "error-404-isvc-c3e9b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c3e9b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:27:10Z" name: error-404-isvc-c3e9b.18c00b70827896d5 namespace: kserve-ci-e2e-test resourceVersion: "25088" uid: d143d657-923d-4b9a-9907-4e89360b7828 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3e9b namespace: kserve-ci-e2e-test resourceVersion: "25050" uid: 75ca1d2d-c77e-490d-9326-b5e7b49ce7de kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c3e9b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:27:10Z" name: error-404-isvc-c3e9b.18c00b708278fce9 namespace: kserve-ci-e2e-test resourceVersion: "25091" uid: 979acf25-39f1-4c17-823f-280531a0155d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3e9b namespace: kserve-ci-e2e-test resourceVersion: "25760" uid: 75ca1d2d-c77e-490d-9326-b5e7b49ce7de kind: Event lastTimestamp: "2026-07-07T15:27:58Z" message: InferenceService [error-404-isvc-c3e9b] is Ready metadata: creationTimestamp: "2026-07-07T15:27:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:27:58Z" name: error-404-isvc-c3e9b.18c00b7bcf9da954 namespace: kserve-ci-e2e-test resourceVersion: "25761" uid: f3a02ca1-0715-4e15-a9a8-8be43e4eaf10 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:16:45.102786Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: 5bd241b8-ee2c-4fc0-9468-6328c4288214 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e4605-predictor-7f46859f89-sq7hs to ip-10-0-128-215.ec2.internal metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:16:45Z" name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs.18c00adef24c0593 namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: 12d8be35-8ff5-45e5-8a55-bfec0a3f5440 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs namespace: kserve-ci-e2e-test resourceVersion: "19867" uid: 5bd241b8-ee2c-4fc0-9468-6328c4288214 kind: Event lastTimestamp: "2026-07-07T15:16:45Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:16:45Z" name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs.18c00adf0c05a4db namespace: kserve-ci-e2e-test resourceVersion: "19887" uid: 162e044a-8ae1-4e6c-bb92-b8064a2652bd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 5bd241b8-ee2c-4fc0-9468-6328c4288214 kind: Event lastTimestamp: "2026-07-07T15:16:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:45Z" name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs.18c00adf0d87d217 namespace: kserve-ci-e2e-test resourceVersion: "19889" uid: 687b5c6c-e92f-4e02-9b46-1b6e4729f291 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 5bd241b8-ee2c-4fc0-9468-6328c4288214 kind: Event lastTimestamp: "2026-07-07T15:16:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:45Z" name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs.18c00adf107feebe namespace: kserve-ci-e2e-test resourceVersion: "19891" uid: a190f95b-52bb-4a2d-a6f3-beec7e6f2da7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 5bd241b8-ee2c-4fc0-9468-6328c4288214 kind: Event lastTimestamp: "2026-07-07T15:16:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:45Z" name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs.18c00adf10f592de namespace: kserve-ci-e2e-test resourceVersion: "19892" uid: 7a9d6aec-f6a7-4804-ac7e-d976f392a374 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 5bd241b8-ee2c-4fc0-9468-6328c4288214 kind: Event lastTimestamp: "2026-07-07T15:16:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:45Z" name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs.18c00adf1100d14a namespace: kserve-ci-e2e-test resourceVersion: "19893" uid: 01aa4c4d-e174-4a36-b9e1-e90353087d21 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 5bd241b8-ee2c-4fc0-9468-6328c4288214 kind: Event lastTimestamp: "2026-07-07T15:16:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:45Z" name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs.18c00adf13b8dac5 namespace: kserve-ci-e2e-test resourceVersion: "19895" uid: e688b804-a4e7-4c4a-9d77-cfecf3b22780 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 5bd241b8-ee2c-4fc0-9468-6328c4288214 kind: Event lastTimestamp: "2026-07-07T15:16:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:45Z" name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs.18c00adf142c5ad0 namespace: kserve-ci-e2e-test resourceVersion: "19896" uid: 92ecda8f-ec3e-40fd-9551-14cd2cea29d7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:16:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 5bd241b8-ee2c-4fc0-9468-6328c4288214 kind: Event lastTimestamp: "2026-07-07T15:17:23Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:23Z" name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs.18c00adf76bb28fc namespace: kserve-ci-e2e-test resourceVersion: "20411" uid: 5d93f327-fed7-498a-8cb9-387d23d11960 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 5bd241b8-ee2c-4fc0-9468-6328c4288214 kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:55Z" name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs.18c00aef4c9d408e namespace: kserve-ci-e2e-test resourceVersion: "20836" uid: c7a2fa08-1bb0-4d19-ab30-922ac92f7dfb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 5bd241b8-ee2c-4fc0-9468-6328c4288214 kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:55Z" name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs.18c00aef4c9dd83a namespace: kserve-ci-e2e-test resourceVersion: "20837" uid: 6db40f46-48c3-42e1-8190-51d2f1a9e2ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 5bd241b8-ee2c-4fc0-9468-6328c4288214 kind: Event lastTimestamp: "2026-07-07T15:17:58Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:17:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:58Z" name: error-404-isvc-e4605-predictor-7f46859f89-sq7hs.18c00aefff4cc167 namespace: kserve-ci-e2e-test resourceVersion: "20895" uid: f6954ee1-1e9d-4d17-bf41-10c07a339d82 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e4605-predictor-7f46859f89 namespace: kserve-ci-e2e-test resourceVersion: "19846" uid: 66bb09db-0013-4182-a52a-f2b2aab73e81 kind: Event lastTimestamp: "2026-07-07T15:16:45Z" message: 'Created pod: error-404-isvc-e4605-predictor-7f46859f89-sq7hs' metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:16:45Z" name: error-404-isvc-e4605-predictor-7f46859f89.18c00adef12e90aa namespace: kserve-ci-e2e-test resourceVersion: "19859" uid: 208abf7e-29fb-4d0e-b747-50d84bd31be1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e4605-predictor namespace: kserve-ci-e2e-test resourceVersion: "19842" uid: 45ac5b42-e72a-4564-9550-963a460f0e69 kind: Event lastTimestamp: "2026-07-07T15:16:45Z" message: Scaled up replica set error-404-isvc-e4605-predictor-7f46859f89 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:16:45Z" name: error-404-isvc-e4605-predictor.18c00adeecf08702 namespace: kserve-ci-e2e-test resourceVersion: "19850" uid: c41b33f7-d666-4b28-9852-ec75ccad2b94 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:17:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e4605-predictor namespace: kserve-ci-e2e-test resourceVersion: "19845" uid: 7a2160dd-a25d-4c5d-8c62-1dc38beb106a kind: Event lastTimestamp: "2026-07-07T15:17:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:17:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:30Z" name: error-404-isvc-e4605-predictor.18c00ae26b724ceb namespace: kserve-ci-e2e-test resourceVersion: "20472" uid: e85649c2-4b3b-4d15-8187-69a2b2a7b142 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:17:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e4605-predictor namespace: kserve-ci-e2e-test resourceVersion: "19845" uid: 7a2160dd-a25d-4c5d-8c62-1dc38beb106a kind: Event lastTimestamp: "2026-07-07T15:17:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:17:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:30Z" name: error-404-isvc-e4605-predictor.18c00ae26becd021 namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: 5e3f6504-5d19-49a8-b739-0eacc188c2e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e4605-predictor namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: 7a2160dd-a25d-4c5d-8c62-1dc38beb106a kind: Event lastTimestamp: "2026-07-07T15:17:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:45Z" name: error-404-isvc-e4605-predictor.18c00aece863ca3a namespace: kserve-ci-e2e-test resourceVersion: "20637" uid: d77fca2f-c240-4cf9-a875-960e98cbf1bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e4605-predictor namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: 7a2160dd-a25d-4c5d-8c62-1dc38beb106a kind: Event lastTimestamp: "2026-07-07T15:17:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:45Z" name: error-404-isvc-e4605-predictor.18c00aece8c7d23f namespace: kserve-ci-e2e-test resourceVersion: "20638" uid: 6bf3d306-4cd1-40a6-8140-0d67d924bc82 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e4605 namespace: kserve-ci-e2e-test resourceVersion: "19827" uid: e753feba-c1a1-423a-8407-a4056eee1fb4 kind: Event lastTimestamp: "2026-07-07T15:17:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:17:33Z" name: error-404-isvc-e4605.18c00adeee277392 namespace: kserve-ci-e2e-test resourceVersion: "20513" uid: faf3570b-deb1-4402-b9bd-317b25b3a87f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e4605 namespace: kserve-ci-e2e-test resourceVersion: "19827" uid: e753feba-c1a1-423a-8407-a4056eee1fb4 kind: Event lastTimestamp: "2026-07-07T15:16:45Z" message: 'Failed to update status for InferenceService "error-404-isvc-e4605": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e4605": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:16:45Z" name: error-404-isvc-e4605.18c00adeef9fb122 namespace: kserve-ci-e2e-test resourceVersion: "19858" uid: 7c11d75e-b6a8-4a37-b765-a0314856c2fd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e4605 namespace: kserve-ci-e2e-test resourceVersion: "19827" uid: e753feba-c1a1-423a-8407-a4056eee1fb4 kind: Event lastTimestamp: "2026-07-07T15:16:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e4605": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:16:45Z" name: error-404-isvc-e4605.18c00adeef9ff41c namespace: kserve-ci-e2e-test resourceVersion: "19861" uid: d2e91ae0-c05a-4e58-bc68-eac50c3f6c72 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e4605 namespace: kserve-ci-e2e-test resourceVersion: "20511" uid: e753feba-c1a1-423a-8407-a4056eee1fb4 kind: Event lastTimestamp: "2026-07-07T15:17:33Z" message: InferenceService [error-404-isvc-e4605] is Ready metadata: creationTimestamp: "2026-07-07T15:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:17:33Z" name: error-404-isvc-e4605.18c00aea32a3686f namespace: kserve-ci-e2e-test resourceVersion: "20512" uid: e389f933-94d5-4fe4-8c41-cb62c6821e96 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:05:20.007076Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg namespace: kserve-ci-e2e-test resourceVersion: "13959" uid: 42f87be0-c8dd-48dc-b20e-d05e5c34750b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg to ip-10-0-128-215.ec2.internal metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:05:20Z" name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg.18c00a3f6f6900e7 namespace: kserve-ci-e2e-test resourceVersion: "13964" uid: 6c7e3984-af0b-4aee-a424-c9ba94e6002e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg namespace: kserve-ci-e2e-test resourceVersion: "13969" uid: 42f87be0-c8dd-48dc-b20e-d05e5c34750b kind: Event lastTimestamp: "2026-07-07T15:05:20Z" message: Add eth0 [10.134.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:05:20Z" name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg.18c00a3f89dd5774 namespace: kserve-ci-e2e-test resourceVersion: "14031" uid: 606b7c52-e934-4915-a522-3c8e43d8a4a0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg namespace: kserve-ci-e2e-test resourceVersion: "13961" uid: 42f87be0-c8dd-48dc-b20e-d05e5c34750b kind: Event lastTimestamp: "2026-07-07T15:05:20Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:20Z" name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg.18c00a3f8b84ef5c namespace: kserve-ci-e2e-test resourceVersion: "14034" uid: 067964bd-17e3-4132-81c2-364aa7378342 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg namespace: kserve-ci-e2e-test resourceVersion: "13961" uid: 42f87be0-c8dd-48dc-b20e-d05e5c34750b kind: Event lastTimestamp: "2026-07-07T15:05:33Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1699" in 12.911s (12.911s including waiting). Image size: 1334700636 bytes.' metadata: creationTimestamp: "2026-07-07T15:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:33Z" name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg.18c00a428d22096f namespace: kserve-ci-e2e-test resourceVersion: "14172" uid: 30f8f93c-54ef-418b-ab4b-f60833dd11ff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg namespace: kserve-ci-e2e-test resourceVersion: "13961" uid: 42f87be0-c8dd-48dc-b20e-d05e5c34750b kind: Event lastTimestamp: "2026-07-07T15:05:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:33Z" name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg.18c00a4290605c95 namespace: kserve-ci-e2e-test resourceVersion: "14175" uid: 11640a14-8aaf-491c-b05e-6b431022ee58 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg namespace: kserve-ci-e2e-test resourceVersion: "13961" uid: 42f87be0-c8dd-48dc-b20e-d05e5c34750b kind: Event lastTimestamp: "2026-07-07T15:05:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:33Z" name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg.18c00a4290cfae8b namespace: kserve-ci-e2e-test resourceVersion: "14176" uid: 31cca351-3c6c-43c8-89cd-7d6a003450ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg namespace: kserve-ci-e2e-test resourceVersion: "13961" uid: 42f87be0-c8dd-48dc-b20e-d05e5c34750b kind: Event lastTimestamp: "2026-07-07T15:05:33Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T15:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:33Z" name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg.18c00a4290dea26f namespace: kserve-ci-e2e-test resourceVersion: "14178" uid: 00732972-e0b2-4625-97f8-befd76f4d781 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg namespace: kserve-ci-e2e-test resourceVersion: "13961" uid: 42f87be0-c8dd-48dc-b20e-d05e5c34750b kind: Event lastTimestamp: "2026-07-07T15:05:36Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.701s (2.701s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T15:05:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:36Z" name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg.18c00a4331eae88e namespace: kserve-ci-e2e-test resourceVersion: "14225" uid: 4c9b0cd8-11e2-4eda-b963-069338a68c1c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg namespace: kserve-ci-e2e-test resourceVersion: "13961" uid: 42f87be0-c8dd-48dc-b20e-d05e5c34750b kind: Event lastTimestamp: "2026-07-07T15:05:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:05:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:36Z" name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg.18c00a4334b77e13 namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: d1448297-5931-428f-9514-d9f368e23cbf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg namespace: kserve-ci-e2e-test resourceVersion: "13961" uid: 42f87be0-c8dd-48dc-b20e-d05e5c34750b kind: Event lastTimestamp: "2026-07-07T15:05:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:05:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:36Z" name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg.18c00a4335229e63 namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 907f7c6a-37bf-4acb-a5c7-35d7e3db2b28 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:05:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg namespace: kserve-ci-e2e-test resourceVersion: "13961" uid: 42f87be0-c8dd-48dc-b20e-d05e5c34750b kind: Event lastTimestamp: "2026-07-07T15:06:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:05:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:12Z" name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg.18c00a435bbbf599 namespace: kserve-ci-e2e-test resourceVersion: "14563" uid: 900055cc-fb08-47fe-9c27-d1c8865ecac6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg namespace: kserve-ci-e2e-test resourceVersion: "13961" uid: 42f87be0-c8dd-48dc-b20e-d05e5c34750b kind: Event lastTimestamp: "2026-07-07T15:06:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:54Z" name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg.18c00a556e161466 namespace: kserve-ci-e2e-test resourceVersion: "15056" uid: f9d836cf-de5e-4112-b793-6e65e9ff0f8f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg namespace: kserve-ci-e2e-test resourceVersion: "13961" uid: 42f87be0-c8dd-48dc-b20e-d05e5c34750b kind: Event lastTimestamp: "2026-07-07T15:06:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:54Z" name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg.18c00a556e174ab9 namespace: kserve-ci-e2e-test resourceVersion: "15057" uid: 9cb36942-9ca9-4c7c-a4c1-18735635e0ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg namespace: kserve-ci-e2e-test resourceVersion: "13961" uid: 42f87be0-c8dd-48dc-b20e-d05e5c34750b kind: Event lastTimestamp: "2026-07-07T15:06:58Z" message: 'Readiness probe failed: Get "https://10.134.0.17:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-07-07T15:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:58Z" name: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg.18c00a5673a127f4 namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: b9682fbc-c1c3-43bd-85d6-079d31bd1fa1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f85ce-predictor-859cc5b5d7 namespace: kserve-ci-e2e-test resourceVersion: "13953" uid: 780efd35-9b0f-4e53-a4b0-039fc0cd4ad1 kind: Event lastTimestamp: "2026-07-07T15:05:19Z" message: 'Created pod: error-404-isvc-f85ce-predictor-859cc5b5d7-qbbgg' metadata: creationTimestamp: "2026-07-07T15:05:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:05:19Z" name: error-404-isvc-f85ce-predictor-859cc5b5d7.18c00a3f6eadd17a namespace: kserve-ci-e2e-test resourceVersion: "13960" uid: a4bb5cd7-23ad-463b-9981-fca5bc3b8877 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f85ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "13950" uid: 9ca35510-8ed4-45cd-806d-3fb69e5ce954 kind: Event lastTimestamp: "2026-07-07T15:05:19Z" message: Scaled up replica set error-404-isvc-f85ce-predictor-859cc5b5d7 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:05:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:05:19Z" name: error-404-isvc-f85ce-predictor.18c00a3f6c92b154 namespace: kserve-ci-e2e-test resourceVersion: "13954" uid: ddbd88e0-4825-47a4-84a8-1f9f3e648942 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:05:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f85ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: 8b6bf21f-8680-49b2-b0c5-376b10559798 kind: Event lastTimestamp: "2026-07-07T15:06:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:05Z" name: error-404-isvc-f85ce-predictor.18c00a42eb918602 namespace: kserve-ci-e2e-test resourceVersion: "14501" uid: 0928ad13-74a6-4ad7-83f3-ab381a328471 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:05:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f85ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: 8b6bf21f-8680-49b2-b0c5-376b10559798 kind: Event lastTimestamp: "2026-07-07T15:06:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:05Z" name: error-404-isvc-f85ce-predictor.18c00a42ec0b2a42 namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: ceaaea10-06a0-40fa-89bf-02e8431ddcb6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:06:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f85ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "14205" uid: 8b6bf21f-8680-49b2-b0c5-376b10559798 kind: Event lastTimestamp: "2026-07-07T15:06:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:50Z" name: error-404-isvc-f85ce-predictor.18c00a4d6916c79c namespace: kserve-ci-e2e-test resourceVersion: "14911" uid: 37f4dcc0-ed82-4204-b153-b98a517c1df1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:06:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f85ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "14205" uid: 8b6bf21f-8680-49b2-b0c5-376b10559798 kind: Event lastTimestamp: "2026-07-07T15:06:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:50Z" name: error-404-isvc-f85ce-predictor.18c00a4d69a2b4ca namespace: kserve-ci-e2e-test resourceVersion: "14913" uid: 2a4ea878-fd0a-4d63-96e5-22c442aabeee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:05:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f85ce namespace: kserve-ci-e2e-test resourceVersion: "13934" uid: 2e55f6e7-191a-4c4d-84b6-c5264c161cb1 kind: Event lastTimestamp: "2026-07-07T15:06:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:05:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:06:23Z" name: error-404-isvc-f85ce.18c00a3f6ccb6a0b namespace: kserve-ci-e2e-test resourceVersion: "14671" uid: 43483ad6-2b7c-4d79-9e66-72cf37af44ee reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f85ce namespace: kserve-ci-e2e-test resourceVersion: "14665" uid: 2e55f6e7-191a-4c4d-84b6-c5264c161cb1 kind: Event lastTimestamp: "2026-07-07T15:06:23Z" message: InferenceService [error-404-isvc-f85ce] is Ready metadata: creationTimestamp: "2026-07-07T15:06:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:06:23Z" name: error-404-isvc-f85ce.18c00a4e2a4af8d5 namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: 64ce74e7-10aa-443c-bb49-ef904701c2ed reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:05:19.905704Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk namespace: kserve-ci-e2e-test resourceVersion: "13937" uid: d8f6481c-fd34-449e-8005-872e9669d5cc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk to ip-10-0-130-158.ec2.internal metadata: creationTimestamp: "2026-07-07T15:05:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:05:19Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk.18c00a3f695e2e57 namespace: kserve-ci-e2e-test resourceVersion: "13941" uid: 0a56f316-b92b-4dcc-80cc-f66012d58ce6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk namespace: kserve-ci-e2e-test resourceVersion: "13939" uid: d8f6481c-fd34-449e-8005-872e9669d5cc kind: Event lastTimestamp: "2026-07-07T15:05:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:20Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk.18c00a3f74c21f3a namespace: kserve-ci-e2e-test resourceVersion: "13987" uid: 0d55e4d0-ae7d-4a0f-87ea-808271bff4ef reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk namespace: kserve-ci-e2e-test resourceVersion: "13951" uid: d8f6481c-fd34-449e-8005-872e9669d5cc kind: Event lastTimestamp: "2026-07-07T15:05:20Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:05:20Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk.18c00a3fa67a2c3d namespace: kserve-ci-e2e-test resourceVersion: "14055" uid: 49e80b9d-79c3-4acb-b06c-c4740b65ffc9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk namespace: kserve-ci-e2e-test resourceVersion: "13939" uid: d8f6481c-fd34-449e-8005-872e9669d5cc kind: Event lastTimestamp: "2026-07-07T15:05:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:20Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk.18c00a3fa7ff9a98 namespace: kserve-ci-e2e-test resourceVersion: "14058" uid: bfef10df-13bf-4fbd-885a-ed8161f13c00 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk namespace: kserve-ci-e2e-test resourceVersion: "13939" uid: d8f6481c-fd34-449e-8005-872e9669d5cc kind: Event lastTimestamp: "2026-07-07T15:05:21Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:21Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk.18c00a3faacace80 namespace: kserve-ci-e2e-test resourceVersion: "14059" uid: 1a9e74fb-2f3e-44a7-a09e-fdc95a25f62d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk namespace: kserve-ci-e2e-test resourceVersion: "13939" uid: d8f6481c-fd34-449e-8005-872e9669d5cc kind: Event lastTimestamp: "2026-07-07T15:05:21Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:21Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk.18c00a3fab3c4ede namespace: kserve-ci-e2e-test resourceVersion: "14060" uid: a7758ff4-4f84-4b5b-a80b-5b9546faa41b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk namespace: kserve-ci-e2e-test resourceVersion: "13939" uid: d8f6481c-fd34-449e-8005-872e9669d5cc kind: Event lastTimestamp: "2026-07-07T15:05:24Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" metadata: creationTimestamp: "2026-07-07T15:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:24Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk.18c00a407ff2ca1d namespace: kserve-ci-e2e-test resourceVersion: "14101" uid: 09f225e8-17f9-41bd-983d-3b83f0e41896 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk namespace: kserve-ci-e2e-test resourceVersion: "13939" uid: d8f6481c-fd34-449e-8005-872e9669d5cc kind: Event lastTimestamp: "2026-07-07T15:05:40Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" in 15.956s (15.957s including waiting). Image size: 1555863462 bytes.' metadata: creationTimestamp: "2026-07-07T15:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:40Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk.18c00a44370fee51 namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 9cdebf4e-39e1-4d6b-b674-46f7443ecaf5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk namespace: kserve-ci-e2e-test resourceVersion: "13939" uid: d8f6481c-fd34-449e-8005-872e9669d5cc kind: Event lastTimestamp: "2026-07-07T15:05:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:40Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk.18c00a443c292d0b namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: ffc783c5-efef-4504-99fd-2ae29b8034b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk namespace: kserve-ci-e2e-test resourceVersion: "13939" uid: d8f6481c-fd34-449e-8005-872e9669d5cc kind: Event lastTimestamp: "2026-07-07T15:05:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:40Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk.18c00a443cb2d942 namespace: kserve-ci-e2e-test resourceVersion: "14274" uid: f70b31a2-ce58-4a25-8fed-6d6fb0504e86 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk namespace: kserve-ci-e2e-test resourceVersion: "13939" uid: d8f6481c-fd34-449e-8005-872e9669d5cc kind: Event lastTimestamp: "2026-07-07T15:05:40Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T15:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:40Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk.18c00a443cc86355 namespace: kserve-ci-e2e-test resourceVersion: "14275" uid: aa1ded6c-c5b9-4244-9232-0ad0056eaed0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk namespace: kserve-ci-e2e-test resourceVersion: "13939" uid: d8f6481c-fd34-449e-8005-872e9669d5cc kind: Event lastTimestamp: "2026-07-07T15:05:51Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 10.933s (10.933s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T15:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:51Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk.18c00a46c8756fac namespace: kserve-ci-e2e-test resourceVersion: "14380" uid: b8ad8b1e-72f7-429f-9e69-d1e2124bc045 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk namespace: kserve-ci-e2e-test resourceVersion: "13939" uid: d8f6481c-fd34-449e-8005-872e9669d5cc kind: Event lastTimestamp: "2026-07-07T15:05:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:51Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk.18c00a46cdcacd95 namespace: kserve-ci-e2e-test resourceVersion: "14381" uid: 1e815fd4-4a15-419e-9964-5afe25deb07f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk namespace: kserve-ci-e2e-test resourceVersion: "13939" uid: d8f6481c-fd34-449e-8005-872e9669d5cc kind: Event lastTimestamp: "2026-07-07T15:05:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:51Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk.18c00a46ceba5d96 namespace: kserve-ci-e2e-test resourceVersion: "14382" uid: e08fce46-856f-48b9-9a86-9a7acc8b49d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-07T15:05:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk namespace: kserve-ci-e2e-test resourceVersion: "13939" uid: d8f6481c-fd34-449e-8005-872e9669d5cc kind: Event lastTimestamp: "2026-07-07T15:06:48Z" message: 'Readiness probe failed: dial tcp 10.133.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:05:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:48Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk.18c00a470e76da56 namespace: kserve-ci-e2e-test resourceVersion: "14896" uid: 9073655e-b27a-421f-86c5-cfa4fafb2ca3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk namespace: kserve-ci-e2e-test resourceVersion: "13939" uid: d8f6481c-fd34-449e-8005-872e9669d5cc kind: Event lastTimestamp: "2026-07-07T15:07:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:29Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk.18c00a5db279dfa6 namespace: kserve-ci-e2e-test resourceVersion: "15533" uid: 6add364d-a1dc-426f-88e7-7f9cdea113cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk namespace: kserve-ci-e2e-test resourceVersion: "13939" uid: d8f6481c-fd34-449e-8005-872e9669d5cc kind: Event lastTimestamp: "2026-07-07T15:07:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:29Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk.18c00a5db27b28c2 namespace: kserve-ci-e2e-test resourceVersion: "15534" uid: 75c071ee-af0d-4cee-ad91-799d1b72e027 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk namespace: kserve-ci-e2e-test resourceVersion: "13939" uid: d8f6481c-fd34-449e-8005-872e9669d5cc kind: Event lastTimestamp: "2026-07-07T15:07:33Z" message: 'Readiness probe failed: Get "https://10.133.0.20:8643/healthz": dial tcp 10.133.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:07:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:33Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk.18c00a5e92c834b6 namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: bfec38d7-d10e-4120-b984-c2496e2db9f1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-766fdfb6bb namespace: kserve-ci-e2e-test resourceVersion: "13927" uid: 4c1f0c92-0007-4f5a-90d3-da5bc421e73f kind: Event lastTimestamp: "2026-07-07T15:05:19Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-766fdfb6bb-wbdsk' metadata: creationTimestamp: "2026-07-07T15:05:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:05:19Z" name: isvc-sklearn-graph-1-predictor-766fdfb6bb.18c00a3f68ee271d namespace: kserve-ci-e2e-test resourceVersion: "13938" uid: 66781362-0b86-485d-834b-294271db45a6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: 30380574-adaa-4980-95a5-128df32fb2a9 kind: Event lastTimestamp: "2026-07-07T15:05:19Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-766fdfb6bb from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:05:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:05:19Z" name: isvc-sklearn-graph-1-predictor.18c00a3f65c49118 namespace: kserve-ci-e2e-test resourceVersion: "13928" uid: e8e6bde2-1e74-4e5a-8e81-54c2fe651106 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:05:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13926" uid: 58bd1e9e-8e76-4908-98fa-5f369f55efe5 kind: Event lastTimestamp: "2026-07-07T15:06:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:19Z" name: isvc-sklearn-graph-1-predictor.18c00a42e47690d1 namespace: kserve-ci-e2e-test resourceVersion: "14619" uid: e3b6ffae-4008-4630-9da2-b8a0bcd8efd2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:05:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13926" uid: 58bd1e9e-8e76-4908-98fa-5f369f55efe5 kind: Event lastTimestamp: "2026-07-07T15:06:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:19Z" name: isvc-sklearn-graph-1-predictor.18c00a42e4fc364f namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 1123d5dd-f655-4dbb-b74b-a544db608209 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:06:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14200" uid: 58bd1e9e-8e76-4908-98fa-5f369f55efe5 kind: Event lastTimestamp: "2026-07-07T15:07:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:20Z" name: isvc-sklearn-graph-1-predictor.18c00a49e3594c2e namespace: kserve-ci-e2e-test resourceVersion: "15356" uid: 9f68d52d-05f8-4615-932f-84673704dd90 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:06:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14200" uid: 58bd1e9e-8e76-4908-98fa-5f369f55efe5 kind: Event lastTimestamp: "2026-07-07T15:07:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:20Z" name: isvc-sklearn-graph-1-predictor.18c00a49e4002f7f namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 10b9d7b1-5936-4362-a8c7-565fe04969d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-07T15:05:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13916" uid: 9fd8092c-ab73-4466-8a0f-bc36231ffb50 kind: Event lastTimestamp: "2026-07-07T15:06:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:05:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:06:59Z" name: isvc-sklearn-graph-1.18c00a3f65c45a57 namespace: kserve-ci-e2e-test resourceVersion: "15150" uid: 4e52ee0f-418d-4d34-8cc1-d342d1037fef reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15146" uid: 9fd8092c-ab73-4466-8a0f-bc36231ffb50 kind: Event lastTimestamp: "2026-07-07T15:06:58Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-07T15:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:06:58Z" name: isvc-sklearn-graph-1.18c00a5677d6b30d namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: 0cc5b912-43b8-474f-936c-fa9227c9cfad reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:05:20.410160Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b namespace: kserve-ci-e2e-test resourceVersion: "14017" uid: 32a0b1b1-74a5-456d-a03f-d13fbb47ece7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b to ip-10-0-128-215.ec2.internal metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:05:20Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b.18c00a3f876f9cdc namespace: kserve-ci-e2e-test resourceVersion: "14022" uid: dbbe0111-b95a-4d09-b4a6-8c2015cafa08 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b namespace: kserve-ci-e2e-test resourceVersion: "14029" uid: 32a0b1b1-74a5-456d-a03f-d13fbb47ece7 kind: Event lastTimestamp: "2026-07-07T15:05:20Z" message: Add eth0 [10.134.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:05:20Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b.18c00a3fa1385ad6 namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 9a1a6aac-4321-4b8b-8386-609eb03b4e2e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: 32a0b1b1-74a5-456d-a03f-d13fbb47ece7 kind: Event lastTimestamp: "2026-07-07T15:05:20Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:20Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b.18c00a3fa2a87421 namespace: kserve-ci-e2e-test resourceVersion: "14052" uid: f4b3cf95-5d37-439d-9925-af7ca2be7ee0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: 32a0b1b1-74a5-456d-a03f-d13fbb47ece7 kind: Event lastTimestamp: "2026-07-07T15:05:33Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" in 12.463s (12.463s including waiting). Image size: 301881278 bytes.' metadata: creationTimestamp: "2026-07-07T15:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:33Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b.18c00a42898f1270 namespace: kserve-ci-e2e-test resourceVersion: "14170" uid: 8ea80371-f018-489c-a34b-5cd7498e506a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: 32a0b1b1-74a5-456d-a03f-d13fbb47ece7 kind: Event lastTimestamp: "2026-07-07T15:05:33Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:33Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b.18c00a428d1191d5 namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: 275c72f0-c1da-41a6-9685-0f733e8e9410 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: 32a0b1b1-74a5-456d-a03f-d13fbb47ece7 kind: Event lastTimestamp: "2026-07-07T15:05:33Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:33Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b.18c00a428d8b7f1b namespace: kserve-ci-e2e-test resourceVersion: "14173" uid: 8d97d463-4f1d-4928-9476-b05c71db9069 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: 32a0b1b1-74a5-456d-a03f-d13fbb47ece7 kind: Event lastTimestamp: "2026-07-07T15:05:37Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" metadata: creationTimestamp: "2026-07-07T15:05:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:37Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b.18c00a4397a0f1f7 namespace: kserve-ci-e2e-test resourceVersion: "14250" uid: 78a8389c-34c4-4b55-aa4e-22b4b4757072 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: 32a0b1b1-74a5-456d-a03f-d13fbb47ece7 kind: Event lastTimestamp: "2026-07-07T15:05:44Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" in 6.472s (6.472s including waiting). Image size: 1555863462 bytes.' metadata: creationTimestamp: "2026-07-07T15:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:44Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b.18c00a45196ae439 namespace: kserve-ci-e2e-test resourceVersion: "14308" uid: ad06b6e6-7fbf-49a2-af9e-dac8cf294a53 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: 32a0b1b1-74a5-456d-a03f-d13fbb47ece7 kind: Event lastTimestamp: "2026-07-07T15:05:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:44Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b.18c00a451c646f82 namespace: kserve-ci-e2e-test resourceVersion: "14309" uid: e8e8305d-0446-42fd-80c9-a74cf8ae45fa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: 32a0b1b1-74a5-456d-a03f-d13fbb47ece7 kind: Event lastTimestamp: "2026-07-07T15:05:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:44Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b.18c00a451cd48139 namespace: kserve-ci-e2e-test resourceVersion: "14310" uid: afea4d9c-674f-4a96-9166-02d21fbd821a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: 32a0b1b1-74a5-456d-a03f-d13fbb47ece7 kind: Event lastTimestamp: "2026-07-07T15:05:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:44Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b.18c00a451cea83f7 namespace: kserve-ci-e2e-test resourceVersion: "14311" uid: b54c7095-5afc-468e-9237-319107ecda13 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: 32a0b1b1-74a5-456d-a03f-d13fbb47ece7 kind: Event lastTimestamp: "2026-07-07T15:05:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:44Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b.18c00a451ffce993 namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: 06a586bd-76c3-4acb-a9f3-84afdc47cbdf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: 32a0b1b1-74a5-456d-a03f-d13fbb47ece7 kind: Event lastTimestamp: "2026-07-07T15:05:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:44Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b.18c00a4520691844 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: a1c29d7c-f8b0-4afd-86f3-52ef7bc54423 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T15:05:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: 32a0b1b1-74a5-456d-a03f-d13fbb47ece7 kind: Event lastTimestamp: "2026-07-07T15:06:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:05:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:58Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b.18c00a4576a85be5 namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: bdd212f1-2ab0-4384-b97c-a160e2702f18 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: 32a0b1b1-74a5-456d-a03f-d13fbb47ece7 kind: Event lastTimestamp: "2026-07-07T15:07:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:29Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b.18c00a5dae61eda4 namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: 4407b4dd-9237-40e0-b020-0ccf04bf36e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: 32a0b1b1-74a5-456d-a03f-d13fbb47ece7 kind: Event lastTimestamp: "2026-07-07T15:07:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:29Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b.18c00a5dae631e99 namespace: kserve-ci-e2e-test resourceVersion: "15506" uid: 5a293084-77d6-496c-8ede-d16e48360fd2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: 32a0b1b1-74a5-456d-a03f-d13fbb47ece7 kind: Event lastTimestamp: "2026-07-07T15:07:31Z" message: 'Readiness probe failed: Get "https://10.134.0.18:8643/healthz": dial tcp 10.134.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:31Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b.18c00a5e2512ce2c namespace: kserve-ci-e2e-test resourceVersion: "15627" uid: 5b06a41a-28ac-4d8e-85aa-28b23e84e75e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-767d6d7f5d namespace: kserve-ci-e2e-test resourceVersion: "14009" uid: 6add9c00-2bd7-4c05-a5c5-9f133fa7a56e kind: Event lastTimestamp: "2026-07-07T15:05:20Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-767d6d7f5d-xkw4b' metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:05:20Z" name: isvc-sklearn-graph-2-predictor-767d6d7f5d.18c00a3f86e00d8f namespace: kserve-ci-e2e-test resourceVersion: "14018" uid: 04147120-163e-4f60-ab01-03078de586b1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: 502ff0f1-0271-40c9-a821-fe07a42afe4b kind: Event lastTimestamp: "2026-07-07T15:05:20Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-767d6d7f5d from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:05:20Z" name: isvc-sklearn-graph-2-predictor.18c00a3f7f96a472 namespace: kserve-ci-e2e-test resourceVersion: "14010" uid: 50571aca-04ca-4182-b4d8-e75484bdf2ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:05:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14008" uid: c4440008-9b8e-4c6e-9b98-b235a8b6900f kind: Event lastTimestamp: "2026-07-07T15:06:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:05:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:05Z" name: isvc-sklearn-graph-2-predictor.18c00a42fe61f993 namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: 670faee1-817a-46f8-9ede-3349225d1faf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:05:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14008" uid: c4440008-9b8e-4c6e-9b98-b235a8b6900f kind: Event lastTimestamp: "2026-07-07T15:06:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:05:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:05Z" name: isvc-sklearn-graph-2-predictor.18c00a42fede99a1 namespace: kserve-ci-e2e-test resourceVersion: "14511" uid: bf8929d6-b1cc-426d-8a4c-91bf60060f2a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:06:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14216" uid: c4440008-9b8e-4c6e-9b98-b235a8b6900f kind: Event lastTimestamp: "2026-07-07T15:07:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:20Z" name: isvc-sklearn-graph-2-predictor.18c00a4d7c4c4625 namespace: kserve-ci-e2e-test resourceVersion: "15363" uid: d2e686f8-a480-47f1-8f87-1c7f47b16de2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:06:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14216" uid: c4440008-9b8e-4c6e-9b98-b235a8b6900f kind: Event lastTimestamp: "2026-07-07T15:07:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:20Z" name: isvc-sklearn-graph-2-predictor.18c00a4d7cfee2f2 namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 13c6a2bf-3ed8-48ec-afb6-83be8a0a065f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-07T15:05:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13991" uid: 9b557138-e9c1-45bc-98f2-f47461be6c7e kind: Event lastTimestamp: "2026-07-07T15:07:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:07:09Z" name: isvc-sklearn-graph-2.18c00a3f7f9827ed namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 418ede4d-4cca-40cd-a3c6-c65a9b0e99af reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15238" uid: 9b557138-e9c1-45bc-98f2-f47461be6c7e kind: Event lastTimestamp: "2026-07-07T15:07:09Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-07T15:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:07:09Z" name: isvc-sklearn-graph-2.18c00a58d39c95d2 namespace: kserve-ci-e2e-test resourceVersion: "15242" uid: 7c330efd-7882-4046-9923-8a28b8c52df4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:05:20.220782Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6 namespace: kserve-ci-e2e-test resourceVersion: "13992" uid: 07a6e774-7566-403d-bcba-c2399f6e7c62 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-bd2c6 to ip-10-0-130-158.ec2.internal metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:05:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6.18c00a3f7c25d308 namespace: kserve-ci-e2e-test resourceVersion: "13997" uid: 7ec2fe7e-0717-4f34-81e0-d98a45ae96e8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6 namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 07a6e774-7566-403d-bcba-c2399f6e7c62 kind: Event lastTimestamp: "2026-07-07T15:05:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6.18c00a3f86e2817e namespace: kserve-ci-e2e-test resourceVersion: "14019" uid: 3744b5e6-d8d5-428f-9e41-7136ceff4895 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:21Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6 namespace: kserve-ci-e2e-test resourceVersion: "14003" uid: 07a6e774-7566-403d-bcba-c2399f6e7c62 kind: Event lastTimestamp: "2026-07-07T15:05:21Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:05:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6.18c00a3fb9885090 namespace: kserve-ci-e2e-test resourceVersion: "14063" uid: 93c63ad4-06f4-4a1a-9272-6424612e2a70 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6 namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 07a6e774-7566-403d-bcba-c2399f6e7c62 kind: Event lastTimestamp: "2026-07-07T15:05:21Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6.18c00a3fbb1fe319 namespace: kserve-ci-e2e-test resourceVersion: "14066" uid: d9e0213a-c847-4962-aff2-adf4c93e42cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6 namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 07a6e774-7566-403d-bcba-c2399f6e7c62 kind: Event lastTimestamp: "2026-07-07T15:05:21Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6.18c00a3fbe63ff26 namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: bcf6457c-59db-438c-a9ea-c1d8078a08d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6 namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 07a6e774-7566-403d-bcba-c2399f6e7c62 kind: Event lastTimestamp: "2026-07-07T15:05:21Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6.18c00a3fbed8a4e5 namespace: kserve-ci-e2e-test resourceVersion: "14069" uid: 97962ed0-0f1c-4f4f-9fbb-9b453aca8068 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6 namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 07a6e774-7566-403d-bcba-c2399f6e7c62 kind: Event lastTimestamp: "2026-07-07T15:05:25Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-07T15:05:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:25Z" name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6.18c00a40bc33a56c namespace: kserve-ci-e2e-test resourceVersion: "14114" uid: 64d2a245-d8da-4ba9-8d38-aa9ada2ac8bc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6 namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 07a6e774-7566-403d-bcba-c2399f6e7c62 kind: Event lastTimestamp: "2026-07-07T15:05:49Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 23.744s (23.744s including waiting). Image size: 1305547511 bytes.' metadata: creationTimestamp: "2026-07-07T15:05:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:49Z" name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6.18c00a464379d980 namespace: kserve-ci-e2e-test resourceVersion: "14354" uid: cae291b1-2adb-49d7-83a9-2a8afdafdb25 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6 namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 07a6e774-7566-403d-bcba-c2399f6e7c62 kind: Event lastTimestamp: "2026-07-07T15:05:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:05:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:49Z" name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6.18c00a464a175921 namespace: kserve-ci-e2e-test resourceVersion: "14355" uid: a5dbd74e-71da-472e-9e49-4903022574e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6 namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 07a6e774-7566-403d-bcba-c2399f6e7c62 kind: Event lastTimestamp: "2026-07-07T15:05:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:05:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:49Z" name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6.18c00a464b4968f7 namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: bd9d27e6-c3e9-4f24-8b55-ae9ea6adfe34 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6 namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 07a6e774-7566-403d-bcba-c2399f6e7c62 kind: Event lastTimestamp: "2026-07-07T15:05:49Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T15:05:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:49Z" name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6.18c00a464b5a6e14 namespace: kserve-ci-e2e-test resourceVersion: "14357" uid: d846a138-4058-41f8-a7a4-fb73de559580 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6 namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 07a6e774-7566-403d-bcba-c2399f6e7c62 kind: Event lastTimestamp: "2026-07-07T15:05:51Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.252s (2.252s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T15:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:51Z" name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6.18c00a46d19f8fbf namespace: kserve-ci-e2e-test resourceVersion: "14384" uid: 9064dbfb-249a-4831-81f0-7fe868a5b3b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6 namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 07a6e774-7566-403d-bcba-c2399f6e7c62 kind: Event lastTimestamp: "2026-07-07T15:05:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:51Z" name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6.18c00a46d78f96a3 namespace: kserve-ci-e2e-test resourceVersion: "14386" uid: 1c38383c-5a53-4965-87b5-da4d5505527f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6 namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 07a6e774-7566-403d-bcba-c2399f6e7c62 kind: Event lastTimestamp: "2026-07-07T15:05:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:51Z" name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6.18c00a46d8dbb26a namespace: kserve-ci-e2e-test resourceVersion: "14387" uid: 80b445e2-9226-4b33-a70f-a57823625232 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-07T15:05:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6 namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 07a6e774-7566-403d-bcba-c2399f6e7c62 kind: Event lastTimestamp: "2026-07-07T15:06:48Z" message: 'Readiness probe failed: dial tcp 10.133.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:05:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:48Z" name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6.18c00a470e76a9ab namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: c8e5d89c-9aba-48bc-9bc2-97a8904787a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6 namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 07a6e774-7566-403d-bcba-c2399f6e7c62 kind: Event lastTimestamp: "2026-07-07T15:07:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:30Z" name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6.18c00a5dc2a4bf88 namespace: kserve-ci-e2e-test resourceVersion: "15583" uid: b9856010-145d-4a2c-a178-16975ce06799 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6 namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 07a6e774-7566-403d-bcba-c2399f6e7c62 kind: Event lastTimestamp: "2026-07-07T15:07:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:30Z" name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6.18c00a5dc2a5358d namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: 1a57b454-f117-452e-85f1-6f941c48fce9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6 namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 07a6e774-7566-403d-bcba-c2399f6e7c62 kind: Event lastTimestamp: "2026-07-07T15:07:33Z" message: 'Readiness probe failed: Get "https://10.133.0.21:8643/healthz": dial tcp 10.133.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:07:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:33Z" name: isvc-xgboost-graph-predictor-555fc69d56-bd2c6.18c00a5e92c8584d namespace: kserve-ci-e2e-test resourceVersion: "15645" uid: c3d13a2b-34ed-43c9-ba37-d6b07419af4d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "13980" uid: f8766c01-cbc1-4338-b692-a9930e9b62c5 kind: Event lastTimestamp: "2026-07-07T15:05:20Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-bd2c6' metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:05:20Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c00a3f7b62b066 namespace: kserve-ci-e2e-test resourceVersion: "13993" uid: 0a96c371-bcda-4520-a796-1ef7ae2734ef reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: 93ad21c8-e06c-4388-a344-bd9d82038e25 kind: Event lastTimestamp: "2026-07-07T15:05:20Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:05:20Z" name: isvc-xgboost-graph-predictor.18c00a3f73f8ba73 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: f08f2692-3fd9-4756-aff7-f9909c7a4723 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:05:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13982" uid: a440225c-5aaa-4c60-af23-cbdae82280c0 kind: Event lastTimestamp: "2026-07-07T15:06:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:05:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:05Z" name: isvc-xgboost-graph-predictor.18c00a42f309cf0d namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: df53c28e-1773-4529-9769-ae429a77f98c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:05:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13982" uid: a440225c-5aaa-4c60-af23-cbdae82280c0 kind: Event lastTimestamp: "2026-07-07T15:06:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:05:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:05Z" name: isvc-xgboost-graph-predictor.18c00a42f395f19b namespace: kserve-ci-e2e-test resourceVersion: "14507" uid: 6e457610-88d6-400c-8940-1e362eea1972 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:06:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14210" uid: a440225c-5aaa-4c60-af23-cbdae82280c0 kind: Event lastTimestamp: "2026-07-07T15:07:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:20Z" name: isvc-xgboost-graph-predictor.18c00a4d7047b5c4 namespace: kserve-ci-e2e-test resourceVersion: "15361" uid: 42d6b0ac-eeb9-4378-a6be-8d2dac4a1484 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:06:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14210" uid: a440225c-5aaa-4c60-af23-cbdae82280c0 kind: Event lastTimestamp: "2026-07-07T15:07:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:20Z" name: isvc-xgboost-graph-predictor.18c00a4d71725b31 namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 7d302a88-c31c-445b-99e7-7ff6edb25235 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-07T15:05:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13963" uid: e6a5211a-7f6f-473e-8e3b-9e7aae2e29cf kind: Event lastTimestamp: "2026-07-07T15:06:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:06:58Z" name: isvc-xgboost-graph.18c00a3f74922b83 namespace: kserve-ci-e2e-test resourceVersion: "15149" uid: 01d9200d-dd29-4cfe-a258-9b0f9e1f2968 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: e6a5211a-7f6f-473e-8e3b-9e7aae2e29cf kind: Event lastTimestamp: "2026-07-07T15:06:58Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-07T15:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:06:58Z" name: isvc-xgboost-graph.18c00a5673e4e1a8 namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: 59de0722-21a9-4a87-8417-90159bfee75b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:07:19.722278Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-b8bd544f6-6mjp5 namespace: kserve-ci-e2e-test resourceVersion: "15337" uid: 5623fdcd-4046-4c6b-8998-ad5e92d1880d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-b8bd544f6-6mjp5 to ip-10-0-130-158.ec2.internal metadata: creationTimestamp: "2026-07-07T15:07:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:07:19Z" name: model-chainer-b8bd544f6-6mjp5.18c00a5b4efdefc7 namespace: kserve-ci-e2e-test resourceVersion: "15342" uid: dc918adc-e778-4243-adaf-904329c6cd25 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:19Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-b8bd544f6-6mjp5 namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: 5623fdcd-4046-4c6b-8998-ad5e92d1880d kind: Event lastTimestamp: "2026-07-07T15:07:19Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T15:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:19Z" name: model-chainer-b8bd544f6-6mjp5.18c00a5b5b64ce19 namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 3f365f57-6dae-4e5a-9618-f19935eeeb85 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:20Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-b8bd544f6-6mjp5 namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: 5623fdcd-4046-4c6b-8998-ad5e92d1880d kind: Event lastTimestamp: "2026-07-07T15:07:20Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:07:20Z" name: model-chainer-b8bd544f6-6mjp5.18c00a5b8c17c361 namespace: kserve-ci-e2e-test resourceVersion: "15367" uid: 11912864-e5fa-4c48-b153-7f62dfed3fec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b8bd544f6-6mjp5 namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: 5623fdcd-4046-4c6b-8998-ad5e92d1880d kind: Event lastTimestamp: "2026-07-07T15:07:20Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:b3cb5f47c4ea9d876e5490bc83b15040508a0aac484b7e9d11de2faf2dbcdfe4" metadata: creationTimestamp: "2026-07-07T15:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:20Z" name: model-chainer-b8bd544f6-6mjp5.18c00a5b8dd97a48 namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 2e62812b-4da7-4fda-b3ce-f6d8bc4ce08e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b8bd544f6-6mjp5 namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: 5623fdcd-4046-4c6b-8998-ad5e92d1880d kind: Event lastTimestamp: "2026-07-07T15:07:22Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:b3cb5f47c4ea9d876e5490bc83b15040508a0aac484b7e9d11de2faf2dbcdfe4" in 2.157s (2.157s including waiting). Image size: 218072672 bytes.' metadata: creationTimestamp: "2026-07-07T15:07:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:22Z" name: model-chainer-b8bd544f6-6mjp5.18c00a5c0e7001ba namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: f191f5ce-64cc-4426-a683-f7de84d594e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b8bd544f6-6mjp5 namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: 5623fdcd-4046-4c6b-8998-ad5e92d1880d kind: Event lastTimestamp: "2026-07-07T15:07:22Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-07T15:07:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:22Z" name: model-chainer-b8bd544f6-6mjp5.18c00a5c110da1c6 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: ae521814-7a28-4c74-8c45-c7c70f730a17 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b8bd544f6-6mjp5 namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: 5623fdcd-4046-4c6b-8998-ad5e92d1880d kind: Event lastTimestamp: "2026-07-07T15:07:22Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-07T15:07:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:22Z" name: model-chainer-b8bd544f6-6mjp5.18c00a5c116df651 namespace: kserve-ci-e2e-test resourceVersion: "15388" uid: f97fa69a-6fa1-4613-9460-7036a9f185d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b8bd544f6-6mjp5 namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: 5623fdcd-4046-4c6b-8998-ad5e92d1880d kind: Event lastTimestamp: "2026-07-07T15:07:29Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-07T15:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:29Z" name: model-chainer-b8bd544f6-6mjp5.18c00a5da5e74416 namespace: kserve-ci-e2e-test resourceVersion: "15474" uid: 270a0ea5-16ec-418f-af56-9776f72011ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:07:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b8bd544f6-6mjp5 namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: 5623fdcd-4046-4c6b-8998-ad5e92d1880d kind: Event lastTimestamp: "2026-07-07T15:07:59Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:07:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:59Z" name: model-chainer-b8bd544f6-6mjp5.18c00a5ea83a55b0 namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 1b83da3d-af9c-4371-bd8b-4751ff7b68d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-b8bd544f6 namespace: kserve-ci-e2e-test resourceVersion: "15332" uid: ad82e4db-05ad-4214-8a17-7d96912bb19f kind: Event lastTimestamp: "2026-07-07T15:07:19Z" message: 'Created pod: model-chainer-b8bd544f6-6mjp5' metadata: creationTimestamp: "2026-07-07T15:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:19Z" name: model-chainer-b8bd544f6.18c00a5b4e612a09 namespace: kserve-ci-e2e-test resourceVersion: "15339" uid: 276022a8-aa98-4ccb-8e33-64fad811418c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: a5503445-18b9-4431-aa55-0b4a6218b449 kind: Event lastTimestamp: "2026-07-07T15:07:19Z" message: Scaled up replica set model-chainer-b8bd544f6 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:19Z" name: model-chainer.18c00a5b4cbc28d1 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 81c6e3a3-977e-444f-99e7-576181a4c215 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15349" uid: aa3ddb31-bad1-444d-9ca5-84b23c32c006 kind: Event lastTimestamp: "2026-07-07T15:07:19Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-07T15:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:07:19Z" name: model-chainer.18c00a5b546fce29 namespace: kserve-ci-e2e-test resourceVersion: "15351" uid: 88183bc1-9ecf-4336-a660-bf21258d846b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: aa3ddb31-bad1-444d-9ca5-84b23c32c006 kind: Event lastTimestamp: "2026-07-07T15:07:19Z" message: 'Failed to update status for InferenceGraph "model-chainer": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:07:19Z" name: model-chainer.18c00a5b57c27e83 namespace: kserve-ci-e2e-test resourceVersion: "15352" uid: 4b4ed2ff-4b48-44e4-9651-090469470826 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: aa3ddb31-bad1-444d-9ca5-84b23c32c006 kind: Event lastTimestamp: "2026-07-07T15:07:19Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:07:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:07:19Z" name: model-chainer.18c00a5b57c2d657 namespace: kserve-ci-e2e-test resourceVersion: "15353" uid: 289284e5-04e2-42ff-aa14-9bdffa680606 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:08:29.983096Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8996c-6b8f8959c8-dl6dm namespace: kserve-ci-e2e-test resourceVersion: "16185" uid: b38ebfe9-843d-4965-8da5-7fc858a0b4eb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-8996c-6b8f8959c8-dl6dm to ip-10-0-130-158.ec2.internal metadata: creationTimestamp: "2026-07-07T15:08:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:08:29Z" name: sequence-graph-8996c-6b8f8959c8-dl6dm.18c00a6baadcf112 namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: 3e9f81f9-16ec-49dd-9c80-79ff1b09e175 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:30Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8996c-6b8f8959c8-dl6dm namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: b38ebfe9-843d-4965-8da5-7fc858a0b4eb kind: Event lastTimestamp: "2026-07-07T15:08:30Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:08:30Z" name: sequence-graph-8996c-6b8f8959c8-dl6dm.18c00a6bc4303bc9 namespace: kserve-ci-e2e-test resourceVersion: "16208" uid: 7f79fbe3-92a0-4b39-83ca-7bed5c5e6b65 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8996c} kind: Pod name: sequence-graph-8996c-6b8f8959c8-dl6dm namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: b38ebfe9-843d-4965-8da5-7fc858a0b4eb kind: Event lastTimestamp: "2026-07-07T15:08:30Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b3cb5f47c4ea9d876e5490bc83b15040508a0aac484b7e9d11de2faf2dbcdfe4" already present on machine metadata: creationTimestamp: "2026-07-07T15:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:30Z" name: sequence-graph-8996c-6b8f8959c8-dl6dm.18c00a6bc5d3d8c7 namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: 8335254d-5cd2-41a4-8dac-d050c97dd38d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8996c} kind: Pod name: sequence-graph-8996c-6b8f8959c8-dl6dm namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: b38ebfe9-843d-4965-8da5-7fc858a0b4eb kind: Event lastTimestamp: "2026-07-07T15:08:30Z" message: 'Created container: sequence-graph-8996c' metadata: creationTimestamp: "2026-07-07T15:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:30Z" name: sequence-graph-8996c-6b8f8959c8-dl6dm.18c00a6bc88b978f namespace: kserve-ci-e2e-test resourceVersion: "16212" uid: 5ad9338a-7689-4f60-a1f7-f61f937506c7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8996c} kind: Pod name: sequence-graph-8996c-6b8f8959c8-dl6dm namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: b38ebfe9-843d-4965-8da5-7fc858a0b4eb kind: Event lastTimestamp: "2026-07-07T15:08:30Z" message: Started container sequence-graph-8996c metadata: creationTimestamp: "2026-07-07T15:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:30Z" name: sequence-graph-8996c-6b8f8959c8-dl6dm.18c00a6bc8fa0bf4 namespace: kserve-ci-e2e-test resourceVersion: "16213" uid: 0903b23e-37c9-4b47-8b81-e19275e7f642 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8996c} kind: Pod name: sequence-graph-8996c-6b8f8959c8-dl6dm namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: b38ebfe9-843d-4965-8da5-7fc858a0b4eb kind: Event lastTimestamp: "2026-07-07T15:16:44Z" message: Stopping container sequence-graph-8996c metadata: creationTimestamp: "2026-07-07T15:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:44Z" name: sequence-graph-8996c-6b8f8959c8-dl6dm.18c00adeda22d28e namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: bec951b3-5f48-45fc-9aa6-d16e7b2d5bdb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:16:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8996c} kind: Pod name: sequence-graph-8996c-6b8f8959c8-dl6dm namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: b38ebfe9-843d-4965-8da5-7fc858a0b4eb kind: Event lastTimestamp: "2026-07-07T15:17:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:12Z" name: sequence-graph-8996c-6b8f8959c8-dl6dm.18c00adf79f659e4 namespace: kserve-ci-e2e-test resourceVersion: "20180" uid: 71d6ba73-ec40-40b5-a560-04c6fc54feb9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-8996c-6b8f8959c8 namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: a77ae987-15cc-4758-94e2-96a07823e709 kind: Event lastTimestamp: "2026-07-07T15:08:29Z" message: 'Created pod: sequence-graph-8996c-6b8f8959c8-dl6dm' metadata: creationTimestamp: "2026-07-07T15:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:29Z" name: sequence-graph-8996c-6b8f8959c8.18c00a6baa280333 namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: 1ac073c0-9719-460d-b91f-0589477e2553 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-8996c namespace: kserve-ci-e2e-test resourceVersion: "16180" uid: e562fc81-00cd-414e-9926-098a3b1de782 kind: Event lastTimestamp: "2026-07-07T15:08:29Z" message: Scaled up replica set sequence-graph-8996c-6b8f8959c8 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:29Z" name: sequence-graph-8996c.18c00a6ba8aa98c7 namespace: kserve-ci-e2e-test resourceVersion: "16183" uid: 65c0bd4b-0616-4732-be7e-046e90682c66 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-8996c namespace: kserve-ci-e2e-test resourceVersion: "16199" uid: 365acd18-70aa-4236-8261-55c1ee6ce71a kind: Event lastTimestamp: "2026-07-07T15:08:30Z" message: InferenceGraph [sequence-graph-8996c] is Ready metadata: creationTimestamp: "2026-07-07T15:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:08:30Z" name: sequence-graph-8996c.18c00a6bae0ac079 namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: 44c4cbd4-25af-4213-84bd-5139abb5f47d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:08:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8996c namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: bebeb7ec-0d3b-432e-b572-b86845869855 kind: Event lastTimestamp: "2026-07-07T15:09:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:09:14Z" name: sequence-graph-8996c.18c00a6f272463c1 namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: 013d253a-b3a2-4d14-b692-415272ccd78e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:08:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8996c namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: bebeb7ec-0d3b-432e-b572-b86845869855 kind: Event lastTimestamp: "2026-07-07T15:09:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:09:14Z" name: sequence-graph-8996c.18c00a6f2778cc5f namespace: kserve-ci-e2e-test resourceVersion: "16538" uid: ca1acbeb-6654-49b3-b1d2-e2dd3dd964cd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:18:55.252046Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a9c06-867dcbbc69-72thf namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: e903711c-e517-4c31-ab7c-108c7340d474 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-a9c06-867dcbbc69-72thf to ip-10-0-130-158.ec2.internal metadata: creationTimestamp: "2026-07-07T15:18:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:18:55Z" name: sequence-graph-a9c06-867dcbbc69-72thf.18c00afd3fcbf3ea namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 2688eb01-15b0-4a4c-877a-67aecf3561c8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:55Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a9c06-867dcbbc69-72thf namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: e903711c-e517-4c31-ab7c-108c7340d474 kind: Event lastTimestamp: "2026-07-07T15:18:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-a9c06-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T15:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:55Z" name: sequence-graph-a9c06-867dcbbc69-72thf.18c00afd49ac24b4 namespace: kserve-ci-e2e-test resourceVersion: "21443" uid: 52ed4b11-e6b5-4748-b00d-f4c7b811d9f6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:56Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a9c06-867dcbbc69-72thf namespace: kserve-ci-e2e-test resourceVersion: "21434" uid: e903711c-e517-4c31-ab7c-108c7340d474 kind: Event lastTimestamp: "2026-07-07T15:18:56Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:18:56Z" name: sequence-graph-a9c06-867dcbbc69-72thf.18c00afd7d21fb30 namespace: kserve-ci-e2e-test resourceVersion: "21455" uid: 955efb66-8d2f-4f59-bf0b-445bce558bab reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a9c06} kind: Pod name: sequence-graph-a9c06-867dcbbc69-72thf namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: e903711c-e517-4c31-ab7c-108c7340d474 kind: Event lastTimestamp: "2026-07-07T15:18:56Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b3cb5f47c4ea9d876e5490bc83b15040508a0aac484b7e9d11de2faf2dbcdfe4" already present on machine metadata: creationTimestamp: "2026-07-07T15:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:56Z" name: sequence-graph-a9c06-867dcbbc69-72thf.18c00afd7e891d52 namespace: kserve-ci-e2e-test resourceVersion: "21458" uid: 0ccd72a5-f0f9-41de-b774-100394dae599 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a9c06} kind: Pod name: sequence-graph-a9c06-867dcbbc69-72thf namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: e903711c-e517-4c31-ab7c-108c7340d474 kind: Event lastTimestamp: "2026-07-07T15:18:56Z" message: 'Created container: sequence-graph-a9c06' metadata: creationTimestamp: "2026-07-07T15:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:56Z" name: sequence-graph-a9c06-867dcbbc69-72thf.18c00afd8179a6cd namespace: kserve-ci-e2e-test resourceVersion: "21459" uid: 67d1c4fd-dd20-49cf-9f41-2001162b9916 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a9c06} kind: Pod name: sequence-graph-a9c06-867dcbbc69-72thf namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: e903711c-e517-4c31-ab7c-108c7340d474 kind: Event lastTimestamp: "2026-07-07T15:18:56Z" message: Started container sequence-graph-a9c06 metadata: creationTimestamp: "2026-07-07T15:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:56Z" name: sequence-graph-a9c06-867dcbbc69-72thf.18c00afd81ea636d namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: 61fd313d-dbf8-4de1-b65b-7640d9132bc0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a9c06} kind: Pod name: sequence-graph-a9c06-867dcbbc69-72thf namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: e903711c-e517-4c31-ab7c-108c7340d474 kind: Event lastTimestamp: "2026-07-07T15:27:09Z" message: Stopping container sequence-graph-a9c06 metadata: creationTimestamp: "2026-07-07T15:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:09Z" name: sequence-graph-a9c06-867dcbbc69-72thf.18c00b7066ec790b namespace: kserve-ci-e2e-test resourceVersion: "25010" uid: 135b3694-b1d6-4c61-a009-ef8e2fc81177 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a9c06} kind: Pod name: sequence-graph-a9c06-867dcbbc69-72thf namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: e903711c-e517-4c31-ab7c-108c7340d474 kind: Event lastTimestamp: "2026-07-07T15:27:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:37Z" name: sequence-graph-a9c06-867dcbbc69-72thf.18c00b711371d4b1 namespace: kserve-ci-e2e-test resourceVersion: "25411" uid: efd9b78a-0d7e-43b0-a95f-60980392d2ee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-a9c06-867dcbbc69 namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: 1bf14060-de24-4072-9eab-d2de10269ad1 kind: Event lastTimestamp: "2026-07-07T15:18:55Z" message: 'Created pod: sequence-graph-a9c06-867dcbbc69-72thf' metadata: creationTimestamp: "2026-07-07T15:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:55Z" name: sequence-graph-a9c06-867dcbbc69.18c00afd3f68340c namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: a151912f-3dd6-4e0a-9dad-679099213492 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-a9c06 namespace: kserve-ci-e2e-test resourceVersion: "21421" uid: 42450241-ce66-4e04-916e-70107fb91f5e kind: Event lastTimestamp: "2026-07-07T15:18:55Z" message: Scaled up replica set sequence-graph-a9c06-867dcbbc69 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:55Z" name: sequence-graph-a9c06.18c00afd3e40a40f namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: e1e01873-2791-444d-8fc8-aafb99400d2f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-a9c06 namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: 443c04b7-90a8-40d8-ae10-d7eb14f9c1e3 kind: Event lastTimestamp: "2026-07-07T15:18:55Z" message: InferenceGraph [sequence-graph-a9c06] is Ready metadata: creationTimestamp: "2026-07-07T15:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:18:55Z" name: sequence-graph-a9c06.18c00afd4743a41b namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 343a12db-a529-48f9-ad34-8af643f0e3a5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a9c06 namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: 98474c7b-2775-430e-ba6b-1cb2d0c330c9 kind: Event lastTimestamp: "2026-07-07T15:19:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:19:10Z" name: sequence-graph-a9c06.18c00b00bcec13f3 namespace: kserve-ci-e2e-test resourceVersion: "21572" uid: 7b6fa857-7be6-409f-be20-6184f11b8372 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a9c06 namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: 98474c7b-2775-430e-ba6b-1cb2d0c330c9 kind: Event lastTimestamp: "2026-07-07T15:19:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:19:10Z" name: sequence-graph-a9c06.18c00b00bd5c48bb namespace: kserve-ci-e2e-test resourceVersion: "21573" uid: 66f4759e-0004-4cce-a7a6-4f9b5c08768d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a9c06 namespace: kserve-ci-e2e-test resourceVersion: "21571" uid: 98474c7b-2775-430e-ba6b-1cb2d0c330c9 kind: Event lastTimestamp: "2026-07-07T15:19:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:19:25Z" name: sequence-graph-a9c06.18c00b043c4c1ce3 namespace: kserve-ci-e2e-test resourceVersion: "21671" uid: 0cd672f0-01fd-4469-a428-595e685d6885 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a9c06 namespace: kserve-ci-e2e-test resourceVersion: "21571" uid: 98474c7b-2775-430e-ba6b-1cb2d0c330c9 kind: Event lastTimestamp: "2026-07-07T15:19:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:19:25Z" name: sequence-graph-a9c06.18c00b043cc5880d namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: dc836257-2b76-4cb7-8537-3b88cd6b251e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:17:44.922129Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e4605-6f67fbb78f-8gb4g namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 3b8ff846-ce7a-4130-bb6b-de6e50651965 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-e4605-6f67fbb78f-8gb4g to ip-10-0-130-158.ec2.internal metadata: creationTimestamp: "2026-07-07T15:17:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:17:44Z" name: sequence-graph-e4605-6f67fbb78f-8gb4g.18c00aecdfcec7ed namespace: kserve-ci-e2e-test resourceVersion: "20621" uid: 9f4afc80-14da-42ae-b20e-b453b3861fe3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:45Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e4605-6f67fbb78f-8gb4g namespace: kserve-ci-e2e-test resourceVersion: "20624" uid: 3b8ff846-ce7a-4130-bb6b-de6e50651965 kind: Event lastTimestamp: "2026-07-07T15:17:45Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:17:45Z" name: sequence-graph-e4605-6f67fbb78f-8gb4g.18c00aecf9799e3f namespace: kserve-ci-e2e-test resourceVersion: "20640" uid: 0824c395-f58c-4942-8309-41eeb8d3e981 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e4605} kind: Pod name: sequence-graph-e4605-6f67fbb78f-8gb4g namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 3b8ff846-ce7a-4130-bb6b-de6e50651965 kind: Event lastTimestamp: "2026-07-07T15:17:45Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b3cb5f47c4ea9d876e5490bc83b15040508a0aac484b7e9d11de2faf2dbcdfe4" already present on machine metadata: creationTimestamp: "2026-07-07T15:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:45Z" name: sequence-graph-e4605-6f67fbb78f-8gb4g.18c00aecfac0eb74 namespace: kserve-ci-e2e-test resourceVersion: "20642" uid: 11d1674d-229d-4a49-8940-874baac589b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e4605} kind: Pod name: sequence-graph-e4605-6f67fbb78f-8gb4g namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 3b8ff846-ce7a-4130-bb6b-de6e50651965 kind: Event lastTimestamp: "2026-07-07T15:17:45Z" message: 'Created container: sequence-graph-e4605' metadata: creationTimestamp: "2026-07-07T15:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:45Z" name: sequence-graph-e4605-6f67fbb78f-8gb4g.18c00aecfdb4cf7f namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: 106e0393-8a49-4711-af49-a7047c6936f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e4605} kind: Pod name: sequence-graph-e4605-6f67fbb78f-8gb4g namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 3b8ff846-ce7a-4130-bb6b-de6e50651965 kind: Event lastTimestamp: "2026-07-07T15:17:45Z" message: Started container sequence-graph-e4605 metadata: creationTimestamp: "2026-07-07T15:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:45Z" name: sequence-graph-e4605-6f67fbb78f-8gb4g.18c00aecfe254c39 namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: 6df81e9b-4d5a-4c09-b5b3-97432cdb0ddb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e4605} kind: Pod name: sequence-graph-e4605-6f67fbb78f-8gb4g namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 3b8ff846-ce7a-4130-bb6b-de6e50651965 kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: Stopping container sequence-graph-e4605 metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:55Z" name: sequence-graph-e4605-6f67fbb78f-8gb4g.18c00aef39250272 namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: 20982ec6-6180-4457-a050-cc0dd3527b64 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:17:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e4605} kind: Pod name: sequence-graph-e4605-6f67fbb78f-8gb4g namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 3b8ff846-ce7a-4130-bb6b-de6e50651965 kind: Event lastTimestamp: "2026-07-07T15:18:21Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:21Z" name: sequence-graph-e4605-6f67fbb78f-8gb4g.18c00aef8f2d93c5 namespace: kserve-ci-e2e-test resourceVersion: "21137" uid: 3c86f414-6753-4368-af67-b73f28b384ab reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-e4605-6f67fbb78f namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: 8e06ab65-4237-44dd-bdc2-c61fd8946570 kind: Event lastTimestamp: "2026-07-07T15:17:44Z" message: 'Created pod: sequence-graph-e4605-6f67fbb78f-8gb4g' metadata: creationTimestamp: "2026-07-07T15:17:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:44Z" name: sequence-graph-e4605-6f67fbb78f.18c00aecdf5c7085 namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 21840559-f8a6-482a-bf74-dfce8bf35e98 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-e4605 namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: 3b6ca101-d389-4fdd-8325-f579124d0129 kind: Event lastTimestamp: "2026-07-07T15:17:44Z" message: Scaled up replica set sequence-graph-e4605-6f67fbb78f from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:17:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:44Z" name: sequence-graph-e4605.18c00aecde759472 namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: 89dda178-f8b2-46a7-8627-c3f4bed13031 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-e4605 namespace: kserve-ci-e2e-test resourceVersion: "20631" uid: 604932bb-c40d-450e-9793-f3b82aac2b00 kind: Event lastTimestamp: "2026-07-07T15:17:45Z" message: InferenceGraph [sequence-graph-e4605] is Ready metadata: creationTimestamp: "2026-07-07T15:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:17:45Z" name: sequence-graph-e4605.18c00aece5190d48 namespace: kserve-ci-e2e-test resourceVersion: "20634" uid: 40882442-5ab1-4dfe-83f6-257bf253a049 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:28:44.850249Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-73e96-7b66c9667c-dkq8v namespace: kserve-ci-e2e-test resourceVersion: "26484" uid: 5ab77e72-be5f-4536-9acd-21e3d767e0f6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-73e96-7b66c9667c-dkq8v to ip-10-0-130-158.ec2.internal metadata: creationTimestamp: "2026-07-07T15:28:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:28:44Z" name: splitter-graph-73e96-7b66c9667c-dkq8v.18c00b8686969d93 namespace: kserve-ci-e2e-test resourceVersion: "26490" uid: 71513fee-8201-419e-b268-a5a0055b805b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:45Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-73e96-7b66c9667c-dkq8v namespace: kserve-ci-e2e-test resourceVersion: "26485" uid: 5ab77e72-be5f-4536-9acd-21e3d767e0f6 kind: Event lastTimestamp: "2026-07-07T15:28:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-73e96-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T15:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:28:45Z" name: splitter-graph-73e96-7b66c9667c-dkq8v.18c00b8691702fb2 namespace: kserve-ci-e2e-test resourceVersion: "26505" uid: 436771a8-3199-47d8-b573-248744cfe998 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:45Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-73e96-7b66c9667c-dkq8v namespace: kserve-ci-e2e-test resourceVersion: "26494" uid: 5ab77e72-be5f-4536-9acd-21e3d767e0f6 kind: Event lastTimestamp: "2026-07-07T15:28:45Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:28:45Z" name: splitter-graph-73e96-7b66c9667c-dkq8v.18c00b86c3c86f5a namespace: kserve-ci-e2e-test resourceVersion: "26513" uid: 3b98eb8d-7f8f-427e-bf5e-1178e4678120 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-73e96} kind: Pod name: splitter-graph-73e96-7b66c9667c-dkq8v namespace: kserve-ci-e2e-test resourceVersion: "26485" uid: 5ab77e72-be5f-4536-9acd-21e3d767e0f6 kind: Event lastTimestamp: "2026-07-07T15:28:45Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b3cb5f47c4ea9d876e5490bc83b15040508a0aac484b7e9d11de2faf2dbcdfe4" already present on machine metadata: creationTimestamp: "2026-07-07T15:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:28:45Z" name: splitter-graph-73e96-7b66c9667c-dkq8v.18c00b86c5308dce namespace: kserve-ci-e2e-test resourceVersion: "26515" uid: 7ad9e990-527e-4912-9aea-04c62d544802 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-73e96} kind: Pod name: splitter-graph-73e96-7b66c9667c-dkq8v namespace: kserve-ci-e2e-test resourceVersion: "26485" uid: 5ab77e72-be5f-4536-9acd-21e3d767e0f6 kind: Event lastTimestamp: "2026-07-07T15:28:45Z" message: 'Created container: splitter-graph-73e96' metadata: creationTimestamp: "2026-07-07T15:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:28:45Z" name: splitter-graph-73e96-7b66c9667c-dkq8v.18c00b86c81ef009 namespace: kserve-ci-e2e-test resourceVersion: "26516" uid: b7c14b85-e8c5-4fc2-80de-a7772b9e6430 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-73e96} kind: Pod name: splitter-graph-73e96-7b66c9667c-dkq8v namespace: kserve-ci-e2e-test resourceVersion: "26485" uid: 5ab77e72-be5f-4536-9acd-21e3d767e0f6 kind: Event lastTimestamp: "2026-07-07T15:28:45Z" message: Started container splitter-graph-73e96 metadata: creationTimestamp: "2026-07-07T15:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:28:45Z" name: splitter-graph-73e96-7b66c9667c-dkq8v.18c00b86c890c8c4 namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: 19c476e9-fd24-470e-8c69-67895dd35365 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:36:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-73e96} kind: Pod name: splitter-graph-73e96-7b66c9667c-dkq8v namespace: kserve-ci-e2e-test resourceVersion: "26485" uid: 5ab77e72-be5f-4536-9acd-21e3d767e0f6 kind: Event lastTimestamp: "2026-07-07T15:36:59Z" message: Stopping container splitter-graph-73e96 metadata: creationTimestamp: "2026-07-07T15:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:36:59Z" name: splitter-graph-73e96-7b66c9667c-dkq8v.18c00bf9b36d361d namespace: kserve-ci-e2e-test resourceVersion: "29950" uid: 1bf95a6a-4c4b-4505-99e5-776ccab344a1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:37:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-73e96} kind: Pod name: splitter-graph-73e96-7b66c9667c-dkq8v namespace: kserve-ci-e2e-test resourceVersion: "26485" uid: 5ab77e72-be5f-4536-9acd-21e3d767e0f6 kind: Event lastTimestamp: "2026-07-07T15:37:27Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:37:27Z" name: splitter-graph-73e96-7b66c9667c-dkq8v.18c00bfa7eee27c5 namespace: kserve-ci-e2e-test resourceVersion: "30190" uid: 80a3d57c-cf78-4707-832a-f120d27147bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-73e96-7b66c9667c namespace: kserve-ci-e2e-test resourceVersion: "26479" uid: c9efac2f-d322-4dc3-bad6-75956ecb1f32 kind: Event lastTimestamp: "2026-07-07T15:28:44Z" message: 'Created pod: splitter-graph-73e96-7b66c9667c-dkq8v' metadata: creationTimestamp: "2026-07-07T15:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:28:44Z" name: splitter-graph-73e96-7b66c9667c.18c00b8686323bd0 namespace: kserve-ci-e2e-test resourceVersion: "26486" uid: 18b7410f-7a7a-498b-af6f-06ffe857e7fd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-73e96 namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: 2eeda09d-6007-48d4-83ec-8fc052fe7134 kind: Event lastTimestamp: "2026-07-07T15:28:44Z" message: Scaled up replica set splitter-graph-73e96-7b66c9667c from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:28:44Z" name: splitter-graph-73e96.18c00b86852bcec8 namespace: kserve-ci-e2e-test resourceVersion: "26480" uid: 13249e4f-d302-49f5-8edb-554e99058aea reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-73e96 namespace: kserve-ci-e2e-test resourceVersion: "26497" uid: fd362f24-5da9-4d98-8d3c-f5542fc558bf kind: Event lastTimestamp: "2026-07-07T15:28:44Z" message: InferenceGraph [splitter-graph-73e96] is Ready metadata: creationTimestamp: "2026-07-07T15:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:28:44Z" name: splitter-graph-73e96.18c00b868b856b23 namespace: kserve-ci-e2e-test resourceVersion: "26499" uid: 4d0a550e-faa9-45bf-ad2e-f1a1fd233df1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-73e96 namespace: kserve-ci-e2e-test resourceVersion: "26472" uid: fd362f24-5da9-4d98-8d3c-f5542fc558bf kind: Event lastTimestamp: "2026-07-07T15:28:44Z" message: 'Failed to update status for InferenceGraph "splitter-graph-73e96": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-73e96": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:28:44Z" name: splitter-graph-73e96.18c00b868e85e222 namespace: kserve-ci-e2e-test resourceVersion: "26500" uid: b1aa897f-40e2-4f7e-874f-8ca5cc8f70e3 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-73e96 namespace: kserve-ci-e2e-test resourceVersion: "26472" uid: fd362f24-5da9-4d98-8d3c-f5542fc558bf kind: Event lastTimestamp: "2026-07-07T15:28:44Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-73e96": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:28:44Z" name: splitter-graph-73e96.18c00b868e8628ef namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: 60c6435d-14ce-48f5-8503-aa8dd417784b reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:28:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-73e96 namespace: kserve-ci-e2e-test resourceVersion: "26478" uid: ed2e7b9a-b27c-4560-b248-349b62b3deec kind: Event lastTimestamp: "2026-07-07T15:29:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:29:29Z" name: splitter-graph-73e96.18c00b8a04fa53aa namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: e49e4ed2-23bd-4900-b545-a404cfc90ffb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:28:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-73e96 namespace: kserve-ci-e2e-test resourceVersion: "26478" uid: ed2e7b9a-b27c-4560-b248-349b62b3deec kind: Event lastTimestamp: "2026-07-07T15:29:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:29:29Z" name: splitter-graph-73e96.18c00b8a05a382ed namespace: kserve-ci-e2e-test resourceVersion: "26836" uid: 8259c861-f5ea-4a2a-b8c5-5181e81ce026 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:27:34.510552Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-761e8-d6767b7cc-qgk52 namespace: kserve-ci-e2e-test resourceVersion: "25365" uid: 94940e5d-2041-4d76-a7d5-3ad4a9d10b42 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-761e8-d6767b7cc-qgk52 to ip-10-0-130-158.ec2.internal metadata: creationTimestamp: "2026-07-07T15:27:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:27:34Z" name: splitter-graph-761e8-d6767b7cc-qgk52.18c00b762603f8f8 namespace: kserve-ci-e2e-test resourceVersion: "25371" uid: 791dadb6-5d60-434a-aeae-f5f24b3463a3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:34Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-761e8-d6767b7cc-qgk52 namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: 94940e5d-2041-4d76-a7d5-3ad4a9d10b42 kind: Event lastTimestamp: "2026-07-07T15:27:34Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:27:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:27:34Z" name: splitter-graph-761e8-d6767b7cc-qgk52.18c00b76400e132d namespace: kserve-ci-e2e-test resourceVersion: "25388" uid: b2c6c6ed-1479-4ec0-8716-507e1b7a8389 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-761e8} kind: Pod name: splitter-graph-761e8-d6767b7cc-qgk52 namespace: kserve-ci-e2e-test resourceVersion: "25369" uid: 94940e5d-2041-4d76-a7d5-3ad4a9d10b42 kind: Event lastTimestamp: "2026-07-07T15:27:34Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b3cb5f47c4ea9d876e5490bc83b15040508a0aac484b7e9d11de2faf2dbcdfe4" already present on machine metadata: creationTimestamp: "2026-07-07T15:27:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:35Z" name: splitter-graph-761e8-d6767b7cc-qgk52.18c00b764314b6bd namespace: kserve-ci-e2e-test resourceVersion: "25390" uid: 51973a97-fd84-43ea-a9a8-eae05f92df88 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-761e8} kind: Pod name: splitter-graph-761e8-d6767b7cc-qgk52 namespace: kserve-ci-e2e-test resourceVersion: "25369" uid: 94940e5d-2041-4d76-a7d5-3ad4a9d10b42 kind: Event lastTimestamp: "2026-07-07T15:27:35Z" message: 'Created container: splitter-graph-761e8' metadata: creationTimestamp: "2026-07-07T15:27:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:35Z" name: splitter-graph-761e8-d6767b7cc-qgk52.18c00b7645dfd066 namespace: kserve-ci-e2e-test resourceVersion: "25391" uid: e8026a93-249b-4bc7-86d3-ef4085ffdf48 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-761e8} kind: Pod name: splitter-graph-761e8-d6767b7cc-qgk52 namespace: kserve-ci-e2e-test resourceVersion: "25369" uid: 94940e5d-2041-4d76-a7d5-3ad4a9d10b42 kind: Event lastTimestamp: "2026-07-07T15:27:35Z" message: Started container splitter-graph-761e8 metadata: creationTimestamp: "2026-07-07T15:27:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:35Z" name: splitter-graph-761e8-d6767b7cc-qgk52.18c00b7646577882 namespace: kserve-ci-e2e-test resourceVersion: "25392" uid: f0190d04-c2ee-406c-a582-24e407c64ef4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-761e8} kind: Pod name: splitter-graph-761e8-d6767b7cc-qgk52 namespace: kserve-ci-e2e-test resourceVersion: "25369" uid: 94940e5d-2041-4d76-a7d5-3ad4a9d10b42 kind: Event lastTimestamp: "2026-07-07T15:27:44Z" message: Stopping container splitter-graph-761e8 metadata: creationTimestamp: "2026-07-07T15:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:44Z" name: splitter-graph-761e8-d6767b7cc-qgk52.18c00b787e5b63e4 namespace: kserve-ci-e2e-test resourceVersion: "25492" uid: 0898a09c-802f-4c54-875c-ed6e403ab953 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-761e8} kind: Pod name: splitter-graph-761e8-d6767b7cc-qgk52 namespace: kserve-ci-e2e-test resourceVersion: "25369" uid: 94940e5d-2041-4d76-a7d5-3ad4a9d10b42 kind: Event lastTimestamp: "2026-07-07T15:28:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:28:11Z" name: splitter-graph-761e8-d6767b7cc-qgk52.18c00b78fa39de28 namespace: kserve-ci-e2e-test resourceVersion: "25896" uid: 8e2626eb-3320-4c23-963e-994f60f82345 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-158.ec2.internal source: component: kubelet host: ip-10-0-130-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-761e8-d6767b7cc namespace: kserve-ci-e2e-test resourceVersion: "25361" uid: 9a36dbd5-84df-4092-997b-5a3c87b211ff kind: Event lastTimestamp: "2026-07-07T15:27:34Z" message: 'Created pod: splitter-graph-761e8-d6767b7cc-qgk52' metadata: creationTimestamp: "2026-07-07T15:27:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:34Z" name: splitter-graph-761e8-d6767b7cc.18c00b76251c3151 namespace: kserve-ci-e2e-test resourceVersion: "25366" uid: b6909ca2-0455-4011-9536-dfd6fd483330 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-761e8 namespace: kserve-ci-e2e-test resourceVersion: "25359" uid: 50f8089a-1e6d-46f1-83c7-927599150e42 kind: Event lastTimestamp: "2026-07-07T15:27:34Z" message: Scaled up replica set splitter-graph-761e8-d6767b7cc from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:27:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:34Z" name: splitter-graph-761e8.18c00b7623ef20e4 namespace: kserve-ci-e2e-test resourceVersion: "25364" uid: 5acfc792-d678-41b0-bf22-cdbc509f2946 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-761e8 namespace: kserve-ci-e2e-test resourceVersion: "25380" uid: 0aa23aea-4030-42e2-a0ef-779c00393579 kind: Event lastTimestamp: "2026-07-07T15:27:34Z" message: InferenceGraph [splitter-graph-761e8] is Ready metadata: creationTimestamp: "2026-07-07T15:27:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:27:34Z" name: splitter-graph-761e8.18c00b762efa0240 namespace: kserve-ci-e2e-test resourceVersion: "25382" uid: f9c2f536-0d70-410e-9a58-87e1dcdd745b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:06:54.163607Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs namespace: kserve-ci-e2e-test resourceVersion: "14988" uid: df197fac-db99-4475-b945-3b9f0073656d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs to ip-10-0-134-178.ec2.internal metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:06:54Z" name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs.18c00a555b93b036 namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: 303f0174-3a79-46a4-a76a-1f90784896bc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:54Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: df197fac-db99-4475-b945-3b9f0073656d kind: Event lastTimestamp: "2026-07-07T15:06:54Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:06:54Z" name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs.18c00a5575dda436 namespace: kserve-ci-e2e-test resourceVersion: "15064" uid: 47cee4a5-adf1-4fda-b11e-2dd534ad6849 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: df197fac-db99-4475-b945-3b9f0073656d kind: Event lastTimestamp: "2026-07-07T15:06:54Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:54Z" name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs.18c00a557753893d namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: c3c207e0-892f-4df3-87a3-be5d62a51754 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: df197fac-db99-4475-b945-3b9f0073656d kind: Event lastTimestamp: "2026-07-07T15:06:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:54Z" name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs.18c00a557a9e9c7d namespace: kserve-ci-e2e-test resourceVersion: "15069" uid: 7431bae1-57c7-4f09-b9c1-61826c28fcff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: df197fac-db99-4475-b945-3b9f0073656d kind: Event lastTimestamp: "2026-07-07T15:06:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:54Z" name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs.18c00a557b22ad94 namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 9a294241-461b-4ce1-b9a1-461afe25dbe2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: df197fac-db99-4475-b945-3b9f0073656d kind: Event lastTimestamp: "2026-07-07T15:06:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:54Z" name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs.18c00a557b2c21a1 namespace: kserve-ci-e2e-test resourceVersion: "15071" uid: 8dc16d44-9bba-45ce-9a2b-4e80fa0adede reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: df197fac-db99-4475-b945-3b9f0073656d kind: Event lastTimestamp: "2026-07-07T15:06:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:54Z" name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs.18c00a557dc528ff namespace: kserve-ci-e2e-test resourceVersion: "15072" uid: 3cfee8b8-3d0d-412f-95fb-682346202cf7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: df197fac-db99-4475-b945-3b9f0073656d kind: Event lastTimestamp: "2026-07-07T15:06:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:54Z" name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs.18c00a557e373845 namespace: kserve-ci-e2e-test resourceVersion: "15073" uid: bf68694b-1aa3-4505-bcf3-f9654eded799 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:06:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: df197fac-db99-4475-b945-3b9f0073656d kind: Event lastTimestamp: "2026-07-07T15:07:32Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:06:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:32Z" name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs.18c00a55d6d3ba4d namespace: kserve-ci-e2e-test resourceVersion: "15629" uid: dce5fed6-e801-43cd-9cd8-193917b4d0f4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: df197fac-db99-4475-b945-3b9f0073656d kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:09Z" name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs.18c00ad696db82c4 namespace: kserve-ci-e2e-test resourceVersion: "19412" uid: d191eeaf-7cb6-4ba1-9b10-c10381998def reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: df197fac-db99-4475-b945-3b9f0073656d kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:09Z" name: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs.18c00ad696dc304b namespace: kserve-ci-e2e-test resourceVersion: "19419" uid: d85e2f38-625b-4af5-b881-ac5902bd8048 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3c9c9-predictor-854b5487bb namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: 2d090be8-14e8-4456-8934-93b9f999879c kind: Event lastTimestamp: "2026-07-07T15:06:54Z" message: 'Created pod: success-200-isvc-3c9c9-predictor-854b5487bb-zz8fs' metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:54Z" name: success-200-isvc-3c9c9-predictor-854b5487bb.18c00a555b15d386 namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: d971fce8-9b1c-48cf-b9ff-8c460b7ef4ad reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3c9c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14975" uid: 423ff2de-9e1a-41ee-8313-330234c69bb0 kind: Event lastTimestamp: "2026-07-07T15:06:54Z" message: Scaled up replica set success-200-isvc-3c9c9-predictor-854b5487bb from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:54Z" name: success-200-isvc-3c9c9-predictor.18c00a555470f8d2 namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: a3c7729b-723c-4338-a3b3-b1b0202131d8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3c9c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14976" uid: c94ce8bc-789e-4f25-8d40-46b93a35915e kind: Event lastTimestamp: "2026-07-07T15:07:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:09Z" name: success-200-isvc-3c9c9-predictor.18c00a58d2f68717 namespace: kserve-ci-e2e-test resourceVersion: "15241" uid: 19cf5492-730f-48fa-8114-d96a322cdb58 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3c9c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14976" uid: c94ce8bc-789e-4f25-8d40-46b93a35915e kind: Event lastTimestamp: "2026-07-07T15:07:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:09Z" name: success-200-isvc-3c9c9-predictor.18c00a58d3b5e52c namespace: kserve-ci-e2e-test resourceVersion: "15243" uid: dde15ed6-f443-4a0b-b0b2-ce11a87037b6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:07:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3c9c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "15239" uid: c94ce8bc-789e-4f25-8d40-46b93a35915e kind: Event lastTimestamp: "2026-07-07T15:08:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:07:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:09Z" name: success-200-isvc-3c9c9-predictor.18c00a5c5348eace namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: 02e95905-d57e-4819-a30b-0011f2ff7dbd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:07:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3c9c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "15239" uid: c94ce8bc-789e-4f25-8d40-46b93a35915e kind: Event lastTimestamp: "2026-07-07T15:08:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:07:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:09Z" name: success-200-isvc-3c9c9-predictor.18c00a5c53e89778 namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: a47060a7-acda-4dd0-b2d1-0167d5202011 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:06:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3c9c9 namespace: kserve-ci-e2e-test resourceVersion: "14969" uid: d9ab1dc0-be1e-4492-b484-3f4bff0af846 kind: Event lastTimestamp: "2026-07-07T15:07:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:07:42Z" name: success-200-isvc-3c9c9.18c00a5554a04ec6 namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: d0f6ed72-a4fc-4c6f-a3a9-246ad83780d8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3c9c9 namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: d9ab1dc0-be1e-4492-b484-3f4bff0af846 kind: Event lastTimestamp: "2026-07-07T15:07:42Z" message: InferenceService [success-200-isvc-3c9c9] is Ready metadata: creationTimestamp: "2026-07-07T15:07:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:07:42Z" name: success-200-isvc-3c9c9.18c00a6092b1020b namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: 0b59506a-19b7-45d7-87d6-10035381a6cd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:16:09.208685Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc namespace: kserve-ci-e2e-test resourceVersion: "19403" uid: 8014b647-4160-4398-b762-80112c6978fe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc to ip-10-0-134-178.ec2.internal metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:16:09Z" name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc.18c00ad696d75975 namespace: kserve-ci-e2e-test resourceVersion: "19414" uid: 96590a7f-af16-48eb-acaa-d9c2019a0a3f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 8014b647-4160-4398-b762-80112c6978fe kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-542f9-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:09Z" name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc.18c00ad6a19f36f6 namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: e131ca06-435d-4740-a421-548ec3a9154a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc namespace: kserve-ci-e2e-test resourceVersion: "19425" uid: 8014b647-4160-4398-b762-80112c6978fe kind: Event lastTimestamp: "2026-07-07T15:16:11Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:16:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:16:11Z" name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc.18c00ad70a96a1bf namespace: kserve-ci-e2e-test resourceVersion: "19491" uid: 96a036f5-1a34-453f-8fe7-9fef522f557a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 8014b647-4160-4398-b762-80112c6978fe kind: Event lastTimestamp: "2026-07-07T15:16:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:16:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:11Z" name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc.18c00ad70c00a429 namespace: kserve-ci-e2e-test resourceVersion: "19493" uid: 7e76b9f0-bd00-41a7-a569-1316c59e457c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 8014b647-4160-4398-b762-80112c6978fe kind: Event lastTimestamp: "2026-07-07T15:16:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:16:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:11Z" name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc.18c00ad70f35049f namespace: kserve-ci-e2e-test resourceVersion: "19495" uid: 57db60d2-decc-479f-8b98-0dbcd69137ca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 8014b647-4160-4398-b762-80112c6978fe kind: Event lastTimestamp: "2026-07-07T15:16:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:16:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:11Z" name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc.18c00ad70fa5f682 namespace: kserve-ci-e2e-test resourceVersion: "19496" uid: f64f1a5c-1005-4427-a427-4b71995f14bd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 8014b647-4160-4398-b762-80112c6978fe kind: Event lastTimestamp: "2026-07-07T15:16:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:16:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:11Z" name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc.18c00ad70fafe0d2 namespace: kserve-ci-e2e-test resourceVersion: "19497" uid: 945b576c-6351-4422-bd51-50ffc4e13f23 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 8014b647-4160-4398-b762-80112c6978fe kind: Event lastTimestamp: "2026-07-07T15:16:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:16:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:11Z" name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc.18c00ad712e714ae namespace: kserve-ci-e2e-test resourceVersion: "19501" uid: e67ced79-971f-4e27-9154-518338dabd39 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 8014b647-4160-4398-b762-80112c6978fe kind: Event lastTimestamp: "2026-07-07T15:16:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:16:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:11Z" name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc.18c00ad713514825 namespace: kserve-ci-e2e-test resourceVersion: "19502" uid: 056db1a7-95c7-42bb-8677-7feecfdf9296 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:16:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 8014b647-4160-4398-b762-80112c6978fe kind: Event lastTimestamp: "2026-07-07T15:16:47Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:16:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:47Z" name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc.18c00ad736f64afc namespace: kserve-ci-e2e-test resourceVersion: "19925" uid: 5b5895fe-be7c-485b-a491-5e9231336a20 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 8014b647-4160-4398-b762-80112c6978fe kind: Event lastTimestamp: "2026-07-07T15:17:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:17:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:19Z" name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc.18c00ae6fdad2d78 namespace: kserve-ci-e2e-test resourceVersion: "20328" uid: 4ee504d5-2422-487e-a0a1-8fcfeb4be2eb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 8014b647-4160-4398-b762-80112c6978fe kind: Event lastTimestamp: "2026-07-07T15:17:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:17:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:19Z" name: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc.18c00ae6fdadb308 namespace: kserve-ci-e2e-test resourceVersion: "20331" uid: 38e612d3-220a-4287-9bcc-0a2e5fd2a93b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-542f9-predictor-67b4c56d5b namespace: kserve-ci-e2e-test resourceVersion: "19382" uid: 18376269-08b0-4513-9d81-2f0e9ed070be kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: 'Created pod: success-200-isvc-542f9-predictor-67b4c56d5b-4mbqc' metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:16:09Z" name: success-200-isvc-542f9-predictor-67b4c56d5b.18c00ad695e8c4c1 namespace: kserve-ci-e2e-test resourceVersion: "19409" uid: ce9c18a0-8601-4362-b252-6d64f9d2aba5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-542f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19380" uid: 4580b267-711e-4d43-acfc-6bf2029691cf kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: Scaled up replica set success-200-isvc-542f9-predictor-67b4c56d5b from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:16:09Z" name: success-200-isvc-542f9-predictor.18c00ad6913efd28 namespace: kserve-ci-e2e-test resourceVersion: "19384" uid: 24c4e209-8bb5-4fb4-80d7-8d3c987323f3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:16:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-542f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19381" uid: 9906aecb-d1ed-4df4-985d-263abca2bce5 kind: Event lastTimestamp: "2026-07-07T15:16:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:16:39Z" name: success-200-isvc-542f9-predictor.18c00ada103e7e19 namespace: kserve-ci-e2e-test resourceVersion: "19719" uid: 50d6f826-e08b-4e64-bac0-e3032fddbf04 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:16:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-542f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19381" uid: 9906aecb-d1ed-4df4-985d-263abca2bce5 kind: Event lastTimestamp: "2026-07-07T15:16:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:16:39Z" name: success-200-isvc-542f9-predictor.18c00ada108743e3 namespace: kserve-ci-e2e-test resourceVersion: "19720" uid: 13377dba-3da2-4ef7-94a2-ed9056a09ffe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:16:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-542f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19605" uid: 9906aecb-d1ed-4df4-985d-263abca2bce5 kind: Event lastTimestamp: "2026-07-07T15:17:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:09Z" name: success-200-isvc-542f9-predictor.18c00ae10ec5dc0d namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: 52c68dec-1cde-4de8-b8da-e397cfbdceb8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:16:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-542f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "19605" uid: 9906aecb-d1ed-4df4-985d-263abca2bce5 kind: Event lastTimestamp: "2026-07-07T15:17:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:09Z" name: success-200-isvc-542f9-predictor.18c00ae10f1c166c namespace: kserve-ci-e2e-test resourceVersion: "20126" uid: 9a330940-2dc9-45a3-96f3-095ad8761fc5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-542f9 namespace: kserve-ci-e2e-test resourceVersion: "19372" uid: db1b65f0-3585-46d2-9126-7f0e3a3fb15b kind: Event lastTimestamp: "2026-07-07T15:16:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:16:58Z" name: success-200-isvc-542f9.18c00ad69144719a namespace: kserve-ci-e2e-test resourceVersion: "20032" uid: 149dff31-8f95-4d51-81bb-73be4d412100 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-542f9 namespace: kserve-ci-e2e-test resourceVersion: "19372" uid: db1b65f0-3585-46d2-9126-7f0e3a3fb15b kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: 'Failed to update status for InferenceService "success-200-isvc-542f9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-542f9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:16:09Z" name: success-200-isvc-542f9.18c00ad691cbe105 namespace: kserve-ci-e2e-test resourceVersion: "19386" uid: bf65a51a-3d93-4af8-8612-aa5b2afc3141 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-542f9 namespace: kserve-ci-e2e-test resourceVersion: "19372" uid: db1b65f0-3585-46d2-9126-7f0e3a3fb15b kind: Event lastTimestamp: "2026-07-07T15:16:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-542f9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:16:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:16:09Z" name: success-200-isvc-542f9.18c00ad691cc3ae8 namespace: kserve-ci-e2e-test resourceVersion: "19387" uid: c7f7a3e5-fe3e-407c-aa93-74e7cb51bf1d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-542f9 namespace: kserve-ci-e2e-test resourceVersion: "20030" uid: db1b65f0-3585-46d2-9126-7f0e3a3fb15b kind: Event lastTimestamp: "2026-07-07T15:16:58Z" message: InferenceService [success-200-isvc-542f9] is Ready metadata: creationTimestamp: "2026-07-07T15:16:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:16:58Z" name: success-200-isvc-542f9.18c00ae1f3af8464 namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: d5cc14b0-e48d-4dbd-a7e6-384532d6f768 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:17:19.577779Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: f4b8ce98-14b0-4949-8976-836e72025179 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5aeac-predictor-74845fc44b-94nkv to ip-10-0-134-178.ec2.internal metadata: creationTimestamp: "2026-07-07T15:17:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:17:19Z" name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv.18c00ae6f92a8c5c namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: 3cf70a79-a995-4ad4-8185-0ac770c49adc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:20Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv namespace: kserve-ci-e2e-test resourceVersion: "20311" uid: f4b8ce98-14b0-4949-8976-836e72025179 kind: Event lastTimestamp: "2026-07-07T15:17:20Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:17:20Z" name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv.18c00ae7128a3585 namespace: kserve-ci-e2e-test resourceVersion: "20357" uid: 0016a19c-ca09-4a71-97dc-3d3e45231744 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: f4b8ce98-14b0-4949-8976-836e72025179 kind: Event lastTimestamp: "2026-07-07T15:17:20Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:20Z" name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv.18c00ae7140e8737 namespace: kserve-ci-e2e-test resourceVersion: "20359" uid: 40bb140d-eef5-4687-860a-3c457fe2fc73 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: f4b8ce98-14b0-4949-8976-836e72025179 kind: Event lastTimestamp: "2026-07-07T15:17:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:20Z" name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv.18c00ae7172ece1a namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: 68e2c166-54cc-4283-97dc-100fab346b11 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: f4b8ce98-14b0-4949-8976-836e72025179 kind: Event lastTimestamp: "2026-07-07T15:17:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:20Z" name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv.18c00ae7179bb090 namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 79f7acb9-ea4f-4b63-9417-880382e36234 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: f4b8ce98-14b0-4949-8976-836e72025179 kind: Event lastTimestamp: "2026-07-07T15:17:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:20Z" name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv.18c00ae717a4aa27 namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: 4c40fc2b-5cc7-463b-aca5-ed164f19a9c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: f4b8ce98-14b0-4949-8976-836e72025179 kind: Event lastTimestamp: "2026-07-07T15:17:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:20Z" name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv.18c00ae71a56c4a5 namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: c5b4567a-51cc-44f7-9501-965313d43ded reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: f4b8ce98-14b0-4949-8976-836e72025179 kind: Event lastTimestamp: "2026-07-07T15:17:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:20Z" name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv.18c00ae71ac423a4 namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: 2730ebe6-2c80-4daa-ae58-8a31ef9500e3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T15:17:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: f4b8ce98-14b0-4949-8976-836e72025179 kind: Event lastTimestamp: "2026-07-07T15:26:37Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:17:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:37Z" name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv.18c00ae75bbe0d3f namespace: kserve-ci-e2e-test resourceVersion: "24741" uid: bf36359e-af5d-4938-9eb6-8b38afbdc7ad reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: f4b8ce98-14b0-4949-8976-836e72025179 kind: Event lastTimestamp: "2026-07-07T15:26:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:34Z" name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv.18c00b68381789e2 namespace: kserve-ci-e2e-test resourceVersion: "24650" uid: 6c6f2461-faf3-4a22-b46b-8d5855fff272 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: f4b8ce98-14b0-4949-8976-836e72025179 kind: Event lastTimestamp: "2026-07-07T15:26:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:34Z" name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv.18c00b683817facc namespace: kserve-ci-e2e-test resourceVersion: "24646" uid: 54767a86-e729-4a18-8b36-92fc4495f799 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: f4b8ce98-14b0-4949-8976-836e72025179 kind: Event lastTimestamp: "2026-07-07T15:26:37Z" message: 'Readiness probe failed: Get "https://10.132.0.42:8643/healthz": dial tcp 10.132.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:37Z" name: success-200-isvc-5aeac-predictor-74845fc44b-94nkv.18c00b68d033eba6 namespace: kserve-ci-e2e-test resourceVersion: "24740" uid: a32b5fb5-1c63-41c4-8c5b-386ed00f1cff reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5aeac-predictor-74845fc44b namespace: kserve-ci-e2e-test resourceVersion: "20278" uid: cb15ca37-014d-4dbc-9303-c1751abb14cb kind: Event lastTimestamp: "2026-07-07T15:17:19Z" message: 'Created pod: success-200-isvc-5aeac-predictor-74845fc44b-94nkv' metadata: creationTimestamp: "2026-07-07T15:17:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:19Z" name: success-200-isvc-5aeac-predictor-74845fc44b.18c00ae6f809d4da namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: ccab6cca-0b2e-4b89-af80-836808699e61 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5aeac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20273" uid: 83fbc6c4-7341-4c9a-9d40-0828fb6e4d60 kind: Event lastTimestamp: "2026-07-07T15:17:19Z" message: Scaled up replica set success-200-isvc-5aeac-predictor-74845fc44b from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:17:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:19Z" name: success-200-isvc-5aeac-predictor.18c00ae6f2bd0f33 namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: 55dc518d-6fcc-4165-93d0-a02755fe00e2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:17:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5aeac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: 2b633f20-c174-49be-83c7-1ee5be12db9d kind: Event lastTimestamp: "2026-07-07T15:18:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:17:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:34Z" name: success-200-isvc-5aeac-predictor.18c00aea71184fc8 namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 254a1d91-d76f-4aba-a562-4d508c5ac06f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:17:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5aeac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: 2b633f20-c174-49be-83c7-1ee5be12db9d kind: Event lastTimestamp: "2026-07-07T15:18:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:17:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:34Z" name: success-200-isvc-5aeac-predictor.18c00aea718a3fe5 namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: d25af65b-b906-4ca3-b514-fe53d882e10f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:17:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5aeac namespace: kserve-ci-e2e-test resourceVersion: "20258" uid: 1a67072a-9efb-4a04-abcb-3d4119905f82 kind: Event lastTimestamp: "2026-07-07T15:18:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:17:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:18:07Z" name: success-200-isvc-5aeac.18c00ae6f3148ed3 namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 8b7c3b3f-e3e1-4f1d-9d2d-9385d1e9c61b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5aeac namespace: kserve-ci-e2e-test resourceVersion: "20258" uid: 1a67072a-9efb-4a04-abcb-3d4119905f82 kind: Event lastTimestamp: "2026-07-07T15:17:19Z" message: 'Failed to update status for InferenceService "success-200-isvc-5aeac": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5aeac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:17:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:17:19Z" name: success-200-isvc-5aeac.18c00ae6f507739c namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: cddc45e4-fc6c-413c-bd98-29f840808008 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5aeac namespace: kserve-ci-e2e-test resourceVersion: "20258" uid: 1a67072a-9efb-4a04-abcb-3d4119905f82 kind: Event lastTimestamp: "2026-07-07T15:17:19Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5aeac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:17:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:17:19Z" name: success-200-isvc-5aeac.18c00ae6f507cdc8 namespace: kserve-ci-e2e-test resourceVersion: "20288" uid: ae4b69d4-8675-41a7-a9b1-a4b81e526226 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5aeac namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: 1a67072a-9efb-4a04-abcb-3d4119905f82 kind: Event lastTimestamp: "2026-07-07T15:18:07Z" message: InferenceService [success-200-isvc-5aeac] is Ready metadata: creationTimestamp: "2026-07-07T15:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:18:07Z" name: success-200-isvc-5aeac.18c00af21a4cf934 namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: e21b7456-a5e0-4174-8b78-14e6f274e831 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:27:44.845715Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c namespace: kserve-ci-e2e-test resourceVersion: "25531" uid: 3fe24f63-5886-46fc-ab27-a003d32f4ef7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-73e96-predictor-755bfdcbd-qkn2c to ip-10-0-134-178.ec2.internal metadata: creationTimestamp: "2026-07-07T15:27:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:27:44Z" name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c.18c00b788e0a204b namespace: kserve-ci-e2e-test resourceVersion: "25536" uid: ae6c8bcb-041b-498d-925a-0aae9b53e681 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c namespace: kserve-ci-e2e-test resourceVersion: "25534" uid: 3fe24f63-5886-46fc-ab27-a003d32f4ef7 kind: Event lastTimestamp: "2026-07-07T15:27:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-73e96-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T15:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:45Z" name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c.18c00b7898c20762 namespace: kserve-ci-e2e-test resourceVersion: "25579" uid: 12cdd0fa-f8cf-4995-aa17-9c6bfe2eb67b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c namespace: kserve-ci-e2e-test resourceVersion: "25544" uid: 3fe24f63-5886-46fc-ab27-a003d32f4ef7 kind: Event lastTimestamp: "2026-07-07T15:27:45Z" message: Add eth0 [10.132.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:27:45Z" name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c.18c00b78cb9f57e5 namespace: kserve-ci-e2e-test resourceVersion: "25600" uid: c2e9abbe-6c08-43b1-9117-89f264be690a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c namespace: kserve-ci-e2e-test resourceVersion: "25534" uid: 3fe24f63-5886-46fc-ab27-a003d32f4ef7 kind: Event lastTimestamp: "2026-07-07T15:27:45Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:45Z" name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c.18c00b78cdfb0cf6 namespace: kserve-ci-e2e-test resourceVersion: "25602" uid: 219a4b07-fa12-47b5-beaa-259960c63850 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c namespace: kserve-ci-e2e-test resourceVersion: "25534" uid: 3fe24f63-5886-46fc-ab27-a003d32f4ef7 kind: Event lastTimestamp: "2026-07-07T15:27:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:45Z" name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c.18c00b78d16324f9 namespace: kserve-ci-e2e-test resourceVersion: "25604" uid: 8b3870cb-4820-4658-9b43-58d8ca7b1104 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c namespace: kserve-ci-e2e-test resourceVersion: "25534" uid: 3fe24f63-5886-46fc-ab27-a003d32f4ef7 kind: Event lastTimestamp: "2026-07-07T15:27:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:45Z" name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c.18c00b78d1d34fe8 namespace: kserve-ci-e2e-test resourceVersion: "25606" uid: 2eabd080-40af-4cbc-a198-480246b828fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c namespace: kserve-ci-e2e-test resourceVersion: "25534" uid: 3fe24f63-5886-46fc-ab27-a003d32f4ef7 kind: Event lastTimestamp: "2026-07-07T15:27:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:45Z" name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c.18c00b78d1dbd815 namespace: kserve-ci-e2e-test resourceVersion: "25607" uid: 05670006-2d21-4633-8a88-8a5b8bfe68cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c namespace: kserve-ci-e2e-test resourceVersion: "25534" uid: 3fe24f63-5886-46fc-ab27-a003d32f4ef7 kind: Event lastTimestamp: "2026-07-07T15:27:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:46Z" name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c.18c00b78d51fec3e namespace: kserve-ci-e2e-test resourceVersion: "25609" uid: 8d35fa01-a592-41b7-bb84-3a57ae54d0a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c namespace: kserve-ci-e2e-test resourceVersion: "25534" uid: 3fe24f63-5886-46fc-ab27-a003d32f4ef7 kind: Event lastTimestamp: "2026-07-07T15:27:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:46Z" name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c.18c00b78d58cb355 namespace: kserve-ci-e2e-test resourceVersion: "25610" uid: 4159297e-da1f-4140-aa80-0a2c1b712c1c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T15:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c namespace: kserve-ci-e2e-test resourceVersion: "25534" uid: 3fe24f63-5886-46fc-ab27-a003d32f4ef7 kind: Event lastTimestamp: "2026-07-07T15:37:02Z" message: 'Readiness probe failed: dial tcp 10.132.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:37:02Z" name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c.18c00b78e0f7fa7b namespace: kserve-ci-e2e-test resourceVersion: "30003" uid: cd1843d9-240e-4e00-8f57-be24c0d6d810 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:36:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c namespace: kserve-ci-e2e-test resourceVersion: "25534" uid: 3fe24f63-5886-46fc-ab27-a003d32f4ef7 kind: Event lastTimestamp: "2026-07-07T15:36:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:36:59Z" name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c.18c00bf9bd96ff8d namespace: kserve-ci-e2e-test resourceVersion: "29968" uid: eb4c757d-37cd-408d-a34c-d6fb8bab8eb5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:36:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c namespace: kserve-ci-e2e-test resourceVersion: "25534" uid: 3fe24f63-5886-46fc-ab27-a003d32f4ef7 kind: Event lastTimestamp: "2026-07-07T15:36:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:36:59Z" name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c.18c00bf9bd97657f namespace: kserve-ci-e2e-test resourceVersion: "29970" uid: be14b944-7f6a-496f-9cb4-ba751e983b4f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:37:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c namespace: kserve-ci-e2e-test resourceVersion: "25534" uid: 3fe24f63-5886-46fc-ab27-a003d32f4ef7 kind: Event lastTimestamp: "2026-07-07T15:37:02Z" message: 'Readiness probe failed: Get "https://10.132.0.46:8643/healthz": dial tcp 10.132.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:37:02Z" name: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c.18c00bfa556646d5 namespace: kserve-ci-e2e-test resourceVersion: "30002" uid: 095f2acf-aba2-45f6-8dda-5c977e98ff4e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-73e96-predictor-755bfdcbd namespace: kserve-ci-e2e-test resourceVersion: "25524" uid: c1d2405e-b3e4-4835-bbee-0e1a15007713 kind: Event lastTimestamp: "2026-07-07T15:27:44Z" message: 'Created pod: success-200-isvc-73e96-predictor-755bfdcbd-qkn2c' metadata: creationTimestamp: "2026-07-07T15:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:44Z" name: success-200-isvc-73e96-predictor-755bfdcbd.18c00b788d4d5636 namespace: kserve-ci-e2e-test resourceVersion: "25533" uid: 8ed8fa8c-a0d7-4c02-8302-4e4205afcfcd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-73e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "25520" uid: 8e8ef54b-7cd8-41f4-8b8a-761e3b95b516 kind: Event lastTimestamp: "2026-07-07T15:27:44Z" message: Scaled up replica set success-200-isvc-73e96-predictor-755bfdcbd from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:44Z" name: success-200-isvc-73e96-predictor.18c00b788a583339 namespace: kserve-ci-e2e-test resourceVersion: "25526" uid: d79aee41-282c-4953-9b7d-ebadb23efd99 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:27:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-73e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "25522" uid: 3196f0f5-a8d4-4894-acdb-57412370979a kind: Event lastTimestamp: "2026-07-07T15:28:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:27:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:28:29Z" name: success-200-isvc-73e96-predictor.18c00b7c0821bb0d namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: 65b1ffd3-880e-4aa9-a929-3797cddde649 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:27:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-73e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "25522" uid: 3196f0f5-a8d4-4894-acdb-57412370979a kind: Event lastTimestamp: "2026-07-07T15:28:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:27:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:28:29Z" name: success-200-isvc-73e96-predictor.18c00b7c08ed3e29 namespace: kserve-ci-e2e-test resourceVersion: "26338" uid: 1f1cf2a1-3cba-4945-a4f8-132fa347ce6a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:28:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-73e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "25768" uid: 3196f0f5-a8d4-4894-acdb-57412370979a kind: Event lastTimestamp: "2026-07-07T15:28:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:28:59Z" name: success-200-isvc-73e96-predictor.18c00b86859c6caf namespace: kserve-ci-e2e-test resourceVersion: "26624" uid: 471641be-fa41-4cdc-9cf6-441477f86b1b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:28:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-73e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "25768" uid: 3196f0f5-a8d4-4894-acdb-57412370979a kind: Event lastTimestamp: "2026-07-07T15:28:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:28:59Z" name: success-200-isvc-73e96-predictor.18c00b86862f2161 namespace: kserve-ci-e2e-test resourceVersion: "26625" uid: 486d2842-a90a-4573-bc93-010b33de37da reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:27:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-73e96 namespace: kserve-ci-e2e-test resourceVersion: "25498" uid: 25c6bf81-f954-48b8-8539-84cbc018f7a0 kind: Event lastTimestamp: "2026-07-07T15:28:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:28:32Z" name: success-200-isvc-73e96.18c00b7889db1d46 namespace: kserve-ci-e2e-test resourceVersion: "26366" uid: 15677f11-5430-4cc2-a45e-6398dfe74295 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-73e96 namespace: kserve-ci-e2e-test resourceVersion: "25498" uid: 25c6bf81-f954-48b8-8539-84cbc018f7a0 kind: Event lastTimestamp: "2026-07-07T15:27:44Z" message: 'Failed to update status for InferenceService "success-200-isvc-73e96": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-73e96": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:27:44Z" name: success-200-isvc-73e96.18c00b788a88e98b namespace: kserve-ci-e2e-test resourceVersion: "25528" uid: 3a295408-ba68-4bf9-9f8e-9e208522358d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-73e96 namespace: kserve-ci-e2e-test resourceVersion: "25498" uid: 25c6bf81-f954-48b8-8539-84cbc018f7a0 kind: Event lastTimestamp: "2026-07-07T15:27:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-73e96": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:27:44Z" name: success-200-isvc-73e96.18c00b788a895561 namespace: kserve-ci-e2e-test resourceVersion: "25529" uid: efc2cb04-f1e3-49ba-a289-d15b67905192 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-73e96 namespace: kserve-ci-e2e-test resourceVersion: "26364" uid: 25c6bf81-f954-48b8-8539-84cbc018f7a0 kind: Event lastTimestamp: "2026-07-07T15:28:32Z" message: InferenceService [success-200-isvc-73e96] is Ready metadata: creationTimestamp: "2026-07-07T15:28:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:28:32Z" name: success-200-isvc-73e96.18c00b839ceb533b namespace: kserve-ci-e2e-test resourceVersion: "26365" uid: 318538b4-b6de-460f-bc05-e80233d39a5d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:26:34.685449Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-761e8-predictor-6d97467478-f9qnw namespace: kserve-ci-e2e-test resourceVersion: "24637" uid: e6ee6064-4c5d-411b-b0e5-57823a6a9fa9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-761e8-predictor-6d97467478-f9qnw to ip-10-0-134-178.ec2.internal metadata: creationTimestamp: "2026-07-07T15:26:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:26:34Z" name: success-200-isvc-761e8-predictor-6d97467478-f9qnw.18c00b683829622e namespace: kserve-ci-e2e-test resourceVersion: "24648" uid: fe871bb2-ebc2-46c4-a5a1-e56ee99fd1ff reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:35Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-761e8-predictor-6d97467478-f9qnw namespace: kserve-ci-e2e-test resourceVersion: "24662" uid: e6ee6064-4c5d-411b-b0e5-57823a6a9fa9 kind: Event lastTimestamp: "2026-07-07T15:26:35Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:26:35Z" name: success-200-isvc-761e8-predictor-6d97467478-f9qnw.18c00b68523e859b namespace: kserve-ci-e2e-test resourceVersion: "24698" uid: 3f14dfc9-470c-4010-add4-65dbd6e6d7e2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-761e8-predictor-6d97467478-f9qnw namespace: kserve-ci-e2e-test resourceVersion: "24640" uid: e6ee6064-4c5d-411b-b0e5-57823a6a9fa9 kind: Event lastTimestamp: "2026-07-07T15:26:35Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:35Z" name: success-200-isvc-761e8-predictor-6d97467478-f9qnw.18c00b6854094f3e namespace: kserve-ci-e2e-test resourceVersion: "24700" uid: be109d17-35a6-433a-9b0f-898dc3807cd4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-761e8-predictor-6d97467478-f9qnw namespace: kserve-ci-e2e-test resourceVersion: "24640" uid: e6ee6064-4c5d-411b-b0e5-57823a6a9fa9 kind: Event lastTimestamp: "2026-07-07T15:26:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:35Z" name: success-200-isvc-761e8-predictor-6d97467478-f9qnw.18c00b685799e7f1 namespace: kserve-ci-e2e-test resourceVersion: "24701" uid: 946f8c48-2b45-4651-9562-c7bb8e1e261c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-761e8-predictor-6d97467478-f9qnw namespace: kserve-ci-e2e-test resourceVersion: "24640" uid: e6ee6064-4c5d-411b-b0e5-57823a6a9fa9 kind: Event lastTimestamp: "2026-07-07T15:26:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:35Z" name: success-200-isvc-761e8-predictor-6d97467478-f9qnw.18c00b68580688f9 namespace: kserve-ci-e2e-test resourceVersion: "24703" uid: 629e6625-6dc8-4a56-b4b8-a17825808b9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-761e8-predictor-6d97467478-f9qnw namespace: kserve-ci-e2e-test resourceVersion: "24640" uid: e6ee6064-4c5d-411b-b0e5-57823a6a9fa9 kind: Event lastTimestamp: "2026-07-07T15:26:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:35Z" name: success-200-isvc-761e8-predictor-6d97467478-f9qnw.18c00b68580f486a namespace: kserve-ci-e2e-test resourceVersion: "24704" uid: 1e3d5cac-ec3e-4c68-b9ee-16b207fa0fe9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-761e8-predictor-6d97467478-f9qnw namespace: kserve-ci-e2e-test resourceVersion: "24640" uid: e6ee6064-4c5d-411b-b0e5-57823a6a9fa9 kind: Event lastTimestamp: "2026-07-07T15:26:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:35Z" name: success-200-isvc-761e8-predictor-6d97467478-f9qnw.18c00b685a7fe5af namespace: kserve-ci-e2e-test resourceVersion: "24707" uid: ca3669e4-a8b5-4406-8143-da6e5f1d783c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-761e8-predictor-6d97467478-f9qnw namespace: kserve-ci-e2e-test resourceVersion: "24640" uid: e6ee6064-4c5d-411b-b0e5-57823a6a9fa9 kind: Event lastTimestamp: "2026-07-07T15:26:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:26:35Z" name: success-200-isvc-761e8-predictor-6d97467478-f9qnw.18c00b685af209b8 namespace: kserve-ci-e2e-test resourceVersion: "24708" uid: bf5f6b10-32b2-4b0e-818d-aabe09a377c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:26:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-761e8-predictor-6d97467478-f9qnw namespace: kserve-ci-e2e-test resourceVersion: "24640" uid: e6ee6064-4c5d-411b-b0e5-57823a6a9fa9 kind: Event lastTimestamp: "2026-07-07T15:27:11Z" message: 'Readiness probe failed: dial tcp 10.132.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:11Z" name: success-200-isvc-761e8-predictor-6d97467478-f9qnw.18c00b687fecaf50 namespace: kserve-ci-e2e-test resourceVersion: "25141" uid: c68012c1-2352-407f-81ba-7182a580e578 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-761e8-predictor-6d97467478-f9qnw namespace: kserve-ci-e2e-test resourceVersion: "24640" uid: e6ee6064-4c5d-411b-b0e5-57823a6a9fa9 kind: Event lastTimestamp: "2026-07-07T15:27:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:44Z" name: success-200-isvc-761e8-predictor-6d97467478-f9qnw.18c00b7895449614 namespace: kserve-ci-e2e-test resourceVersion: "25566" uid: 3a26f966-8bee-49a9-ac34-174bbaff842c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-761e8-predictor-6d97467478-f9qnw namespace: kserve-ci-e2e-test resourceVersion: "24640" uid: e6ee6064-4c5d-411b-b0e5-57823a6a9fa9 kind: Event lastTimestamp: "2026-07-07T15:27:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:44Z" name: success-200-isvc-761e8-predictor-6d97467478-f9qnw.18c00b789544efc3 namespace: kserve-ci-e2e-test resourceVersion: "25568" uid: e152b519-18a9-4d7a-93ee-8e127e543be9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-761e8-predictor-6d97467478-f9qnw namespace: kserve-ci-e2e-test resourceVersion: "24640" uid: e6ee6064-4c5d-411b-b0e5-57823a6a9fa9 kind: Event lastTimestamp: "2026-07-07T15:27:46Z" message: 'Readiness probe failed: Get "https://10.132.0.44:8643/healthz": dial tcp 10.132.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:46Z" name: success-200-isvc-761e8-predictor-6d97467478-f9qnw.18c00b7908260fca namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 508622f8-c39d-4c55-bea6-ed9304f804b2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-761e8-predictor-6d97467478 namespace: kserve-ci-e2e-test resourceVersion: "24615" uid: eefe6b1e-158d-4a14-9337-832cc859726e kind: Event lastTimestamp: "2026-07-07T15:26:34Z" message: 'Created pod: success-200-isvc-761e8-predictor-6d97467478-f9qnw' metadata: creationTimestamp: "2026-07-07T15:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:26:34Z" name: success-200-isvc-761e8-predictor-6d97467478.18c00b6836f4d89e namespace: kserve-ci-e2e-test resourceVersion: "24641" uid: 5f6a9a8b-7fbb-4d12-ae7e-cabf7aeca18a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-761e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24609" uid: 58cb9ca2-d719-471c-b0f2-1a27262499be kind: Event lastTimestamp: "2026-07-07T15:26:34Z" message: Scaled up replica set success-200-isvc-761e8-predictor-6d97467478 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:26:34Z" name: success-200-isvc-761e8-predictor.18c00b682e3d0ff8 namespace: kserve-ci-e2e-test resourceVersion: "24618" uid: 83e2907b-a125-410f-9d5f-f612ff5a9b8a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:26:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-761e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24612" uid: 14c9de7c-7a09-4d46-a070-edf823feb9b2 kind: Event lastTimestamp: "2026-07-07T15:27:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:04Z" name: success-200-isvc-761e8-predictor.18c00b6baa8800ac namespace: kserve-ci-e2e-test resourceVersion: "24954" uid: 38ea1663-e1a8-4f0f-b77b-0c9873fd1e30 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:26:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-761e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24612" uid: 14c9de7c-7a09-4d46-a070-edf823feb9b2 kind: Event lastTimestamp: "2026-07-07T15:27:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:04Z" name: success-200-isvc-761e8-predictor.18c00b6bab1616ae namespace: kserve-ci-e2e-test resourceVersion: "24955" uid: 4564ac1e-1e21-441a-b121-e4cad4ad4a74 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:27:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-761e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24837" uid: 14c9de7c-7a09-4d46-a070-edf823feb9b2 kind: Event lastTimestamp: "2026-07-07T15:27:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:34Z" name: success-200-isvc-761e8-predictor.18c00b72a9473c76 namespace: kserve-ci-e2e-test resourceVersion: "25377" uid: 5179bf25-0680-4d0e-93d8-b619f58716a7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:27:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-761e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24837" uid: 14c9de7c-7a09-4d46-a070-edf823feb9b2 kind: Event lastTimestamp: "2026-07-07T15:27:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:34Z" name: success-200-isvc-761e8-predictor.18c00b72a9bed053 namespace: kserve-ci-e2e-test resourceVersion: "25378" uid: 23965f83-a48f-4f51-b2d9-96193da902e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:26:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-761e8 namespace: kserve-ci-e2e-test resourceVersion: "24603" uid: bbb84d1c-1790-4ffc-b519-86c8a439de48 kind: Event lastTimestamp: "2026-07-07T15:27:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:27:22Z" name: success-200-isvc-761e8.18c00b682cbba9df namespace: kserve-ci-e2e-test resourceVersion: "25250" uid: 3bcb0f88-5b5b-4ad5-9983-7e75d7125960 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-761e8 namespace: kserve-ci-e2e-test resourceVersion: "24603" uid: bbb84d1c-1790-4ffc-b519-86c8a439de48 kind: Event lastTimestamp: "2026-07-07T15:26:34Z" message: 'Failed to update status for InferenceService "success-200-isvc-761e8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-761e8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:26:34Z" name: success-200-isvc-761e8.18c00b682e4635c4 namespace: kserve-ci-e2e-test resourceVersion: "24616" uid: afac5012-f450-4a7d-9106-c82acf18343f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:26:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-761e8 namespace: kserve-ci-e2e-test resourceVersion: "24603" uid: bbb84d1c-1790-4ffc-b519-86c8a439de48 kind: Event lastTimestamp: "2026-07-07T15:26:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-761e8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:26:34Z" name: success-200-isvc-761e8.18c00b682e4687c7 namespace: kserve-ci-e2e-test resourceVersion: "24619" uid: 7f01c997-7740-43a3-b07d-8dc34e81468f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-761e8 namespace: kserve-ci-e2e-test resourceVersion: "25246" uid: bbb84d1c-1790-4ffc-b519-86c8a439de48 kind: Event lastTimestamp: "2026-07-07T15:27:22Z" message: InferenceService [success-200-isvc-761e8] is Ready metadata: creationTimestamp: "2026-07-07T15:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:27:22Z" name: success-200-isvc-761e8.18c00b733eb46f64 namespace: kserve-ci-e2e-test resourceVersion: "25248" uid: 94afb55c-17be-47b5-8380-9c96a884be6d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:07:29.953696Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw namespace: kserve-ci-e2e-test resourceVersion: "15515" uid: 135de414-c12c-40d5-ba2a-1ba5614d3e18 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8996c-predictor-bb4b8bbb-462gw to ip-10-0-134-178.ec2.internal metadata: creationTimestamp: "2026-07-07T15:07:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:07:29Z" name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw.18c00a5db0d51c57 namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: 28f4bb2e-ef9d-4c1f-b78d-372f37cae59f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 135de414-c12c-40d5-ba2a-1ba5614d3e18 kind: Event lastTimestamp: "2026-07-07T15:07:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-8996c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T15:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:30Z" name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw.18c00a5dbdea861f namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: fbc94feb-9144-4af2-9723-3de9bdf0e9d7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw namespace: kserve-ci-e2e-test resourceVersion: "15532" uid: 135de414-c12c-40d5-ba2a-1ba5614d3e18 kind: Event lastTimestamp: "2026-07-07T15:07:30Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:07:30Z" name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw.18c00a5dee445e3b namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 3d474b3b-f61d-4c50-9691-2b85c514293b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 135de414-c12c-40d5-ba2a-1ba5614d3e18 kind: Event lastTimestamp: "2026-07-07T15:07:31Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:31Z" name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw.18c00a5defae4797 namespace: kserve-ci-e2e-test resourceVersion: "15605" uid: 8be2c3c9-19f5-49b0-a188-7c175a57ec13 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 135de414-c12c-40d5-ba2a-1ba5614d3e18 kind: Event lastTimestamp: "2026-07-07T15:07:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:31Z" name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw.18c00a5df2c5ff5e namespace: kserve-ci-e2e-test resourceVersion: "15606" uid: 9a3ed0f2-084a-4c5c-87e7-86ea2becca0b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 135de414-c12c-40d5-ba2a-1ba5614d3e18 kind: Event lastTimestamp: "2026-07-07T15:07:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:31Z" name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw.18c00a5df3329315 namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: 015cb065-660f-426d-bf2d-db60b6f511ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 135de414-c12c-40d5-ba2a-1ba5614d3e18 kind: Event lastTimestamp: "2026-07-07T15:07:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:31Z" name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw.18c00a5df39261d2 namespace: kserve-ci-e2e-test resourceVersion: "15608" uid: d2753943-140a-4391-9be7-4d4fad5c1ec5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 135de414-c12c-40d5-ba2a-1ba5614d3e18 kind: Event lastTimestamp: "2026-07-07T15:07:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:31Z" name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw.18c00a5df634882d namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: 8e0e49e6-45f8-4735-ade2-925f8f572df5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 135de414-c12c-40d5-ba2a-1ba5614d3e18 kind: Event lastTimestamp: "2026-07-07T15:07:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:31Z" name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw.18c00a5df6a22139 namespace: kserve-ci-e2e-test resourceVersion: "15610" uid: c16b455d-b5d9-4d82-abda-fabc0ecbf674 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T15:07:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 135de414-c12c-40d5-ba2a-1ba5614d3e18 kind: Event lastTimestamp: "2026-07-07T15:16:47Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:47Z" name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw.18c00a5e0761b2a7 namespace: kserve-ci-e2e-test resourceVersion: "19917" uid: 885114f2-dc06-4a0a-b9f1-505dc86bb1fe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 135de414-c12c-40d5-ba2a-1ba5614d3e18 kind: Event lastTimestamp: "2026-07-07T15:16:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:44Z" name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw.18c00adee62a9414 namespace: kserve-ci-e2e-test resourceVersion: "19825" uid: 0f298dcc-4096-4f77-8e3d-9fb969ee5122 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 135de414-c12c-40d5-ba2a-1ba5614d3e18 kind: Event lastTimestamp: "2026-07-07T15:16:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:44Z" name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw.18c00adee62b3ab2 namespace: kserve-ci-e2e-test resourceVersion: "19828" uid: ed02de8f-a2b5-4a62-8c97-9b322a772a5a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 135de414-c12c-40d5-ba2a-1ba5614d3e18 kind: Event lastTimestamp: "2026-07-07T15:16:47Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": dial tcp 10.132.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:47Z" name: success-200-isvc-8996c-predictor-bb4b8bbb-462gw.18c00adf7bc7b69d namespace: kserve-ci-e2e-test resourceVersion: "19916" uid: a6ba7fd4-2afc-4567-9ddc-3c8952c903b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8996c-predictor-bb4b8bbb namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: 7e7a2e5f-e465-42dc-aeff-4df2a80d8503 kind: Event lastTimestamp: "2026-07-07T15:07:29Z" message: 'Created pod: success-200-isvc-8996c-predictor-bb4b8bbb-462gw' metadata: creationTimestamp: "2026-07-07T15:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:29Z" name: success-200-isvc-8996c-predictor-bb4b8bbb.18c00a5db04ecd71 namespace: kserve-ci-e2e-test resourceVersion: "15518" uid: 5cf0ae39-ccf7-4dec-b5a9-5ecc291c2193 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8996c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15498" uid: 47250e72-1cc8-42a3-a867-01cdab4c7b55 kind: Event lastTimestamp: "2026-07-07T15:07:29Z" message: Scaled up replica set success-200-isvc-8996c-predictor-bb4b8bbb from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:29Z" name: success-200-isvc-8996c-predictor.18c00a5daea09518 namespace: kserve-ci-e2e-test resourceVersion: "15503" uid: c4abe9f6-1d95-4620-8067-181bd4fb741f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:07:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8996c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15501" uid: 3c3e4828-2846-44d5-a340-1b47784fb7ce kind: Event lastTimestamp: "2026-07-07T15:08:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:14Z" name: success-200-isvc-8996c-predictor.18c00a612e08446a namespace: kserve-ci-e2e-test resourceVersion: "16044" uid: 511f0a9f-a63f-4d86-b715-d77d8ba7554e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:07:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8996c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15501" uid: 3c3e4828-2846-44d5-a340-1b47784fb7ce kind: Event lastTimestamp: "2026-07-07T15:08:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:07:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:14Z" name: success-200-isvc-8996c-predictor.18c00a612eaa41e7 namespace: kserve-ci-e2e-test resourceVersion: "16045" uid: 598cde4c-a60f-4024-b39a-5c94d56a7b38 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:07:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8996c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15756" uid: 3c3e4828-2846-44d5-a340-1b47784fb7ce kind: Event lastTimestamp: "2026-07-07T15:08:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:45Z" name: success-200-isvc-8996c-predictor.18c00a64ad7311b7 namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: e9b468cf-f01a-4952-86d1-71e5c7c1bded reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:07:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8996c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15756" uid: 3c3e4828-2846-44d5-a340-1b47784fb7ce kind: Event lastTimestamp: "2026-07-07T15:08:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:45Z" name: success-200-isvc-8996c-predictor.18c00a64adc69134 namespace: kserve-ci-e2e-test resourceVersion: "16329" uid: 65b9fc97-101c-4adc-822f-a80ad1bebb50 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:07:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8996c namespace: kserve-ci-e2e-test resourceVersion: "15480" uid: c1132f7a-caa1-4bad-aa94-fd8e076efdfc kind: Event lastTimestamp: "2026-07-07T15:08:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:08:17Z" name: success-200-isvc-8996c.18c00a5daf249d02 namespace: kserve-ci-e2e-test resourceVersion: "16072" uid: fe517074-a384-4ae5-aedc-0df3f396e593 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8996c namespace: kserve-ci-e2e-test resourceVersion: "16070" uid: c1132f7a-caa1-4bad-aa94-fd8e076efdfc kind: Event lastTimestamp: "2026-07-07T15:08:17Z" message: InferenceService [success-200-isvc-8996c] is Ready metadata: creationTimestamp: "2026-07-07T15:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:08:17Z" name: success-200-isvc-8996c.18c00a68c57da195 namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 41d4309a-e4d7-444f-83f0-e46a953e54f8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:17:55.163434Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8 namespace: kserve-ci-e2e-test resourceVersion: "20777" uid: 6fca471a-5cc7-4a75-aa8b-5be08533e7ea kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8 to ip-10-0-134-178.ec2.internal metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:17:55Z" name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8.18c00aef423c83b3 namespace: kserve-ci-e2e-test resourceVersion: "20786" uid: 03c6c003-09d5-4afd-97f3-142177c0d401 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8 namespace: kserve-ci-e2e-test resourceVersion: "20795" uid: 6fca471a-5cc7-4a75-aa8b-5be08533e7ea kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:17:55Z" name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8.18c00aef5c6df72b namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 5e5bb569-1a89-4124-a00e-51585b396c44 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8 namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: 6fca471a-5cc7-4a75-aa8b-5be08533e7ea kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:55Z" name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8.18c00aef5e09b76b namespace: kserve-ci-e2e-test resourceVersion: "20849" uid: 69af00c9-585e-4a82-88e1-24e0619f9fb7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8 namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: 6fca471a-5cc7-4a75-aa8b-5be08533e7ea kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:55Z" name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8.18c00aef60dc13f0 namespace: kserve-ci-e2e-test resourceVersion: "20851" uid: dbd11423-b93c-49f7-ad42-46d2efc1a78b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8 namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: 6fca471a-5cc7-4a75-aa8b-5be08533e7ea kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:55Z" name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8.18c00aef616c0dac namespace: kserve-ci-e2e-test resourceVersion: "20852" uid: 558bc1e0-9dd8-46af-9478-5933e6efd027 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8 namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: 6fca471a-5cc7-4a75-aa8b-5be08533e7ea kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:55Z" name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8.18c00aef617545fd namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: 4f3f94e8-4128-478e-a66c-81b27391bd16 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8 namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: 6fca471a-5cc7-4a75-aa8b-5be08533e7ea kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:55Z" name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8.18c00aef642fe373 namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: 0126fb7a-07fb-47c6-b6f5-1aca65a6a0e3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8 namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: 6fca471a-5cc7-4a75-aa8b-5be08533e7ea kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:55Z" name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8.18c00aef649d6858 namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 64b2625f-c906-4813-9c5a-f61a742339aa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T15:17:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8 namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: 6fca471a-5cc7-4a75-aa8b-5be08533e7ea kind: Event lastTimestamp: "2026-07-07T15:27:12Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:12Z" name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8.18c00aef8bcca01a namespace: kserve-ci-e2e-test resourceVersion: "25150" uid: c58eb1d8-53f8-4a2c-9b4d-cd6b1d712de5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8 namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: 6fca471a-5cc7-4a75-aa8b-5be08533e7ea kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:10Z" name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8.18c00b70807bf06c namespace: kserve-ci-e2e-test resourceVersion: "25080" uid: 051d737f-33c7-47ea-91bb-dfe952e45452 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8 namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: 6fca471a-5cc7-4a75-aa8b-5be08533e7ea kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:10Z" name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8.18c00b70807c7a87 namespace: kserve-ci-e2e-test resourceVersion: "25083" uid: b41bd8d0-ddab-4f90-bf02-5d213b11687c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8 namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: 6fca471a-5cc7-4a75-aa8b-5be08533e7ea kind: Event lastTimestamp: "2026-07-07T15:27:12Z" message: 'Readiness probe failed: Get "https://10.132.0.43:8643/healthz": dial tcp 10.132.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:12Z" name: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8.18c00b710037fb8a namespace: kserve-ci-e2e-test resourceVersion: "25149" uid: f79053af-6512-450f-8be4-2c06e2dc386a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a9c06-predictor-bdb4cb5fd namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: ab0481d9-da00-4b3a-98d1-12599ec7f450 kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: 'Created pod: success-200-isvc-a9c06-predictor-bdb4cb5fd-dg4q8' metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:55Z" name: success-200-isvc-a9c06-predictor-bdb4cb5fd.18c00aef41b98b15 namespace: kserve-ci-e2e-test resourceVersion: "20781" uid: 1ebf6269-676a-4771-8fcc-2fde8546bd64 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a9c06-predictor namespace: kserve-ci-e2e-test resourceVersion: "20758" uid: c12cd69d-f7d4-4782-a6a8-d5533b80a9dc kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: Scaled up replica set success-200-isvc-a9c06-predictor-bdb4cb5fd from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:55Z" name: success-200-isvc-a9c06-predictor.18c00aef3f529710 namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: bc516eb1-6ccb-4b91-a909-a60d9009c128 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:18:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9c06-predictor namespace: kserve-ci-e2e-test resourceVersion: "20760" uid: 8faf4df0-2cce-4bbe-913e-d15285699553 kind: Event lastTimestamp: "2026-07-07T15:18:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:18:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:25Z" name: success-200-isvc-a9c06-predictor.18c00af2bdcfb224 namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: 2b7e7f62-38c8-4054-b0d8-a63cbb270db4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:18:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9c06-predictor namespace: kserve-ci-e2e-test resourceVersion: "20760" uid: 8faf4df0-2cce-4bbe-913e-d15285699553 kind: Event lastTimestamp: "2026-07-07T15:18:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:18:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:25Z" name: success-200-isvc-a9c06-predictor.18c00af2be4dee59 namespace: kserve-ci-e2e-test resourceVersion: "21167" uid: c02bf5de-9b8e-4ee6-bfe2-d352911093fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:18:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9c06-predictor namespace: kserve-ci-e2e-test resourceVersion: "21014" uid: 8faf4df0-2cce-4bbe-913e-d15285699553 kind: Event lastTimestamp: "2026-07-07T15:18:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:18:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:55Z" name: success-200-isvc-a9c06-predictor.18c00af9bca9d1b4 namespace: kserve-ci-e2e-test resourceVersion: "21417" uid: 21b911de-c5f8-4487-ae2d-c7fe90e61e68 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:18:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9c06-predictor namespace: kserve-ci-e2e-test resourceVersion: "21014" uid: 8faf4df0-2cce-4bbe-913e-d15285699553 kind: Event lastTimestamp: "2026-07-07T15:18:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:18:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:55Z" name: success-200-isvc-a9c06-predictor.18c00af9bd20e688 namespace: kserve-ci-e2e-test resourceVersion: "21420" uid: 26cedb30-510c-4418-bbd3-95c6a4e8c2c6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a9c06 namespace: kserve-ci-e2e-test resourceVersion: "20749" uid: 9416bbc2-cf0b-4649-938e-d355d00b0481 kind: Event lastTimestamp: "2026-07-07T15:18:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:18:42Z" name: success-200-isvc-a9c06.18c00aef3f5cc574 namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: 94c936d9-319c-43b5-a305-da13dbeb8dc1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a9c06 namespace: kserve-ci-e2e-test resourceVersion: "21318" uid: 9416bbc2-cf0b-4649-938e-d355d00b0481 kind: Event lastTimestamp: "2026-07-07T15:18:42Z" message: InferenceService [success-200-isvc-a9c06] is Ready metadata: creationTimestamp: "2026-07-07T15:18:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:18:42Z" name: success-200-isvc-a9c06.18c00afa489e05fe namespace: kserve-ci-e2e-test resourceVersion: "21319" uid: d9f3e708-1603-4625-b0e3-85ec29b8d18f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:27:10.166382Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj namespace: kserve-ci-e2e-test resourceVersion: "25049" uid: bac4cca2-eea1-4e92-a48c-d8ba22d11f0f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj to ip-10-0-134-178.ec2.internal metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:27:10Z" name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj.18c00b707afd89b7 namespace: kserve-ci-e2e-test resourceVersion: "25053" uid: 339f9f6f-030f-4feb-9d3f-41c87115474f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj namespace: kserve-ci-e2e-test resourceVersion: "25067" uid: bac4cca2-eea1-4e92-a48c-d8ba22d11f0f kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:27:10Z" name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj.18c00b70951cf9a4 namespace: kserve-ci-e2e-test resourceVersion: "25111" uid: f34b815a-4dd6-44f0-bc75-842f0e21ada0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj namespace: kserve-ci-e2e-test resourceVersion: "25051" uid: bac4cca2-eea1-4e92-a48c-d8ba22d11f0f kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:10Z" name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj.18c00b7096df12ff namespace: kserve-ci-e2e-test resourceVersion: "25113" uid: 73751c18-de6a-4a50-9f64-be826f19ca5a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj namespace: kserve-ci-e2e-test resourceVersion: "25051" uid: bac4cca2-eea1-4e92-a48c-d8ba22d11f0f kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:10Z" name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj.18c00b7099fef856 namespace: kserve-ci-e2e-test resourceVersion: "25114" uid: 5a546780-3769-4919-bd67-0918b2974659 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj namespace: kserve-ci-e2e-test resourceVersion: "25051" uid: bac4cca2-eea1-4e92-a48c-d8ba22d11f0f kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:10Z" name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj.18c00b709a6ad86a namespace: kserve-ci-e2e-test resourceVersion: "25115" uid: 373e16a0-ae95-4993-83ee-e43a6082f3a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj namespace: kserve-ci-e2e-test resourceVersion: "25051" uid: bac4cca2-eea1-4e92-a48c-d8ba22d11f0f kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:10Z" name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj.18c00b709a7373ae namespace: kserve-ci-e2e-test resourceVersion: "25116" uid: 7031a3b6-0915-4240-9f71-4078dd78a77e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj namespace: kserve-ci-e2e-test resourceVersion: "25051" uid: bac4cca2-eea1-4e92-a48c-d8ba22d11f0f kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:10Z" name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj.18c00b709d2b2301 namespace: kserve-ci-e2e-test resourceVersion: "25117" uid: 85729cb5-81e8-4f5a-86ee-9bc6b8ca3599 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj namespace: kserve-ci-e2e-test resourceVersion: "25051" uid: bac4cca2-eea1-4e92-a48c-d8ba22d11f0f kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:10Z" name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj.18c00b709db4e287 namespace: kserve-ci-e2e-test resourceVersion: "25119" uid: ecff5c4f-76d4-481e-a627-ad84fdd1c99d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj namespace: kserve-ci-e2e-test resourceVersion: "25051" uid: bac4cca2-eea1-4e92-a48c-d8ba22d11f0f kind: Event lastTimestamp: "2026-07-07T15:27:48Z" message: 'Readiness probe failed: dial tcp 10.132.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:27:48Z" name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj.18c00b70ec4f7ae7 namespace: kserve-ci-e2e-test resourceVersion: "25647" uid: a77ff6d0-2d55-4ff0-98d7-eb7315980535 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:44:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj namespace: kserve-ci-e2e-test resourceVersion: "25051" uid: bac4cca2-eea1-4e92-a48c-d8ba22d11f0f kind: Event lastTimestamp: "2026-07-07T15:44:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:44:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:44:29Z" name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj.18c00c627b674dfa namespace: kserve-ci-e2e-test resourceVersion: "33070" uid: e18258bc-a17d-4c9e-8136-a192f029427c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:44:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj namespace: kserve-ci-e2e-test resourceVersion: "25051" uid: bac4cca2-eea1-4e92-a48c-d8ba22d11f0f kind: Event lastTimestamp: "2026-07-07T15:44:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:44:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:44:29Z" name: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj.18c00c627b67f3a2 namespace: kserve-ci-e2e-test resourceVersion: "33071" uid: d96bcf1b-2d4d-421b-a4cd-517d96fdb527 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c3e9b-predictor-8557bd9dd4 namespace: kserve-ci-e2e-test resourceVersion: "25040" uid: 2cfbfd21-1a23-4eb3-a0f2-4d6d7e5211bb kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: 'Created pod: success-200-isvc-c3e9b-predictor-8557bd9dd4-9r7lj' metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:10Z" name: success-200-isvc-c3e9b-predictor-8557bd9dd4.18c00b707a7bf145 namespace: kserve-ci-e2e-test resourceVersion: "25052" uid: 4f6a5595-e75e-47e1-b955-bf68c24dc519 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c3e9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25038" uid: cdfb15da-e5b0-46ca-882e-b61dba03a8dc kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: Scaled up replica set success-200-isvc-c3e9b-predictor-8557bd9dd4 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:10Z" name: success-200-isvc-c3e9b-predictor.18c00b70759442ed namespace: kserve-ci-e2e-test resourceVersion: "25042" uid: 3110c950-2136-4d5d-8c74-d9ef067d5292 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:27:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3e9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25039" uid: cd05de1b-dfb1-40d7-afde-7f6b95bd623f kind: Event lastTimestamp: "2026-07-07T15:27:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:27:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:40Z" name: success-200-isvc-c3e9b-predictor.18c00b73f522eda9 namespace: kserve-ci-e2e-test resourceVersion: "25433" uid: a7d2c7cc-e8af-4afe-afc4-8612f04a1843 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:27:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3e9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25039" uid: cd05de1b-dfb1-40d7-afde-7f6b95bd623f kind: Event lastTimestamp: "2026-07-07T15:27:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:27:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:27:40Z" name: success-200-isvc-c3e9b-predictor.18c00b73f61d4eaa namespace: kserve-ci-e2e-test resourceVersion: "25434" uid: 70db5f77-4ba6-44de-b2a9-eefdf636da04 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:27:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3e9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25284" uid: cd05de1b-dfb1-40d7-afde-7f6b95bd623f kind: Event lastTimestamp: "2026-07-07T15:28:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:28:25Z" name: success-200-isvc-c3e9b-predictor.18c00b7af4b9df6e namespace: kserve-ci-e2e-test resourceVersion: "26304" uid: cfebab73-3822-4ee6-a4ff-247d3dd28ad9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:27:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3e9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25284" uid: cd05de1b-dfb1-40d7-afde-7f6b95bd623f kind: Event lastTimestamp: "2026-07-07T15:28:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:28:25Z" name: success-200-isvc-c3e9b-predictor.18c00b7af533fd8b namespace: kserve-ci-e2e-test resourceVersion: "26305" uid: 9e1038f0-21a3-426e-95b3-8f4d2f9a6883 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3e9b namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: 70b3d67f-448d-4337-960d-38834f8c7177 kind: Event lastTimestamp: "2026-07-07T15:27:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:27:58Z" name: success-200-isvc-c3e9b.18c00b707645fb35 namespace: kserve-ci-e2e-test resourceVersion: "25749" uid: e0cae86d-f9e9-4d6f-a883-05a606512455 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3e9b namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: 70b3d67f-448d-4337-960d-38834f8c7177 kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: 'Failed to update status for InferenceService "success-200-isvc-c3e9b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c3e9b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:27:10Z" name: success-200-isvc-c3e9b.18c00b7076fdc5f8 namespace: kserve-ci-e2e-test resourceVersion: "25046" uid: 705c17e2-5663-4db2-9b57-367b2ab6f60a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3e9b namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: 70b3d67f-448d-4337-960d-38834f8c7177 kind: Event lastTimestamp: "2026-07-07T15:27:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c3e9b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:27:10Z" name: success-200-isvc-c3e9b.18c00b7076fe15e7 namespace: kserve-ci-e2e-test resourceVersion: "25047" uid: 7c0d8c17-4835-4093-ba61-e59b8296e011 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:27:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3e9b namespace: kserve-ci-e2e-test resourceVersion: "25747" uid: 70b3d67f-448d-4337-960d-38834f8c7177 kind: Event lastTimestamp: "2026-07-07T15:27:58Z" message: InferenceService [success-200-isvc-c3e9b] is Ready metadata: creationTimestamp: "2026-07-07T15:27:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:27:58Z" name: success-200-isvc-c3e9b.18c00b7ba97d238a namespace: kserve-ci-e2e-test resourceVersion: "25748" uid: 984474b9-5e91-4ac0-a896-bf1e0ec63142 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:16:44.921670Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: eb042c00-7d75-463c-baa9-1df984077702 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw to ip-10-0-134-178.ec2.internal metadata: creationTimestamp: "2026-07-07T15:16:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:16:44Z" name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw.18c00adee78047b2 namespace: kserve-ci-e2e-test resourceVersion: "19834" uid: fba32d05-fd8a-49d0-9559-2a1593eac6e2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw namespace: kserve-ci-e2e-test resourceVersion: "19839" uid: eb042c00-7d75-463c-baa9-1df984077702 kind: Event lastTimestamp: "2026-07-07T15:16:45Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:16:45Z" name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw.18c00adf014635cd namespace: kserve-ci-e2e-test resourceVersion: "19875" uid: bf54d9f6-ba37-426a-99c0-1688a37010db reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw namespace: kserve-ci-e2e-test resourceVersion: "19831" uid: eb042c00-7d75-463c-baa9-1df984077702 kind: Event lastTimestamp: "2026-07-07T15:16:45Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:45Z" name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw.18c00adf02dc464d namespace: kserve-ci-e2e-test resourceVersion: "19878" uid: bfa7a410-f79b-4b23-b256-65ca5803076a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw namespace: kserve-ci-e2e-test resourceVersion: "19831" uid: eb042c00-7d75-463c-baa9-1df984077702 kind: Event lastTimestamp: "2026-07-07T15:16:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:45Z" name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw.18c00adf05ebd121 namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: dc55190a-c559-4a2a-a8e6-2d842c059773 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw namespace: kserve-ci-e2e-test resourceVersion: "19831" uid: eb042c00-7d75-463c-baa9-1df984077702 kind: Event lastTimestamp: "2026-07-07T15:16:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:45Z" name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw.18c00adf0659cb46 namespace: kserve-ci-e2e-test resourceVersion: "19880" uid: 08d2b924-630c-4a71-b16f-20c03f8d39f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw namespace: kserve-ci-e2e-test resourceVersion: "19831" uid: eb042c00-7d75-463c-baa9-1df984077702 kind: Event lastTimestamp: "2026-07-07T15:16:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:45Z" name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw.18c00adf0662e1e6 namespace: kserve-ci-e2e-test resourceVersion: "19881" uid: 18552dee-8732-4f4c-99af-6bf4c2c5f85d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw namespace: kserve-ci-e2e-test resourceVersion: "19831" uid: eb042c00-7d75-463c-baa9-1df984077702 kind: Event lastTimestamp: "2026-07-07T15:16:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:45Z" name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw.18c00adf0934cb66 namespace: kserve-ci-e2e-test resourceVersion: "19885" uid: fe1a9436-b8f8-49dc-a0de-ad8a0d787263 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw namespace: kserve-ci-e2e-test resourceVersion: "19831" uid: eb042c00-7d75-463c-baa9-1df984077702 kind: Event lastTimestamp: "2026-07-07T15:16:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:45Z" name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw.18c00adf09a7b767 namespace: kserve-ci-e2e-test resourceVersion: "19886" uid: 28592b73-2fdc-4bfc-9559-12af92314a3c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:16:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw namespace: kserve-ci-e2e-test resourceVersion: "19831" uid: eb042c00-7d75-463c-baa9-1df984077702 kind: Event lastTimestamp: "2026-07-07T15:17:22Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:22Z" name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw.18c00adf2b65938a namespace: kserve-ci-e2e-test resourceVersion: "20397" uid: 270f736d-6acf-438d-8f04-0340c14ca3ee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw namespace: kserve-ci-e2e-test resourceVersion: "19831" uid: eb042c00-7d75-463c-baa9-1df984077702 kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:55Z" name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw.18c00aef42139b49 namespace: kserve-ci-e2e-test resourceVersion: "20784" uid: e7159ea5-29b6-489f-8324-918b0dd941b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw namespace: kserve-ci-e2e-test resourceVersion: "19831" uid: eb042c00-7d75-463c-baa9-1df984077702 kind: Event lastTimestamp: "2026-07-07T15:17:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:55Z" name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw.18c00aef42145cf8 namespace: kserve-ci-e2e-test resourceVersion: "20787" uid: 942129aa-9fa6-4254-9f62-32416a4abfc8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw namespace: kserve-ci-e2e-test resourceVersion: "19831" uid: eb042c00-7d75-463c-baa9-1df984077702 kind: Event lastTimestamp: "2026-07-07T15:17:57Z" message: 'Readiness probe failed: Get "https://10.132.0.41:8643/healthz": dial tcp 10.132.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:17:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:57Z" name: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw.18c00aefb3882ed8 namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: 1a508bfc-7dea-4d93-b011-94f467ab9dcb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e4605-predictor-5dcbcbf488 namespace: kserve-ci-e2e-test resourceVersion: "19809" uid: 420f986d-1bb3-4ea5-8416-ce009701e29b kind: Event lastTimestamp: "2026-07-07T15:16:44Z" message: 'Created pod: success-200-isvc-e4605-predictor-5dcbcbf488-2vwnw' metadata: creationTimestamp: "2026-07-07T15:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:16:44Z" name: success-200-isvc-e4605-predictor-5dcbcbf488.18c00adee6cbec18 namespace: kserve-ci-e2e-test resourceVersion: "19829" uid: 357b0dea-b17d-4bdf-82f0-7f0f4bfcf710 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e4605-predictor namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: 8ada3369-910d-4602-809a-05e2d50e4b4d kind: Event lastTimestamp: "2026-07-07T15:16:44Z" message: Scaled up replica set success-200-isvc-e4605-predictor-5dcbcbf488 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:16:44Z" name: success-200-isvc-e4605-predictor.18c00adee36b15a2 namespace: kserve-ci-e2e-test resourceVersion: "19813" uid: d0ecbb2c-58cb-4826-b9c4-d92a87bb1379 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:16:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e4605-predictor namespace: kserve-ci-e2e-test resourceVersion: "19807" uid: c3b86ec1-3365-4919-9a76-833b79c91b45 kind: Event lastTimestamp: "2026-07-07T15:17:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:29Z" name: success-200-isvc-e4605-predictor.18c00ae26210fbc3 namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: 8e2a6509-1caa-4170-b152-dd0b786a1fb0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:16:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e4605-predictor namespace: kserve-ci-e2e-test resourceVersion: "19807" uid: c3b86ec1-3365-4919-9a76-833b79c91b45 kind: Event lastTimestamp: "2026-07-07T15:17:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:29Z" name: success-200-isvc-e4605-predictor.18c00ae2627f4fdd namespace: kserve-ci-e2e-test resourceVersion: "20471" uid: 25a37e55-5427-44c7-92a2-66f5f55d6d9f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:17:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e4605-predictor namespace: kserve-ci-e2e-test resourceVersion: "20050" uid: c3b86ec1-3365-4919-9a76-833b79c91b45 kind: Event lastTimestamp: "2026-07-07T15:17:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:17:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:44Z" name: success-200-isvc-e4605-predictor.18c00ae5e1b07329 namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: 1453bee9-9a75-468f-893c-fffae50bfffb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:17:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e4605-predictor namespace: kserve-ci-e2e-test resourceVersion: "20050" uid: c3b86ec1-3365-4919-9a76-833b79c91b45 kind: Event lastTimestamp: "2026-07-07T15:17:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:17:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:44Z" name: success-200-isvc-e4605-predictor.18c00ae5e2355535 namespace: kserve-ci-e2e-test resourceVersion: "20628" uid: 0c477121-d06b-4557-9b0e-fc71421e69a9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:16:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e4605 namespace: kserve-ci-e2e-test resourceVersion: "19781" uid: 5a03245a-8ace-4c79-9707-fdf0503405e4 kind: Event lastTimestamp: "2026-07-07T15:17:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:17:32Z" name: success-200-isvc-e4605.18c00adee3cf9141 namespace: kserve-ci-e2e-test resourceVersion: "20498" uid: bcb8543a-232e-454f-a54a-3c980604baeb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e4605 namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 5a03245a-8ace-4c79-9707-fdf0503405e4 kind: Event lastTimestamp: "2026-07-07T15:17:32Z" message: InferenceService [success-200-isvc-e4605] is Ready metadata: creationTimestamp: "2026-07-07T15:17:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:17:32Z" name: success-200-isvc-e4605.18c00ae9e6d82258 namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: e445871b-5401-49eb-b091-78fbacc330eb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:05:19.602788Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: 2448e304-2d49-4f37-bdf8-f5598d6e4698 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f85ce-predictor-6578cf6d78-c57pb to ip-10-0-134-178.ec2.internal metadata: creationTimestamp: "2026-07-07T15:05:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:05:19Z" name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb.18c00a3f57500a07 namespace: kserve-ci-e2e-test resourceVersion: "13909" uid: 102e5e9d-7d87-46d0-bdae-31dce0dc3a55 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:20Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb namespace: kserve-ci-e2e-test resourceVersion: "13912" uid: 2448e304-2d49-4f37-bdf8-f5598d6e4698 kind: Event lastTimestamp: "2026-07-07T15:05:20Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:05:20Z" name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb.18c00a3f71d95f88 namespace: kserve-ci-e2e-test resourceVersion: "13972" uid: 80d4c425-288d-442a-9dc5-db9662cb68d6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb namespace: kserve-ci-e2e-test resourceVersion: "13907" uid: 2448e304-2d49-4f37-bdf8-f5598d6e4698 kind: Event lastTimestamp: "2026-07-07T15:05:20Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" metadata: creationTimestamp: "2026-07-07T15:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:20Z" name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb.18c00a3f73635359 namespace: kserve-ci-e2e-test resourceVersion: "13981" uid: 8363ddf1-3bb5-4228-800f-a6978d94c907 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb namespace: kserve-ci-e2e-test resourceVersion: "13907" uid: 2448e304-2d49-4f37-bdf8-f5598d6e4698 kind: Event lastTimestamp: "2026-07-07T15:05:33Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1699" in 13.379s (13.379s including waiting). Image size: 1335613540 bytes.' metadata: creationTimestamp: "2026-07-07T15:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:33Z" name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb.18c00a4290e03d5f namespace: kserve-ci-e2e-test resourceVersion: "14177" uid: f9895a44-7e7b-4c80-9ef6-255ec6f94a23 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb namespace: kserve-ci-e2e-test resourceVersion: "13907" uid: 2448e304-2d49-4f37-bdf8-f5598d6e4698 kind: Event lastTimestamp: "2026-07-07T15:05:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:33Z" name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb.18c00a4294f8751f namespace: kserve-ci-e2e-test resourceVersion: "14179" uid: 32185b83-c93f-46fd-9eb6-8209066631c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb namespace: kserve-ci-e2e-test resourceVersion: "13907" uid: 2448e304-2d49-4f37-bdf8-f5598d6e4698 kind: Event lastTimestamp: "2026-07-07T15:05:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:33Z" name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb.18c00a429586ed87 namespace: kserve-ci-e2e-test resourceVersion: "14180" uid: 74e25bea-ee62-419b-9f53-63d866cfcad1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb namespace: kserve-ci-e2e-test resourceVersion: "13907" uid: 2448e304-2d49-4f37-bdf8-f5598d6e4698 kind: Event lastTimestamp: "2026-07-07T15:05:33Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T15:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:33Z" name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb.18c00a429594e5fb namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 3dbfc1cd-017f-4714-b31e-47ed134d9513 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb namespace: kserve-ci-e2e-test resourceVersion: "13907" uid: 2448e304-2d49-4f37-bdf8-f5598d6e4698 kind: Event lastTimestamp: "2026-07-07T15:05:35Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.393s (2.393s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T15:05:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:35Z" name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb.18c00a432438ee52 namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 37437c83-69d1-4508-8a58-16f2e032c252 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb namespace: kserve-ci-e2e-test resourceVersion: "13907" uid: 2448e304-2d49-4f37-bdf8-f5598d6e4698 kind: Event lastTimestamp: "2026-07-07T15:05:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:05:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:35Z" name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb.18c00a4327612558 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: c305865b-3c34-4b00-aba3-e023d034580a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb namespace: kserve-ci-e2e-test resourceVersion: "13907" uid: 2448e304-2d49-4f37-bdf8-f5598d6e4698 kind: Event lastTimestamp: "2026-07-07T15:05:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:05:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:05:35Z" name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb.18c00a4327f194f3 namespace: kserve-ci-e2e-test resourceVersion: "14223" uid: e0b76444-a05a-47cc-a3a5-eefc2b7f0a14 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:05:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb namespace: kserve-ci-e2e-test resourceVersion: "13907" uid: 2448e304-2d49-4f37-bdf8-f5598d6e4698 kind: Event lastTimestamp: "2026-07-07T15:06:12Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:05:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:12Z" name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb.18c00a435b96b0f2 namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: 71c72b09-6d12-4cfd-b4ad-60557fad25e1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb namespace: kserve-ci-e2e-test resourceVersion: "13907" uid: 2448e304-2d49-4f37-bdf8-f5598d6e4698 kind: Event lastTimestamp: "2026-07-07T15:06:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:54Z" name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb.18c00a5566139aac namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 06146525-19d7-426b-8a3e-cef13069dc36 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb namespace: kserve-ci-e2e-test resourceVersion: "13907" uid: 2448e304-2d49-4f37-bdf8-f5598d6e4698 kind: Event lastTimestamp: "2026-07-07T15:06:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:54Z" name: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb.18c00a5566144e64 namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: 536e9a12-22c5-4463-b73a-cbc60c660157 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f85ce-predictor-6578cf6d78 namespace: kserve-ci-e2e-test resourceVersion: "13891" uid: 1a2eb604-7d23-43a0-9ac6-d8fea2fb3143 kind: Event lastTimestamp: "2026-07-07T15:05:19Z" message: 'Created pod: success-200-isvc-f85ce-predictor-6578cf6d78-c57pb' metadata: creationTimestamp: "2026-07-07T15:05:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:05:19Z" name: success-200-isvc-f85ce-predictor-6578cf6d78.18c00a3f569bc15c namespace: kserve-ci-e2e-test resourceVersion: "13906" uid: bac1e3e8-28b8-4a39-9803-ae344678689d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:05:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f85ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "13888" uid: d2996b5a-1747-4cc5-96cd-eace89e316df kind: Event lastTimestamp: "2026-07-07T15:05:19Z" message: Scaled up replica set success-200-isvc-f85ce-predictor-6578cf6d78 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:05:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:05:19Z" name: success-200-isvc-f85ce-predictor.18c00a3f50224dff namespace: kserve-ci-e2e-test resourceVersion: "13893" uid: 9cda2327-d980-4578-80e4-657bbb4ded31 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:05:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f85ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "13901" uid: 9b996539-023f-4c43-9840-5f4f0b1edc3f kind: Event lastTimestamp: "2026-07-07T15:06:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:04Z" name: success-200-isvc-f85ce-predictor.18c00a42d58541b3 namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 157e1d6f-a9b3-4902-8d2c-a3f8d3cee70d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:05:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f85ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "13901" uid: 9b996539-023f-4c43-9840-5f4f0b1edc3f kind: Event lastTimestamp: "2026-07-07T15:06:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:04Z" name: success-200-isvc-f85ce-predictor.18c00a42d6014617 namespace: kserve-ci-e2e-test resourceVersion: "14495" uid: 5ecf734b-c090-4620-b5aa-3f69f435683c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:06:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f85ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 9b996539-023f-4c43-9840-5f4f0b1edc3f kind: Event lastTimestamp: "2026-07-07T15:06:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:06:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:34Z" name: success-200-isvc-f85ce-predictor.18c00a4d54038a96 namespace: kserve-ci-e2e-test resourceVersion: "14753" uid: d01d9975-2086-4801-89c1-4c313929abe2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:06:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f85ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 9b996539-023f-4c43-9840-5f4f0b1edc3f kind: Event lastTimestamp: "2026-07-07T15:06:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:06:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:34Z" name: success-200-isvc-f85ce-predictor.18c00a4d548a26cb namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: 716d271e-9d44-49b6-91e7-58de2ac6e2be reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:05:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f85ce namespace: kserve-ci-e2e-test resourceVersion: "13883" uid: 435692c5-2258-4fa5-849b-39da9caf7315 kind: Event lastTimestamp: "2026-07-07T15:06:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:05:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:06:23Z" name: success-200-isvc-f85ce.18c00a3f56090abf namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 54171cf5-7cb3-4de3-962b-7cf5d11e465e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f85ce namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: 435692c5-2258-4fa5-849b-39da9caf7315 kind: Event lastTimestamp: "2026-07-07T15:06:23Z" message: InferenceService [success-200-isvc-f85ce] is Ready metadata: creationTimestamp: "2026-07-07T15:06:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:06:23Z" name: success-200-isvc-f85ce.18c00a4e216c8089 namespace: kserve-ci-e2e-test resourceVersion: "14661" uid: ed0f91ad-7368-454d-9684-aeb31bf6cbd8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:07:54.109809Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3c9c9-7bd8bf8867-hjp6q namespace: kserve-ci-e2e-test resourceVersion: "15846" uid: 86140778-5169-4d13-bad6-258d806b5763 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-3c9c9-7bd8bf8867-hjp6q to ip-10-0-128-215.ec2.internal metadata: creationTimestamp: "2026-07-07T15:07:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:07:54Z" name: switch-graph-3c9c9-7bd8bf8867-hjp6q.18c00a6350a61caa namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: 1561a604-1936-44d7-aa99-9cdc7f54b60d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:54Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3c9c9-7bd8bf8867-hjp6q namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 86140778-5169-4d13-bad6-258d806b5763 kind: Event lastTimestamp: "2026-07-07T15:07:54Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:07:54Z" name: switch-graph-3c9c9-7bd8bf8867-hjp6q.18c00a636a56dd75 namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: 4e4cbe97-6995-4754-b24d-d5c8794dc694 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3c9c9} kind: Pod name: switch-graph-3c9c9-7bd8bf8867-hjp6q namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: 86140778-5169-4d13-bad6-258d806b5763 kind: Event lastTimestamp: "2026-07-07T15:07:54Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:b3cb5f47c4ea9d876e5490bc83b15040508a0aac484b7e9d11de2faf2dbcdfe4" metadata: creationTimestamp: "2026-07-07T15:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:54Z" name: switch-graph-3c9c9-7bd8bf8867-hjp6q.18c00a636bb65fe0 namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: e08ae1fd-4a85-4fed-9179-df03cdf143db reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3c9c9} kind: Pod name: switch-graph-3c9c9-7bd8bf8867-hjp6q namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: 86140778-5169-4d13-bad6-258d806b5763 kind: Event lastTimestamp: "2026-07-07T15:07:56Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:b3cb5f47c4ea9d876e5490bc83b15040508a0aac484b7e9d11de2faf2dbcdfe4" in 2.296s (2.296s including waiting). Image size: 218072672 bytes.' metadata: creationTimestamp: "2026-07-07T15:07:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:56Z" name: switch-graph-3c9c9-7bd8bf8867-hjp6q.18c00a63f49c87e7 namespace: kserve-ci-e2e-test resourceVersion: "15885" uid: 45156829-8261-46b7-b3e7-800d8e7db253 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3c9c9} kind: Pod name: switch-graph-3c9c9-7bd8bf8867-hjp6q namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: 86140778-5169-4d13-bad6-258d806b5763 kind: Event lastTimestamp: "2026-07-07T15:07:56Z" message: 'Created container: switch-graph-3c9c9' metadata: creationTimestamp: "2026-07-07T15:07:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:56Z" name: switch-graph-3c9c9-7bd8bf8867-hjp6q.18c00a63f8756415 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 5f6d6c79-3b4f-42a8-8492-77a724aed386 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3c9c9} kind: Pod name: switch-graph-3c9c9-7bd8bf8867-hjp6q namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: 86140778-5169-4d13-bad6-258d806b5763 kind: Event lastTimestamp: "2026-07-07T15:07:56Z" message: Started container switch-graph-3c9c9 metadata: creationTimestamp: "2026-07-07T15:07:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:56Z" name: switch-graph-3c9c9-7bd8bf8867-hjp6q.18c00a63f8e3a686 namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: ab6b1d63-fd39-4b89-9326-2eb11f93c468 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:16:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3c9c9} kind: Pod name: switch-graph-3c9c9-7bd8bf8867-hjp6q namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: 86140778-5169-4d13-bad6-258d806b5763 kind: Event lastTimestamp: "2026-07-07T15:16:08Z" message: Stopping container switch-graph-3c9c9 metadata: creationTimestamp: "2026-07-07T15:16:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:08Z" name: switch-graph-3c9c9-7bd8bf8867-hjp6q.18c00ad6826e99b9 namespace: kserve-ci-e2e-test resourceVersion: "19365" uid: dc3034df-320c-4d02-b579-b6aa3efe2715 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:16:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3c9c9} kind: Pod name: switch-graph-3c9c9-7bd8bf8867-hjp6q namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: 86140778-5169-4d13-bad6-258d806b5763 kind: Event lastTimestamp: "2026-07-07T15:16:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:16:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:16:38Z" name: switch-graph-3c9c9-7bd8bf8867-hjp6q.18c00ad7916df133 namespace: kserve-ci-e2e-test resourceVersion: "19714" uid: a2de108f-c3af-4f25-b76a-7c90c4ebae9e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-3c9c9-7bd8bf8867 namespace: kserve-ci-e2e-test resourceVersion: "15841" uid: 97d25361-c369-4606-8790-817d6cd05cd4 kind: Event lastTimestamp: "2026-07-07T15:07:54Z" message: 'Created pod: switch-graph-3c9c9-7bd8bf8867-hjp6q' metadata: creationTimestamp: "2026-07-07T15:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:54Z" name: switch-graph-3c9c9-7bd8bf8867.18c00a635010b42c namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: 55c8eea7-1e5d-4c24-aef6-fe1c41e177ff reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-3c9c9 namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: 696f1f16-e0d0-4393-92a4-3944c89de1f1 kind: Event lastTimestamp: "2026-07-07T15:07:54Z" message: Scaled up replica set switch-graph-3c9c9-7bd8bf8867 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:54Z" name: switch-graph-3c9c9.18c00a634e43ec4f namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 5e82fa52-d209-4296-9a83-c28fedc43572 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-3c9c9 namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: 20036166-9400-4553-9841-c31dc2fc60cb kind: Event lastTimestamp: "2026-07-07T15:07:54Z" message: InferenceGraph [switch-graph-3c9c9] is Ready metadata: creationTimestamp: "2026-07-07T15:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:07:54Z" name: switch-graph-3c9c9.18c00a6354fc2fc5 namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 08917002-3823-48a2-8d90-530c5ad3d319 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-3c9c9 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: 20036166-9400-4553-9841-c31dc2fc60cb kind: Event lastTimestamp: "2026-07-07T15:07:54Z" message: 'Failed to update status for InferenceGraph "switch-graph-3c9c9": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-3c9c9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:07:54Z" name: switch-graph-3c9c9.18c00a63596dcf9c namespace: kserve-ci-e2e-test resourceVersion: "15864" uid: c77a2b4b-f0d8-4d55-be27-1cc15e7a7570 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-3c9c9 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: 20036166-9400-4553-9841-c31dc2fc60cb kind: Event lastTimestamp: "2026-07-07T15:07:54Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-3c9c9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:07:54Z" name: switch-graph-3c9c9.18c00a63596e38d9 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: bea1eca1-4994-45c8-b389-922d607ae88f reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:08:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-3c9c9 namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: 4dee77a2-0bba-44d9-a8ec-2f34dbeeac51 kind: Event lastTimestamp: "2026-07-07T15:08:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:39Z" name: switch-graph-3c9c9.18c00a66cdbfb922 namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 49b6d1b0-e6f3-4125-97ce-a7b9801f7629 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:08:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-3c9c9 namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: 4dee77a2-0bba-44d9-a8ec-2f34dbeeac51 kind: Event lastTimestamp: "2026-07-07T15:08:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:39Z" name: switch-graph-3c9c9.18c00a66ce6b6f5d namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: 9fd1d711-9048-4e93-959b-d02f049dcc41 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:28:10.176707Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-c3e9b-7f8b6f4d7d-5twgt namespace: kserve-ci-e2e-test resourceVersion: "25860" uid: 03391917-8310-4234-b9f4-2d3d2a425067 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-c3e9b-7f8b6f4d7d-5twgt to ip-10-0-128-215.ec2.internal metadata: creationTimestamp: "2026-07-07T15:28:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:28:10Z" name: switch-graph-c3e9b-7f8b6f4d7d-5twgt.18c00b7e73e26ac6 namespace: kserve-ci-e2e-test resourceVersion: "25865" uid: 892d9726-0176-4a07-b3d0-1b1e53ca6c93 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:10Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-c3e9b-7f8b6f4d7d-5twgt namespace: kserve-ci-e2e-test resourceVersion: "25873" uid: 03391917-8310-4234-b9f4-2d3d2a425067 kind: Event lastTimestamp: "2026-07-07T15:28:10Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:28:10Z" name: switch-graph-c3e9b-7f8b6f4d7d-5twgt.18c00b7e8e0359aa namespace: kserve-ci-e2e-test resourceVersion: "25885" uid: 6de24fc1-5e65-40a7-9e7f-2ff8d9c24b17 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c3e9b} kind: Pod name: switch-graph-c3e9b-7f8b6f4d7d-5twgt namespace: kserve-ci-e2e-test resourceVersion: "25863" uid: 03391917-8310-4234-b9f4-2d3d2a425067 kind: Event lastTimestamp: "2026-07-07T15:28:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b3cb5f47c4ea9d876e5490bc83b15040508a0aac484b7e9d11de2faf2dbcdfe4" already present on machine metadata: creationTimestamp: "2026-07-07T15:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:28:10Z" name: switch-graph-c3e9b-7f8b6f4d7d-5twgt.18c00b7e8f65c41e namespace: kserve-ci-e2e-test resourceVersion: "25887" uid: f95a26a0-fa0f-480a-99d8-9717dbe65171 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c3e9b} kind: Pod name: switch-graph-c3e9b-7f8b6f4d7d-5twgt namespace: kserve-ci-e2e-test resourceVersion: "25863" uid: 03391917-8310-4234-b9f4-2d3d2a425067 kind: Event lastTimestamp: "2026-07-07T15:28:10Z" message: 'Created container: switch-graph-c3e9b' metadata: creationTimestamp: "2026-07-07T15:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:28:10Z" name: switch-graph-c3e9b-7f8b6f4d7d-5twgt.18c00b7e9265943d namespace: kserve-ci-e2e-test resourceVersion: "25888" uid: 0b794a3d-24b0-4ceb-8c5e-0b77e06b0c74 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c3e9b} kind: Pod name: switch-graph-c3e9b-7f8b6f4d7d-5twgt namespace: kserve-ci-e2e-test resourceVersion: "25863" uid: 03391917-8310-4234-b9f4-2d3d2a425067 kind: Event lastTimestamp: "2026-07-07T15:28:10Z" message: Started container switch-graph-c3e9b metadata: creationTimestamp: "2026-07-07T15:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:28:10Z" name: switch-graph-c3e9b-7f8b6f4d7d-5twgt.18c00b7e92e6003b namespace: kserve-ci-e2e-test resourceVersion: "25890" uid: 1174cf7c-e122-4691-8dea-c2427a4d62c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:44:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c3e9b} kind: Pod name: switch-graph-c3e9b-7f8b6f4d7d-5twgt namespace: kserve-ci-e2e-test resourceVersion: "25863" uid: 03391917-8310-4234-b9f4-2d3d2a425067 kind: Event lastTimestamp: "2026-07-07T15:44:29Z" message: Stopping container switch-graph-c3e9b metadata: creationTimestamp: "2026-07-07T15:44:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:44:29Z" name: switch-graph-c3e9b-7f8b6f4d7d-5twgt.18c00c6270b89baa namespace: kserve-ci-e2e-test resourceVersion: "33054" uid: f5118126-db8f-4002-90f9-bdf48b3f5a2f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:44:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c3e9b} kind: Pod name: switch-graph-c3e9b-7f8b6f4d7d-5twgt namespace: kserve-ci-e2e-test resourceVersion: "25863" uid: 03391917-8310-4234-b9f4-2d3d2a425067 kind: Event lastTimestamp: "2026-07-07T15:44:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:44:51Z" name: switch-graph-c3e9b-7f8b6f4d7d-5twgt.18c00c6305ea0b1e namespace: kserve-ci-e2e-test resourceVersion: "33290" uid: f6c34f0c-5f0b-4e08-b075-ec4d91ce5646 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-215.ec2.internal source: component: kubelet host: ip-10-0-128-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-c3e9b-7f8b6f4d7d namespace: kserve-ci-e2e-test resourceVersion: "25856" uid: b3102c5f-ccc3-4d2d-af8d-6fcc39afb409 kind: Event lastTimestamp: "2026-07-07T15:28:10Z" message: 'Created pod: switch-graph-c3e9b-7f8b6f4d7d-5twgt' metadata: creationTimestamp: "2026-07-07T15:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:28:10Z" name: switch-graph-c3e9b-7f8b6f4d7d.18c00b7e71a63ae5 namespace: kserve-ci-e2e-test resourceVersion: "25862" uid: 73a90c03-0d27-4e7c-81f5-f9485f96ed00 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-c3e9b namespace: kserve-ci-e2e-test resourceVersion: "25853" uid: f57be29a-b625-43dd-b8af-d067af208217 kind: Event lastTimestamp: "2026-07-07T15:28:10Z" message: Scaled up replica set switch-graph-c3e9b-7f8b6f4d7d from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:28:10Z" name: switch-graph-c3e9b.18c00b7e704b61d0 namespace: kserve-ci-e2e-test resourceVersion: "25859" uid: 7f40a4f2-54ad-46d6-b2fb-e0bf6d5e1607 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-c3e9b namespace: kserve-ci-e2e-test resourceVersion: "25876" uid: 494686ee-3c7b-4900-b81e-107b99836df8 kind: Event lastTimestamp: "2026-07-07T15:28:10Z" message: InferenceGraph [switch-graph-c3e9b] is Ready metadata: creationTimestamp: "2026-07-07T15:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:28:10Z" name: switch-graph-c3e9b.18c00b7e7ec4b70b namespace: kserve-ci-e2e-test resourceVersion: "25878" uid: 9f0b99d3-9762-4827-a2f6-52a5699aa3fd reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-c3e9b namespace: kserve-ci-e2e-test resourceVersion: "25855" uid: 666dd1bd-2d00-44fc-8fe3-f558ee4ec9c8 kind: Event lastTimestamp: "2026-07-07T15:28:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:28:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:28:25Z" name: switch-graph-c3e9b.18c00b81efa2403b namespace: kserve-ci-e2e-test resourceVersion: "26301" uid: 5a5015f4-399e-43ee-870b-b9bbcf2be1a7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-c3e9b namespace: kserve-ci-e2e-test resourceVersion: "25855" uid: 666dd1bd-2d00-44fc-8fe3-f558ee4ec9c8 kind: Event lastTimestamp: "2026-07-07T15:28:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:28:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:28:25Z" name: switch-graph-c3e9b.18c00b81f00f1e12 namespace: kserve-ci-e2e-test resourceVersion: "26303" uid: 62fd0908-3e2e-40f1-8ff5-63f5e7825e7a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-c3e9b namespace: kserve-ci-e2e-test resourceVersion: "26302" uid: 666dd1bd-2d00-44fc-8fe3-f558ee4ec9c8 kind: Event lastTimestamp: "2026-07-07T15:28:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:28:40Z" name: switch-graph-c3e9b.18c00b856f020dcc namespace: kserve-ci-e2e-test resourceVersion: "26433" uid: 37b48ad1-b5ef-4198-804c-737fe314a4f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:28:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-c3e9b namespace: kserve-ci-e2e-test resourceVersion: "26302" uid: 666dd1bd-2d00-44fc-8fe3-f558ee4ec9c8 kind: Event lastTimestamp: "2026-07-07T15:28:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:28:40Z" name: switch-graph-c3e9b.18c00b856f79dbbf namespace: kserve-ci-e2e-test resourceVersion: "26434" uid: f6c28048-ae29-4dcb-bf47-e50738686e14 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:06:39.574821Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f85ce-7d456c44cd-fj847 namespace: kserve-ci-e2e-test resourceVersion: "14810" uid: 9cf7b141-0758-4ef9-85f0-a69102495a37 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-f85ce-7d456c44cd-fj847 to ip-10-0-134-178.ec2.internal metadata: creationTimestamp: "2026-07-07T15:06:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:06:39Z" name: switch-graph-f85ce-7d456c44cd-fj847.18c00a51f60456f2 namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: b1beb9f2-1a8a-4ebc-b30c-c2c3cd244e42 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-584bd5fd6b-hmqtn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:39Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f85ce-7d456c44cd-fj847 namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: 9cf7b141-0758-4ef9-85f0-a69102495a37 kind: Event lastTimestamp: "2026-07-07T15:06:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-f85ce-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:39Z" name: switch-graph-f85ce-7d456c44cd-fj847.18c00a51ffd7526a namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: 35f55875-6e55-439b-a750-8ed3f6d902ae reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:40Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f85ce-7d456c44cd-fj847 namespace: kserve-ci-e2e-test resourceVersion: "14817" uid: 9cf7b141-0758-4ef9-85f0-a69102495a37 kind: Event lastTimestamp: "2026-07-07T15:06:40Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:06:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:06:40Z" name: switch-graph-f85ce-7d456c44cd-fj847.18c00a5233431746 namespace: kserve-ci-e2e-test resourceVersion: "14833" uid: 3f09ee58-4a45-4109-928a-04fec933ac80 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f85ce} kind: Pod name: switch-graph-f85ce-7d456c44cd-fj847 namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: 9cf7b141-0758-4ef9-85f0-a69102495a37 kind: Event lastTimestamp: "2026-07-07T15:06:40Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:b3cb5f47c4ea9d876e5490bc83b15040508a0aac484b7e9d11de2faf2dbcdfe4" metadata: creationTimestamp: "2026-07-07T15:06:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:40Z" name: switch-graph-f85ce-7d456c44cd-fj847.18c00a5234f5f6ed namespace: kserve-ci-e2e-test resourceVersion: "14835" uid: edc325dc-699c-4bb1-bb33-f630c9331c6e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f85ce} kind: Pod name: switch-graph-f85ce-7d456c44cd-fj847 namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: 9cf7b141-0758-4ef9-85f0-a69102495a37 kind: Event lastTimestamp: "2026-07-07T15:06:45Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:b3cb5f47c4ea9d876e5490bc83b15040508a0aac484b7e9d11de2faf2dbcdfe4" in 4.91s (4.91s including waiting). Image size: 218072672 bytes.' metadata: creationTimestamp: "2026-07-07T15:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:45Z" name: switch-graph-f85ce-7d456c44cd-fj847.18c00a5359ab4b71 namespace: kserve-ci-e2e-test resourceVersion: "14869" uid: a311ebc2-9891-452b-be05-e3ceef67b94a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f85ce} kind: Pod name: switch-graph-f85ce-7d456c44cd-fj847 namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: 9cf7b141-0758-4ef9-85f0-a69102495a37 kind: Event lastTimestamp: "2026-07-07T15:06:45Z" message: 'Created container: switch-graph-f85ce' metadata: creationTimestamp: "2026-07-07T15:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:45Z" name: switch-graph-f85ce-7d456c44cd-fj847.18c00a535c894ce8 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: c4aa9986-137d-43cb-bb48-77b4f9db4158 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f85ce} kind: Pod name: switch-graph-f85ce-7d456c44cd-fj847 namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: 9cf7b141-0758-4ef9-85f0-a69102495a37 kind: Event lastTimestamp: "2026-07-07T15:06:45Z" message: Started container switch-graph-f85ce metadata: creationTimestamp: "2026-07-07T15:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:45Z" name: switch-graph-f85ce-7d456c44cd-fj847.18c00a535cf2654a namespace: kserve-ci-e2e-test resourceVersion: "14871" uid: 7d80da75-c6a1-44bd-a2c4-eacd6c24cfe2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f85ce} kind: Pod name: switch-graph-f85ce-7d456c44cd-fj847 namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: 9cf7b141-0758-4ef9-85f0-a69102495a37 kind: Event lastTimestamp: "2026-07-07T15:06:53Z" message: Stopping container switch-graph-f85ce metadata: creationTimestamp: "2026-07-07T15:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:53Z" name: switch-graph-f85ce-7d456c44cd-fj847.18c00a554884d39c namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: c77205df-46a9-476c-a3a7-8a0011178afe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:06:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f85ce} kind: Pod name: switch-graph-f85ce-7d456c44cd-fj847 namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: 9cf7b141-0758-4ef9-85f0-a69102495a37 kind: Event lastTimestamp: "2026-07-07T15:07:22Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:06:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:22Z" name: switch-graph-f85ce-7d456c44cd-fj847.18c00a560fcc185a namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: 5e18b358-d314-4486-985a-1281968a3c16 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-178.ec2.internal source: component: kubelet host: ip-10-0-134-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-f85ce-7d456c44cd namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: 5af161a4-e7d3-4782-b11a-012fbcb425a7 kind: Event lastTimestamp: "2026-07-07T15:06:39Z" message: 'Created pod: switch-graph-f85ce-7d456c44cd-fj847' metadata: creationTimestamp: "2026-07-07T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:39Z" name: switch-graph-f85ce-7d456c44cd.18c00a51f57adcbd namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: b1c3eee2-7d26-485c-a487-53e0a6ff5c71 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-f85ce namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: a863cb7b-bcaa-4003-96a1-83f21ef6f97b kind: Event lastTimestamp: "2026-07-07T15:06:39Z" message: Scaled up replica set switch-graph-f85ce-7d456c44cd from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:39Z" name: switch-graph-f85ce.18c00a51f498ae3c namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: d4eacfd9-9399-4ae2-91bf-f4da9fe74bab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-f85ce namespace: kserve-ci-e2e-test resourceVersion: "14822" uid: 4ce24cbd-25eb-4bb4-a573-89a6134873ed kind: Event lastTimestamp: "2026-07-07T15:06:39Z" message: InferenceGraph [switch-graph-f85ce] is Ready metadata: creationTimestamp: "2026-07-07T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:06:39Z" name: switch-graph-f85ce.18c00a51fb1bf21a namespace: kserve-ci-e2e-test resourceVersion: "14824" uid: bff9e73e-0c61-4f7a-95b2-7a07313f42ff reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "33323"