---
apiVersion: apps/v1
items:
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2026-07-08T07:14:55Z"
    generation: 1
    labels:
      app.kubernetes.io/component: llminferenceservice-workload
      app.kubernetes.io/name: auth-disabled-test
      app.kubernetes.io/part-of: llminferenceservice
      kserve.io/component: workload
      llm-d.ai/role: both
    managedFields:
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:labels:
            .: {}
            f:app.kubernetes.io/component: {}
            f:app.kubernetes.io/name: {}
            f:app.kubernetes.io/part-of: {}
            f:kserve.io/component: {}
            f:llm-d.ai/role: {}
          f:ownerReferences:
            .: {}
            k:{"uid":"ce7c4c80-56a3-44d8-a8c1-056804de3ef9"}: {}
        f:spec:
          f:progressDeadlineSeconds: {}
          f:replicas: {}
          f:revisionHistoryLimit: {}
          f:selector: {}
          f:strategy:
            f:rollingUpdate:
              .: {}
              f:maxSurge: {}
              f:maxUnavailable: {}
            f:type: {}
          f:template:
            f:metadata:
              f:labels:
                .: {}
                f:app.kubernetes.io/component: {}
                f:app.kubernetes.io/name: {}
                f:app.kubernetes.io/part-of: {}
                f:kserve.io/component: {}
                f:llm-d.ai/role: {}
            f:spec:
              f:containers:
                k:{"name":"main"}:
                  .: {}
                  f:command: {}
                  f:env:
                    .: {}
                    k:{"name":"HF_HUB_CACHE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HOME"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"TORCHINDUCTOR_CACHE_DIR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"USER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_CPU_KVCACHE_SPACE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_ENABLE_V1_MULTIPROCESSING"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_LOGGING_LEVEL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:lifecycle:
                    .: {}
                    f:preStop:
                      .: {}
                      f:exec:
                        .: {}
                        f:command: {}
                  f:livenessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:name: {}
                  f:ports:
                    .: {}
                    k:{"containerPort":8000,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:protocol: {}
                  f:readinessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:securityContext:
                    .: {}
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      .: {}
                      f:drop: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsNonRoot: {}
                    f:seccompProfile:
                      .: {}
                      f:type: {}
                  f:startupProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/dev/shm"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/home"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
                    k:{"mountPath":"/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/tmp"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/kserve/tls"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
              f:dnsPolicy: {}
              f:initContainers:
                .: {}
                k:{"name":"storage-initializer"}:
                  .: {}
                  f:args: {}
                  f:env:
                    .: {}
                    k:{"name":"AWS_ACCESS_KEY_ID"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"AWS_CA_BUNDLE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_CA_BUNDLE_CONFIGMAP"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_ENDPOINT_URL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_SECRET_ACCESS_KEY"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"HF_HUB_ENABLE_HF_TRANSFER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_HIGH_PERFORMANCE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_NUM_CONCURRENT_RANGE_GETS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_ENDPOINT"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_USE_HTTPS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_VERIFY_SSL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:name: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
              f:restartPolicy: {}
              f:schedulerName: {}
              f:securityContext: {}
              f:terminationGracePeriodSeconds: {}
              f:volumes:
                .: {}
                k:{"name":"dshm"}:
                  .: {}
                  f:emptyDir:
                    .: {}
                    f:medium: {}
                    f:sizeLimit: {}
                  f:name: {}
                k:{"name":"home"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"kserve-provision-location"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"model-cache"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tls-certs"}:
                  .: {}
                  f:name: {}
                  f:secret:
                    .: {}
                    f:defaultMode: {}
                    f:secretName: {}
                k:{"name":"tmp-dir"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
      manager: manager
      operation: Update
      time: "2026-07-08T07:14:55Z"
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:deployment.kubernetes.io/revision: {}
        f:status:
          f:availableReplicas: {}
          f:conditions:
            .: {}
            k:{"type":"Available"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"Progressing"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:observedGeneration: {}
          f:readyReplicas: {}
          f:replicas: {}
          f:updatedReplicas: {}
      manager: kube-controller-manager
      operation: Update
      subresource: status
      time: "2026-07-08T07:16:55Z"
    name: auth-disabled-test-kserve
    namespace: kserve-ci-e2e-test
    ownerReferences:
    - apiVersion: serving.kserve.io/v1alpha2
      blockOwnerDeletion: true
      controller: true
      kind: LLMInferenceService
      name: auth-disabled-test
      uid: ce7c4c80-56a3-44d8-a8c1-056804de3ef9
    resourceVersion: "30565"
    uid: 0b2c84e7-86b7-43e7-b596-ad815acebd6b
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app.kubernetes.io/component: llminferenceservice-workload
        app.kubernetes.io/name: auth-disabled-test
        app.kubernetes.io/part-of: llminferenceservice
        kserve.io/component: workload
        llm-d.ai/role: both
    strategy:
      rollingUpdate:
        maxSurge: 25%
        maxUnavailable: 25%
      type: RollingUpdate
    template:
      metadata:
        labels:
          app.kubernetes.io/component: llminferenceservice-workload
          app.kubernetes.io/name: auth-disabled-test
          app.kubernetes.io/part-of: llminferenceservice
          kserve.io/component: workload
          llm-d.ai/role: both
      spec:
        containers:
        - command:
          - /bin/bash
          - -c
          - |-
            if [ -f /etc/profile.d/ibm-aiu-setup.sh ]; then
              source /etc/profile.d/ibm-aiu-setup.sh
            fi

            if [ "$KSERVE_INFER_ROCE" = "true" ]; then
              echo "Trying to infer RoCE configs ... "
              grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
              grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null

              cat /proc/driver/nvidia/params

              KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}

              echo "[Infer RoCE] Discovering active HCAs ..."
              active_hcas=()
              # Loop through all mlx5 devices found in sysfs
              for hca_dir in /sys/class/infiniband/mlx5_*; do
                  # Ensure it's a directory before proceeding
                  if [ -d "$hca_dir" ]; then
                      hca_name=$(basename "$hca_dir")
                      port_state_file="$hca_dir/ports/1/state" # Assume port 1
                      type_file="$hca_dir/ports/1/gid_attrs/types/*"

                      echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
                      if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
                          echo "[Infer RoCE] Found active HCA: $hca_name"
                          active_hcas+=("$hca_name")
                      else
                          echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
                      fi
                  fi
              done

              # Check if we found any active HCAs
              if [ ${#active_hcas[@]} -gt 0 ]; then
                  # Join the array elements with a comma
                  hca_port_pairs=()
                  for hca in "${active_hcas[@]}"; do
                    hca_port_pairs+=("${hca}:1")
                  done

                  active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
                  hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
                  echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
                  export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
                  export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
                  export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}

                  echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
                  echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
                  echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
              else
                  echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
              fi

              if [ ${#active_hcas[@]} -gt 0 ]; then
                  echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."

                  # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
                  declare -A gid_index_count
                  declare -A hca_gid_index

                  for hca_name in "${active_hcas[@]}"; do
                      echo "[Infer RoCE] Processing HCA: ${hca_name}"

                      # Find all RoCE v2 IPv4 GIDs for this HCA and count by index
                      for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
                          if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
                              idx=$(basename "$tpath")
                              gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
                              # Check for IPv4 GID (contains ffff:)
                              if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
                                  gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
                                  echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
                                  hca_gid_index["${hca_name}"]="${idx}"
                                  gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
                                  break  # Use first found IPv4 GID per HCA
                              fi
                          fi
                      done
                  done

                  # Find the most common GID index (most likely to be consistent across nodes)
                  best_gid_index=""
                  max_count=0
                  for idx in "${!gid_index_count[@]}"; do
                      count=${gid_index_count["${idx}"]}
                      echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
                      if [ $count -gt $max_count ]; then
                          max_count=$count
                          best_gid_index="$idx"
                      fi
                  done

                  # Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
                  if [ ${#gid_index_count[@]} -gt 1 ]; then
                      echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
                      # If there's a tie, prefer index 3 as it's most common in SR-IOV setups
                      if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
                          best_gid_index="3"
                          echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
                      fi
                  fi

                  # Check if GID_INDEX is already set via environment variables
                  if [ -n "${NCCL_IB_GID_INDEX}" ]; then
                      echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
                  elif [ -n "$best_gid_index" ]; then
                      echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"

                      export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}

                      echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
                  else
                      echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
                  fi
              else
                  echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
              fi
            fi

            # --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
            # Older versions still need the blanket --disable-uvicorn-access-log.
            ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
            VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
            echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
            if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
              ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
            fi
            echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"

            # --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
            SHUTDOWN_TIMEOUT_ARGS=""
            if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
              SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout 40"
            fi

            eval "exec vllm serve /mnt/models \
              --served-model-name "facebook/opt-125m" "publishers/kserve-ci-e2e-test/models/facebook/opt-125m" \
              --port 8000 \
              ${ACCESS_LOG_ARGS} \
              ${SHUTDOWN_TIMEOUT_ARGS} \
              --enable-ssl-refresh \
              --ssl-certfile /var/run/kserve/tls/tls.crt \
              --ssl-keyfile /var/run/kserve/tls/tls.key \
              ${VLLM_ADDITIONAL_ARGS} \
              $@"
          - --
          env:
          - name: HOME
            value: /home
          - name: VLLM_LOGGING_LEVEL
            value: DEBUG
          - name: VLLM_CPU_KVCACHE_SPACE
            value: "1"
          - name: VLLM_ENABLE_V1_MULTIPROCESSING
            value: "0"
          - name: USER
            value: nonroot
          - name: TORCHINDUCTOR_CACHE_DIR
            value: /tmp/torchinductor-cache
          - name: HF_HUB_CACHE
            value: /models
          image: public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.19.0
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - /bin/sleep
                - "15"
          livenessProbe:
            failureThreshold: 10
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: main
          ports:
          - containerPort: 8000
            protocol: TCP
          readinessProbe:
            failureThreshold: 2
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 1
            successThreshold: 1
            timeoutSeconds: 1
          resources:
            limits:
              cpu: "2"
              memory: 7Gi
            requests:
              cpu: 200m
              memory: 2Gi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
            runAsNonRoot: true
            seccompProfile:
              type: RuntimeDefault
          startupProbe:
            failureThreshold: 60
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /home
            name: home
          - mountPath: /tmp
            name: tmp-dir
          - mountPath: /dev/shm
            name: dshm
          - mountPath: /models
            name: model-cache
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
          - mountPath: /mnt/models
            name: kserve-provision-location
            readOnly: true
        dnsPolicy: ClusterFirst
        initContainers:
        - args:
          - hf://facebook/opt-125m
          - /mnt/models
          env:
          - name: AWS_ACCESS_KEY_ID
            valueFrom:
              secretKeyRef:
                key: AWS_ACCESS_KEY_ID
                name: seaweedfs-s3-creds
          - name: AWS_SECRET_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                key: AWS_SECRET_ACCESS_KEY
                name: seaweedfs-s3-creds
          - name: S3_USE_HTTPS
            value: "0"
          - name: S3_ENDPOINT
            value: s3-service.kserve:8333
          - name: AWS_ENDPOINT_URL
            value: http://s3-service.kserve:8333
          - name: S3_VERIFY_SSL
            value: "0"
          - name: AWS_CA_BUNDLE
            value: /etc/ssl/custom-certs/cabundle.crt
          - name: AWS_CA_BUNDLE_CONFIGMAP
            value: odh-kserve-custom-ca-bundle
          - name: HF_HUB_ENABLE_HF_TRANSFER
            value: "1"
          - name: HF_XET_HIGH_PERFORMANCE
            value: "1"
          - name: HF_XET_NUM_CONCURRENT_RANGE_GETS
            value: "8"
          image: quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773
          imagePullPolicy: IfNotPresent
          name: storage-initializer
          resources:
            limits:
              cpu: "1"
              memory: 24Gi
            requests:
              cpu: 100m
              memory: 100Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /mnt/models
            name: kserve-provision-location
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        terminationGracePeriodSeconds: 60
        volumes:
        - emptyDir: {}
          name: home
        - emptyDir:
            medium: Memory
            sizeLimit: 1Gi
          name: dshm
        - emptyDir: {}
          name: model-cache
        - emptyDir: {}
          name: tmp-dir
        - name: tls-certs
          secret:
            defaultMode: 420
            secretName: auth-disabled-test-kserve-self-signed-certs
        - emptyDir: {}
          name: kserve-provision-location
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2026-07-08T07:16:55Z"
      lastUpdateTime: "2026-07-08T07:16:55Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    - lastTransitionTime: "2026-07-08T07:14:55Z"
      lastUpdateTime: "2026-07-08T07:16:55Z"
      message: ReplicaSet "auth-disabled-test-kserve-68f5d8959c" has successfully
        progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2026-07-08T07:14:55Z"
    generation: 1
    labels:
      app.kubernetes.io/component: llminferenceservice-router-scheduler
      app.kubernetes.io/name: auth-disabled-test
      app.kubernetes.io/part-of: llminferenceservice
    managedFields:
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:labels:
            .: {}
            f:app.kubernetes.io/component: {}
            f:app.kubernetes.io/name: {}
            f:app.kubernetes.io/part-of: {}
          f:ownerReferences:
            .: {}
            k:{"uid":"ce7c4c80-56a3-44d8-a8c1-056804de3ef9"}: {}
        f:spec:
          f:progressDeadlineSeconds: {}
          f:replicas: {}
          f:revisionHistoryLimit: {}
          f:selector: {}
          f:strategy:
            f:type: {}
          f:template:
            f:metadata:
              f:annotations:
                .: {}
                f:app.kubernetes.io/version: {}
                f:certificates.kserve.io/expiration-v2: {}
              f:labels:
                .: {}
                f:app.kubernetes.io/component: {}
                f:app.kubernetes.io/name: {}
                f:app.kubernetes.io/part-of: {}
            f:spec:
              f:containers:
                k:{"name":"main"}:
                  .: {}
                  f:args: {}
                  f:command: {}
                  f:env:
                    .: {}
                    k:{"name":"SSL_CERT_DIR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:lifecycle:
                    .: {}
                    f:preStop:
                      .: {}
                      f:exec:
                        .: {}
                        f:command: {}
                  f:livenessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:grpc:
                      .: {}
                      f:port: {}
                      f:service: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:name: {}
                  f:ports:
                    .: {}
                    k:{"containerPort":5557,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9002,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9003,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9090,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                  f:readinessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:grpc:
                      .: {}
                      f:port: {}
                      f:service: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources:
                    .: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:securityContext:
                    .: {}
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      .: {}
                      f:drop: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsNonRoot: {}
                    f:seccompProfile:
                      .: {}
                      f:type: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/tmp/tokenizer"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/kserve/tls"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
              f:dnsPolicy: {}
              f:initContainers:
                .: {}
                k:{"name":"storage-initializer"}:
                  .: {}
                  f:args: {}
                  f:env:
                    .: {}
                    k:{"name":"AWS_ACCESS_KEY_ID"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"AWS_CA_BUNDLE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_CA_BUNDLE_CONFIGMAP"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_ENDPOINT_URL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_SECRET_ACCESS_KEY"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"HF_HUB_ENABLE_HF_TRANSFER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_HIGH_PERFORMANCE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_NUM_CONCURRENT_RANGE_GETS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_ENDPOINT"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_USE_HTTPS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_VERIFY_SSL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"STORAGE_ALLOW_PATTERNS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:name: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
              f:restartPolicy: {}
              f:schedulerName: {}
              f:securityContext: {}
              f:serviceAccount: {}
              f:serviceAccountName: {}
              f:terminationGracePeriodSeconds: {}
              f:volumes:
                .: {}
                k:{"name":"kserve-provision-location"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tls-certs"}:
                  .: {}
                  f:name: {}
                  f:secret:
                    .: {}
                    f:defaultMode: {}
                    f:secretName: {}
                k:{"name":"tokenizer-cache"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tokenizer-tmp"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tokenizer-uds"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
      manager: manager
      operation: Update
      time: "2026-07-08T07:14:55Z"
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:deployment.kubernetes.io/revision: {}
        f:status:
          f:availableReplicas: {}
          f:conditions:
            .: {}
            k:{"type":"Available"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"Progressing"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:observedGeneration: {}
          f:readyReplicas: {}
          f:replicas: {}
          f:updatedReplicas: {}
      manager: kube-controller-manager
      operation: Update
      subresource: status
      time: "2026-07-08T07:15:30Z"
    name: auth-disabled-test-kserve-router-scheduler
    namespace: kserve-ci-e2e-test
    ownerReferences:
    - apiVersion: serving.kserve.io/v1alpha2
      blockOwnerDeletion: true
      controller: true
      kind: LLMInferenceService
      name: auth-disabled-test
      uid: ce7c4c80-56a3-44d8-a8c1-056804de3ef9
    resourceVersion: "29135"
    uid: 2d3259ac-03bd-42ae-b0d8-f179406fe26d
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app.kubernetes.io/component: llminferenceservice-router-scheduler
        app.kubernetes.io/name: auth-disabled-test
        app.kubernetes.io/part-of: llminferenceservice
    strategy:
      type: Recreate
    template:
      metadata:
        annotations:
          app.kubernetes.io/version: 0.9.0
          certificates.kserve.io/expiration-v2: "true"
        labels:
          app.kubernetes.io/component: llminferenceservice-router-scheduler
          app.kubernetes.io/name: auth-disabled-test
          app.kubernetes.io/part-of: llminferenceservice
      spec:
        containers:
        - args:
          - --config-text
          - |
            apiVersion: inference.networking.x-k8s.io/v1alpha1
            kind: EndpointPickerConfig
            plugins:
            - type: single-profile-handler
            - type: queue-scorer
            - type: prefix-cache-scorer
            - type: max-score-picker
            schedulingProfiles:
            - name: default
              plugins:
              - pluginRef: queue-scorer
                weight: 2
              - pluginRef: prefix-cache-scorer
                weight: 3
              - pluginRef: max-score-picker
          command:
          - /app/epp
          - --pool-name
          - auth-disabled-test-inference-pool
          - --pool-namespace
          - kserve-ci-e2e-test
          - --zap-encoder
          - json
          - --grpc-port
          - "9002"
          - --grpc-health-port
          - "9003"
          - --enable-cert-reload=true
          - --secure-serving=true
          - --model-server-metrics-scheme=https
          - --cert-path=/var/run/kserve/tls
          env:
          - name: SSL_CERT_DIR
            value: /var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs
          image: ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0-rc.2
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - /bin/sleep
                - "15"
          livenessProbe:
            failureThreshold: 3
            grpc:
              port: 9003
              service: liveness
            initialDelaySeconds: 5
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: main
          ports:
          - containerPort: 9002
            name: grpc
            protocol: TCP
          - containerPort: 9003
            name: grpc-health
            protocol: TCP
          - containerPort: 9090
            name: metrics
            protocol: TCP
          - containerPort: 5557
            name: zmq
            protocol: TCP
          readinessProbe:
            failureThreshold: 3
            grpc:
              port: 9003
              service: readiness
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          resources:
            requests:
              cpu: 256m
              memory: 500Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
            runAsNonRoot: true
            seccompProfile:
              type: RuntimeDefault
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
          - mountPath: /tmp/tokenizer
            name: tokenizer-uds
        dnsPolicy: ClusterFirst
        initContainers:
        - args:
          - hf://facebook/opt-125m
          - /mnt/models
          env:
          - name: AWS_ACCESS_KEY_ID
            valueFrom:
              secretKeyRef:
                key: AWS_ACCESS_KEY_ID
                name: seaweedfs-s3-creds
          - name: AWS_SECRET_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                key: AWS_SECRET_ACCESS_KEY
                name: seaweedfs-s3-creds
          - name: S3_USE_HTTPS
            value: "0"
          - name: S3_ENDPOINT
            value: s3-service.kserve:8333
          - name: AWS_ENDPOINT_URL
            value: http://s3-service.kserve:8333
          - name: S3_VERIFY_SSL
            value: "0"
          - name: AWS_CA_BUNDLE
            value: /etc/ssl/custom-certs/cabundle.crt
          - name: AWS_CA_BUNDLE_CONFIGMAP
            value: odh-kserve-custom-ca-bundle
          - name: HF_HUB_ENABLE_HF_TRANSFER
            value: "1"
          - name: HF_XET_HIGH_PERFORMANCE
            value: "1"
          - name: HF_XET_NUM_CONCURRENT_RANGE_GETS
            value: "8"
          - name: STORAGE_ALLOW_PATTERNS
            value: '["tokenizer.json", "tokenizer_config.json", "special_tokens_map.json",
              "vocab.json", "merges.txt", "config.json", "generation_config.json"]'
          image: quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773
          imagePullPolicy: IfNotPresent
          name: storage-initializer
          resources:
            limits:
              cpu: "1"
              memory: 24Gi
            requests:
              cpu: 100m
              memory: 100Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /mnt/models
            name: kserve-provision-location
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        serviceAccount: auth-disabled-test-epp-sa
        serviceAccountName: auth-disabled-test-epp-sa
        terminationGracePeriodSeconds: 60
        volumes:
        - name: tls-certs
          secret:
            defaultMode: 420
            secretName: auth-disabled-test-kserve-self-signed-certs
        - emptyDir: {}
          name: tokenizer-uds
        - emptyDir: {}
          name: tokenizer-tmp
        - emptyDir: {}
          name: tokenizer-cache
        - emptyDir: {}
          name: kserve-provision-location
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2026-07-08T07:15:30Z"
      lastUpdateTime: "2026-07-08T07:15:30Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    - lastTransitionTime: "2026-07-08T07:14:55Z"
      lastUpdateTime: "2026-07-08T07:15:30Z"
      message: ReplicaSet "auth-disabled-test-kserve-router-scheduler-5898f785fb"
        has successfully progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2026-07-08T07:38:21Z"
    generation: 1
    labels:
      app.kubernetes.io/component: llminferenceservice-workload
      app.kubernetes.io/name: custom-route-timeout-test
      app.kubernetes.io/part-of: llminferenceservice
      kserve.io/component: workload
      llm-d.ai/role: both
    managedFields:
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:labels:
            .: {}
            f:app.kubernetes.io/component: {}
            f:app.kubernetes.io/name: {}
            f:app.kubernetes.io/part-of: {}
            f:kserve.io/component: {}
            f:llm-d.ai/role: {}
          f:ownerReferences:
            .: {}
            k:{"uid":"5fd98af4-ea51-4e8c-95c6-4ab08c991e2c"}: {}
        f:spec:
          f:progressDeadlineSeconds: {}
          f:replicas: {}
          f:revisionHistoryLimit: {}
          f:selector: {}
          f:strategy:
            f:rollingUpdate:
              .: {}
              f:maxSurge: {}
              f:maxUnavailable: {}
            f:type: {}
          f:template:
            f:metadata:
              f:labels:
                .: {}
                f:app.kubernetes.io/component: {}
                f:app.kubernetes.io/name: {}
                f:app.kubernetes.io/part-of: {}
                f:kserve.io/component: {}
                f:llm-d.ai/role: {}
            f:spec:
              f:containers:
                k:{"name":"main"}:
                  .: {}
                  f:command: {}
                  f:env:
                    .: {}
                    k:{"name":"HF_HUB_CACHE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HOME"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"TORCHINDUCTOR_CACHE_DIR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"USER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_CPU_KVCACHE_SPACE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_ENABLE_V1_MULTIPROCESSING"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_LOGGING_LEVEL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:lifecycle:
                    .: {}
                    f:preStop:
                      .: {}
                      f:exec:
                        .: {}
                        f:command: {}
                  f:livenessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:name: {}
                  f:ports:
                    .: {}
                    k:{"containerPort":8000,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:protocol: {}
                  f:readinessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:securityContext:
                    .: {}
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      .: {}
                      f:drop: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsNonRoot: {}
                    f:seccompProfile:
                      .: {}
                      f:type: {}
                  f:startupProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/dev/shm"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/home"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
                    k:{"mountPath":"/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/tmp"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/kserve/tls"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
              f:dnsPolicy: {}
              f:initContainers:
                .: {}
                k:{"name":"storage-initializer"}:
                  .: {}
                  f:args: {}
                  f:env:
                    .: {}
                    k:{"name":"AWS_ACCESS_KEY_ID"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"AWS_CA_BUNDLE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_CA_BUNDLE_CONFIGMAP"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_ENDPOINT_URL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_SECRET_ACCESS_KEY"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"HF_HUB_ENABLE_HF_TRANSFER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_HIGH_PERFORMANCE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_NUM_CONCURRENT_RANGE_GETS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_ENDPOINT"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_USE_HTTPS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_VERIFY_SSL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:name: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
              f:restartPolicy: {}
              f:schedulerName: {}
              f:securityContext: {}
              f:terminationGracePeriodSeconds: {}
              f:volumes:
                .: {}
                k:{"name":"dshm"}:
                  .: {}
                  f:emptyDir:
                    .: {}
                    f:medium: {}
                    f:sizeLimit: {}
                  f:name: {}
                k:{"name":"home"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"kserve-provision-location"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"model-cache"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tls-certs"}:
                  .: {}
                  f:name: {}
                  f:secret:
                    .: {}
                    f:defaultMode: {}
                    f:secretName: {}
                k:{"name":"tmp-dir"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
      manager: manager
      operation: Update
      time: "2026-07-08T07:38:21Z"
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:deployment.kubernetes.io/revision: {}
        f:status:
          f:availableReplicas: {}
          f:conditions:
            .: {}
            k:{"type":"Available"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"Progressing"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:observedGeneration: {}
          f:readyReplicas: {}
          f:replicas: {}
          f:updatedReplicas: {}
      manager: kube-controller-manager
      operation: Update
      subresource: status
      time: "2026-07-08T07:40:32Z"
    name: custom-route-timeout-test-kserve
    namespace: kserve-ci-e2e-test
    ownerReferences:
    - apiVersion: serving.kserve.io/v1alpha2
      blockOwnerDeletion: true
      controller: true
      kind: LLMInferenceService
      name: custom-route-timeout-test
      uid: 5fd98af4-ea51-4e8c-95c6-4ab08c991e2c
    resourceVersion: "53251"
    uid: 25fdd25b-bdc0-494d-9fd9-091157de6d8c
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app.kubernetes.io/component: llminferenceservice-workload
        app.kubernetes.io/name: custom-route-timeout-test
        app.kubernetes.io/part-of: llminferenceservice
        kserve.io/component: workload
        llm-d.ai/role: both
    strategy:
      rollingUpdate:
        maxSurge: 25%
        maxUnavailable: 25%
      type: RollingUpdate
    template:
      metadata:
        labels:
          app.kubernetes.io/component: llminferenceservice-workload
          app.kubernetes.io/name: custom-route-timeout-test
          app.kubernetes.io/part-of: llminferenceservice
          kserve.io/component: workload
          llm-d.ai/role: both
      spec:
        containers:
        - command:
          - /bin/bash
          - -c
          - |-
            if [ -f /etc/profile.d/ibm-aiu-setup.sh ]; then
              source /etc/profile.d/ibm-aiu-setup.sh
            fi

            if [ "$KSERVE_INFER_ROCE" = "true" ]; then
              echo "Trying to infer RoCE configs ... "
              grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
              grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null

              cat /proc/driver/nvidia/params

              KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}

              echo "[Infer RoCE] Discovering active HCAs ..."
              active_hcas=()
              # Loop through all mlx5 devices found in sysfs
              for hca_dir in /sys/class/infiniband/mlx5_*; do
                  # Ensure it's a directory before proceeding
                  if [ -d "$hca_dir" ]; then
                      hca_name=$(basename "$hca_dir")
                      port_state_file="$hca_dir/ports/1/state" # Assume port 1
                      type_file="$hca_dir/ports/1/gid_attrs/types/*"

                      echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
                      if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
                          echo "[Infer RoCE] Found active HCA: $hca_name"
                          active_hcas+=("$hca_name")
                      else
                          echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
                      fi
                  fi
              done

              # Check if we found any active HCAs
              if [ ${#active_hcas[@]} -gt 0 ]; then
                  # Join the array elements with a comma
                  hca_port_pairs=()
                  for hca in "${active_hcas[@]}"; do
                    hca_port_pairs+=("${hca}:1")
                  done

                  active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
                  hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
                  echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
                  export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
                  export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
                  export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}

                  echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
                  echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
                  echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
              else
                  echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
              fi

              if [ ${#active_hcas[@]} -gt 0 ]; then
                  echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."

                  # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
                  declare -A gid_index_count
                  declare -A hca_gid_index

                  for hca_name in "${active_hcas[@]}"; do
                      echo "[Infer RoCE] Processing HCA: ${hca_name}"

                      # Find all RoCE v2 IPv4 GIDs for this HCA and count by index
                      for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
                          if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
                              idx=$(basename "$tpath")
                              gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
                              # Check for IPv4 GID (contains ffff:)
                              if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
                                  gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
                                  echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
                                  hca_gid_index["${hca_name}"]="${idx}"
                                  gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
                                  break  # Use first found IPv4 GID per HCA
                              fi
                          fi
                      done
                  done

                  # Find the most common GID index (most likely to be consistent across nodes)
                  best_gid_index=""
                  max_count=0
                  for idx in "${!gid_index_count[@]}"; do
                      count=${gid_index_count["${idx}"]}
                      echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
                      if [ $count -gt $max_count ]; then
                          max_count=$count
                          best_gid_index="$idx"
                      fi
                  done

                  # Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
                  if [ ${#gid_index_count[@]} -gt 1 ]; then
                      echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
                      # If there's a tie, prefer index 3 as it's most common in SR-IOV setups
                      if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
                          best_gid_index="3"
                          echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
                      fi
                  fi

                  # Check if GID_INDEX is already set via environment variables
                  if [ -n "${NCCL_IB_GID_INDEX}" ]; then
                      echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
                  elif [ -n "$best_gid_index" ]; then
                      echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"

                      export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}

                      echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
                  else
                      echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
                  fi
              else
                  echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
              fi
            fi

            # --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
            # Older versions still need the blanket --disable-uvicorn-access-log.
            ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
            VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
            echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
            if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
              ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
            fi
            echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"

            # --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
            SHUTDOWN_TIMEOUT_ARGS=""
            if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
              SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout 40"
            fi

            eval "exec vllm serve /mnt/models \
              --served-model-name "facebook/opt-125m" "publishers/kserve-ci-e2e-test/models/facebook/opt-125m" \
              --port 8000 \
              ${ACCESS_LOG_ARGS} \
              ${SHUTDOWN_TIMEOUT_ARGS} \
              --enable-ssl-refresh \
              --ssl-certfile /var/run/kserve/tls/tls.crt \
              --ssl-keyfile /var/run/kserve/tls/tls.key \
              ${VLLM_ADDITIONAL_ARGS} \
              $@"
          - --
          env:
          - name: HOME
            value: /home
          - name: VLLM_LOGGING_LEVEL
            value: DEBUG
          - name: VLLM_CPU_KVCACHE_SPACE
            value: "1"
          - name: VLLM_ENABLE_V1_MULTIPROCESSING
            value: "0"
          - name: USER
            value: nonroot
          - name: TORCHINDUCTOR_CACHE_DIR
            value: /tmp/torchinductor-cache
          - name: HF_HUB_CACHE
            value: /models
          image: public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.19.0
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - /bin/sleep
                - "15"
          livenessProbe:
            failureThreshold: 10
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: main
          ports:
          - containerPort: 8000
            protocol: TCP
          readinessProbe:
            failureThreshold: 2
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 1
            successThreshold: 1
            timeoutSeconds: 1
          resources:
            limits:
              cpu: "2"
              memory: 7Gi
            requests:
              cpu: 200m
              memory: 2Gi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
            runAsNonRoot: true
            seccompProfile:
              type: RuntimeDefault
          startupProbe:
            failureThreshold: 60
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /home
            name: home
          - mountPath: /tmp
            name: tmp-dir
          - mountPath: /dev/shm
            name: dshm
          - mountPath: /models
            name: model-cache
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
          - mountPath: /mnt/models
            name: kserve-provision-location
            readOnly: true
        dnsPolicy: ClusterFirst
        initContainers:
        - args:
          - hf://facebook/opt-125m
          - /mnt/models
          env:
          - name: AWS_ACCESS_KEY_ID
            valueFrom:
              secretKeyRef:
                key: AWS_ACCESS_KEY_ID
                name: seaweedfs-s3-creds
          - name: AWS_SECRET_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                key: AWS_SECRET_ACCESS_KEY
                name: seaweedfs-s3-creds
          - name: S3_USE_HTTPS
            value: "0"
          - name: S3_ENDPOINT
            value: s3-service.kserve:8333
          - name: AWS_ENDPOINT_URL
            value: http://s3-service.kserve:8333
          - name: S3_VERIFY_SSL
            value: "0"
          - name: AWS_CA_BUNDLE
            value: /etc/ssl/custom-certs/cabundle.crt
          - name: AWS_CA_BUNDLE_CONFIGMAP
            value: odh-kserve-custom-ca-bundle
          - name: HF_HUB_ENABLE_HF_TRANSFER
            value: "1"
          - name: HF_XET_HIGH_PERFORMANCE
            value: "1"
          - name: HF_XET_NUM_CONCURRENT_RANGE_GETS
            value: "8"
          image: quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773
          imagePullPolicy: IfNotPresent
          name: storage-initializer
          resources:
            limits:
              cpu: "1"
              memory: 24Gi
            requests:
              cpu: 100m
              memory: 100Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /mnt/models
            name: kserve-provision-location
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        terminationGracePeriodSeconds: 60
        volumes:
        - emptyDir: {}
          name: home
        - emptyDir:
            medium: Memory
            sizeLimit: 1Gi
          name: dshm
        - emptyDir: {}
          name: model-cache
        - emptyDir: {}
          name: tmp-dir
        - name: tls-certs
          secret:
            defaultMode: 420
            secretName: custom-route-timeout-test-kserve-self-signed-certs
        - emptyDir: {}
          name: kserve-provision-location
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2026-07-08T07:40:32Z"
      lastUpdateTime: "2026-07-08T07:40:32Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    - lastTransitionTime: "2026-07-08T07:38:21Z"
      lastUpdateTime: "2026-07-08T07:40:32Z"
      message: ReplicaSet "custom-route-timeout-test-kserve-b6f9dd5f7" has successfully
        progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2026-07-08T07:38:22Z"
    generation: 1
    labels:
      app.kubernetes.io/component: llminferenceservice-router-scheduler
      app.kubernetes.io/name: custom-route-timeout-test
      app.kubernetes.io/part-of: llminferenceservice
    managedFields:
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:labels:
            .: {}
            f:app.kubernetes.io/component: {}
            f:app.kubernetes.io/name: {}
            f:app.kubernetes.io/part-of: {}
          f:ownerReferences:
            .: {}
            k:{"uid":"5fd98af4-ea51-4e8c-95c6-4ab08c991e2c"}: {}
        f:spec:
          f:progressDeadlineSeconds: {}
          f:replicas: {}
          f:revisionHistoryLimit: {}
          f:selector: {}
          f:strategy:
            f:type: {}
          f:template:
            f:metadata:
              f:annotations:
                .: {}
                f:app.kubernetes.io/version: {}
                f:certificates.kserve.io/expiration-v2: {}
              f:labels:
                .: {}
                f:app.kubernetes.io/component: {}
                f:app.kubernetes.io/name: {}
                f:app.kubernetes.io/part-of: {}
            f:spec:
              f:containers:
                k:{"name":"main"}:
                  .: {}
                  f:args: {}
                  f:command: {}
                  f:env:
                    .: {}
                    k:{"name":"SSL_CERT_DIR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:lifecycle:
                    .: {}
                    f:preStop:
                      .: {}
                      f:exec:
                        .: {}
                        f:command: {}
                  f:livenessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:grpc:
                      .: {}
                      f:port: {}
                      f:service: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:name: {}
                  f:ports:
                    .: {}
                    k:{"containerPort":5557,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9002,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9003,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9090,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                  f:readinessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:grpc:
                      .: {}
                      f:port: {}
                      f:service: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources:
                    .: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:securityContext:
                    .: {}
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      .: {}
                      f:drop: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsNonRoot: {}
                    f:seccompProfile:
                      .: {}
                      f:type: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/tmp/tokenizer"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/kserve/tls"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
              f:dnsPolicy: {}
              f:initContainers:
                .: {}
                k:{"name":"storage-initializer"}:
                  .: {}
                  f:args: {}
                  f:env:
                    .: {}
                    k:{"name":"AWS_ACCESS_KEY_ID"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"AWS_CA_BUNDLE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_CA_BUNDLE_CONFIGMAP"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_ENDPOINT_URL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_SECRET_ACCESS_KEY"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"HF_HUB_ENABLE_HF_TRANSFER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_HIGH_PERFORMANCE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_NUM_CONCURRENT_RANGE_GETS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_ENDPOINT"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_USE_HTTPS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_VERIFY_SSL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"STORAGE_ALLOW_PATTERNS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:name: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
              f:restartPolicy: {}
              f:schedulerName: {}
              f:securityContext: {}
              f:serviceAccount: {}
              f:serviceAccountName: {}
              f:terminationGracePeriodSeconds: {}
              f:volumes:
                .: {}
                k:{"name":"kserve-provision-location"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tls-certs"}:
                  .: {}
                  f:name: {}
                  f:secret:
                    .: {}
                    f:defaultMode: {}
                    f:secretName: {}
                k:{"name":"tokenizer-cache"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tokenizer-tmp"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tokenizer-uds"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
      manager: manager
      operation: Update
      time: "2026-07-08T07:38:22Z"
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:deployment.kubernetes.io/revision: {}
        f:status:
          f:availableReplicas: {}
          f:conditions:
            .: {}
            k:{"type":"Available"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"Progressing"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:observedGeneration: {}
          f:readyReplicas: {}
          f:replicas: {}
          f:updatedReplicas: {}
      manager: kube-controller-manager
      operation: Update
      subresource: status
      time: "2026-07-08T07:38:55Z"
    name: custom-route-timeout-test-kserve-router-scheduler
    namespace: kserve-ci-e2e-test
    ownerReferences:
    - apiVersion: serving.kserve.io/v1alpha2
      blockOwnerDeletion: true
      controller: true
      kind: LLMInferenceService
      name: custom-route-timeout-test
      uid: 5fd98af4-ea51-4e8c-95c6-4ab08c991e2c
    resourceVersion: "51955"
    uid: 8ed9a5f3-b302-4058-9e59-c769dedb86cd
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app.kubernetes.io/component: llminferenceservice-router-scheduler
        app.kubernetes.io/name: custom-route-timeout-test
        app.kubernetes.io/part-of: llminferenceservice
    strategy:
      type: Recreate
    template:
      metadata:
        annotations:
          app.kubernetes.io/version: 0.9.0
          certificates.kserve.io/expiration-v2: "true"
        labels:
          app.kubernetes.io/component: llminferenceservice-router-scheduler
          app.kubernetes.io/name: custom-route-timeout-test
          app.kubernetes.io/part-of: llminferenceservice
      spec:
        containers:
        - args:
          - --config-text
          - |
            apiVersion: inference.networking.x-k8s.io/v1alpha1
            kind: EndpointPickerConfig
            plugins:
            - type: single-profile-handler
            - type: queue-scorer
            - type: prefix-cache-scorer
            - type: max-score-picker
            schedulingProfiles:
            - name: default
              plugins:
              - pluginRef: queue-scorer
                weight: 2
              - pluginRef: prefix-cache-scorer
                weight: 3
              - pluginRef: max-score-picker
          command:
          - /app/epp
          - --pool-name
          - custom-route-timeout-test-inference-pool
          - --pool-namespace
          - kserve-ci-e2e-test
          - --zap-encoder
          - json
          - --grpc-port
          - "9002"
          - --grpc-health-port
          - "9003"
          - --enable-cert-reload=true
          - --secure-serving=true
          - --model-server-metrics-scheme=https
          - --cert-path=/var/run/kserve/tls
          env:
          - name: SSL_CERT_DIR
            value: /var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs
          image: ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0-rc.2
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - /bin/sleep
                - "15"
          livenessProbe:
            failureThreshold: 3
            grpc:
              port: 9003
              service: liveness
            initialDelaySeconds: 5
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: main
          ports:
          - containerPort: 9002
            name: grpc
            protocol: TCP
          - containerPort: 9003
            name: grpc-health
            protocol: TCP
          - containerPort: 9090
            name: metrics
            protocol: TCP
          - containerPort: 5557
            name: zmq
            protocol: TCP
          readinessProbe:
            failureThreshold: 3
            grpc:
              port: 9003
              service: readiness
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          resources:
            requests:
              cpu: 256m
              memory: 500Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
            runAsNonRoot: true
            seccompProfile:
              type: RuntimeDefault
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
          - mountPath: /tmp/tokenizer
            name: tokenizer-uds
        dnsPolicy: ClusterFirst
        initContainers:
        - args:
          - hf://facebook/opt-125m
          - /mnt/models
          env:
          - name: AWS_ACCESS_KEY_ID
            valueFrom:
              secretKeyRef:
                key: AWS_ACCESS_KEY_ID
                name: seaweedfs-s3-creds
          - name: AWS_SECRET_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                key: AWS_SECRET_ACCESS_KEY
                name: seaweedfs-s3-creds
          - name: S3_USE_HTTPS
            value: "0"
          - name: S3_ENDPOINT
            value: s3-service.kserve:8333
          - name: AWS_ENDPOINT_URL
            value: http://s3-service.kserve:8333
          - name: S3_VERIFY_SSL
            value: "0"
          - name: AWS_CA_BUNDLE
            value: /etc/ssl/custom-certs/cabundle.crt
          - name: AWS_CA_BUNDLE_CONFIGMAP
            value: odh-kserve-custom-ca-bundle
          - name: HF_HUB_ENABLE_HF_TRANSFER
            value: "1"
          - name: HF_XET_HIGH_PERFORMANCE
            value: "1"
          - name: HF_XET_NUM_CONCURRENT_RANGE_GETS
            value: "8"
          - name: STORAGE_ALLOW_PATTERNS
            value: '["tokenizer.json", "tokenizer_config.json", "special_tokens_map.json",
              "vocab.json", "merges.txt", "config.json", "generation_config.json"]'
          image: quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773
          imagePullPolicy: IfNotPresent
          name: storage-initializer
          resources:
            limits:
              cpu: "1"
              memory: 24Gi
            requests:
              cpu: 100m
              memory: 100Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /mnt/models
            name: kserve-provision-location
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        serviceAccount: custom-route-timeout-test-epp-sa
        serviceAccountName: custom-route-timeout-test-epp-sa
        terminationGracePeriodSeconds: 60
        volumes:
        - name: tls-certs
          secret:
            defaultMode: 420
            secretName: custom-route-timeout-test-kserve-self-signed-certs
        - emptyDir: {}
          name: tokenizer-uds
        - emptyDir: {}
          name: tokenizer-tmp
        - emptyDir: {}
          name: tokenizer-cache
        - emptyDir: {}
          name: kserve-provision-location
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2026-07-08T07:38:55Z"
      lastUpdateTime: "2026-07-08T07:38:55Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    - lastTransitionTime: "2026-07-08T07:38:22Z"
      lastUpdateTime: "2026-07-08T07:38:55Z"
      message: ReplicaSet "custom-route-timeout-test-kserve-router-scheduler-65985dc45b"
        has successfully progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2026-07-08T07:21:01Z"
    generation: 1
    labels:
      app.kubernetes.io/component: llminferenceservice-workload
      app.kubernetes.io/name: llmisvc-model-fb-opt-125m-route-87882a8e
      app.kubernetes.io/part-of: llminferenceservice
      kserve.io/component: workload
      llm-d.ai/role: both
    managedFields:
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:labels:
            .: {}
            f:app.kubernetes.io/component: {}
            f:app.kubernetes.io/name: {}
            f:app.kubernetes.io/part-of: {}
            f:kserve.io/component: {}
            f:llm-d.ai/role: {}
          f:ownerReferences:
            .: {}
            k:{"uid":"565176bb-ec06-4726-bd46-befcc78f8c33"}: {}
        f:spec:
          f:progressDeadlineSeconds: {}
          f:replicas: {}
          f:revisionHistoryLimit: {}
          f:selector: {}
          f:strategy:
            f:rollingUpdate:
              .: {}
              f:maxSurge: {}
              f:maxUnavailable: {}
            f:type: {}
          f:template:
            f:metadata:
              f:labels:
                .: {}
                f:app.kubernetes.io/component: {}
                f:app.kubernetes.io/name: {}
                f:app.kubernetes.io/part-of: {}
                f:kserve.io/component: {}
                f:llm-d.ai/role: {}
            f:spec:
              f:containers:
                k:{"name":"main"}:
                  .: {}
                  f:command: {}
                  f:env:
                    .: {}
                    k:{"name":"HF_HUB_CACHE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HOME"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"TORCHINDUCTOR_CACHE_DIR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"USER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_CPU_KVCACHE_SPACE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_ENABLE_V1_MULTIPROCESSING"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_LOGGING_LEVEL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:lifecycle:
                    .: {}
                    f:preStop:
                      .: {}
                      f:exec:
                        .: {}
                        f:command: {}
                  f:livenessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:name: {}
                  f:ports:
                    .: {}
                    k:{"containerPort":8000,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:protocol: {}
                  f:readinessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:securityContext:
                    .: {}
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      .: {}
                      f:drop: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsNonRoot: {}
                    f:seccompProfile:
                      .: {}
                      f:type: {}
                  f:startupProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/dev/shm"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/home"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
                    k:{"mountPath":"/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/tmp"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/kserve/tls"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
              f:dnsPolicy: {}
              f:initContainers:
                .: {}
                k:{"name":"storage-initializer"}:
                  .: {}
                  f:args: {}
                  f:env:
                    .: {}
                    k:{"name":"AWS_ACCESS_KEY_ID"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"AWS_CA_BUNDLE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_CA_BUNDLE_CONFIGMAP"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_ENDPOINT_URL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_SECRET_ACCESS_KEY"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"HF_HUB_ENABLE_HF_TRANSFER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_HIGH_PERFORMANCE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_NUM_CONCURRENT_RANGE_GETS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_ENDPOINT"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_USE_HTTPS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_VERIFY_SSL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:name: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
              f:restartPolicy: {}
              f:schedulerName: {}
              f:securityContext: {}
              f:terminationGracePeriodSeconds: {}
              f:volumes:
                .: {}
                k:{"name":"dshm"}:
                  .: {}
                  f:emptyDir:
                    .: {}
                    f:medium: {}
                    f:sizeLimit: {}
                  f:name: {}
                k:{"name":"home"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"kserve-provision-location"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"model-cache"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tls-certs"}:
                  .: {}
                  f:name: {}
                  f:secret:
                    .: {}
                    f:defaultMode: {}
                    f:secretName: {}
                k:{"name":"tmp-dir"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
      manager: manager
      operation: Update
      time: "2026-07-08T07:21:01Z"
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:deployment.kubernetes.io/revision: {}
        f:status:
          f:availableReplicas: {}
          f:conditions:
            .: {}
            k:{"type":"Available"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"Progressing"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:observedGeneration: {}
          f:readyReplicas: {}
          f:replicas: {}
          f:updatedReplicas: {}
      manager: kube-controller-manager
      operation: Update
      subresource: status
      time: "2026-07-08T07:23:02Z"
    name: llmisvc-model-fb-opt-125m-route-87882a8e-kserve
    namespace: kserve-ci-e2e-test
    ownerReferences:
    - apiVersion: serving.kserve.io/v1alpha2
      blockOwnerDeletion: true
      controller: true
      kind: LLMInferenceService
      name: llmisvc-model-fb-opt-125m-route-87882a8e
      uid: 565176bb-ec06-4726-bd46-befcc78f8c33
    resourceVersion: "36770"
    uid: 7423aef6-87ce-4e5d-a3be-cb281cd37a44
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app.kubernetes.io/component: llminferenceservice-workload
        app.kubernetes.io/name: llmisvc-model-fb-opt-125m-route-87882a8e
        app.kubernetes.io/part-of: llminferenceservice
        kserve.io/component: workload
        llm-d.ai/role: both
    strategy:
      rollingUpdate:
        maxSurge: 25%
        maxUnavailable: 25%
      type: RollingUpdate
    template:
      metadata:
        labels:
          app.kubernetes.io/component: llminferenceservice-workload
          app.kubernetes.io/name: llmisvc-model-fb-opt-125m-route-87882a8e
          app.kubernetes.io/part-of: llminferenceservice
          kserve.io/component: workload
          llm-d.ai/role: both
      spec:
        containers:
        - command:
          - /bin/bash
          - -c
          - |-
            if [ -f /etc/profile.d/ibm-aiu-setup.sh ]; then
              source /etc/profile.d/ibm-aiu-setup.sh
            fi

            if [ "$KSERVE_INFER_ROCE" = "true" ]; then
              echo "Trying to infer RoCE configs ... "
              grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
              grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null

              cat /proc/driver/nvidia/params

              KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}

              echo "[Infer RoCE] Discovering active HCAs ..."
              active_hcas=()
              # Loop through all mlx5 devices found in sysfs
              for hca_dir in /sys/class/infiniband/mlx5_*; do
                  # Ensure it's a directory before proceeding
                  if [ -d "$hca_dir" ]; then
                      hca_name=$(basename "$hca_dir")
                      port_state_file="$hca_dir/ports/1/state" # Assume port 1
                      type_file="$hca_dir/ports/1/gid_attrs/types/*"

                      echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
                      if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
                          echo "[Infer RoCE] Found active HCA: $hca_name"
                          active_hcas+=("$hca_name")
                      else
                          echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
                      fi
                  fi
              done

              # Check if we found any active HCAs
              if [ ${#active_hcas[@]} -gt 0 ]; then
                  # Join the array elements with a comma
                  hca_port_pairs=()
                  for hca in "${active_hcas[@]}"; do
                    hca_port_pairs+=("${hca}:1")
                  done

                  active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
                  hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
                  echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
                  export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
                  export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
                  export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}

                  echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
                  echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
                  echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
              else
                  echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
              fi

              if [ ${#active_hcas[@]} -gt 0 ]; then
                  echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."

                  # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
                  declare -A gid_index_count
                  declare -A hca_gid_index

                  for hca_name in "${active_hcas[@]}"; do
                      echo "[Infer RoCE] Processing HCA: ${hca_name}"

                      # Find all RoCE v2 IPv4 GIDs for this HCA and count by index
                      for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
                          if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
                              idx=$(basename "$tpath")
                              gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
                              # Check for IPv4 GID (contains ffff:)
                              if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
                                  gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
                                  echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
                                  hca_gid_index["${hca_name}"]="${idx}"
                                  gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
                                  break  # Use first found IPv4 GID per HCA
                              fi
                          fi
                      done
                  done

                  # Find the most common GID index (most likely to be consistent across nodes)
                  best_gid_index=""
                  max_count=0
                  for idx in "${!gid_index_count[@]}"; do
                      count=${gid_index_count["${idx}"]}
                      echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
                      if [ $count -gt $max_count ]; then
                          max_count=$count
                          best_gid_index="$idx"
                      fi
                  done

                  # Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
                  if [ ${#gid_index_count[@]} -gt 1 ]; then
                      echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
                      # If there's a tie, prefer index 3 as it's most common in SR-IOV setups
                      if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
                          best_gid_index="3"
                          echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
                      fi
                  fi

                  # Check if GID_INDEX is already set via environment variables
                  if [ -n "${NCCL_IB_GID_INDEX}" ]; then
                      echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
                  elif [ -n "$best_gid_index" ]; then
                      echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"

                      export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}

                      echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
                  else
                      echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
                  fi
              else
                  echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
              fi
            fi

            # --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
            # Older versions still need the blanket --disable-uvicorn-access-log.
            ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
            VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
            echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
            if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
              ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
            fi
            echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"

            # --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
            SHUTDOWN_TIMEOUT_ARGS=""
            if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
              SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout 40"
            fi

            eval "exec vllm serve /mnt/models \
              --served-model-name "facebook/opt-125m" "publishers/kserve-ci-e2e-test/models/facebook/opt-125m" \
              --port 8000 \
              ${ACCESS_LOG_ARGS} \
              ${SHUTDOWN_TIMEOUT_ARGS} \
              --enable-ssl-refresh \
              --ssl-certfile /var/run/kserve/tls/tls.crt \
              --ssl-keyfile /var/run/kserve/tls/tls.key \
              ${VLLM_ADDITIONAL_ARGS} \
              $@"
          - --
          env:
          - name: HOME
            value: /home
          - name: VLLM_LOGGING_LEVEL
            value: DEBUG
          - name: VLLM_CPU_KVCACHE_SPACE
            value: "1"
          - name: VLLM_ENABLE_V1_MULTIPROCESSING
            value: "0"
          - name: USER
            value: nonroot
          - name: TORCHINDUCTOR_CACHE_DIR
            value: /tmp/torchinductor-cache
          - name: HF_HUB_CACHE
            value: /models
          image: public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.19.0
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - /bin/sleep
                - "15"
          livenessProbe:
            failureThreshold: 10
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: main
          ports:
          - containerPort: 8000
            protocol: TCP
          readinessProbe:
            failureThreshold: 2
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 1
            successThreshold: 1
            timeoutSeconds: 1
          resources:
            limits:
              cpu: "2"
              memory: 7Gi
            requests:
              cpu: 200m
              memory: 2Gi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
            runAsNonRoot: true
            seccompProfile:
              type: RuntimeDefault
          startupProbe:
            failureThreshold: 60
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /home
            name: home
          - mountPath: /tmp
            name: tmp-dir
          - mountPath: /dev/shm
            name: dshm
          - mountPath: /models
            name: model-cache
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
          - mountPath: /mnt/models
            name: kserve-provision-location
            readOnly: true
        dnsPolicy: ClusterFirst
        initContainers:
        - args:
          - hf://facebook/opt-125m
          - /mnt/models
          env:
          - name: AWS_ACCESS_KEY_ID
            valueFrom:
              secretKeyRef:
                key: AWS_ACCESS_KEY_ID
                name: seaweedfs-s3-creds
          - name: AWS_SECRET_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                key: AWS_SECRET_ACCESS_KEY
                name: seaweedfs-s3-creds
          - name: S3_USE_HTTPS
            value: "0"
          - name: S3_ENDPOINT
            value: s3-service.kserve:8333
          - name: AWS_ENDPOINT_URL
            value: http://s3-service.kserve:8333
          - name: S3_VERIFY_SSL
            value: "0"
          - name: AWS_CA_BUNDLE
            value: /etc/ssl/custom-certs/cabundle.crt
          - name: AWS_CA_BUNDLE_CONFIGMAP
            value: odh-kserve-custom-ca-bundle
          - name: HF_HUB_ENABLE_HF_TRANSFER
            value: "1"
          - name: HF_XET_HIGH_PERFORMANCE
            value: "1"
          - name: HF_XET_NUM_CONCURRENT_RANGE_GETS
            value: "8"
          image: quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773
          imagePullPolicy: IfNotPresent
          name: storage-initializer
          resources:
            limits:
              cpu: "1"
              memory: 24Gi
            requests:
              cpu: 100m
              memory: 100Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /mnt/models
            name: kserve-provision-location
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        terminationGracePeriodSeconds: 60
        volumes:
        - emptyDir: {}
          name: home
        - emptyDir:
            medium: Memory
            sizeLimit: 1Gi
          name: dshm
        - emptyDir: {}
          name: model-cache
        - emptyDir: {}
          name: tmp-dir
        - name: tls-certs
          secret:
            defaultMode: 420
            secretName: llmisva690bbc929faec8bc98c767f16c003c1-kserve-self-signed-certs
        - emptyDir: {}
          name: kserve-provision-location
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2026-07-08T07:23:02Z"
      lastUpdateTime: "2026-07-08T07:23:02Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    - lastTransitionTime: "2026-07-08T07:21:01Z"
      lastUpdateTime: "2026-07-08T07:23:02Z"
      message: ReplicaSet "llmisvc-model-fb-opt-125m-route-87882a8e-kserve-5c57b59c67"
        has successfully progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2026-07-08T07:16:26Z"
    generation: 1
    labels:
      app.kubernetes.io/component: llminferenceservice-workload
      app.kubernetes.io/name: llmisvc-model-fb-opt-125m-with-7ca60146
      app.kubernetes.io/part-of: llminferenceservice
      kserve.io/component: workload
      llm-d.ai/role: both
    managedFields:
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:labels:
            .: {}
            f:app.kubernetes.io/component: {}
            f:app.kubernetes.io/name: {}
            f:app.kubernetes.io/part-of: {}
            f:kserve.io/component: {}
            f:llm-d.ai/role: {}
          f:ownerReferences:
            .: {}
            k:{"uid":"43d7de34-80f8-461c-8576-1e03984be0d5"}: {}
        f:spec:
          f:progressDeadlineSeconds: {}
          f:replicas: {}
          f:revisionHistoryLimit: {}
          f:selector: {}
          f:strategy:
            f:rollingUpdate:
              .: {}
              f:maxSurge: {}
              f:maxUnavailable: {}
            f:type: {}
          f:template:
            f:metadata:
              f:labels:
                .: {}
                f:app.kubernetes.io/component: {}
                f:app.kubernetes.io/name: {}
                f:app.kubernetes.io/part-of: {}
                f:kserve.io/component: {}
                f:llm-d.ai/role: {}
            f:spec:
              f:containers:
                k:{"name":"main"}:
                  .: {}
                  f:args: {}
                  f:command: {}
                  f:env:
                    .: {}
                    k:{"name":"HF_HUB_CACHE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HOME"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"TORCHINDUCTOR_CACHE_DIR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"USER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_CPU_KVCACHE_SPACE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_ENABLE_V1_MULTIPROCESSING"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_LOGGING_LEVEL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:lifecycle:
                    .: {}
                    f:preStop:
                      .: {}
                      f:exec:
                        .: {}
                        f:command: {}
                  f:livenessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:name: {}
                  f:ports:
                    .: {}
                    k:{"containerPort":8000,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:protocol: {}
                  f:readinessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:securityContext:
                    .: {}
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      .: {}
                      f:drop: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsNonRoot: {}
                    f:seccompProfile:
                      .: {}
                      f:type: {}
                  f:startupProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/dev/shm"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/home"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/mnt"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
                    k:{"mountPath":"/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/tmp"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/kserve/tls"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
              f:dnsPolicy: {}
              f:initContainers:
                .: {}
                k:{"name":"storage-initializer"}:
                  .: {}
                  f:args: {}
                  f:env:
                    .: {}
                    k:{"name":"AWS_ACCESS_KEY_ID"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"AWS_CA_BUNDLE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_CA_BUNDLE_CONFIGMAP"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_ENDPOINT_URL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_SECRET_ACCESS_KEY"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"CA_BUNDLE_CONFIGMAP_NAME"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"CA_BUNDLE_VOLUME_MOUNT_POINT"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_HUB_ENABLE_HF_TRANSFER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_HIGH_PERFORMANCE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_NUM_CONCURRENT_RANGE_GETS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_ENDPOINT"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_USE_HTTPS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_VERIFY_SSL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:name: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/etc/ssl/custom-certs"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
                    k:{"mountPath":"/mnt"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
              f:restartPolicy: {}
              f:schedulerName: {}
              f:securityContext: {}
              f:terminationGracePeriodSeconds: {}
              f:volumes:
                .: {}
                k:{"name":"cabundle-cert"}:
                  .: {}
                  f:configMap:
                    .: {}
                    f:defaultMode: {}
                    f:name: {}
                  f:name: {}
                k:{"name":"dshm"}:
                  .: {}
                  f:emptyDir:
                    .: {}
                    f:medium: {}
                    f:sizeLimit: {}
                  f:name: {}
                k:{"name":"home"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"kserve-provision-location"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"model-cache"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tls-certs"}:
                  .: {}
                  f:name: {}
                  f:secret:
                    .: {}
                    f:defaultMode: {}
                    f:secretName: {}
                k:{"name":"tmp-dir"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
      manager: manager
      operation: Update
      time: "2026-07-08T07:16:26Z"
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:deployment.kubernetes.io/revision: {}
        f:status:
          f:availableReplicas: {}
          f:conditions:
            .: {}
            k:{"type":"Available"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"Progressing"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:observedGeneration: {}
          f:readyReplicas: {}
          f:replicas: {}
          f:updatedReplicas: {}
      manager: kube-controller-manager
      operation: Update
      subresource: status
      time: "2026-07-08T07:17:56Z"
    name: llmisvc-model-fb-opt-125m-with-7ca60146-kserve
    namespace: kserve-ci-e2e-test
    ownerReferences:
    - apiVersion: serving.kserve.io/v1alpha2
      blockOwnerDeletion: true
      controller: true
      kind: LLMInferenceService
      name: llmisvc-model-fb-opt-125m-with-7ca60146
      uid: 43d7de34-80f8-461c-8576-1e03984be0d5
    resourceVersion: "31662"
    uid: 98638d0f-9512-4509-98e5-14865c1befce
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app.kubernetes.io/component: llminferenceservice-workload
        app.kubernetes.io/name: llmisvc-model-fb-opt-125m-with-7ca60146
        app.kubernetes.io/part-of: llminferenceservice
        kserve.io/component: workload
        llm-d.ai/role: both
    strategy:
      rollingUpdate:
        maxSurge: 25%
        maxUnavailable: 25%
      type: RollingUpdate
    template:
      metadata:
        labels:
          app.kubernetes.io/component: llminferenceservice-workload
          app.kubernetes.io/name: llmisvc-model-fb-opt-125m-with-7ca60146
          app.kubernetes.io/part-of: llminferenceservice
          kserve.io/component: workload
          llm-d.ai/role: both
      spec:
        containers:
        - args:
          - --enable-lora
          - --lora-modules
          - '''{"name":"lora-adapter-1","path":"/mnt/lora/lora-adapter-1"}'''
          - '''{"name":"publishers/kserve-ci-e2e-test/models/lora-adapter-1","path":"/mnt/lora/lora-adapter-1"}'''
          command:
          - /bin/bash
          - -c
          - |-
            if [ -f /etc/profile.d/ibm-aiu-setup.sh ]; then
              source /etc/profile.d/ibm-aiu-setup.sh
            fi

            if [ "$KSERVE_INFER_ROCE" = "true" ]; then
              echo "Trying to infer RoCE configs ... "
              grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
              grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null

              cat /proc/driver/nvidia/params

              KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}

              echo "[Infer RoCE] Discovering active HCAs ..."
              active_hcas=()
              # Loop through all mlx5 devices found in sysfs
              for hca_dir in /sys/class/infiniband/mlx5_*; do
                  # Ensure it's a directory before proceeding
                  if [ -d "$hca_dir" ]; then
                      hca_name=$(basename "$hca_dir")
                      port_state_file="$hca_dir/ports/1/state" # Assume port 1
                      type_file="$hca_dir/ports/1/gid_attrs/types/*"

                      echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
                      if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
                          echo "[Infer RoCE] Found active HCA: $hca_name"
                          active_hcas+=("$hca_name")
                      else
                          echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
                      fi
                  fi
              done

              # Check if we found any active HCAs
              if [ ${#active_hcas[@]} -gt 0 ]; then
                  # Join the array elements with a comma
                  hca_port_pairs=()
                  for hca in "${active_hcas[@]}"; do
                    hca_port_pairs+=("${hca}:1")
                  done

                  active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
                  hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
                  echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
                  export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
                  export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
                  export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}

                  echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
                  echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
                  echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
              else
                  echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
              fi

              if [ ${#active_hcas[@]} -gt 0 ]; then
                  echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."

                  # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
                  declare -A gid_index_count
                  declare -A hca_gid_index

                  for hca_name in "${active_hcas[@]}"; do
                      echo "[Infer RoCE] Processing HCA: ${hca_name}"

                      # Find all RoCE v2 IPv4 GIDs for this HCA and count by index
                      for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
                          if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
                              idx=$(basename "$tpath")
                              gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
                              # Check for IPv4 GID (contains ffff:)
                              if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
                                  gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
                                  echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
                                  hca_gid_index["${hca_name}"]="${idx}"
                                  gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
                                  break  # Use first found IPv4 GID per HCA
                              fi
                          fi
                      done
                  done

                  # Find the most common GID index (most likely to be consistent across nodes)
                  best_gid_index=""
                  max_count=0
                  for idx in "${!gid_index_count[@]}"; do
                      count=${gid_index_count["${idx}"]}
                      echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
                      if [ $count -gt $max_count ]; then
                          max_count=$count
                          best_gid_index="$idx"
                      fi
                  done

                  # Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
                  if [ ${#gid_index_count[@]} -gt 1 ]; then
                      echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
                      # If there's a tie, prefer index 3 as it's most common in SR-IOV setups
                      if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
                          best_gid_index="3"
                          echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
                      fi
                  fi

                  # Check if GID_INDEX is already set via environment variables
                  if [ -n "${NCCL_IB_GID_INDEX}" ]; then
                      echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
                  elif [ -n "$best_gid_index" ]; then
                      echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"

                      export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}

                      echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
                  else
                      echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
                  fi
              else
                  echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
              fi
            fi

            # --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
            # Older versions still need the blanket --disable-uvicorn-access-log.
            ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
            VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
            echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
            if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
              ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
            fi
            echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"

            # --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
            SHUTDOWN_TIMEOUT_ARGS=""
            if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
              SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout 40"
            fi

            eval "exec vllm serve /mnt/models \
              --served-model-name "facebook/opt-125m" "publishers/kserve-ci-e2e-test/models/facebook/opt-125m" \
              --port 8000 \
              ${ACCESS_LOG_ARGS} \
              ${SHUTDOWN_TIMEOUT_ARGS} \
              --enable-ssl-refresh \
              --ssl-certfile /var/run/kserve/tls/tls.crt \
              --ssl-keyfile /var/run/kserve/tls/tls.key \
              ${VLLM_ADDITIONAL_ARGS} \
              $@"
          - --
          env:
          - name: HOME
            value: /home
          - name: VLLM_LOGGING_LEVEL
            value: DEBUG
          - name: VLLM_CPU_KVCACHE_SPACE
            value: "1"
          - name: VLLM_ENABLE_V1_MULTIPROCESSING
            value: "0"
          - name: USER
            value: nonroot
          - name: TORCHINDUCTOR_CACHE_DIR
            value: /tmp/torchinductor-cache
          - name: HF_HUB_CACHE
            value: /models
          image: public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.19.0
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - /bin/sleep
                - "15"
          livenessProbe:
            failureThreshold: 10
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: main
          ports:
          - containerPort: 8000
            protocol: TCP
          readinessProbe:
            failureThreshold: 2
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 1
            successThreshold: 1
            timeoutSeconds: 1
          resources:
            limits:
              cpu: "2"
              memory: 7Gi
            requests:
              cpu: 200m
              memory: 2Gi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
            runAsNonRoot: true
            seccompProfile:
              type: RuntimeDefault
          startupProbe:
            failureThreshold: 60
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /home
            name: home
          - mountPath: /tmp
            name: tmp-dir
          - mountPath: /dev/shm
            name: dshm
          - mountPath: /models
            name: model-cache
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
          - mountPath: /mnt
            name: kserve-provision-location
            readOnly: true
        dnsPolicy: ClusterFirst
        initContainers:
        - args:
          - hf://facebook/opt-125m
          - /mnt/models
          - hf://edbeeching/opt-125m-lora
          - /mnt/lora/lora-adapter-1
          env:
          - name: AWS_ACCESS_KEY_ID
            valueFrom:
              secretKeyRef:
                key: AWS_ACCESS_KEY_ID
                name: seaweedfs-s3-creds
          - name: AWS_SECRET_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                key: AWS_SECRET_ACCESS_KEY
                name: seaweedfs-s3-creds
          - name: S3_USE_HTTPS
            value: "0"
          - name: S3_ENDPOINT
            value: s3-service.kserve:8333
          - name: AWS_ENDPOINT_URL
            value: http://s3-service.kserve:8333
          - name: S3_VERIFY_SSL
            value: "0"
          - name: AWS_CA_BUNDLE
            value: /etc/ssl/custom-certs/cabundle.crt
          - name: AWS_CA_BUNDLE_CONFIGMAP
            value: odh-kserve-custom-ca-bundle
          - name: HF_HUB_ENABLE_HF_TRANSFER
            value: "1"
          - name: HF_XET_HIGH_PERFORMANCE
            value: "1"
          - name: HF_XET_NUM_CONCURRENT_RANGE_GETS
            value: "8"
          - name: CA_BUNDLE_CONFIGMAP_NAME
            value: odh-kserve-custom-ca-bundle
          - name: CA_BUNDLE_VOLUME_MOUNT_POINT
            value: /etc/ssl/custom-certs
          image: quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773
          imagePullPolicy: IfNotPresent
          name: storage-initializer
          resources:
            limits:
              cpu: "1"
              memory: 24Gi
            requests:
              cpu: 100m
              memory: 100Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /mnt
            name: kserve-provision-location
          - mountPath: /etc/ssl/custom-certs
            name: cabundle-cert
            readOnly: true
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        terminationGracePeriodSeconds: 60
        volumes:
        - emptyDir: {}
          name: home
        - emptyDir:
            medium: Memory
            sizeLimit: 1Gi
          name: dshm
        - emptyDir: {}
          name: model-cache
        - emptyDir: {}
          name: tmp-dir
        - name: tls-certs
          secret:
            defaultMode: 420
            secretName: llmisv3e414c2ba058a022dfd694dbcbac5b51-kserve-self-signed-certs
        - emptyDir: {}
          name: kserve-provision-location
        - configMap:
            defaultMode: 420
            name: odh-kserve-custom-ca-bundle
          name: cabundle-cert
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2026-07-08T07:17:56Z"
      lastUpdateTime: "2026-07-08T07:17:56Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    - lastTransitionTime: "2026-07-08T07:16:26Z"
      lastUpdateTime: "2026-07-08T07:17:56Z"
      message: ReplicaSet "llmisvc-model-fb-opt-125m-with-7ca60146-kserve-6694784b54"
        has successfully progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2026-07-08T07:16:26Z"
    generation: 1
    labels:
      app.kubernetes.io/component: llminferenceservice-router-scheduler
      app.kubernetes.io/name: llmisvc-model-fb-opt-125m-with-7ca60146
      app.kubernetes.io/part-of: llminferenceservice
    managedFields:
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:labels:
            .: {}
            f:app.kubernetes.io/component: {}
            f:app.kubernetes.io/name: {}
            f:app.kubernetes.io/part-of: {}
          f:ownerReferences:
            .: {}
            k:{"uid":"43d7de34-80f8-461c-8576-1e03984be0d5"}: {}
        f:spec:
          f:progressDeadlineSeconds: {}
          f:replicas: {}
          f:revisionHistoryLimit: {}
          f:selector: {}
          f:strategy:
            f:type: {}
          f:template:
            f:metadata:
              f:annotations:
                .: {}
                f:app.kubernetes.io/version: {}
                f:certificates.kserve.io/expiration-v2: {}
              f:labels:
                .: {}
                f:app.kubernetes.io/component: {}
                f:app.kubernetes.io/name: {}
                f:app.kubernetes.io/part-of: {}
            f:spec:
              f:containers:
                k:{"name":"main"}:
                  .: {}
                  f:args: {}
                  f:command: {}
                  f:env:
                    .: {}
                    k:{"name":"SSL_CERT_DIR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:lifecycle:
                    .: {}
                    f:preStop:
                      .: {}
                      f:exec:
                        .: {}
                        f:command: {}
                  f:livenessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:grpc:
                      .: {}
                      f:port: {}
                      f:service: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:name: {}
                  f:ports:
                    .: {}
                    k:{"containerPort":5557,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9002,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9003,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9090,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                  f:readinessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:grpc:
                      .: {}
                      f:port: {}
                      f:service: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources:
                    .: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:securityContext:
                    .: {}
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      .: {}
                      f:drop: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsNonRoot: {}
                    f:seccompProfile:
                      .: {}
                      f:type: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/tmp/tokenizer"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/kserve/tls"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
              f:dnsPolicy: {}
              f:initContainers:
                .: {}
                k:{"name":"storage-initializer"}:
                  .: {}
                  f:args: {}
                  f:env:
                    .: {}
                    k:{"name":"AWS_ACCESS_KEY_ID"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"AWS_CA_BUNDLE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_CA_BUNDLE_CONFIGMAP"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_ENDPOINT_URL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_SECRET_ACCESS_KEY"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"HF_HUB_ENABLE_HF_TRANSFER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_HIGH_PERFORMANCE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_NUM_CONCURRENT_RANGE_GETS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_ENDPOINT"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_USE_HTTPS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_VERIFY_SSL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"STORAGE_ALLOW_PATTERNS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:name: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
              f:restartPolicy: {}
              f:schedulerName: {}
              f:securityContext: {}
              f:serviceAccount: {}
              f:serviceAccountName: {}
              f:terminationGracePeriodSeconds: {}
              f:volumes:
                .: {}
                k:{"name":"kserve-provision-location"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tls-certs"}:
                  .: {}
                  f:name: {}
                  f:secret:
                    .: {}
                    f:defaultMode: {}
                    f:secretName: {}
                k:{"name":"tokenizer-cache"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tokenizer-tmp"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tokenizer-uds"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
      manager: manager
      operation: Update
      time: "2026-07-08T07:16:26Z"
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:deployment.kubernetes.io/revision: {}
        f:status:
          f:availableReplicas: {}
          f:conditions:
            .: {}
            k:{"type":"Available"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"Progressing"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:observedGeneration: {}
          f:readyReplicas: {}
          f:replicas: {}
          f:updatedReplicas: {}
      manager: kube-controller-manager
      operation: Update
      subresource: status
      time: "2026-07-08T07:17:00Z"
    name: llmisvc-model-fb-opt-125m-with-7ca60146-kserve-router-scheduler
    namespace: kserve-ci-e2e-test
    ownerReferences:
    - apiVersion: serving.kserve.io/v1alpha2
      blockOwnerDeletion: true
      controller: true
      kind: LLMInferenceService
      name: llmisvc-model-fb-opt-125m-with-7ca60146
      uid: 43d7de34-80f8-461c-8576-1e03984be0d5
    resourceVersion: "30649"
    uid: 07b4f646-3653-426e-9882-1ec59f2580e2
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app.kubernetes.io/component: llminferenceservice-router-scheduler
        app.kubernetes.io/name: llmisvc-model-fb-opt-125m-with-7ca60146
        app.kubernetes.io/part-of: llminferenceservice
    strategy:
      type: Recreate
    template:
      metadata:
        annotations:
          app.kubernetes.io/version: 0.9.0
          certificates.kserve.io/expiration-v2: "true"
        labels:
          app.kubernetes.io/component: llminferenceservice-router-scheduler
          app.kubernetes.io/name: llmisvc-model-fb-opt-125m-with-7ca60146
          app.kubernetes.io/part-of: llminferenceservice
      spec:
        containers:
        - args:
          - --config-text
          - |
            apiVersion: inference.networking.x-k8s.io/v1alpha1
            kind: EndpointPickerConfig
            plugins:
            - type: single-profile-handler
            - type: queue-scorer
            - type: prefix-cache-scorer
            - type: max-score-picker
            - type: lora-affinity-scorer
            schedulingProfiles:
            - name: default
              plugins:
              - pluginRef: lora-affinity-scorer
                weight: 4
              - pluginRef: queue-scorer
                weight: 2
              - pluginRef: prefix-cache-scorer
                weight: 3
              - pluginRef: max-score-picker
          command:
          - /app/epp
          - --pool-name
          - llmisvc-model-fb-opt-125m-with-7ca60146-inference-pool
          - --pool-namespace
          - kserve-ci-e2e-test
          - --zap-encoder
          - json
          - --grpc-port
          - "9002"
          - --grpc-health-port
          - "9003"
          - --enable-cert-reload=true
          - --secure-serving=true
          - --model-server-metrics-scheme=https
          - --cert-path=/var/run/kserve/tls
          env:
          - name: SSL_CERT_DIR
            value: /var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs
          image: ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0-rc.2
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - /bin/sleep
                - "15"
          livenessProbe:
            failureThreshold: 3
            grpc:
              port: 9003
              service: liveness
            initialDelaySeconds: 5
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: main
          ports:
          - containerPort: 9002
            name: grpc
            protocol: TCP
          - containerPort: 9003
            name: grpc-health
            protocol: TCP
          - containerPort: 9090
            name: metrics
            protocol: TCP
          - containerPort: 5557
            name: zmq
            protocol: TCP
          readinessProbe:
            failureThreshold: 3
            grpc:
              port: 9003
              service: readiness
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          resources:
            requests:
              cpu: 256m
              memory: 500Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
            runAsNonRoot: true
            seccompProfile:
              type: RuntimeDefault
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
          - mountPath: /tmp/tokenizer
            name: tokenizer-uds
        dnsPolicy: ClusterFirst
        initContainers:
        - args:
          - hf://facebook/opt-125m
          - /mnt/models
          env:
          - name: AWS_ACCESS_KEY_ID
            valueFrom:
              secretKeyRef:
                key: AWS_ACCESS_KEY_ID
                name: seaweedfs-s3-creds
          - name: AWS_SECRET_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                key: AWS_SECRET_ACCESS_KEY
                name: seaweedfs-s3-creds
          - name: S3_USE_HTTPS
            value: "0"
          - name: S3_ENDPOINT
            value: s3-service.kserve:8333
          - name: AWS_ENDPOINT_URL
            value: http://s3-service.kserve:8333
          - name: S3_VERIFY_SSL
            value: "0"
          - name: AWS_CA_BUNDLE
            value: /etc/ssl/custom-certs/cabundle.crt
          - name: AWS_CA_BUNDLE_CONFIGMAP
            value: odh-kserve-custom-ca-bundle
          - name: HF_HUB_ENABLE_HF_TRANSFER
            value: "1"
          - name: HF_XET_HIGH_PERFORMANCE
            value: "1"
          - name: HF_XET_NUM_CONCURRENT_RANGE_GETS
            value: "8"
          - name: STORAGE_ALLOW_PATTERNS
            value: '["tokenizer.json", "tokenizer_config.json", "special_tokens_map.json",
              "vocab.json", "merges.txt", "config.json", "generation_config.json"]'
          image: quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773
          imagePullPolicy: IfNotPresent
          name: storage-initializer
          resources:
            limits:
              cpu: "1"
              memory: 24Gi
            requests:
              cpu: 100m
              memory: 100Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /mnt/models
            name: kserve-provision-location
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        serviceAccount: llmisvc-model-fb-opt-125m-with-7ca60146-epp-sa
        serviceAccountName: llmisvc-model-fb-opt-125m-with-7ca60146-epp-sa
        terminationGracePeriodSeconds: 60
        volumes:
        - name: tls-certs
          secret:
            defaultMode: 420
            secretName: llmisv3e414c2ba058a022dfd694dbcbac5b51-kserve-self-signed-certs
        - emptyDir: {}
          name: tokenizer-uds
        - emptyDir: {}
          name: tokenizer-tmp
        - emptyDir: {}
          name: tokenizer-cache
        - emptyDir: {}
          name: kserve-provision-location
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2026-07-08T07:17:00Z"
      lastUpdateTime: "2026-07-08T07:17:00Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    - lastTransitionTime: "2026-07-08T07:16:26Z"
      lastUpdateTime: "2026-07-08T07:17:00Z"
      message: ReplicaSet "llmisvc-model-fb-opt-125m-with-7ca60146-kserve-router-scheduler-6588f4987"
        has successfully progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2026-07-08T07:33:12Z"
    generation: 1
    labels:
      app.kubernetes.io/component: llminferenceservice-workload
      app.kubernetes.io/name: llmisvc-model-fb-opt-125m-with-ba4d693a
      app.kubernetes.io/part-of: llminferenceservice
      kserve.io/component: workload
      llm-d.ai/role: both
    managedFields:
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:labels:
            .: {}
            f:app.kubernetes.io/component: {}
            f:app.kubernetes.io/name: {}
            f:app.kubernetes.io/part-of: {}
            f:kserve.io/component: {}
            f:llm-d.ai/role: {}
          f:ownerReferences:
            .: {}
            k:{"uid":"79fa8820-032c-4f6b-aef9-01746be29eab"}: {}
        f:spec:
          f:progressDeadlineSeconds: {}
          f:replicas: {}
          f:revisionHistoryLimit: {}
          f:selector: {}
          f:strategy:
            f:rollingUpdate:
              .: {}
              f:maxSurge: {}
              f:maxUnavailable: {}
            f:type: {}
          f:template:
            f:metadata:
              f:labels:
                .: {}
                f:app.kubernetes.io/component: {}
                f:app.kubernetes.io/name: {}
                f:app.kubernetes.io/part-of: {}
                f:kserve.io/component: {}
                f:llm-d.ai/role: {}
            f:spec:
              f:containers:
                k:{"name":"main"}:
                  .: {}
                  f:args: {}
                  f:command: {}
                  f:env:
                    .: {}
                    k:{"name":"HF_HUB_CACHE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HOME"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"TORCHINDUCTOR_CACHE_DIR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"USER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_CPU_KVCACHE_SPACE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_ENABLE_V1_MULTIPROCESSING"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_LOGGING_LEVEL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:lifecycle:
                    .: {}
                    f:preStop:
                      .: {}
                      f:exec:
                        .: {}
                        f:command: {}
                  f:livenessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:name: {}
                  f:ports:
                    .: {}
                    k:{"containerPort":8000,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:protocol: {}
                  f:readinessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:securityContext:
                    .: {}
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      .: {}
                      f:drop: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsNonRoot: {}
                    f:seccompProfile:
                      .: {}
                      f:type: {}
                  f:startupProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/dev/shm"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/home"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/mnt"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
                    k:{"mountPath":"/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/tmp"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/kserve/tls"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
              f:dnsPolicy: {}
              f:initContainers:
                .: {}
                k:{"name":"storage-initializer"}:
                  .: {}
                  f:args: {}
                  f:env:
                    .: {}
                    k:{"name":"AWS_ACCESS_KEY_ID"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"AWS_CA_BUNDLE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_CA_BUNDLE_CONFIGMAP"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_ENDPOINT_URL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_SECRET_ACCESS_KEY"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"CA_BUNDLE_CONFIGMAP_NAME"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"CA_BUNDLE_VOLUME_MOUNT_POINT"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_HUB_ENABLE_HF_TRANSFER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_HIGH_PERFORMANCE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_NUM_CONCURRENT_RANGE_GETS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_ENDPOINT"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_USE_HTTPS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_VERIFY_SSL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:name: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/etc/ssl/custom-certs"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
                    k:{"mountPath":"/mnt"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
              f:restartPolicy: {}
              f:schedulerName: {}
              f:securityContext: {}
              f:terminationGracePeriodSeconds: {}
              f:volumes:
                .: {}
                k:{"name":"cabundle-cert"}:
                  .: {}
                  f:configMap:
                    .: {}
                    f:defaultMode: {}
                    f:name: {}
                  f:name: {}
                k:{"name":"dshm"}:
                  .: {}
                  f:emptyDir:
                    .: {}
                    f:medium: {}
                    f:sizeLimit: {}
                  f:name: {}
                k:{"name":"home"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"kserve-provision-location"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"model-cache"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tls-certs"}:
                  .: {}
                  f:name: {}
                  f:secret:
                    .: {}
                    f:defaultMode: {}
                    f:secretName: {}
                k:{"name":"tmp-dir"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
      manager: manager
      operation: Update
      time: "2026-07-08T07:33:12Z"
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:deployment.kubernetes.io/revision: {}
        f:status:
          f:availableReplicas: {}
          f:conditions:
            .: {}
            k:{"type":"Available"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"Progressing"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:observedGeneration: {}
          f:readyReplicas: {}
          f:replicas: {}
          f:updatedReplicas: {}
      manager: kube-controller-manager
      operation: Update
      subresource: status
      time: "2026-07-08T07:34:32Z"
    name: llmisvc-model-fb-opt-125m-with-ba4d693a-kserve
    namespace: kserve-ci-e2e-test
    ownerReferences:
    - apiVersion: serving.kserve.io/v1alpha2
      blockOwnerDeletion: true
      controller: true
      kind: LLMInferenceService
      name: llmisvc-model-fb-opt-125m-with-ba4d693a
      uid: 79fa8820-032c-4f6b-aef9-01746be29eab
    resourceVersion: "47883"
    uid: 208cf454-2b70-42ba-9bb3-2c453664c1f8
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app.kubernetes.io/component: llminferenceservice-workload
        app.kubernetes.io/name: llmisvc-model-fb-opt-125m-with-ba4d693a
        app.kubernetes.io/part-of: llminferenceservice
        kserve.io/component: workload
        llm-d.ai/role: both
    strategy:
      rollingUpdate:
        maxSurge: 25%
        maxUnavailable: 25%
      type: RollingUpdate
    template:
      metadata:
        labels:
          app.kubernetes.io/component: llminferenceservice-workload
          app.kubernetes.io/name: llmisvc-model-fb-opt-125m-with-ba4d693a
          app.kubernetes.io/part-of: llminferenceservice
          kserve.io/component: workload
          llm-d.ai/role: both
      spec:
        containers:
        - args:
          - --enable-lora
          - --lora-modules
          - '''{"name":"lora-adapter-1","path":"/mnt/lora/lora-adapter-1"}'''
          - '''{"name":"publishers/kserve-ci-e2e-test/models/lora-adapter-1","path":"/mnt/lora/lora-adapter-1"}'''
          command:
          - /bin/bash
          - -c
          - |-
            if [ -f /etc/profile.d/ibm-aiu-setup.sh ]; then
              source /etc/profile.d/ibm-aiu-setup.sh
            fi

            if [ "$KSERVE_INFER_ROCE" = "true" ]; then
              echo "Trying to infer RoCE configs ... "
              grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
              grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null

              cat /proc/driver/nvidia/params

              KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}

              echo "[Infer RoCE] Discovering active HCAs ..."
              active_hcas=()
              # Loop through all mlx5 devices found in sysfs
              for hca_dir in /sys/class/infiniband/mlx5_*; do
                  # Ensure it's a directory before proceeding
                  if [ -d "$hca_dir" ]; then
                      hca_name=$(basename "$hca_dir")
                      port_state_file="$hca_dir/ports/1/state" # Assume port 1
                      type_file="$hca_dir/ports/1/gid_attrs/types/*"

                      echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
                      if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
                          echo "[Infer RoCE] Found active HCA: $hca_name"
                          active_hcas+=("$hca_name")
                      else
                          echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
                      fi
                  fi
              done

              # Check if we found any active HCAs
              if [ ${#active_hcas[@]} -gt 0 ]; then
                  # Join the array elements with a comma
                  hca_port_pairs=()
                  for hca in "${active_hcas[@]}"; do
                    hca_port_pairs+=("${hca}:1")
                  done

                  active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
                  hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
                  echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
                  export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
                  export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
                  export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}

                  echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
                  echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
                  echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
              else
                  echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
              fi

              if [ ${#active_hcas[@]} -gt 0 ]; then
                  echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."

                  # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
                  declare -A gid_index_count
                  declare -A hca_gid_index

                  for hca_name in "${active_hcas[@]}"; do
                      echo "[Infer RoCE] Processing HCA: ${hca_name}"

                      # Find all RoCE v2 IPv4 GIDs for this HCA and count by index
                      for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
                          if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
                              idx=$(basename "$tpath")
                              gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
                              # Check for IPv4 GID (contains ffff:)
                              if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
                                  gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
                                  echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
                                  hca_gid_index["${hca_name}"]="${idx}"
                                  gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
                                  break  # Use first found IPv4 GID per HCA
                              fi
                          fi
                      done
                  done

                  # Find the most common GID index (most likely to be consistent across nodes)
                  best_gid_index=""
                  max_count=0
                  for idx in "${!gid_index_count[@]}"; do
                      count=${gid_index_count["${idx}"]}
                      echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
                      if [ $count -gt $max_count ]; then
                          max_count=$count
                          best_gid_index="$idx"
                      fi
                  done

                  # Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
                  if [ ${#gid_index_count[@]} -gt 1 ]; then
                      echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
                      # If there's a tie, prefer index 3 as it's most common in SR-IOV setups
                      if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
                          best_gid_index="3"
                          echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
                      fi
                  fi

                  # Check if GID_INDEX is already set via environment variables
                  if [ -n "${NCCL_IB_GID_INDEX}" ]; then
                      echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
                  elif [ -n "$best_gid_index" ]; then
                      echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"

                      export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}

                      echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
                  else
                      echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
                  fi
              else
                  echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
              fi
            fi

            # --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
            # Older versions still need the blanket --disable-uvicorn-access-log.
            ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
            VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
            echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
            if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
              ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
            fi
            echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"

            # --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
            SHUTDOWN_TIMEOUT_ARGS=""
            if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
              SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout 40"
            fi

            eval "exec vllm serve /mnt/models \
              --served-model-name "facebook/opt-125m" "publishers/kserve-ci-e2e-test/models/facebook/opt-125m" \
              --port 8000 \
              ${ACCESS_LOG_ARGS} \
              ${SHUTDOWN_TIMEOUT_ARGS} \
              --enable-ssl-refresh \
              --ssl-certfile /var/run/kserve/tls/tls.crt \
              --ssl-keyfile /var/run/kserve/tls/tls.key \
              ${VLLM_ADDITIONAL_ARGS} \
              $@"
          - --
          env:
          - name: HOME
            value: /home
          - name: VLLM_LOGGING_LEVEL
            value: DEBUG
          - name: VLLM_CPU_KVCACHE_SPACE
            value: "1"
          - name: VLLM_ENABLE_V1_MULTIPROCESSING
            value: "0"
          - name: USER
            value: nonroot
          - name: TORCHINDUCTOR_CACHE_DIR
            value: /tmp/torchinductor-cache
          - name: HF_HUB_CACHE
            value: /models
          image: public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.19.0
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - /bin/sleep
                - "15"
          livenessProbe:
            failureThreshold: 10
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: main
          ports:
          - containerPort: 8000
            protocol: TCP
          readinessProbe:
            failureThreshold: 2
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 1
            successThreshold: 1
            timeoutSeconds: 1
          resources:
            limits:
              cpu: "2"
              memory: 7Gi
            requests:
              cpu: 200m
              memory: 2Gi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
            runAsNonRoot: true
            seccompProfile:
              type: RuntimeDefault
          startupProbe:
            failureThreshold: 60
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /home
            name: home
          - mountPath: /tmp
            name: tmp-dir
          - mountPath: /dev/shm
            name: dshm
          - mountPath: /models
            name: model-cache
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
          - mountPath: /mnt
            name: kserve-provision-location
            readOnly: true
        dnsPolicy: ClusterFirst
        initContainers:
        - args:
          - hf://facebook/opt-125m
          - /mnt/models
          - hf://edbeeching/opt-125m-lora
          - /mnt/lora/lora-adapter-1
          env:
          - name: AWS_ACCESS_KEY_ID
            valueFrom:
              secretKeyRef:
                key: AWS_ACCESS_KEY_ID
                name: seaweedfs-s3-creds
          - name: AWS_SECRET_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                key: AWS_SECRET_ACCESS_KEY
                name: seaweedfs-s3-creds
          - name: S3_USE_HTTPS
            value: "0"
          - name: S3_ENDPOINT
            value: s3-service.kserve:8333
          - name: AWS_ENDPOINT_URL
            value: http://s3-service.kserve:8333
          - name: S3_VERIFY_SSL
            value: "0"
          - name: AWS_CA_BUNDLE
            value: /etc/ssl/custom-certs/cabundle.crt
          - name: AWS_CA_BUNDLE_CONFIGMAP
            value: odh-kserve-custom-ca-bundle
          - name: HF_HUB_ENABLE_HF_TRANSFER
            value: "1"
          - name: HF_XET_HIGH_PERFORMANCE
            value: "1"
          - name: HF_XET_NUM_CONCURRENT_RANGE_GETS
            value: "8"
          - name: CA_BUNDLE_CONFIGMAP_NAME
            value: odh-kserve-custom-ca-bundle
          - name: CA_BUNDLE_VOLUME_MOUNT_POINT
            value: /etc/ssl/custom-certs
          image: quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773
          imagePullPolicy: IfNotPresent
          name: storage-initializer
          resources:
            limits:
              cpu: "1"
              memory: 24Gi
            requests:
              cpu: 100m
              memory: 100Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /mnt
            name: kserve-provision-location
          - mountPath: /etc/ssl/custom-certs
            name: cabundle-cert
            readOnly: true
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        terminationGracePeriodSeconds: 60
        volumes:
        - emptyDir: {}
          name: home
        - emptyDir:
            medium: Memory
            sizeLimit: 1Gi
          name: dshm
        - emptyDir: {}
          name: model-cache
        - emptyDir: {}
          name: tmp-dir
        - name: tls-certs
          secret:
            defaultMode: 420
            secretName: llmisv77ff2528d3e9b4972cd9335229fce9f0-kserve-self-signed-certs
        - emptyDir: {}
          name: kserve-provision-location
        - configMap:
            defaultMode: 420
            name: odh-kserve-custom-ca-bundle
          name: cabundle-cert
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2026-07-08T07:34:32Z"
      lastUpdateTime: "2026-07-08T07:34:32Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    - lastTransitionTime: "2026-07-08T07:33:12Z"
      lastUpdateTime: "2026-07-08T07:34:32Z"
      message: ReplicaSet "llmisvc-model-fb-opt-125m-with-ba4d693a-kserve-55585bf4d9"
        has successfully progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2026-07-08T07:33:12Z"
    generation: 1
    labels:
      app.kubernetes.io/component: llminferenceservice-router-scheduler
      app.kubernetes.io/name: llmisvc-model-fb-opt-125m-with-ba4d693a
      app.kubernetes.io/part-of: llminferenceservice
    managedFields:
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:labels:
            .: {}
            f:app.kubernetes.io/component: {}
            f:app.kubernetes.io/name: {}
            f:app.kubernetes.io/part-of: {}
          f:ownerReferences:
            .: {}
            k:{"uid":"79fa8820-032c-4f6b-aef9-01746be29eab"}: {}
        f:spec:
          f:progressDeadlineSeconds: {}
          f:replicas: {}
          f:revisionHistoryLimit: {}
          f:selector: {}
          f:strategy:
            f:type: {}
          f:template:
            f:metadata:
              f:annotations:
                .: {}
                f:app.kubernetes.io/version: {}
                f:certificates.kserve.io/expiration-v2: {}
              f:labels:
                .: {}
                f:app.kubernetes.io/component: {}
                f:app.kubernetes.io/name: {}
                f:app.kubernetes.io/part-of: {}
            f:spec:
              f:containers:
                k:{"name":"main"}:
                  .: {}
                  f:args: {}
                  f:command: {}
                  f:env:
                    .: {}
                    k:{"name":"SSL_CERT_DIR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:lifecycle:
                    .: {}
                    f:preStop:
                      .: {}
                      f:exec:
                        .: {}
                        f:command: {}
                  f:livenessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:grpc:
                      .: {}
                      f:port: {}
                      f:service: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:name: {}
                  f:ports:
                    .: {}
                    k:{"containerPort":5557,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9002,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9003,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9090,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                  f:readinessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:grpc:
                      .: {}
                      f:port: {}
                      f:service: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources:
                    .: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:securityContext:
                    .: {}
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      .: {}
                      f:drop: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsNonRoot: {}
                    f:seccompProfile:
                      .: {}
                      f:type: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/tmp/tokenizer"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/kserve/tls"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
              f:dnsPolicy: {}
              f:initContainers:
                .: {}
                k:{"name":"storage-initializer"}:
                  .: {}
                  f:args: {}
                  f:env:
                    .: {}
                    k:{"name":"AWS_ACCESS_KEY_ID"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"AWS_CA_BUNDLE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_CA_BUNDLE_CONFIGMAP"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_ENDPOINT_URL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_SECRET_ACCESS_KEY"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"HF_HUB_ENABLE_HF_TRANSFER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_HIGH_PERFORMANCE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_NUM_CONCURRENT_RANGE_GETS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_ENDPOINT"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_USE_HTTPS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_VERIFY_SSL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"STORAGE_ALLOW_PATTERNS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:name: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
              f:restartPolicy: {}
              f:schedulerName: {}
              f:securityContext: {}
              f:serviceAccount: {}
              f:serviceAccountName: {}
              f:terminationGracePeriodSeconds: {}
              f:volumes:
                .: {}
                k:{"name":"kserve-provision-location"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tls-certs"}:
                  .: {}
                  f:name: {}
                  f:secret:
                    .: {}
                    f:defaultMode: {}
                    f:secretName: {}
                k:{"name":"tokenizer-cache"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tokenizer-tmp"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tokenizer-uds"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
      manager: manager
      operation: Update
      time: "2026-07-08T07:33:12Z"
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:deployment.kubernetes.io/revision: {}
        f:status:
          f:availableReplicas: {}
          f:conditions:
            .: {}
            k:{"type":"Available"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"Progressing"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:observedGeneration: {}
          f:readyReplicas: {}
          f:replicas: {}
          f:updatedReplicas: {}
      manager: kube-controller-manager
      operation: Update
      subresource: status
      time: "2026-07-08T07:33:47Z"
    name: llmisvc-model-fb-opt-125m-with-ba4d693a-kserve-router-scheduler
    namespace: kserve-ci-e2e-test
    ownerReferences:
    - apiVersion: serving.kserve.io/v1alpha2
      blockOwnerDeletion: true
      controller: true
      kind: LLMInferenceService
      name: llmisvc-model-fb-opt-125m-with-ba4d693a
      uid: 79fa8820-032c-4f6b-aef9-01746be29eab
    resourceVersion: "47222"
    uid: 7bb204bb-a6c2-4eac-a0f7-f85970789783
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app.kubernetes.io/component: llminferenceservice-router-scheduler
        app.kubernetes.io/name: llmisvc-model-fb-opt-125m-with-ba4d693a
        app.kubernetes.io/part-of: llminferenceservice
    strategy:
      type: Recreate
    template:
      metadata:
        annotations:
          app.kubernetes.io/version: 0.9.0
          certificates.kserve.io/expiration-v2: "true"
        labels:
          app.kubernetes.io/component: llminferenceservice-router-scheduler
          app.kubernetes.io/name: llmisvc-model-fb-opt-125m-with-ba4d693a
          app.kubernetes.io/part-of: llminferenceservice
      spec:
        containers:
        - args:
          - --config-text
          - |
            apiVersion: inference.networking.x-k8s.io/v1alpha1
            kind: EndpointPickerConfig
            plugins:
            - type: single-profile-handler
            - type: queue-scorer
            - type: prefix-cache-scorer
            - type: max-score-picker
            - type: lora-affinity-scorer
            schedulingProfiles:
            - name: default
              plugins:
              - pluginRef: lora-affinity-scorer
                weight: 4
              - pluginRef: queue-scorer
                weight: 2
              - pluginRef: prefix-cache-scorer
                weight: 3
              - pluginRef: max-score-picker
          command:
          - /app/epp
          - --pool-name
          - llmisvc-model-fb-opt-125m-with-ba4d693a-inference-pool
          - --pool-namespace
          - kserve-ci-e2e-test
          - --zap-encoder
          - json
          - --grpc-port
          - "9002"
          - --grpc-health-port
          - "9003"
          - --enable-cert-reload=true
          - --secure-serving=true
          - --model-server-metrics-scheme=https
          - --cert-path=/var/run/kserve/tls
          env:
          - name: SSL_CERT_DIR
            value: /var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs
          image: ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0-rc.2
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - /bin/sleep
                - "15"
          livenessProbe:
            failureThreshold: 3
            grpc:
              port: 9003
              service: liveness
            initialDelaySeconds: 5
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: main
          ports:
          - containerPort: 9002
            name: grpc
            protocol: TCP
          - containerPort: 9003
            name: grpc-health
            protocol: TCP
          - containerPort: 9090
            name: metrics
            protocol: TCP
          - containerPort: 5557
            name: zmq
            protocol: TCP
          readinessProbe:
            failureThreshold: 3
            grpc:
              port: 9003
              service: readiness
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          resources:
            requests:
              cpu: 256m
              memory: 500Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
            runAsNonRoot: true
            seccompProfile:
              type: RuntimeDefault
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
          - mountPath: /tmp/tokenizer
            name: tokenizer-uds
        dnsPolicy: ClusterFirst
        initContainers:
        - args:
          - hf://facebook/opt-125m
          - /mnt/models
          env:
          - name: AWS_ACCESS_KEY_ID
            valueFrom:
              secretKeyRef:
                key: AWS_ACCESS_KEY_ID
                name: seaweedfs-s3-creds
          - name: AWS_SECRET_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                key: AWS_SECRET_ACCESS_KEY
                name: seaweedfs-s3-creds
          - name: S3_USE_HTTPS
            value: "0"
          - name: S3_ENDPOINT
            value: s3-service.kserve:8333
          - name: AWS_ENDPOINT_URL
            value: http://s3-service.kserve:8333
          - name: S3_VERIFY_SSL
            value: "0"
          - name: AWS_CA_BUNDLE
            value: /etc/ssl/custom-certs/cabundle.crt
          - name: AWS_CA_BUNDLE_CONFIGMAP
            value: odh-kserve-custom-ca-bundle
          - name: HF_HUB_ENABLE_HF_TRANSFER
            value: "1"
          - name: HF_XET_HIGH_PERFORMANCE
            value: "1"
          - name: HF_XET_NUM_CONCURRENT_RANGE_GETS
            value: "8"
          - name: STORAGE_ALLOW_PATTERNS
            value: '["tokenizer.json", "tokenizer_config.json", "special_tokens_map.json",
              "vocab.json", "merges.txt", "config.json", "generation_config.json"]'
          image: quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773
          imagePullPolicy: IfNotPresent
          name: storage-initializer
          resources:
            limits:
              cpu: "1"
              memory: 24Gi
            requests:
              cpu: 100m
              memory: 100Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /mnt/models
            name: kserve-provision-location
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        serviceAccount: llmisvc-model-fb-opt-125m-with-ba4d693a-epp-sa
        serviceAccountName: llmisvc-model-fb-opt-125m-with-ba4d693a-epp-sa
        terminationGracePeriodSeconds: 60
        volumes:
        - name: tls-certs
          secret:
            defaultMode: 420
            secretName: llmisv77ff2528d3e9b4972cd9335229fce9f0-kserve-self-signed-certs
        - emptyDir: {}
          name: tokenizer-uds
        - emptyDir: {}
          name: tokenizer-tmp
        - emptyDir: {}
          name: tokenizer-cache
        - emptyDir: {}
          name: kserve-provision-location
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2026-07-08T07:33:47Z"
      lastUpdateTime: "2026-07-08T07:33:47Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    - lastTransitionTime: "2026-07-08T07:33:12Z"
      lastUpdateTime: "2026-07-08T07:33:47Z"
      message: ReplicaSet "llmisvc-model-fb-opt-125m-with-ba4d693a-kserve-router-scheduler-5d6c88c6f4"
        has successfully progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2026-07-08T07:54:26Z"
    generation: 1
    labels:
      app.kubernetes.io/component: llminferenceservice-router-scheduler
      app.kubernetes.io/name: llmisvc-model-pvc-router-manage-2577e794
      app.kubernetes.io/part-of: llminferenceservice
    managedFields:
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:labels:
            .: {}
            f:app.kubernetes.io/component: {}
            f:app.kubernetes.io/name: {}
            f:app.kubernetes.io/part-of: {}
          f:ownerReferences:
            .: {}
            k:{"uid":"f79149f0-62e1-4029-a269-01f75473f7f7"}: {}
        f:spec:
          f:progressDeadlineSeconds: {}
          f:replicas: {}
          f:revisionHistoryLimit: {}
          f:selector: {}
          f:strategy:
            f:type: {}
          f:template:
            f:metadata:
              f:annotations:
                .: {}
                f:app.kubernetes.io/version: {}
                f:certificates.kserve.io/expiration-v2: {}
              f:labels:
                .: {}
                f:app.kubernetes.io/component: {}
                f:app.kubernetes.io/name: {}
                f:app.kubernetes.io/part-of: {}
            f:spec:
              f:containers:
                k:{"name":"main"}:
                  .: {}
                  f:args: {}
                  f:command: {}
                  f:env:
                    .: {}
                    k:{"name":"SSL_CERT_DIR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:lifecycle:
                    .: {}
                    f:preStop:
                      .: {}
                      f:exec:
                        .: {}
                        f:command: {}
                  f:livenessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:grpc:
                      .: {}
                      f:port: {}
                      f:service: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:name: {}
                  f:ports:
                    .: {}
                    k:{"containerPort":5557,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9002,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9003,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9090,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                  f:readinessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:grpc:
                      .: {}
                      f:port: {}
                      f:service: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources:
                    .: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:securityContext:
                    .: {}
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      .: {}
                      f:drop: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsNonRoot: {}
                    f:seccompProfile:
                      .: {}
                      f:type: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/tmp/tokenizer"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/kserve/tls"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
              f:dnsPolicy: {}
              f:restartPolicy: {}
              f:schedulerName: {}
              f:securityContext: {}
              f:serviceAccount: {}
              f:serviceAccountName: {}
              f:terminationGracePeriodSeconds: {}
              f:volumes:
                .: {}
                k:{"name":"kserve-pvc-source"}:
                  .: {}
                  f:name: {}
                  f:persistentVolumeClaim:
                    .: {}
                    f:claimName: {}
                k:{"name":"tls-certs"}:
                  .: {}
                  f:name: {}
                  f:secret:
                    .: {}
                    f:defaultMode: {}
                    f:secretName: {}
                k:{"name":"tokenizer-cache"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tokenizer-tmp"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tokenizer-uds"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
      manager: manager
      operation: Update
      time: "2026-07-08T07:54:26Z"
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:deployment.kubernetes.io/revision: {}
        f:status:
          f:availableReplicas: {}
          f:conditions:
            .: {}
            k:{"type":"Available"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"Progressing"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:observedGeneration: {}
          f:readyReplicas: {}
          f:replicas: {}
          f:updatedReplicas: {}
      manager: kube-controller-manager
      operation: Update
      subresource: status
      time: "2026-07-08T07:54:58Z"
    name: llmisvc1dc4269d1ada5f2d28562215d180c57f-kserve-router-scheduler
    namespace: kserve-ci-e2e-test
    ownerReferences:
    - apiVersion: serving.kserve.io/v1alpha2
      blockOwnerDeletion: true
      controller: true
      kind: LLMInferenceService
      name: llmisvc-model-pvc-router-manage-2577e794
      uid: f79149f0-62e1-4029-a269-01f75473f7f7
    resourceVersion: "65133"
    uid: c97d0f34-4565-43ed-b20b-7e28ac516716
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app.kubernetes.io/component: llminferenceservice-router-scheduler
        app.kubernetes.io/name: llmisvc-model-pvc-router-manage-2577e794
        app.kubernetes.io/part-of: llminferenceservice
    strategy:
      type: Recreate
    template:
      metadata:
        annotations:
          app.kubernetes.io/version: 0.9.0
          certificates.kserve.io/expiration-v2: "true"
        labels:
          app.kubernetes.io/component: llminferenceservice-router-scheduler
          app.kubernetes.io/name: llmisvc-model-pvc-router-manage-2577e794
          app.kubernetes.io/part-of: llminferenceservice
      spec:
        containers:
        - args:
          - --config-text
          - |
            apiVersion: inference.networking.x-k8s.io/v1alpha1
            kind: EndpointPickerConfig
            plugins:
            - type: single-profile-handler
            - type: queue-scorer
            - type: prefix-cache-scorer
            - type: max-score-picker
            schedulingProfiles:
            - name: default
              plugins:
              - pluginRef: queue-scorer
                weight: 2
              - pluginRef: prefix-cache-scorer
                weight: 3
              - pluginRef: max-score-picker
          command:
          - /app/epp
          - --pool-name
          - llmisvc-model-pvc-router-manage-2577e794-inference-pool
          - --pool-namespace
          - kserve-ci-e2e-test
          - --zap-encoder
          - json
          - --grpc-port
          - "9002"
          - --grpc-health-port
          - "9003"
          - --enable-cert-reload=true
          - --secure-serving=true
          - --model-server-metrics-scheme=https
          - --cert-path=/var/run/kserve/tls
          env:
          - name: SSL_CERT_DIR
            value: /var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs
          image: ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0-rc.2
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - /bin/sleep
                - "15"
          livenessProbe:
            failureThreshold: 3
            grpc:
              port: 9003
              service: liveness
            initialDelaySeconds: 5
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: main
          ports:
          - containerPort: 9002
            name: grpc
            protocol: TCP
          - containerPort: 9003
            name: grpc-health
            protocol: TCP
          - containerPort: 9090
            name: metrics
            protocol: TCP
          - containerPort: 5557
            name: zmq
            protocol: TCP
          readinessProbe:
            failureThreshold: 3
            grpc:
              port: 9003
              service: readiness
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          resources:
            requests:
              cpu: 256m
              memory: 500Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
            runAsNonRoot: true
            seccompProfile:
              type: RuntimeDefault
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
          - mountPath: /tmp/tokenizer
            name: tokenizer-uds
        dnsPolicy: ClusterFirst
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        serviceAccount: llmisvc-model-pvc-router-manage-2577e794-epp-sa
        serviceAccountName: llmisvc-model-pvc-router-manage-2577e794-epp-sa
        terminationGracePeriodSeconds: 60
        volumes:
        - name: tls-certs
          secret:
            defaultMode: 420
            secretName: llmisv1dc4269d1ada5f2d28562215d180c57f-kserve-self-signed-certs
        - emptyDir: {}
          name: tokenizer-uds
        - emptyDir: {}
          name: tokenizer-tmp
        - emptyDir: {}
          name: tokenizer-cache
        - name: kserve-pvc-source
          persistentVolumeClaim:
            claimName: e2e-pvc-model-storage
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2026-07-08T07:54:58Z"
      lastUpdateTime: "2026-07-08T07:54:58Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    - lastTransitionTime: "2026-07-08T07:54:26Z"
      lastUpdateTime: "2026-07-08T07:54:58Z"
      message: ReplicaSet "llmisvc1dc4269d1ada5f2d28562215d180c57f-kserve-router-scheduler-7dbcb75dbc"
        has successfully progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2026-07-08T07:21:02Z"
    generation: 1
    labels:
      app.kubernetes.io/component: llminferenceservice-router-scheduler
      app.kubernetes.io/name: llmisvc-model-fb-opt-125m-route-87882a8e
      app.kubernetes.io/part-of: llminferenceservice
    managedFields:
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:labels:
            .: {}
            f:app.kubernetes.io/component: {}
            f:app.kubernetes.io/name: {}
            f:app.kubernetes.io/part-of: {}
          f:ownerReferences:
            .: {}
            k:{"uid":"565176bb-ec06-4726-bd46-befcc78f8c33"}: {}
        f:spec:
          f:progressDeadlineSeconds: {}
          f:replicas: {}
          f:revisionHistoryLimit: {}
          f:selector: {}
          f:strategy:
            f:type: {}
          f:template:
            f:metadata:
              f:annotations:
                .: {}
                f:app.kubernetes.io/version: {}
                f:certificates.kserve.io/expiration-v2: {}
              f:labels:
                .: {}
                f:app.kubernetes.io/component: {}
                f:app.kubernetes.io/name: {}
                f:app.kubernetes.io/part-of: {}
            f:spec:
              f:containers:
                k:{"name":"main"}:
                  .: {}
                  f:args: {}
                  f:command: {}
                  f:env:
                    .: {}
                    k:{"name":"SSL_CERT_DIR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:lifecycle:
                    .: {}
                    f:preStop:
                      .: {}
                      f:exec:
                        .: {}
                        f:command: {}
                  f:livenessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:grpc:
                      .: {}
                      f:port: {}
                      f:service: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:name: {}
                  f:ports:
                    .: {}
                    k:{"containerPort":5557,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9002,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9003,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9090,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                  f:readinessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:grpc:
                      .: {}
                      f:port: {}
                      f:service: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources:
                    .: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:securityContext:
                    .: {}
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      .: {}
                      f:drop: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsNonRoot: {}
                    f:seccompProfile:
                      .: {}
                      f:type: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/tmp/tokenizer"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/kserve/tls"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
              f:dnsPolicy: {}
              f:initContainers:
                .: {}
                k:{"name":"storage-initializer"}:
                  .: {}
                  f:args: {}
                  f:env:
                    .: {}
                    k:{"name":"AWS_ACCESS_KEY_ID"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"AWS_CA_BUNDLE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_CA_BUNDLE_CONFIGMAP"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_ENDPOINT_URL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_SECRET_ACCESS_KEY"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"HF_HUB_ENABLE_HF_TRANSFER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_HIGH_PERFORMANCE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_NUM_CONCURRENT_RANGE_GETS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_ENDPOINT"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_USE_HTTPS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_VERIFY_SSL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"STORAGE_ALLOW_PATTERNS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:name: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
              f:restartPolicy: {}
              f:schedulerName: {}
              f:securityContext: {}
              f:serviceAccount: {}
              f:serviceAccountName: {}
              f:terminationGracePeriodSeconds: {}
              f:volumes:
                .: {}
                k:{"name":"kserve-provision-location"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tls-certs"}:
                  .: {}
                  f:name: {}
                  f:secret:
                    .: {}
                    f:defaultMode: {}
                    f:secretName: {}
                k:{"name":"tokenizer-cache"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tokenizer-tmp"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tokenizer-uds"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
      manager: manager
      operation: Update
      time: "2026-07-08T07:21:02Z"
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:deployment.kubernetes.io/revision: {}
        f:status:
          f:availableReplicas: {}
          f:conditions:
            .: {}
            k:{"type":"Available"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"Progressing"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:observedGeneration: {}
          f:readyReplicas: {}
          f:replicas: {}
          f:updatedReplicas: {}
      manager: kube-controller-manager
      operation: Update
      subresource: status
      time: "2026-07-08T07:21:35Z"
    name: llmisvca690bbc929faec8bc98c767f16c003c1-kserve-router-scheduler
    namespace: kserve-ci-e2e-test
    ownerReferences:
    - apiVersion: serving.kserve.io/v1alpha2
      blockOwnerDeletion: true
      controller: true
      kind: LLMInferenceService
      name: llmisvc-model-fb-opt-125m-route-87882a8e
      uid: 565176bb-ec06-4726-bd46-befcc78f8c33
    resourceVersion: "35403"
    uid: 43b917fe-7e17-488b-831f-fafec25f64b0
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app.kubernetes.io/component: llminferenceservice-router-scheduler
        app.kubernetes.io/name: llmisvc-model-fb-opt-125m-route-87882a8e
        app.kubernetes.io/part-of: llminferenceservice
    strategy:
      type: Recreate
    template:
      metadata:
        annotations:
          app.kubernetes.io/version: 0.9.0
          certificates.kserve.io/expiration-v2: "true"
        labels:
          app.kubernetes.io/component: llminferenceservice-router-scheduler
          app.kubernetes.io/name: llmisvc-model-fb-opt-125m-route-87882a8e
          app.kubernetes.io/part-of: llminferenceservice
      spec:
        containers:
        - args:
          - --config-text
          - |
            apiVersion: inference.networking.x-k8s.io/v1alpha1
            kind: EndpointPickerConfig
            plugins:
            - type: single-profile-handler
            - type: queue-scorer
            - type: prefix-cache-scorer
            - type: max-score-picker
            schedulingProfiles:
            - name: default
              plugins:
              - pluginRef: queue-scorer
                weight: 2
              - pluginRef: prefix-cache-scorer
                weight: 3
              - pluginRef: max-score-picker
          command:
          - /app/epp
          - --pool-name
          - llmisvc-model-fb-opt-125m-route-87882a8e-inference-pool
          - --pool-namespace
          - kserve-ci-e2e-test
          - --zap-encoder
          - json
          - --grpc-port
          - "9002"
          - --grpc-health-port
          - "9003"
          - --enable-cert-reload=true
          - --secure-serving=true
          - --model-server-metrics-scheme=https
          - --cert-path=/var/run/kserve/tls
          env:
          - name: SSL_CERT_DIR
            value: /var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs
          image: ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0-rc.2
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - /bin/sleep
                - "15"
          livenessProbe:
            failureThreshold: 3
            grpc:
              port: 9003
              service: liveness
            initialDelaySeconds: 5
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: main
          ports:
          - containerPort: 9002
            name: grpc
            protocol: TCP
          - containerPort: 9003
            name: grpc-health
            protocol: TCP
          - containerPort: 9090
            name: metrics
            protocol: TCP
          - containerPort: 5557
            name: zmq
            protocol: TCP
          readinessProbe:
            failureThreshold: 3
            grpc:
              port: 9003
              service: readiness
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          resources:
            requests:
              cpu: 256m
              memory: 500Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
            runAsNonRoot: true
            seccompProfile:
              type: RuntimeDefault
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
          - mountPath: /tmp/tokenizer
            name: tokenizer-uds
        dnsPolicy: ClusterFirst
        initContainers:
        - args:
          - hf://facebook/opt-125m
          - /mnt/models
          env:
          - name: AWS_ACCESS_KEY_ID
            valueFrom:
              secretKeyRef:
                key: AWS_ACCESS_KEY_ID
                name: seaweedfs-s3-creds
          - name: AWS_SECRET_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                key: AWS_SECRET_ACCESS_KEY
                name: seaweedfs-s3-creds
          - name: S3_USE_HTTPS
            value: "0"
          - name: S3_ENDPOINT
            value: s3-service.kserve:8333
          - name: AWS_ENDPOINT_URL
            value: http://s3-service.kserve:8333
          - name: S3_VERIFY_SSL
            value: "0"
          - name: AWS_CA_BUNDLE
            value: /etc/ssl/custom-certs/cabundle.crt
          - name: AWS_CA_BUNDLE_CONFIGMAP
            value: odh-kserve-custom-ca-bundle
          - name: HF_HUB_ENABLE_HF_TRANSFER
            value: "1"
          - name: HF_XET_HIGH_PERFORMANCE
            value: "1"
          - name: HF_XET_NUM_CONCURRENT_RANGE_GETS
            value: "8"
          - name: STORAGE_ALLOW_PATTERNS
            value: '["tokenizer.json", "tokenizer_config.json", "special_tokens_map.json",
              "vocab.json", "merges.txt", "config.json", "generation_config.json"]'
          image: quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773
          imagePullPolicy: IfNotPresent
          name: storage-initializer
          resources:
            limits:
              cpu: "1"
              memory: 24Gi
            requests:
              cpu: 100m
              memory: 100Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /mnt/models
            name: kserve-provision-location
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        serviceAccount: llmisvc-model-fb-opt-125m-route-87882a8e-epp-sa
        serviceAccountName: llmisvc-model-fb-opt-125m-route-87882a8e-epp-sa
        terminationGracePeriodSeconds: 60
        volumes:
        - name: tls-certs
          secret:
            defaultMode: 420
            secretName: llmisva690bbc929faec8bc98c767f16c003c1-kserve-self-signed-certs
        - emptyDir: {}
          name: tokenizer-uds
        - emptyDir: {}
          name: tokenizer-tmp
        - emptyDir: {}
          name: tokenizer-cache
        - emptyDir: {}
          name: kserve-provision-location
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2026-07-08T07:21:35Z"
      lastUpdateTime: "2026-07-08T07:21:35Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    - lastTransitionTime: "2026-07-08T07:21:02Z"
      lastUpdateTime: "2026-07-08T07:21:35Z"
      message: ReplicaSet "llmisvca690bbc929faec8bc98c767f16c003c1-kserve-router-scheduler-599658ff65"
        has successfully progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2026-07-08T07:07:58Z"
    generation: 1
    labels:
      gateway.istio.io/managed: openshift.io-gateway-controller-v1
      gateway.networking.k8s.io/gateway-name: router-gateway-1
    managedFields:
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations: {}
          f:labels:
            f:gateway.istio.io/managed: {}
            f:gateway.networking.k8s.io/gateway-name: {}
          f:ownerReferences:
            k:{"uid":"b9e516c8-3eb0-4bc2-8b12-7f29f4f24e92"}: {}
        f:spec:
          f:selector: {}
          f:template:
            f:metadata:
              f:annotations:
                f:istio.io/rev: {}
                f:prometheus.io/path: {}
                f:prometheus.io/port: {}
                f:prometheus.io/scrape: {}
              f:labels:
                f:gateway.istio.io/managed: {}
                f:gateway.networking.k8s.io/gateway-name: {}
                f:service.istio.io/canonical-name: {}
                f:service.istio.io/canonical-revision: {}
                f:sidecar.istio.io/inject: {}
            f:spec:
              f:containers:
                k:{"name":"istio-proxy"}:
                  .: {}
                  f:args: {}
                  f:env:
                    k:{"name":"CA_ADDR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"GOMAXPROCS"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        f:resourceFieldRef: {}
                    k:{"name":"GOMEMLIMIT"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        f:resourceFieldRef: {}
                    k:{"name":"HOST_IP"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        f:fieldRef: {}
                    k:{"name":"INSTANCE_IP"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        f:fieldRef: {}
                    k:{"name":"ISTIO_CPU_LIMIT"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        f:resourceFieldRef: {}
                    k:{"name":"ISTIO_META_APP_CONTAINERS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"ISTIO_META_CLUSTER_ID"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"ISTIO_META_INTERCEPTION_MODE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"ISTIO_META_MESH_ID"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"ISTIO_META_NODE_NAME"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        f:fieldRef: {}
                    k:{"name":"ISTIO_META_OWNER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"ISTIO_META_POD_PORTS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"ISTIO_META_WORKLOAD_NAME"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"PILOT_CERT_PROVIDER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"POD_NAME"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        f:fieldRef: {}
                    k:{"name":"POD_NAMESPACE"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        f:fieldRef: {}
                    k:{"name":"PROXY_CONFIG"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"SERVICE_ACCOUNT"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        f:fieldRef: {}
                    k:{"name":"TRUST_DOMAIN"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:name: {}
                  f:ports:
                    k:{"containerPort":15020,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":15021,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":15090,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                  f:readinessProbe:
                    f:failureThreshold: {}
                    f:httpGet:
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources:
                    f:limits:
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      f:cpu: {}
                      f:memory: {}
                  f:securityContext:
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      f:drop: {}
                    f:privileged: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsGroup: {}
                    f:runAsNonRoot: {}
                    f:runAsUser: {}
                  f:startupProbe:
                    f:failureThreshold: {}
                    f:httpGet:
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:volumeMounts:
                    k:{"mountPath":"/etc/istio/pod"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/etc/istio/proxy"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/lib/istio/data"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/secrets/credential-uds"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/secrets/istio"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/secrets/tokens"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/secrets/workload-spiffe-credentials"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/secrets/workload-spiffe-uds"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
              f:securityContext:
                f:sysctls: {}
              f:serviceAccountName: {}
              f:volumes:
                k:{"name":"credential-socket"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"istio-data"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"istio-envoy"}:
                  .: {}
                  f:emptyDir:
                    f:medium: {}
                  f:name: {}
                k:{"name":"istio-podinfo"}:
                  .: {}
                  f:downwardAPI:
                    f:items: {}
                  f:name: {}
                k:{"name":"istio-token"}:
                  .: {}
                  f:name: {}
                  f:projected:
                    f:sources: {}
                k:{"name":"istiod-ca-cert"}:
                  .: {}
                  f:configMap:
                    f:name: {}
                  f:name: {}
                k:{"name":"workload-certs"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"workload-socket"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
      manager: openshift.io/gateway-controller/v1
      operation: Apply
      time: "2026-07-08T07:07:58Z"
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:deployment.kubernetes.io/revision: {}
        f:status:
          f:availableReplicas: {}
          f:conditions:
            .: {}
            k:{"type":"Available"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"Progressing"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:observedGeneration: {}
          f:readyReplicas: {}
          f:replicas: {}
          f:updatedReplicas: {}
      manager: kube-controller-manager
      operation: Update
      subresource: status
      time: "2026-07-08T07:19:00Z"
    name: router-gateway-1-openshift-default
    namespace: kserve-ci-e2e-test
    ownerReferences:
    - apiVersion: gateway.networking.k8s.io/v1beta1
      kind: Gateway
      name: router-gateway-1
      uid: b9e516c8-3eb0-4bc2-8b12-7f29f4f24e92
    resourceVersion: "32813"
    uid: 3a62b7e4-d965-4759-a221-966bfd1ea2ae
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        gateway.networking.k8s.io/gateway-name: router-gateway-1
    strategy:
      rollingUpdate:
        maxSurge: 25%
        maxUnavailable: 25%
      type: RollingUpdate
    template:
      metadata:
        annotations:
          istio.io/rev: openshift-gateway
          prometheus.io/path: /stats/prometheus
          prometheus.io/port: "15020"
          prometheus.io/scrape: "true"
        labels:
          gateway.istio.io/managed: istio.io-gateway-controller
          gateway.networking.k8s.io/gateway-name: router-gateway-1
          service.istio.io/canonical-name: router-gateway-1-openshift-default
          service.istio.io/canonical-revision: latest
          sidecar.istio.io/inject: "false"
      spec:
        containers:
        - args:
          - proxy
          - router
          - --domain
          - $(POD_NAMESPACE).svc.cluster.local
          - --proxyLogLevel
          - warning
          - --proxyComponentLogLevel
          - misc:error
          - --log_output_level
          - default:info
          env:
          - name: PILOT_CERT_PROVIDER
            value: istiod
          - name: CA_ADDR
            value: istiod-openshift-gateway.openshift-ingress.svc:15012
          - name: POD_NAME
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: metadata.name
          - name: POD_NAMESPACE
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: metadata.namespace
          - name: INSTANCE_IP
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: status.podIP
          - name: SERVICE_ACCOUNT
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: spec.serviceAccountName
          - name: HOST_IP
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: status.hostIP
          - name: ISTIO_CPU_LIMIT
            valueFrom:
              resourceFieldRef:
                divisor: "0"
                resource: limits.cpu
          - name: PROXY_CONFIG
            value: |
              {"discoveryAddress":"istiod-openshift-gateway.openshift-ingress.svc:15012","proxyHeaders":{"server":{"disabled":true},"envoyDebugHeaders":{"disabled":true},"metadataExchangeHeaders":{"mode":"IN_MESH"}}}
          - name: ISTIO_META_POD_PORTS
            value: '[]'
          - name: ISTIO_META_APP_CONTAINERS
          - name: GOMEMLIMIT
            valueFrom:
              resourceFieldRef:
                divisor: "0"
                resource: limits.memory
          - name: GOMAXPROCS
            valueFrom:
              resourceFieldRef:
                divisor: "0"
                resource: limits.cpu
          - name: ISTIO_META_CLUSTER_ID
            value: Kubernetes
          - name: ISTIO_META_NODE_NAME
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: spec.nodeName
          - name: ISTIO_META_INTERCEPTION_MODE
            value: REDIRECT
          - name: ISTIO_META_WORKLOAD_NAME
            value: router-gateway-1-openshift-default
          - name: ISTIO_META_OWNER
            value: kubernetes://apis/apps/v1/namespaces/kserve-ci-e2e-test/deployments/router-gateway-1-openshift-default
          - name: ISTIO_META_MESH_ID
            value: cluster.local
          - name: TRUST_DOMAIN
            value: cluster.local
          image: registry.redhat.io/openshift-service-mesh/istio-proxyv2-rhel9@sha256:7d15cebf9b62f3f235c0eab5158ac8ff2fda86a1d193490dc94c301402c99da8
          imagePullPolicy: IfNotPresent
          name: istio-proxy
          ports:
          - containerPort: 15020
            name: metrics
            protocol: TCP
          - containerPort: 15021
            name: status-port
            protocol: TCP
          - containerPort: 15090
            name: http-envoy-prom
            protocol: TCP
          readinessProbe:
            failureThreshold: 4
            httpGet:
              path: /healthz/ready
              port: 15021
              scheme: HTTP
            periodSeconds: 15
            successThreshold: 1
            timeoutSeconds: 1
          resources:
            limits:
              cpu: "2"
              memory: 1Gi
            requests:
              cpu: 100m
              memory: 128Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            privileged: false
            readOnlyRootFilesystem: true
            runAsGroup: 1000699999
            runAsNonRoot: true
            runAsUser: 1000699999
          startupProbe:
            failureThreshold: 30
            httpGet:
              path: /healthz/ready
              port: 15021
              scheme: HTTP
            initialDelaySeconds: 1
            periodSeconds: 1
            successThreshold: 1
            timeoutSeconds: 1
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
          - mountPath: /var/run/secrets/workload-spiffe-uds
            name: workload-socket
          - mountPath: /var/run/secrets/credential-uds
            name: credential-socket
          - mountPath: /var/run/secrets/workload-spiffe-credentials
            name: workload-certs
          - mountPath: /var/run/secrets/istio
            name: istiod-ca-cert
          - mountPath: /var/lib/istio/data
            name: istio-data
          - mountPath: /etc/istio/proxy
            name: istio-envoy
          - mountPath: /var/run/secrets/tokens
            name: istio-token
          - mountPath: /etc/istio/pod
            name: istio-podinfo
        dnsPolicy: ClusterFirst
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext:
          sysctls:
          - name: net.ipv4.ip_unprivileged_port_start
            value: "0"
        serviceAccount: router-gateway-1-openshift-default
        serviceAccountName: router-gateway-1-openshift-default
        terminationGracePeriodSeconds: 30
        volumes:
        - emptyDir: {}
          name: workload-socket
        - emptyDir: {}
          name: credential-socket
        - emptyDir: {}
          name: workload-certs
        - emptyDir:
            medium: Memory
          name: istio-envoy
        - emptyDir: {}
          name: istio-data
        - downwardAPI:
            defaultMode: 420
            items:
            - fieldRef:
                apiVersion: v1
                fieldPath: metadata.labels
              path: labels
            - fieldRef:
                apiVersion: v1
                fieldPath: metadata.annotations
              path: annotations
          name: istio-podinfo
        - name: istio-token
          projected:
            defaultMode: 420
            sources:
            - serviceAccountToken:
                audience: istio-ca
                expirationSeconds: 43200
                path: istio-token
        - configMap:
            defaultMode: 420
            name: openshift-gw-ca-root-cert
          name: istiod-ca-cert
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2026-07-08T07:07:58Z"
      lastUpdateTime: "2026-07-08T07:08:03Z"
      message: ReplicaSet "router-gateway-1-openshift-default-799f46c59b" has successfully
        progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    - lastTransitionTime: "2026-07-08T07:19:00Z"
      lastUpdateTime: "2026-07-08T07:19:00Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2026-07-08T08:18:27Z"
    generation: 1
    labels:
      gateway.istio.io/managed: openshift.io-gateway-controller-v1
      gateway.networking.k8s.io/gateway-name: router-gateway-2
    managedFields:
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations: {}
          f:labels:
            f:gateway.istio.io/managed: {}
            f:gateway.networking.k8s.io/gateway-name: {}
          f:ownerReferences:
            k:{"uid":"bad4569c-993d-4e23-874f-add0a3f9fcbb"}: {}
        f:spec:
          f:selector: {}
          f:template:
            f:metadata:
              f:annotations:
                f:istio.io/rev: {}
                f:prometheus.io/path: {}
                f:prometheus.io/port: {}
                f:prometheus.io/scrape: {}
              f:labels:
                f:gateway.istio.io/managed: {}
                f:gateway.networking.k8s.io/gateway-name: {}
                f:service.istio.io/canonical-name: {}
                f:service.istio.io/canonical-revision: {}
                f:sidecar.istio.io/inject: {}
            f:spec:
              f:containers:
                k:{"name":"istio-proxy"}:
                  .: {}
                  f:args: {}
                  f:env:
                    k:{"name":"CA_ADDR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"GOMAXPROCS"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        f:resourceFieldRef: {}
                    k:{"name":"GOMEMLIMIT"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        f:resourceFieldRef: {}
                    k:{"name":"HOST_IP"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        f:fieldRef: {}
                    k:{"name":"INSTANCE_IP"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        f:fieldRef: {}
                    k:{"name":"ISTIO_CPU_LIMIT"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        f:resourceFieldRef: {}
                    k:{"name":"ISTIO_META_APP_CONTAINERS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"ISTIO_META_CLUSTER_ID"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"ISTIO_META_INTERCEPTION_MODE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"ISTIO_META_MESH_ID"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"ISTIO_META_NODE_NAME"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        f:fieldRef: {}
                    k:{"name":"ISTIO_META_OWNER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"ISTIO_META_POD_PORTS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"ISTIO_META_WORKLOAD_NAME"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"PILOT_CERT_PROVIDER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"POD_NAME"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        f:fieldRef: {}
                    k:{"name":"POD_NAMESPACE"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        f:fieldRef: {}
                    k:{"name":"PROXY_CONFIG"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"SERVICE_ACCOUNT"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        f:fieldRef: {}
                    k:{"name":"TRUST_DOMAIN"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:name: {}
                  f:ports:
                    k:{"containerPort":15020,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":15021,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":15090,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                  f:readinessProbe:
                    f:failureThreshold: {}
                    f:httpGet:
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources:
                    f:limits:
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      f:cpu: {}
                      f:memory: {}
                  f:securityContext:
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      f:drop: {}
                    f:privileged: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsGroup: {}
                    f:runAsNonRoot: {}
                    f:runAsUser: {}
                  f:startupProbe:
                    f:failureThreshold: {}
                    f:httpGet:
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:volumeMounts:
                    k:{"mountPath":"/etc/istio/pod"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/etc/istio/proxy"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/lib/istio/data"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/secrets/credential-uds"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/secrets/istio"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/secrets/tokens"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/secrets/workload-spiffe-credentials"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/secrets/workload-spiffe-uds"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
              f:securityContext:
                f:sysctls: {}
              f:serviceAccountName: {}
              f:volumes:
                k:{"name":"credential-socket"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"istio-data"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"istio-envoy"}:
                  .: {}
                  f:emptyDir:
                    f:medium: {}
                  f:name: {}
                k:{"name":"istio-podinfo"}:
                  .: {}
                  f:downwardAPI:
                    f:items: {}
                  f:name: {}
                k:{"name":"istio-token"}:
                  .: {}
                  f:name: {}
                  f:projected:
                    f:sources: {}
                k:{"name":"istiod-ca-cert"}:
                  .: {}
                  f:configMap:
                    f:name: {}
                  f:name: {}
                k:{"name":"workload-certs"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"workload-socket"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
      manager: openshift.io/gateway-controller/v1
      operation: Apply
      time: "2026-07-08T08:18:28Z"
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:deployment.kubernetes.io/revision: {}
        f:status:
          f:availableReplicas: {}
          f:conditions:
            .: {}
            k:{"type":"Available"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"Progressing"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:observedGeneration: {}
          f:readyReplicas: {}
          f:replicas: {}
          f:updatedReplicas: {}
      manager: kube-controller-manager
      operation: Update
      subresource: status
      time: "2026-07-08T08:18:50Z"
    name: router-gateway-2-openshift-default
    namespace: kserve-ci-e2e-test
    ownerReferences:
    - apiVersion: gateway.networking.k8s.io/v1beta1
      kind: Gateway
      name: router-gateway-2
      uid: bad4569c-993d-4e23-874f-add0a3f9fcbb
    resourceVersion: "85056"
    uid: 4554083c-2685-4065-81de-1f9e7c9bcc02
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        gateway.networking.k8s.io/gateway-name: router-gateway-2
    strategy:
      rollingUpdate:
        maxSurge: 25%
        maxUnavailable: 25%
      type: RollingUpdate
    template:
      metadata:
        annotations:
          istio.io/rev: openshift-gateway
          prometheus.io/path: /stats/prometheus
          prometheus.io/port: "15020"
          prometheus.io/scrape: "true"
        labels:
          gateway.istio.io/managed: istio.io-gateway-controller
          gateway.networking.k8s.io/gateway-name: router-gateway-2
          service.istio.io/canonical-name: router-gateway-2-openshift-default
          service.istio.io/canonical-revision: latest
          sidecar.istio.io/inject: "false"
      spec:
        containers:
        - args:
          - proxy
          - router
          - --domain
          - $(POD_NAMESPACE).svc.cluster.local
          - --proxyLogLevel
          - warning
          - --proxyComponentLogLevel
          - misc:error
          - --log_output_level
          - default:info
          env:
          - name: PILOT_CERT_PROVIDER
            value: istiod
          - name: CA_ADDR
            value: istiod-openshift-gateway.openshift-ingress.svc:15012
          - name: POD_NAME
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: metadata.name
          - name: POD_NAMESPACE
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: metadata.namespace
          - name: INSTANCE_IP
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: status.podIP
          - name: SERVICE_ACCOUNT
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: spec.serviceAccountName
          - name: HOST_IP
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: status.hostIP
          - name: ISTIO_CPU_LIMIT
            valueFrom:
              resourceFieldRef:
                divisor: "0"
                resource: limits.cpu
          - name: PROXY_CONFIG
            value: |
              {"discoveryAddress":"istiod-openshift-gateway.openshift-ingress.svc:15012","proxyHeaders":{"server":{"disabled":true},"envoyDebugHeaders":{"disabled":true},"metadataExchangeHeaders":{"mode":"IN_MESH"}}}
          - name: ISTIO_META_POD_PORTS
            value: '[]'
          - name: ISTIO_META_APP_CONTAINERS
          - name: GOMEMLIMIT
            valueFrom:
              resourceFieldRef:
                divisor: "0"
                resource: limits.memory
          - name: GOMAXPROCS
            valueFrom:
              resourceFieldRef:
                divisor: "0"
                resource: limits.cpu
          - name: ISTIO_META_CLUSTER_ID
            value: Kubernetes
          - name: ISTIO_META_NODE_NAME
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: spec.nodeName
          - name: ISTIO_META_INTERCEPTION_MODE
            value: REDIRECT
          - name: ISTIO_META_WORKLOAD_NAME
            value: router-gateway-2-openshift-default
          - name: ISTIO_META_OWNER
            value: kubernetes://apis/apps/v1/namespaces/kserve-ci-e2e-test/deployments/router-gateway-2-openshift-default
          - name: ISTIO_META_MESH_ID
            value: cluster.local
          - name: TRUST_DOMAIN
            value: cluster.local
          image: registry.redhat.io/openshift-service-mesh/istio-proxyv2-rhel9@sha256:7d15cebf9b62f3f235c0eab5158ac8ff2fda86a1d193490dc94c301402c99da8
          imagePullPolicy: IfNotPresent
          name: istio-proxy
          ports:
          - containerPort: 15020
            name: metrics
            protocol: TCP
          - containerPort: 15021
            name: status-port
            protocol: TCP
          - containerPort: 15090
            name: http-envoy-prom
            protocol: TCP
          readinessProbe:
            failureThreshold: 4
            httpGet:
              path: /healthz/ready
              port: 15021
              scheme: HTTP
            periodSeconds: 15
            successThreshold: 1
            timeoutSeconds: 1
          resources:
            limits:
              cpu: "2"
              memory: 1Gi
            requests:
              cpu: 100m
              memory: 128Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            privileged: false
            readOnlyRootFilesystem: true
            runAsGroup: 1000699999
            runAsNonRoot: true
            runAsUser: 1000699999
          startupProbe:
            failureThreshold: 30
            httpGet:
              path: /healthz/ready
              port: 15021
              scheme: HTTP
            initialDelaySeconds: 1
            periodSeconds: 1
            successThreshold: 1
            timeoutSeconds: 1
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
          - mountPath: /var/run/secrets/workload-spiffe-uds
            name: workload-socket
          - mountPath: /var/run/secrets/credential-uds
            name: credential-socket
          - mountPath: /var/run/secrets/workload-spiffe-credentials
            name: workload-certs
          - mountPath: /var/run/secrets/istio
            name: istiod-ca-cert
          - mountPath: /var/lib/istio/data
            name: istio-data
          - mountPath: /etc/istio/proxy
            name: istio-envoy
          - mountPath: /var/run/secrets/tokens
            name: istio-token
          - mountPath: /etc/istio/pod
            name: istio-podinfo
        dnsPolicy: ClusterFirst
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext:
          sysctls:
          - name: net.ipv4.ip_unprivileged_port_start
            value: "0"
        serviceAccount: router-gateway-2-openshift-default
        serviceAccountName: router-gateway-2-openshift-default
        terminationGracePeriodSeconds: 30
        volumes:
        - emptyDir: {}
          name: workload-socket
        - emptyDir: {}
          name: credential-socket
        - emptyDir: {}
          name: workload-certs
        - emptyDir:
            medium: Memory
          name: istio-envoy
        - emptyDir: {}
          name: istio-data
        - downwardAPI:
            defaultMode: 420
            items:
            - fieldRef:
                apiVersion: v1
                fieldPath: metadata.labels
              path: labels
            - fieldRef:
                apiVersion: v1
                fieldPath: metadata.annotations
              path: annotations
          name: istio-podinfo
        - name: istio-token
          projected:
            defaultMode: 420
            sources:
            - serviceAccountToken:
                audience: istio-ca
                expirationSeconds: 43200
                path: istio-token
        - configMap:
            defaultMode: 420
            name: openshift-gw-ca-root-cert
          name: istiod-ca-cert
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2026-07-08T08:18:27Z"
      lastUpdateTime: "2026-07-08T08:18:30Z"
      message: ReplicaSet "router-gateway-2-openshift-default-54c789bdc6" has successfully
        progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    - lastTransitionTime: "2026-07-08T08:18:50Z"
      lastUpdateTime: "2026-07-08T08:18:50Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2026-07-08T08:19:02Z"
    generation: 1
    labels:
      app.kubernetes.io/component: llminferenceservice-workload
      app.kubernetes.io/name: router-with-refs-pd-test
      app.kubernetes.io/part-of: llminferenceservice
      kserve.io/component: workload
      llm-d.ai/role: decode
    managedFields:
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:labels:
            .: {}
            f:app.kubernetes.io/component: {}
            f:app.kubernetes.io/name: {}
            f:app.kubernetes.io/part-of: {}
            f:kserve.io/component: {}
            f:llm-d.ai/role: {}
          f:ownerReferences:
            .: {}
            k:{"uid":"5071849d-29aa-4402-acb7-9d9bfa4d0c2a"}: {}
        f:spec:
          f:progressDeadlineSeconds: {}
          f:replicas: {}
          f:revisionHistoryLimit: {}
          f:selector: {}
          f:strategy:
            f:rollingUpdate:
              .: {}
              f:maxSurge: {}
              f:maxUnavailable: {}
            f:type: {}
          f:template:
            f:metadata:
              f:labels:
                .: {}
                f:app.kubernetes.io/component: {}
                f:app.kubernetes.io/name: {}
                f:app.kubernetes.io/part-of: {}
                f:kserve.io/component: {}
                f:llm-d.ai/role: {}
            f:spec:
              f:containers:
                k:{"name":"main"}:
                  .: {}
                  f:command: {}
                  f:env:
                    .: {}
                    k:{"name":"HF_HUB_CACHE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HOME"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"TORCHINDUCTOR_CACHE_DIR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"USER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_CPU_KVCACHE_SPACE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_ENABLE_V1_MULTIPROCESSING"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_LOGGING_LEVEL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:lifecycle:
                    .: {}
                    f:preStop:
                      .: {}
                      f:exec:
                        .: {}
                        f:command: {}
                  f:livenessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:name: {}
                  f:ports:
                    .: {}
                    k:{"containerPort":8001,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:protocol: {}
                  f:readinessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:securityContext:
                    .: {}
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      .: {}
                      f:drop: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsNonRoot: {}
                    f:seccompProfile:
                      .: {}
                      f:type: {}
                  f:startupProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/dev/shm"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/home"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
                    k:{"mountPath":"/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/tmp"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/kserve/tls"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
              f:dnsPolicy: {}
              f:initContainers:
                .: {}
                k:{"name":"llm-d-routing-sidecar"}:
                  .: {}
                  f:command: {}
                  f:env:
                    .: {}
                    k:{"name":"INFERENCE_POOL_NAME"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"INFERENCE_POOL_NAMESPACE"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:fieldRef: {}
                    k:{"name":"SSL_CERT_DIR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:livenessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:name: {}
                  f:ports:
                    .: {}
                    k:{"containerPort":8000,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:protocol: {}
                  f:readinessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources: {}
                  f:restartPolicy: {}
                  f:securityContext:
                    .: {}
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      .: {}
                      f:drop: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsNonRoot: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/var/run/kserve/tls"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
                k:{"name":"storage-initializer"}:
                  .: {}
                  f:args: {}
                  f:env:
                    .: {}
                    k:{"name":"AWS_ACCESS_KEY_ID"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"AWS_CA_BUNDLE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_CA_BUNDLE_CONFIGMAP"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_ENDPOINT_URL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_SECRET_ACCESS_KEY"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"HF_HUB_ENABLE_HF_TRANSFER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_HIGH_PERFORMANCE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_NUM_CONCURRENT_RANGE_GETS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_ENDPOINT"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_USE_HTTPS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_VERIFY_SSL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:name: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
              f:restartPolicy: {}
              f:schedulerName: {}
              f:securityContext: {}
              f:serviceAccount: {}
              f:serviceAccountName: {}
              f:terminationGracePeriodSeconds: {}
              f:volumes:
                .: {}
                k:{"name":"dshm"}:
                  .: {}
                  f:emptyDir:
                    .: {}
                    f:medium: {}
                    f:sizeLimit: {}
                  f:name: {}
                k:{"name":"home"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"kserve-provision-location"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"model-cache"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tls-certs"}:
                  .: {}
                  f:name: {}
                  f:secret:
                    .: {}
                    f:defaultMode: {}
                    f:secretName: {}
                k:{"name":"tmp-dir"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
      manager: manager
      operation: Update
      time: "2026-07-08T08:19:02Z"
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:deployment.kubernetes.io/revision: {}
        f:status:
          f:availableReplicas: {}
          f:conditions:
            .: {}
            k:{"type":"Available"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"Progressing"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:observedGeneration: {}
          f:readyReplicas: {}
          f:replicas: {}
          f:updatedReplicas: {}
      manager: kube-controller-manager
      operation: Update
      subresource: status
      time: "2026-07-08T08:22:22Z"
    name: router-with-refs-pd-test-kserve
    namespace: kserve-ci-e2e-test
    ownerReferences:
    - apiVersion: serving.kserve.io/v1alpha2
      blockOwnerDeletion: true
      controller: true
      kind: LLMInferenceService
      name: router-with-refs-pd-test
      uid: 5071849d-29aa-4402-acb7-9d9bfa4d0c2a
    resourceVersion: "87981"
    uid: 85b0811c-f56e-488a-b66b-319fc75ceb1f
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app.kubernetes.io/component: llminferenceservice-workload
        app.kubernetes.io/name: router-with-refs-pd-test
        app.kubernetes.io/part-of: llminferenceservice
        kserve.io/component: workload
        llm-d.ai/role: decode
    strategy:
      rollingUpdate:
        maxSurge: 25%
        maxUnavailable: 25%
      type: RollingUpdate
    template:
      metadata:
        labels:
          app.kubernetes.io/component: llminferenceservice-workload
          app.kubernetes.io/name: router-with-refs-pd-test
          app.kubernetes.io/part-of: llminferenceservice
          kserve.io/component: workload
          llm-d.ai/role: decode
      spec:
        containers:
        - command:
          - /bin/bash
          - -c
          - |-
            if [ -f /etc/profile.d/ibm-aiu-setup.sh ]; then
              source /etc/profile.d/ibm-aiu-setup.sh
            fi

            if [ "$KSERVE_INFER_ROCE" = "true" ]; then
              echo "Trying to infer RoCE configs ... "
              grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
              grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null

              cat /proc/driver/nvidia/params

              KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}

              echo "[Infer RoCE] Discovering active HCAs ..."
              active_hcas=()
              # Loop through all mlx5 devices found in sysfs
              for hca_dir in /sys/class/infiniband/mlx5_*; do
                  # Ensure it's a directory before proceeding
                  if [ -d "$hca_dir" ]; then
                      hca_name=$(basename "$hca_dir")
                      port_state_file="$hca_dir/ports/1/state" # Assume port 1
                      type_file="$hca_dir/ports/1/gid_attrs/types/*"

                      echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
                      if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
                          echo "[Infer RoCE] Found active HCA: $hca_name"
                          active_hcas+=("$hca_name")
                      else
                          echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
                      fi
                  fi
              done

              # Check if we found any active HCAs
              if [ ${#active_hcas[@]} -gt 0 ]; then
                  # Join the array elements with a comma
                  hca_port_pairs=()
                  for hca in "${active_hcas[@]}"; do
                    hca_port_pairs+=("${hca}:1")
                  done

                  active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
                  hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
                  echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
                  export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
                  export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
                  export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}

                  echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
                  echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
                  echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
              else
                  echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
              fi

              if [ ${#active_hcas[@]} -gt 0 ]; then
                  echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."

                  # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
                  declare -A gid_index_count
                  declare -A hca_gid_index

                  for hca_name in "${active_hcas[@]}"; do
                      echo "[Infer RoCE] Processing HCA: ${hca_name}"

                      # Find all RoCE v2 IPv4 GIDs for this HCA and count by index
                      for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
                          if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
                              idx=$(basename "$tpath")
                              gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
                              # Check for IPv4 GID (contains ffff:)
                              if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
                                  gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
                                  echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
                                  hca_gid_index["${hca_name}"]="${idx}"
                                  gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
                                  break  # Use first found IPv4 GID per HCA
                              fi
                          fi
                      done
                  done

                  # Find the most common GID index (most likely to be consistent across nodes)
                  best_gid_index=""
                  max_count=0
                  for idx in "${!gid_index_count[@]}"; do
                      count=${gid_index_count["${idx}"]}
                      echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
                      if [ $count -gt $max_count ]; then
                          max_count=$count
                          best_gid_index="$idx"
                      fi
                  done

                  # Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
                  if [ ${#gid_index_count[@]} -gt 1 ]; then
                      echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
                      # If there's a tie, prefer index 3 as it's most common in SR-IOV setups
                      if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
                          best_gid_index="3"
                          echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
                      fi
                  fi

                  # Check if GID_INDEX is already set via environment variables
                  if [ -n "${NCCL_IB_GID_INDEX}" ]; then
                      echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
                  elif [ -n "$best_gid_index" ]; then
                      echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"

                      export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}

                      echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
                  else
                      echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
                  fi
              else
                  echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
              fi
            fi

            # --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
            # Older versions still need the blanket --disable-uvicorn-access-log.
            ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
            VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
            echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
            if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
              ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
            fi
            echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"

            # --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
            SHUTDOWN_TIMEOUT_ARGS=""
            if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
              SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout 40"
            fi

            eval "exec vllm serve /mnt/models \
              --served-model-name "facebook/opt-125m" "publishers/kserve-ci-e2e-test/models/facebook/opt-125m" \
              --port 8001 \
              ${ACCESS_LOG_ARGS} \
              ${SHUTDOWN_TIMEOUT_ARGS} \
              --enable-ssl-refresh \
              --ssl-certfile /var/run/kserve/tls/tls.crt \
              --ssl-keyfile /var/run/kserve/tls/tls.key \
              ${VLLM_ADDITIONAL_ARGS} \
              $@"
          - --
          env:
          - name: HOME
            value: /home
          - name: VLLM_LOGGING_LEVEL
            value: DEBUG
          - name: VLLM_CPU_KVCACHE_SPACE
            value: "1"
          - name: VLLM_ENABLE_V1_MULTIPROCESSING
            value: "0"
          - name: USER
            value: nonroot
          - name: TORCHINDUCTOR_CACHE_DIR
            value: /tmp/torchinductor-cache
          - name: HF_HUB_CACHE
            value: /models
          image: public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.19.0
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - /bin/sleep
                - "15"
          livenessProbe:
            failureThreshold: 8
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            initialDelaySeconds: 180
            periodSeconds: 30
            successThreshold: 1
            timeoutSeconds: 30
          name: main
          ports:
          - containerPort: 8001
            protocol: TCP
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 5
          resources:
            limits:
              cpu: "2"
              memory: 7Gi
            requests:
              cpu: 200m
              memory: 2Gi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
            runAsNonRoot: true
            seccompProfile:
              type: RuntimeDefault
          startupProbe:
            failureThreshold: 60
            httpGet:
              path: /health
              port: 8001
              scheme: HTTPS
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /home
            name: home
          - mountPath: /tmp
            name: tmp-dir
          - mountPath: /dev/shm
            name: dshm
          - mountPath: /models
            name: model-cache
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
          - mountPath: /mnt/models
            name: kserve-provision-location
            readOnly: true
        dnsPolicy: ClusterFirst
        initContainers:
        - command:
          - /app/pd-sidecar
          - --port=8000
          - --vllm-port=8001
          - --kv-connector=nixlv2
          - --enable-ssrf-protection=true
          - --pool-group=inference.networking.x-k8s.io
          - --inference-pool=kserve-ci-e2e-test/router-with-refs-pd-test-inference-pool
          - --secure-proxy=true
          - --cert-path=/var/run/kserve/tls
          - --decoder-use-tls=true
          - --prefiller-use-tls=true
          env:
          - name: INFERENCE_POOL_NAMESPACE
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: metadata.namespace
          - name: SSL_CERT_DIR
            value: /var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs
          - name: INFERENCE_POOL_NAME
            value: router-with-refs-pd-test-inference-pool
          image: ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.9.0-rc.2
          imagePullPolicy: IfNotPresent
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10
          name: llm-d-routing-sidecar
          ports:
          - containerPort: 8000
            protocol: TCP
          readinessProbe:
            failureThreshold: 10
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 5
          resources: {}
          restartPolicy: Always
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
            runAsNonRoot: true
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
        - args:
          - hf://facebook/opt-125m
          - /mnt/models
          env:
          - name: AWS_ACCESS_KEY_ID
            valueFrom:
              secretKeyRef:
                key: AWS_ACCESS_KEY_ID
                name: seaweedfs-s3-creds
          - name: AWS_SECRET_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                key: AWS_SECRET_ACCESS_KEY
                name: seaweedfs-s3-creds
          - name: S3_USE_HTTPS
            value: "0"
          - name: S3_ENDPOINT
            value: s3-service.kserve:8333
          - name: AWS_ENDPOINT_URL
            value: http://s3-service.kserve:8333
          - name: S3_VERIFY_SSL
            value: "0"
          - name: AWS_CA_BUNDLE
            value: /etc/ssl/custom-certs/cabundle.crt
          - name: AWS_CA_BUNDLE_CONFIGMAP
            value: odh-kserve-custom-ca-bundle
          - name: HF_HUB_ENABLE_HF_TRANSFER
            value: "1"
          - name: HF_XET_HIGH_PERFORMANCE
            value: "1"
          - name: HF_XET_NUM_CONCURRENT_RANGE_GETS
            value: "8"
          image: quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773
          imagePullPolicy: IfNotPresent
          name: storage-initializer
          resources:
            limits:
              cpu: "1"
              memory: 24Gi
            requests:
              cpu: 100m
              memory: 100Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /mnt/models
            name: kserve-provision-location
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        serviceAccount: router-with-refs-pd-test-kserve
        serviceAccountName: router-with-refs-pd-test-kserve
        terminationGracePeriodSeconds: 60
        volumes:
        - emptyDir: {}
          name: home
        - emptyDir: {}
          name: tmp-dir
        - emptyDir:
            medium: Memory
            sizeLimit: 1Gi
          name: dshm
        - emptyDir: {}
          name: model-cache
        - name: tls-certs
          secret:
            defaultMode: 420
            secretName: router-with-refs-pd-test-kserve-self-signed-certs
        - emptyDir: {}
          name: kserve-provision-location
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2026-07-08T08:22:22Z"
      lastUpdateTime: "2026-07-08T08:22:22Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    - lastTransitionTime: "2026-07-08T08:19:02Z"
      lastUpdateTime: "2026-07-08T08:22:22Z"
      message: ReplicaSet "router-with-refs-pd-test-kserve-6978d748b8" has successfully
        progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2026-07-08T08:19:02Z"
    generation: 1
    labels:
      app.kubernetes.io/component: llminferenceservice-workload-prefill
      app.kubernetes.io/name: router-with-refs-pd-test
      app.kubernetes.io/part-of: llminferenceservice
      kserve.io/component: workload
      llm-d.ai/role: prefill
    managedFields:
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:labels:
            .: {}
            f:app.kubernetes.io/component: {}
            f:app.kubernetes.io/name: {}
            f:app.kubernetes.io/part-of: {}
            f:kserve.io/component: {}
            f:llm-d.ai/role: {}
          f:ownerReferences:
            .: {}
            k:{"uid":"5071849d-29aa-4402-acb7-9d9bfa4d0c2a"}: {}
        f:spec:
          f:progressDeadlineSeconds: {}
          f:replicas: {}
          f:revisionHistoryLimit: {}
          f:selector: {}
          f:strategy:
            f:rollingUpdate:
              .: {}
              f:maxSurge: {}
              f:maxUnavailable: {}
            f:type: {}
          f:template:
            f:metadata:
              f:labels:
                .: {}
                f:app.kubernetes.io/component: {}
                f:app.kubernetes.io/name: {}
                f:app.kubernetes.io/part-of: {}
                f:kserve.io/component: {}
                f:llm-d.ai/role: {}
            f:spec:
              f:containers:
                k:{"name":"main"}:
                  .: {}
                  f:command: {}
                  f:env:
                    .: {}
                    k:{"name":"HF_HUB_CACHE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HOME"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"TORCHINDUCTOR_CACHE_DIR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"USER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_CPU_KVCACHE_SPACE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_ENABLE_V1_MULTIPROCESSING"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_LOGGING_LEVEL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:lifecycle:
                    .: {}
                    f:preStop:
                      .: {}
                      f:exec:
                        .: {}
                        f:command: {}
                  f:livenessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:name: {}
                  f:ports:
                    .: {}
                    k:{"containerPort":8000,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:protocol: {}
                  f:readinessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:securityContext:
                    .: {}
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      .: {}
                      f:drop: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsNonRoot: {}
                    f:seccompProfile:
                      .: {}
                      f:type: {}
                  f:startupProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/dev/shm"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/home"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
                    k:{"mountPath":"/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/tmp"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/kserve/tls"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
              f:dnsPolicy: {}
              f:initContainers:
                .: {}
                k:{"name":"storage-initializer"}:
                  .: {}
                  f:args: {}
                  f:env:
                    .: {}
                    k:{"name":"AWS_ACCESS_KEY_ID"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"AWS_CA_BUNDLE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_CA_BUNDLE_CONFIGMAP"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_ENDPOINT_URL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_SECRET_ACCESS_KEY"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"HF_HUB_ENABLE_HF_TRANSFER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_HIGH_PERFORMANCE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_NUM_CONCURRENT_RANGE_GETS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_ENDPOINT"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_USE_HTTPS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_VERIFY_SSL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:name: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
              f:restartPolicy: {}
              f:schedulerName: {}
              f:securityContext: {}
              f:terminationGracePeriodSeconds: {}
              f:volumes:
                .: {}
                k:{"name":"dshm"}:
                  .: {}
                  f:emptyDir:
                    .: {}
                    f:medium: {}
                    f:sizeLimit: {}
                  f:name: {}
                k:{"name":"home"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"kserve-provision-location"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"model-cache"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tls-certs"}:
                  .: {}
                  f:name: {}
                  f:secret:
                    .: {}
                    f:defaultMode: {}
                    f:secretName: {}
                k:{"name":"tmp-dir"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
      manager: manager
      operation: Update
      time: "2026-07-08T08:19:02Z"
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:deployment.kubernetes.io/revision: {}
        f:status:
          f:availableReplicas: {}
          f:conditions:
            .: {}
            k:{"type":"Available"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"Progressing"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:observedGeneration: {}
          f:readyReplicas: {}
          f:replicas: {}
          f:updatedReplicas: {}
      manager: kube-controller-manager
      operation: Update
      subresource: status
      time: "2026-07-08T08:22:22Z"
    name: router-with-refs-pd-test-kserve-prefill
    namespace: kserve-ci-e2e-test
    ownerReferences:
    - apiVersion: serving.kserve.io/v1alpha2
      blockOwnerDeletion: true
      controller: true
      kind: LLMInferenceService
      name: router-with-refs-pd-test
      uid: 5071849d-29aa-4402-acb7-9d9bfa4d0c2a
    resourceVersion: "87987"
    uid: f8c8c676-24f3-4173-8fd6-561c632e6436
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app.kubernetes.io/component: llminferenceservice-workload-prefill
        app.kubernetes.io/name: router-with-refs-pd-test
        app.kubernetes.io/part-of: llminferenceservice
        kserve.io/component: workload
        llm-d.ai/role: prefill
    strategy:
      rollingUpdate:
        maxSurge: 25%
        maxUnavailable: 25%
      type: RollingUpdate
    template:
      metadata:
        labels:
          app.kubernetes.io/component: llminferenceservice-workload-prefill
          app.kubernetes.io/name: router-with-refs-pd-test
          app.kubernetes.io/part-of: llminferenceservice
          kserve.io/component: workload
          llm-d.ai/role: prefill
      spec:
        containers:
        - command:
          - /bin/bash
          - -c
          - |-
            if [ -f /etc/profile.d/ibm-aiu-setup.sh ]; then
              source /etc/profile.d/ibm-aiu-setup.sh
            fi

            if [ "$KSERVE_INFER_ROCE" = "true" ]; then
              echo "Trying to infer RoCE configs ... "
              grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
              grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null

              cat /proc/driver/nvidia/params

              KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}

              echo "[Infer RoCE] Discovering active HCAs ..."
              active_hcas=()
              # Loop through all mlx5 devices found in sysfs
              for hca_dir in /sys/class/infiniband/mlx5_*; do
                  # Ensure it's a directory before proceeding
                  if [ -d "$hca_dir" ]; then
                      hca_name=$(basename "$hca_dir")
                      port_state_file="$hca_dir/ports/1/state" # Assume port 1
                      type_file="$hca_dir/ports/1/gid_attrs/types/*"

                      echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
                      if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
                          echo "[Infer RoCE] Found active HCA: $hca_name"
                          active_hcas+=("$hca_name")
                      else
                          echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
                      fi
                  fi
              done

              # Check if we found any active HCAs
              if [ ${#active_hcas[@]} -gt 0 ]; then
                  # Join the array elements with a comma
                  hca_port_pairs=()
                  for hca in "${active_hcas[@]}"; do
                    hca_port_pairs+=("${hca}:1")
                  done

                  active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
                  hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
                  echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
                  export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
                  export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
                  export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}

                  echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
                  echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
                  echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
              else
                  echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
              fi

              if [ ${#active_hcas[@]} -gt 0 ]; then
                  echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."

                  # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
                  declare -A gid_index_count
                  declare -A hca_gid_index

                  for hca_name in "${active_hcas[@]}"; do
                      echo "[Infer RoCE] Processing HCA: ${hca_name}"

                      # Find all RoCE v2 IPv4 GIDs for this HCA and count by index
                      for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
                          if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
                              idx=$(basename "$tpath")
                              gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
                              # Check for IPv4 GID (contains ffff:)
                              if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
                                  gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
                                  echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
                                  hca_gid_index["${hca_name}"]="${idx}"
                                  gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
                                  break  # Use first found IPv4 GID per HCA
                              fi
                          fi
                      done
                  done

                  # Find the most common GID index (most likely to be consistent across nodes)
                  best_gid_index=""
                  max_count=0
                  for idx in "${!gid_index_count[@]}"; do
                      count=${gid_index_count["${idx}"]}
                      echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
                      if [ $count -gt $max_count ]; then
                          max_count=$count
                          best_gid_index="$idx"
                      fi
                  done

                  # Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
                  if [ ${#gid_index_count[@]} -gt 1 ]; then
                      echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
                      # If there's a tie, prefer index 3 as it's most common in SR-IOV setups
                      if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
                          best_gid_index="3"
                          echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
                      fi
                  fi

                  # Check if GID_INDEX is already set via environment variables
                  if [ -n "${NCCL_IB_GID_INDEX}" ]; then
                      echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
                  elif [ -n "$best_gid_index" ]; then
                      echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"

                      export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}

                      echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
                  else
                      echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
                  fi
              else
                  echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
              fi
            fi

            # --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
            # Older versions still need the blanket --disable-uvicorn-access-log.
            ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
            VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
            echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
            if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
              ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
            fi
            echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"

            # --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
            SHUTDOWN_TIMEOUT_ARGS=""
            if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
              SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout 40"
            fi

            eval "exec vllm serve /mnt/models \
              --served-model-name "facebook/opt-125m" \
              --port 8000 \
              ${ACCESS_LOG_ARGS} \
              ${SHUTDOWN_TIMEOUT_ARGS} \
              --enable-ssl-refresh \
              --ssl-certfile /var/run/kserve/tls/tls.crt \
              --ssl-keyfile /var/run/kserve/tls/tls.key \
              ${VLLM_ADDITIONAL_ARGS} \
              $@"
          - --
          env:
          - name: HOME
            value: /home
          - name: VLLM_LOGGING_LEVEL
            value: DEBUG
          - name: VLLM_CPU_KVCACHE_SPACE
            value: "1"
          - name: VLLM_ENABLE_V1_MULTIPROCESSING
            value: "0"
          - name: USER
            value: nonroot
          - name: TORCHINDUCTOR_CACHE_DIR
            value: /tmp/torchinductor-cache
          - name: HF_HUB_CACHE
            value: /models
          image: public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.19.0
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - /bin/sleep
                - "15"
          livenessProbe:
            failureThreshold: 8
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            initialDelaySeconds: 180
            periodSeconds: 30
            successThreshold: 1
            timeoutSeconds: 30
          name: main
          ports:
          - containerPort: 8000
            protocol: TCP
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 5
          resources:
            limits:
              cpu: "2"
              memory: 7Gi
            requests:
              cpu: 200m
              memory: 2Gi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
            runAsNonRoot: true
            seccompProfile:
              type: RuntimeDefault
          startupProbe:
            failureThreshold: 60
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
          - mountPath: /home
            name: home
          - mountPath: /tmp
            name: tmp-dir
          - mountPath: /dev/shm
            name: dshm
          - mountPath: /models
            name: model-cache
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
          - mountPath: /mnt/models
            name: kserve-provision-location
            readOnly: true
        dnsPolicy: ClusterFirst
        initContainers:
        - args:
          - hf://facebook/opt-125m
          - /mnt/models
          env:
          - name: AWS_ACCESS_KEY_ID
            valueFrom:
              secretKeyRef:
                key: AWS_ACCESS_KEY_ID
                name: seaweedfs-s3-creds
          - name: AWS_SECRET_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                key: AWS_SECRET_ACCESS_KEY
                name: seaweedfs-s3-creds
          - name: S3_USE_HTTPS
            value: "0"
          - name: S3_ENDPOINT
            value: s3-service.kserve:8333
          - name: AWS_ENDPOINT_URL
            value: http://s3-service.kserve:8333
          - name: S3_VERIFY_SSL
            value: "0"
          - name: AWS_CA_BUNDLE
            value: /etc/ssl/custom-certs/cabundle.crt
          - name: AWS_CA_BUNDLE_CONFIGMAP
            value: odh-kserve-custom-ca-bundle
          - name: HF_HUB_ENABLE_HF_TRANSFER
            value: "1"
          - name: HF_XET_HIGH_PERFORMANCE
            value: "1"
          - name: HF_XET_NUM_CONCURRENT_RANGE_GETS
            value: "8"
          image: quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773
          imagePullPolicy: IfNotPresent
          name: storage-initializer
          resources:
            limits:
              cpu: "1"
              memory: 24Gi
            requests:
              cpu: 100m
              memory: 100Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /mnt/models
            name: kserve-provision-location
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        terminationGracePeriodSeconds: 60
        volumes:
        - emptyDir: {}
          name: home
        - emptyDir: {}
          name: tmp-dir
        - emptyDir:
            medium: Memory
            sizeLimit: 1Gi
          name: dshm
        - emptyDir: {}
          name: model-cache
        - name: tls-certs
          secret:
            defaultMode: 420
            secretName: router-with-refs-pd-test-kserve-self-signed-certs
        - emptyDir: {}
          name: kserve-provision-location
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2026-07-08T08:22:22Z"
      lastUpdateTime: "2026-07-08T08:22:22Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    - lastTransitionTime: "2026-07-08T08:19:02Z"
      lastUpdateTime: "2026-07-08T08:22:22Z"
      message: ReplicaSet "router-with-refs-pd-test-kserve-prefill-56559b5fdf" has
        successfully progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2026-07-08T08:19:03Z"
    generation: 1
    labels:
      app.kubernetes.io/component: llminferenceservice-router-scheduler
      app.kubernetes.io/name: router-with-refs-pd-test
      app.kubernetes.io/part-of: llminferenceservice
    managedFields:
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:labels:
            .: {}
            f:app.kubernetes.io/component: {}
            f:app.kubernetes.io/name: {}
            f:app.kubernetes.io/part-of: {}
          f:ownerReferences:
            .: {}
            k:{"uid":"5071849d-29aa-4402-acb7-9d9bfa4d0c2a"}: {}
        f:spec:
          f:progressDeadlineSeconds: {}
          f:replicas: {}
          f:revisionHistoryLimit: {}
          f:selector: {}
          f:strategy:
            f:type: {}
          f:template:
            f:metadata:
              f:annotations:
                .: {}
                f:app.kubernetes.io/version: {}
                f:certificates.kserve.io/expiration-v2: {}
              f:labels:
                .: {}
                f:app.kubernetes.io/component: {}
                f:app.kubernetes.io/name: {}
                f:app.kubernetes.io/part-of: {}
            f:spec:
              f:containers:
                k:{"name":"main"}:
                  .: {}
                  f:args: {}
                  f:command: {}
                  f:env:
                    .: {}
                    k:{"name":"SSL_CERT_DIR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:lifecycle:
                    .: {}
                    f:preStop:
                      .: {}
                      f:exec:
                        .: {}
                        f:command: {}
                  f:livenessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:grpc:
                      .: {}
                      f:port: {}
                      f:service: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:name: {}
                  f:ports:
                    .: {}
                    k:{"containerPort":5557,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9002,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9003,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9090,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                  f:readinessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:grpc:
                      .: {}
                      f:port: {}
                      f:service: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources:
                    .: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:securityContext:
                    .: {}
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      .: {}
                      f:drop: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsNonRoot: {}
                    f:seccompProfile:
                      .: {}
                      f:type: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/tmp/tokenizer"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/kserve/tls"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
              f:dnsPolicy: {}
              f:initContainers:
                .: {}
                k:{"name":"storage-initializer"}:
                  .: {}
                  f:args: {}
                  f:env:
                    .: {}
                    k:{"name":"AWS_ACCESS_KEY_ID"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"AWS_CA_BUNDLE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_CA_BUNDLE_CONFIGMAP"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_ENDPOINT_URL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_SECRET_ACCESS_KEY"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"HF_HUB_ENABLE_HF_TRANSFER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_HIGH_PERFORMANCE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_NUM_CONCURRENT_RANGE_GETS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_ENDPOINT"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_USE_HTTPS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_VERIFY_SSL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"STORAGE_ALLOW_PATTERNS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:name: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
              f:restartPolicy: {}
              f:schedulerName: {}
              f:securityContext: {}
              f:serviceAccount: {}
              f:serviceAccountName: {}
              f:terminationGracePeriodSeconds: {}
              f:volumes:
                .: {}
                k:{"name":"kserve-provision-location"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tls-certs"}:
                  .: {}
                  f:name: {}
                  f:secret:
                    .: {}
                    f:defaultMode: {}
                    f:secretName: {}
                k:{"name":"tokenizer-cache"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tokenizer-tmp"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tokenizer-uds"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
      manager: manager
      operation: Update
      time: "2026-07-08T08:19:03Z"
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:deployment.kubernetes.io/revision: {}
        f:status:
          f:availableReplicas: {}
          f:conditions:
            .: {}
            k:{"type":"Available"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"Progressing"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:observedGeneration: {}
          f:readyReplicas: {}
          f:replicas: {}
          f:updatedReplicas: {}
      manager: kube-controller-manager
      operation: Update
      subresource: status
      time: "2026-07-08T08:19:37Z"
    name: router-with-refs-pd-test-kserve-router-scheduler
    namespace: kserve-ci-e2e-test
    ownerReferences:
    - apiVersion: serving.kserve.io/v1alpha2
      blockOwnerDeletion: true
      controller: true
      kind: LLMInferenceService
      name: router-with-refs-pd-test
      uid: 5071849d-29aa-4402-acb7-9d9bfa4d0c2a
    resourceVersion: "85899"
    uid: 93d73a57-13c4-434a-a4a8-bc0dfb26fd0f
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app.kubernetes.io/component: llminferenceservice-router-scheduler
        app.kubernetes.io/name: router-with-refs-pd-test
        app.kubernetes.io/part-of: llminferenceservice
    strategy:
      type: Recreate
    template:
      metadata:
        annotations:
          app.kubernetes.io/version: 0.9.0
          certificates.kserve.io/expiration-v2: "true"
        labels:
          app.kubernetes.io/component: llminferenceservice-router-scheduler
          app.kubernetes.io/name: router-with-refs-pd-test
          app.kubernetes.io/part-of: llminferenceservice
      spec:
        containers:
        - args:
          - --config-text
          - |
            apiVersion: inference.networking.x-k8s.io/v1alpha1
            kind: EndpointPickerConfig
            plugins:
            - type: disagg-headers-handler
            - type: prefill-filter
            - type: decode-filter
            - type: queue-scorer
            - type: prefix-cache-scorer
            - type: max-score-picker
            - type: always-disagg-pd-decider
            - parameters:
                deciders:
                  prefill: always-disagg-pd-decider
              type: disagg-profile-handler
            schedulingProfiles:
            - name: prefill
              plugins:
              - pluginRef: prefill-filter
              - pluginRef: queue-scorer
                weight: 2
              - pluginRef: prefix-cache-scorer
                weight: 3
              - pluginRef: max-score-picker
            - name: decode
              plugins:
              - pluginRef: decode-filter
              - pluginRef: queue-scorer
                weight: 2
              - pluginRef: prefix-cache-scorer
                weight: 3
              - pluginRef: max-score-picker
          command:
          - /app/epp
          - --pool-name
          - router-with-refs-pd-test-inference-pool
          - --pool-namespace
          - kserve-ci-e2e-test
          - --zap-encoder
          - json
          - --grpc-port
          - "9002"
          - --grpc-health-port
          - "9003"
          - --enable-cert-reload=true
          - --secure-serving=true
          - --model-server-metrics-scheme=https
          - --cert-path=/var/run/kserve/tls
          env:
          - name: SSL_CERT_DIR
            value: /var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs
          image: ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0-rc.2
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - /bin/sleep
                - "15"
          livenessProbe:
            failureThreshold: 3
            grpc:
              port: 9003
              service: liveness
            initialDelaySeconds: 5
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: main
          ports:
          - containerPort: 9002
            name: grpc
            protocol: TCP
          - containerPort: 9003
            name: grpc-health
            protocol: TCP
          - containerPort: 9090
            name: metrics
            protocol: TCP
          - containerPort: 5557
            name: zmq
            protocol: TCP
          readinessProbe:
            failureThreshold: 3
            grpc:
              port: 9003
              service: readiness
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          resources:
            requests:
              cpu: 256m
              memory: 500Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
            runAsNonRoot: true
            seccompProfile:
              type: RuntimeDefault
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
          - mountPath: /tmp/tokenizer
            name: tokenizer-uds
        dnsPolicy: ClusterFirst
        initContainers:
        - args:
          - hf://facebook/opt-125m
          - /mnt/models
          env:
          - name: AWS_ACCESS_KEY_ID
            valueFrom:
              secretKeyRef:
                key: AWS_ACCESS_KEY_ID
                name: seaweedfs-s3-creds
          - name: AWS_SECRET_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                key: AWS_SECRET_ACCESS_KEY
                name: seaweedfs-s3-creds
          - name: S3_USE_HTTPS
            value: "0"
          - name: S3_ENDPOINT
            value: s3-service.kserve:8333
          - name: AWS_ENDPOINT_URL
            value: http://s3-service.kserve:8333
          - name: S3_VERIFY_SSL
            value: "0"
          - name: AWS_CA_BUNDLE
            value: /etc/ssl/custom-certs/cabundle.crt
          - name: AWS_CA_BUNDLE_CONFIGMAP
            value: odh-kserve-custom-ca-bundle
          - name: HF_HUB_ENABLE_HF_TRANSFER
            value: "1"
          - name: HF_XET_HIGH_PERFORMANCE
            value: "1"
          - name: HF_XET_NUM_CONCURRENT_RANGE_GETS
            value: "8"
          - name: STORAGE_ALLOW_PATTERNS
            value: '["tokenizer.json", "tokenizer_config.json", "special_tokens_map.json",
              "vocab.json", "merges.txt", "config.json", "generation_config.json"]'
          image: quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773
          imagePullPolicy: IfNotPresent
          name: storage-initializer
          resources:
            limits:
              cpu: "1"
              memory: 24Gi
            requests:
              cpu: 100m
              memory: 100Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /mnt/models
            name: kserve-provision-location
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        serviceAccount: router-with-refs-pd-test-epp-sa
        serviceAccountName: router-with-refs-pd-test-epp-sa
        terminationGracePeriodSeconds: 60
        volumes:
        - name: tls-certs
          secret:
            defaultMode: 420
            secretName: router-with-refs-pd-test-kserve-self-signed-certs
        - emptyDir: {}
          name: tokenizer-uds
        - emptyDir: {}
          name: tokenizer-tmp
        - emptyDir: {}
          name: tokenizer-cache
        - emptyDir: {}
          name: kserve-provision-location
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2026-07-08T08:19:37Z"
      lastUpdateTime: "2026-07-08T08:19:37Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    - lastTransitionTime: "2026-07-08T08:19:03Z"
      lastUpdateTime: "2026-07-08T08:19:37Z"
      message: ReplicaSet "router-with-refs-pd-test-kserve-router-scheduler-6ddb8947b5"
        has successfully progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2026-07-08T07:55:58Z"
    generation: 1
    labels:
      app.kubernetes.io/component: llminferenceservice-workload
      app.kubernetes.io/name: router-with-refs-test
      app.kubernetes.io/part-of: llminferenceservice
      kserve.io/component: workload
      llm-d.ai/role: both
    managedFields:
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:labels:
            .: {}
            f:app.kubernetes.io/component: {}
            f:app.kubernetes.io/name: {}
            f:app.kubernetes.io/part-of: {}
            f:kserve.io/component: {}
            f:llm-d.ai/role: {}
          f:ownerReferences:
            .: {}
            k:{"uid":"cb3afc0b-477a-4222-a441-67bfedc83c98"}: {}
        f:spec:
          f:progressDeadlineSeconds: {}
          f:replicas: {}
          f:revisionHistoryLimit: {}
          f:selector: {}
          f:strategy:
            f:rollingUpdate:
              .: {}
              f:maxSurge: {}
              f:maxUnavailable: {}
            f:type: {}
          f:template:
            f:metadata:
              f:labels:
                .: {}
                f:app.kubernetes.io/component: {}
                f:app.kubernetes.io/name: {}
                f:app.kubernetes.io/part-of: {}
                f:kserve.io/component: {}
                f:llm-d.ai/role: {}
            f:spec:
              f:containers:
                k:{"name":"main"}:
                  .: {}
                  f:command: {}
                  f:env:
                    .: {}
                    k:{"name":"HF_HUB_CACHE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HOME"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"TORCHINDUCTOR_CACHE_DIR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"USER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_CPU_KVCACHE_SPACE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_ENABLE_V1_MULTIPROCESSING"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"VLLM_LOGGING_LEVEL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:lifecycle:
                    .: {}
                    f:preStop:
                      .: {}
                      f:exec:
                        .: {}
                        f:command: {}
                  f:livenessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:name: {}
                  f:ports:
                    .: {}
                    k:{"containerPort":8000,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:protocol: {}
                  f:readinessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:securityContext:
                    .: {}
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      .: {}
                      f:drop: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsNonRoot: {}
                    f:seccompProfile:
                      .: {}
                      f:type: {}
                  f:startupProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:httpGet:
                      .: {}
                      f:path: {}
                      f:port: {}
                      f:scheme: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/dev/shm"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/home"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
                    k:{"mountPath":"/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/tmp"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/kserve/tls"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
              f:dnsPolicy: {}
              f:initContainers:
                .: {}
                k:{"name":"storage-initializer"}:
                  .: {}
                  f:args: {}
                  f:env:
                    .: {}
                    k:{"name":"AWS_ACCESS_KEY_ID"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"AWS_CA_BUNDLE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_CA_BUNDLE_CONFIGMAP"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_ENDPOINT_URL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_SECRET_ACCESS_KEY"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"HF_HUB_ENABLE_HF_TRANSFER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_HIGH_PERFORMANCE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_NUM_CONCURRENT_RANGE_GETS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_ENDPOINT"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_USE_HTTPS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_VERIFY_SSL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:name: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
              f:restartPolicy: {}
              f:schedulerName: {}
              f:securityContext: {}
              f:terminationGracePeriodSeconds: {}
              f:volumes:
                .: {}
                k:{"name":"dshm"}:
                  .: {}
                  f:emptyDir:
                    .: {}
                    f:medium: {}
                    f:sizeLimit: {}
                  f:name: {}
                k:{"name":"home"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"kserve-provision-location"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"model-cache"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tls-certs"}:
                  .: {}
                  f:name: {}
                  f:secret:
                    .: {}
                    f:defaultMode: {}
                    f:secretName: {}
                k:{"name":"tmp-dir"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
      manager: manager
      operation: Update
      time: "2026-07-08T07:55:58Z"
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:deployment.kubernetes.io/revision: {}
        f:status:
          f:availableReplicas: {}
          f:conditions:
            .: {}
            k:{"type":"Available"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"Progressing"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:observedGeneration: {}
          f:readyReplicas: {}
          f:replicas: {}
          f:updatedReplicas: {}
      manager: kube-controller-manager
      operation: Update
      subresource: status
      time: "2026-07-08T07:58:09Z"
    name: router-with-refs-test-kserve
    namespace: kserve-ci-e2e-test
    ownerReferences:
    - apiVersion: serving.kserve.io/v1alpha2
      blockOwnerDeletion: true
      controller: true
      kind: LLMInferenceService
      name: router-with-refs-test
      uid: cb3afc0b-477a-4222-a441-67bfedc83c98
    resourceVersion: "67755"
    uid: 2f6d89a0-4f3e-4781-b925-68fde725f45b
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app.kubernetes.io/component: llminferenceservice-workload
        app.kubernetes.io/name: router-with-refs-test
        app.kubernetes.io/part-of: llminferenceservice
        kserve.io/component: workload
        llm-d.ai/role: both
    strategy:
      rollingUpdate:
        maxSurge: 25%
        maxUnavailable: 25%
      type: RollingUpdate
    template:
      metadata:
        labels:
          app.kubernetes.io/component: llminferenceservice-workload
          app.kubernetes.io/name: router-with-refs-test
          app.kubernetes.io/part-of: llminferenceservice
          kserve.io/component: workload
          llm-d.ai/role: both
      spec:
        containers:
        - command:
          - /bin/bash
          - -c
          - |-
            if [ -f /etc/profile.d/ibm-aiu-setup.sh ]; then
              source /etc/profile.d/ibm-aiu-setup.sh
            fi

            if [ "$KSERVE_INFER_ROCE" = "true" ]; then
              echo "Trying to infer RoCE configs ... "
              grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
              grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null

              cat /proc/driver/nvidia/params

              KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}

              echo "[Infer RoCE] Discovering active HCAs ..."
              active_hcas=()
              # Loop through all mlx5 devices found in sysfs
              for hca_dir in /sys/class/infiniband/mlx5_*; do
                  # Ensure it's a directory before proceeding
                  if [ -d "$hca_dir" ]; then
                      hca_name=$(basename "$hca_dir")
                      port_state_file="$hca_dir/ports/1/state" # Assume port 1
                      type_file="$hca_dir/ports/1/gid_attrs/types/*"

                      echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
                      if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
                          echo "[Infer RoCE] Found active HCA: $hca_name"
                          active_hcas+=("$hca_name")
                      else
                          echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
                      fi
                  fi
              done

              # Check if we found any active HCAs
              if [ ${#active_hcas[@]} -gt 0 ]; then
                  # Join the array elements with a comma
                  hca_port_pairs=()
                  for hca in "${active_hcas[@]}"; do
                    hca_port_pairs+=("${hca}:1")
                  done

                  active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
                  hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
                  echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
                  export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
                  export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
                  export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}

                  echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
                  echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
                  echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
              else
                  echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
              fi

              if [ ${#active_hcas[@]} -gt 0 ]; then
                  echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."

                  # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
                  declare -A gid_index_count
                  declare -A hca_gid_index

                  for hca_name in "${active_hcas[@]}"; do
                      echo "[Infer RoCE] Processing HCA: ${hca_name}"

                      # Find all RoCE v2 IPv4 GIDs for this HCA and count by index
                      for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
                          if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
                              idx=$(basename "$tpath")
                              gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
                              # Check for IPv4 GID (contains ffff:)
                              if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
                                  gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
                                  echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
                                  hca_gid_index["${hca_name}"]="${idx}"
                                  gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
                                  break  # Use first found IPv4 GID per HCA
                              fi
                          fi
                      done
                  done

                  # Find the most common GID index (most likely to be consistent across nodes)
                  best_gid_index=""
                  max_count=0
                  for idx in "${!gid_index_count[@]}"; do
                      count=${gid_index_count["${idx}"]}
                      echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
                      if [ $count -gt $max_count ]; then
                          max_count=$count
                          best_gid_index="$idx"
                      fi
                  done

                  # Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
                  if [ ${#gid_index_count[@]} -gt 1 ]; then
                      echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
                      # If there's a tie, prefer index 3 as it's most common in SR-IOV setups
                      if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
                          best_gid_index="3"
                          echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
                      fi
                  fi

                  # Check if GID_INDEX is already set via environment variables
                  if [ -n "${NCCL_IB_GID_INDEX}" ]; then
                      echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
                  elif [ -n "$best_gid_index" ]; then
                      echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"

                      export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}

                      echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
                  else
                      echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
                  fi
              else
                  echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
              fi
            fi

            # --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
            # Older versions still need the blanket --disable-uvicorn-access-log.
            ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
            VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
            echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
            if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
              ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
            fi
            echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"

            # --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
            SHUTDOWN_TIMEOUT_ARGS=""
            if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
              SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout 40"
            fi

            eval "exec vllm serve /mnt/models \
              --served-model-name "facebook/opt-125m" "publishers/kserve-ci-e2e-test/models/facebook/opt-125m" \
              --port 8000 \
              ${ACCESS_LOG_ARGS} \
              ${SHUTDOWN_TIMEOUT_ARGS} \
              --enable-ssl-refresh \
              --ssl-certfile /var/run/kserve/tls/tls.crt \
              --ssl-keyfile /var/run/kserve/tls/tls.key \
              ${VLLM_ADDITIONAL_ARGS} \
              $@"
          - --
          env:
          - name: HOME
            value: /home
          - name: VLLM_LOGGING_LEVEL
            value: DEBUG
          - name: VLLM_CPU_KVCACHE_SPACE
            value: "1"
          - name: VLLM_ENABLE_V1_MULTIPROCESSING
            value: "0"
          - name: USER
            value: nonroot
          - name: TORCHINDUCTOR_CACHE_DIR
            value: /tmp/torchinductor-cache
          - name: HF_HUB_CACHE
            value: /models
          image: public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.19.0
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - /bin/sleep
                - "15"
          livenessProbe:
            failureThreshold: 10
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: main
          ports:
          - containerPort: 8000
            protocol: TCP
          readinessProbe:
            failureThreshold: 2
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 1
            successThreshold: 1
            timeoutSeconds: 1
          resources:
            limits:
              cpu: "2"
              memory: 7Gi
            requests:
              cpu: 200m
              memory: 2Gi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
            runAsNonRoot: true
            seccompProfile:
              type: RuntimeDefault
          startupProbe:
            failureThreshold: 60
            httpGet:
              path: /health
              port: 8000
              scheme: HTTPS
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /home
            name: home
          - mountPath: /tmp
            name: tmp-dir
          - mountPath: /dev/shm
            name: dshm
          - mountPath: /models
            name: model-cache
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
          - mountPath: /mnt/models
            name: kserve-provision-location
            readOnly: true
        dnsPolicy: ClusterFirst
        initContainers:
        - args:
          - hf://facebook/opt-125m
          - /mnt/models
          env:
          - name: AWS_ACCESS_KEY_ID
            valueFrom:
              secretKeyRef:
                key: AWS_ACCESS_KEY_ID
                name: seaweedfs-s3-creds
          - name: AWS_SECRET_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                key: AWS_SECRET_ACCESS_KEY
                name: seaweedfs-s3-creds
          - name: S3_USE_HTTPS
            value: "0"
          - name: S3_ENDPOINT
            value: s3-service.kserve:8333
          - name: AWS_ENDPOINT_URL
            value: http://s3-service.kserve:8333
          - name: S3_VERIFY_SSL
            value: "0"
          - name: AWS_CA_BUNDLE
            value: /etc/ssl/custom-certs/cabundle.crt
          - name: AWS_CA_BUNDLE_CONFIGMAP
            value: odh-kserve-custom-ca-bundle
          - name: HF_HUB_ENABLE_HF_TRANSFER
            value: "1"
          - name: HF_XET_HIGH_PERFORMANCE
            value: "1"
          - name: HF_XET_NUM_CONCURRENT_RANGE_GETS
            value: "8"
          image: quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773
          imagePullPolicy: IfNotPresent
          name: storage-initializer
          resources:
            limits:
              cpu: "1"
              memory: 24Gi
            requests:
              cpu: 100m
              memory: 100Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /mnt/models
            name: kserve-provision-location
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        terminationGracePeriodSeconds: 60
        volumes:
        - emptyDir: {}
          name: home
        - emptyDir:
            medium: Memory
            sizeLimit: 1Gi
          name: dshm
        - emptyDir: {}
          name: model-cache
        - emptyDir: {}
          name: tmp-dir
        - name: tls-certs
          secret:
            defaultMode: 420
            secretName: router-with-refs-test-kserve-self-signed-certs
        - emptyDir: {}
          name: kserve-provision-location
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2026-07-08T07:58:09Z"
      lastUpdateTime: "2026-07-08T07:58:09Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    - lastTransitionTime: "2026-07-08T07:55:58Z"
      lastUpdateTime: "2026-07-08T07:58:09Z"
      message: ReplicaSet "router-with-refs-test-kserve-794b6f7745" has successfully
        progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: apps/v1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
    creationTimestamp: "2026-07-08T07:55:59Z"
    generation: 1
    labels:
      app.kubernetes.io/component: llminferenceservice-router-scheduler
      app.kubernetes.io/name: router-with-refs-test
      app.kubernetes.io/part-of: llminferenceservice
    managedFields:
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:labels:
            .: {}
            f:app.kubernetes.io/component: {}
            f:app.kubernetes.io/name: {}
            f:app.kubernetes.io/part-of: {}
          f:ownerReferences:
            .: {}
            k:{"uid":"cb3afc0b-477a-4222-a441-67bfedc83c98"}: {}
        f:spec:
          f:progressDeadlineSeconds: {}
          f:replicas: {}
          f:revisionHistoryLimit: {}
          f:selector: {}
          f:strategy:
            f:type: {}
          f:template:
            f:metadata:
              f:annotations:
                .: {}
                f:app.kubernetes.io/version: {}
                f:certificates.kserve.io/expiration-v2: {}
              f:labels:
                .: {}
                f:app.kubernetes.io/component: {}
                f:app.kubernetes.io/name: {}
                f:app.kubernetes.io/part-of: {}
            f:spec:
              f:containers:
                k:{"name":"main"}:
                  .: {}
                  f:args: {}
                  f:command: {}
                  f:env:
                    .: {}
                    k:{"name":"SSL_CERT_DIR"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:lifecycle:
                    .: {}
                    f:preStop:
                      .: {}
                      f:exec:
                        .: {}
                        f:command: {}
                  f:livenessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:grpc:
                      .: {}
                      f:port: {}
                      f:service: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:name: {}
                  f:ports:
                    .: {}
                    k:{"containerPort":5557,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9002,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9003,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                    k:{"containerPort":9090,"protocol":"TCP"}:
                      .: {}
                      f:containerPort: {}
                      f:name: {}
                      f:protocol: {}
                  f:readinessProbe:
                    .: {}
                    f:failureThreshold: {}
                    f:grpc:
                      .: {}
                      f:port: {}
                      f:service: {}
                    f:initialDelaySeconds: {}
                    f:periodSeconds: {}
                    f:successThreshold: {}
                    f:timeoutSeconds: {}
                  f:resources:
                    .: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:securityContext:
                    .: {}
                    f:allowPrivilegeEscalation: {}
                    f:capabilities:
                      .: {}
                      f:drop: {}
                    f:readOnlyRootFilesystem: {}
                    f:runAsNonRoot: {}
                    f:seccompProfile:
                      .: {}
                      f:type: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/tmp/tokenizer"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                    k:{"mountPath":"/var/run/kserve/tls"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
                      f:readOnly: {}
              f:dnsPolicy: {}
              f:initContainers:
                .: {}
                k:{"name":"storage-initializer"}:
                  .: {}
                  f:args: {}
                  f:env:
                    .: {}
                    k:{"name":"AWS_ACCESS_KEY_ID"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"AWS_CA_BUNDLE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_CA_BUNDLE_CONFIGMAP"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_ENDPOINT_URL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"AWS_SECRET_ACCESS_KEY"}:
                      .: {}
                      f:name: {}
                      f:valueFrom:
                        .: {}
                        f:secretKeyRef: {}
                    k:{"name":"HF_HUB_ENABLE_HF_TRANSFER"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_HIGH_PERFORMANCE"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"HF_XET_NUM_CONCURRENT_RANGE_GETS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_ENDPOINT"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_USE_HTTPS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"S3_VERIFY_SSL"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                    k:{"name":"STORAGE_ALLOW_PATTERNS"}:
                      .: {}
                      f:name: {}
                      f:value: {}
                  f:image: {}
                  f:imagePullPolicy: {}
                  f:name: {}
                  f:resources:
                    .: {}
                    f:limits:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                    f:requests:
                      .: {}
                      f:cpu: {}
                      f:memory: {}
                  f:terminationMessagePath: {}
                  f:terminationMessagePolicy: {}
                  f:volumeMounts:
                    .: {}
                    k:{"mountPath":"/mnt/models"}:
                      .: {}
                      f:mountPath: {}
                      f:name: {}
              f:restartPolicy: {}
              f:schedulerName: {}
              f:securityContext: {}
              f:serviceAccount: {}
              f:serviceAccountName: {}
              f:terminationGracePeriodSeconds: {}
              f:volumes:
                .: {}
                k:{"name":"kserve-provision-location"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tls-certs"}:
                  .: {}
                  f:name: {}
                  f:secret:
                    .: {}
                    f:defaultMode: {}
                    f:secretName: {}
                k:{"name":"tokenizer-cache"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tokenizer-tmp"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
                k:{"name":"tokenizer-uds"}:
                  .: {}
                  f:emptyDir: {}
                  f:name: {}
      manager: manager
      operation: Update
      time: "2026-07-08T07:55:59Z"
    - apiVersion: apps/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:deployment.kubernetes.io/revision: {}
        f:status:
          f:availableReplicas: {}
          f:conditions:
            .: {}
            k:{"type":"Available"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"Progressing"}:
              .: {}
              f:lastTransitionTime: {}
              f:lastUpdateTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:observedGeneration: {}
          f:readyReplicas: {}
          f:replicas: {}
          f:updatedReplicas: {}
      manager: kube-controller-manager
      operation: Update
      subresource: status
      time: "2026-07-08T07:56:33Z"
    name: router-with-refs-test-kserve-router-scheduler
    namespace: kserve-ci-e2e-test
    ownerReferences:
    - apiVersion: serving.kserve.io/v1alpha2
      blockOwnerDeletion: true
      controller: true
      kind: LLMInferenceService
      name: router-with-refs-test
      uid: cb3afc0b-477a-4222-a441-67bfedc83c98
    resourceVersion: "66504"
    uid: 2beb63eb-bbd9-4f46-b749-e6acf2d4dd89
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app.kubernetes.io/component: llminferenceservice-router-scheduler
        app.kubernetes.io/name: router-with-refs-test
        app.kubernetes.io/part-of: llminferenceservice
    strategy:
      type: Recreate
    template:
      metadata:
        annotations:
          app.kubernetes.io/version: 0.9.0
          certificates.kserve.io/expiration-v2: "true"
        labels:
          app.kubernetes.io/component: llminferenceservice-router-scheduler
          app.kubernetes.io/name: router-with-refs-test
          app.kubernetes.io/part-of: llminferenceservice
      spec:
        containers:
        - args:
          - --config-text
          - |
            apiVersion: inference.networking.x-k8s.io/v1alpha1
            kind: EndpointPickerConfig
            plugins:
            - type: single-profile-handler
            - type: queue-scorer
            - type: prefix-cache-scorer
            - type: max-score-picker
            schedulingProfiles:
            - name: default
              plugins:
              - pluginRef: queue-scorer
                weight: 2
              - pluginRef: prefix-cache-scorer
                weight: 3
              - pluginRef: max-score-picker
          command:
          - /app/epp
          - --pool-name
          - router-with-refs-test-inference-pool
          - --pool-namespace
          - kserve-ci-e2e-test
          - --zap-encoder
          - json
          - --grpc-port
          - "9002"
          - --grpc-health-port
          - "9003"
          - --enable-cert-reload=true
          - --secure-serving=true
          - --model-server-metrics-scheme=https
          - --cert-path=/var/run/kserve/tls
          env:
          - name: SSL_CERT_DIR
            value: /var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs
          image: ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0-rc.2
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                - /bin/sleep
                - "15"
          livenessProbe:
            failureThreshold: 3
            grpc:
              port: 9003
              service: liveness
            initialDelaySeconds: 5
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: main
          ports:
          - containerPort: 9002
            name: grpc
            protocol: TCP
          - containerPort: 9003
            name: grpc-health
            protocol: TCP
          - containerPort: 9090
            name: metrics
            protocol: TCP
          - containerPort: 5557
            name: zmq
            protocol: TCP
          readinessProbe:
            failureThreshold: 3
            grpc:
              port: 9003
              service: readiness
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          resources:
            requests:
              cpu: 256m
              memory: 500Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
            runAsNonRoot: true
            seccompProfile:
              type: RuntimeDefault
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
          - mountPath: /tmp/tokenizer
            name: tokenizer-uds
        dnsPolicy: ClusterFirst
        initContainers:
        - args:
          - hf://facebook/opt-125m
          - /mnt/models
          env:
          - name: AWS_ACCESS_KEY_ID
            valueFrom:
              secretKeyRef:
                key: AWS_ACCESS_KEY_ID
                name: seaweedfs-s3-creds
          - name: AWS_SECRET_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                key: AWS_SECRET_ACCESS_KEY
                name: seaweedfs-s3-creds
          - name: S3_USE_HTTPS
            value: "0"
          - name: S3_ENDPOINT
            value: s3-service.kserve:8333
          - name: AWS_ENDPOINT_URL
            value: http://s3-service.kserve:8333
          - name: S3_VERIFY_SSL
            value: "0"
          - name: AWS_CA_BUNDLE
            value: /etc/ssl/custom-certs/cabundle.crt
          - name: AWS_CA_BUNDLE_CONFIGMAP
            value: odh-kserve-custom-ca-bundle
          - name: HF_HUB_ENABLE_HF_TRANSFER
            value: "1"
          - name: HF_XET_HIGH_PERFORMANCE
            value: "1"
          - name: HF_XET_NUM_CONCURRENT_RANGE_GETS
            value: "8"
          - name: STORAGE_ALLOW_PATTERNS
            value: '["tokenizer.json", "tokenizer_config.json", "special_tokens_map.json",
              "vocab.json", "merges.txt", "config.json", "generation_config.json"]'
          image: quay.io/opendatahub/kserve-storage-initializer@sha256:956644def59861b6c4d288d394adc3ba824655b2a247e0388911f52fbf9bf773
          imagePullPolicy: IfNotPresent
          name: storage-initializer
          resources:
            limits:
              cpu: "1"
              memory: 24Gi
            requests:
              cpu: 100m
              memory: 100Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: FallbackToLogsOnError
          volumeMounts:
          - mountPath: /mnt/models
            name: kserve-provision-location
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        serviceAccount: router-with-refs-test-epp-sa
        serviceAccountName: router-with-refs-test-epp-sa
        terminationGracePeriodSeconds: 60
        volumes:
        - name: tls-certs
          secret:
            defaultMode: 420
            secretName: router-with-refs-test-kserve-self-signed-certs
        - emptyDir: {}
          name: tokenizer-uds
        - emptyDir: {}
          name: tokenizer-tmp
        - emptyDir: {}
          name: tokenizer-cache
        - emptyDir: {}
          name: kserve-provision-location
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2026-07-08T07:56:33Z"
      lastUpdateTime: "2026-07-08T07:56:33Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    - lastTransitionTime: "2026-07-08T07:55:59Z"
      lastUpdateTime: "2026-07-08T07:56:33Z"
      message: ReplicaSet "router-with-refs-test-kserve-router-scheduler-74dd96b744"
        has successfully progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
kind: DeploymentList
metadata:
  resourceVersion: "98434"