---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
  annotations:
    internal.config.kubernetes.io/previousKinds: LLMInferenceServiceConfig
    internal.config.kubernetes.io/previousNames: kserve-config-llm-prefill-template
    internal.config.kubernetes.io/previousNamespaces: opendatahub
    internal.serving.kserve.io/kv-cache-shm-percent-of-cpu: "120"
    serving.kserve.io/well-known-config: "true"
  creationTimestamp: "2026-10-29T21:52:13Z"
  finalizers:
  - serving.kserve.io/llmisvcconfig-finalizer
  generation: 3
  labels:
    platform.opendatahub.io/part-of: kserve
  managedFields:
  - apiVersion: serving.kserve.io/v1alpha2
    fieldsType: FieldsV1
    fieldsV1:
      f:metadata:
        f:annotations:
          f:internal.config.kubernetes.io/previousKinds: {}
          f:internal.config.kubernetes.io/previousNames: {}
          f:internal.config.kubernetes.io/previousNamespaces: {}
          f:internal.serving.kserve.io/kv-cache-shm-percent-of-cpu: {}
          f:serving.kserve.io/well-known-config: {}
        f:labels:
          f:platform.opendatahub.io/part-of: {}
      f:spec:
        f:prefill:
          f:annotations:
            f:serving.kserve.io/model-based-routing-enabled: {}
          f:template:
            f:containers:
              k:{"name":"main"}:
                .: {}
                f:command: {}
                f:env:
                  k:{"name":"HF_HUB_CACHE"}:
                    .: {}
                    f:name: {}
                    f:value: {}
                  k:{"name":"HOME"}:
                    .: {}
                    f:name: {}
                    f:value: {}
                  k:{"name":"KSERVE_KV_TRANSFER_ARGS"}:
                    .: {}
                    f:name: {}
                    f:value: {}
                  k:{"name":"VLLM_LOGGING_LEVEL"}:
                    .: {}
                    f:name: {}
                    f:value: {}
                  k:{"name":"VLLM_NIXL_SIDE_CHANNEL_HOST"}:
                    .: {}
                    f:name: {}
                    f:valueFrom:
                      f:fieldRef: {}
                f:image: {}
                f:imagePullPolicy: {}
                f:lifecycle:
                  f:preStop:
                    f:exec:
                      f:command: {}
                f:livenessProbe:
                  f:failureThreshold: {}
                  f:httpGet:
                    f:path: {}
                    f:port: {}
                    f:scheme: {}
                  f:periodSeconds: {}
                  f:timeoutSeconds: {}
                f:name: {}
                f:ports:
                  k:{"containerPort":8000,"protocol":"TCP"}:
                    .: {}
                    f:containerPort: {}
                    f:protocol: {}
                f:readinessProbe:
                  f:failureThreshold: {}
                  f:httpGet:
                    f:path: {}
                    f:port: {}
                    f:scheme: {}
                  f:periodSeconds: {}
                  f:timeoutSeconds: {}
                f:securityContext:
                  f:allowPrivilegeEscalation: {}
                  f:capabilities:
                    f:drop: {}
                  f:readOnlyRootFilesystem: {}
                  f:runAsNonRoot: {}
                  f:seccompProfile:
                    f:type: {}
                f:startupProbe:
                  f:failureThreshold: {}
                  f:httpGet:
                    f:path: {}
                    f:port: {}
                    f:scheme: {}
                  f:periodSeconds: {}
                f:terminationMessagePath: {}
                f:terminationMessagePolicy: {}
                f:volumeMounts:
                  k:{"mountPath":"/dev/shm"}:
                    .: {}
                    f:mountPath: {}
                    f:name: {}
                  k:{"mountPath":"/home"}:
                    .: {}
                    f:mountPath: {}
                    f:name: {}
                  k:{"mountPath":"/models"}:
                    .: {}
                    f:mountPath: {}
                    f:name: {}
                  k:{"mountPath":"/tmp"}:
                    .: {}
                    f:mountPath: {}
                    f:name: {}
                  k:{"mountPath":"/var/run/kserve/tls"}:
                    .: {}
                    f:mountPath: {}
                    f:name: {}
                    f:readOnly: {}
            f:terminationGracePeriodSeconds: {}
            f:volumes:
              k:{"name":"dshm"}:
                .: {}
                f:emptyDir:
                  f:medium: {}
                  f:sizeLimit: {}
                f:name: {}
              k:{"name":"home"}:
                .: {}
                f:emptyDir: {}
                f:name: {}
              k:{"name":"model-cache"}:
                .: {}
                f:emptyDir: {}
                f:name: {}
              k:{"name":"tls-certs"}:
                .: {}
                f:name: {}
                f:secret:
                  f:secretName: {}
              k:{"name":"tmp-dir"}:
                .: {}
                f:emptyDir: {}
                f:name: {}
    manager: kserve-module-controller
    operation: Apply
    time: "2026-10-29T21:52:36Z"
  - apiVersion: serving.kserve.io/v1alpha2
    fieldsType: FieldsV1
    fieldsV1:
      f:metadata:
        f:finalizers:
          .: {}
          v:"serving.kserve.io/llmisvcconfig-finalizer": {}
      f:spec:
        f:model:
          .: {}
          f:uri: {}
        f:prefill:
          f:template:
            f:containers:
              k:{"name":"main"}:
                f:resources: {}
    manager: manager
    operation: Update
    time: "2026-10-29T21:52:13Z"
  - apiVersion: serving.kserve.io/v1alpha2
    fieldsType: FieldsV1
    fieldsV1:
      f:status:
        .: {}
        f:conditions: {}
        f:observedGeneration: {}
        f:referencedBy: {}
    manager: manager
    operation: Update
    subresource: status
    time: "2026-10-29T22:16:43Z"
  name: v3-6-0-ea-2-kserve-config-llm-prefill-template
  namespace: opendatahub
  resourceVersion: "86452"
  uid: f74b8e8d-c482-49c7-8e99-61ad02e4f0af
spec:
  model:
    uri: ""
  prefill:
    annotations:
      serving.kserve.io/model-based-routing-enabled: "true"
    template:
      containers:
      - command:
        - /bin/bash
        - -c
        - |-
          # Spyre architecture-specific setup for ppc64le/s390x
          if [ -d /opt/ibm/spyre ]; then
            ARCH="$(arch)"
            case "${ARCH}" in
              ppc64le)
                if [ -f /opt/rh/gcc-toolset-14/enable ]; then
                  . /opt/rh/gcc-toolset-14/enable
                  export PATH
                fi
                ;;
              s390x)
                export FLEX_DEVICE=VF
                if [ -f /etc/profile.d/ibm-aiu-setup.sh ]; then
                  source /etc/profile.d/ibm-aiu-setup.sh
                fi
                export TORCH_SENDNN_TEMP_CACHE_DIR=/opt/ibm/spyre/models/cache/
                if [ -n "${AIU_AUTOGEN_SENLIB_CONFIG_FILE:-}" ] && [ -r "${AIU_AUTOGEN_SENLIB_CONFIG_FILE}" ]; then
                  if jq -e '(.SNT_MCI.DCR.MCI_CTRL.ENABLE_RISCV = "0x0") | del(.SNT_MCI.init) | (.METRICS.general.enable = true)' "${AIU_AUTOGEN_SENLIB_CONFIG_FILE}" > "$HOME/.senlib.json" && [ -s "$HOME/.senlib.json" ]; then
                    export SENLIB_DEVEL_CONFIG_FILE="$HOME/.senlib.json"
                  else
                    echo "WARNING: jq failed to process ${AIU_AUTOGEN_SENLIB_CONFIG_FILE}, skipping SENLIB config generation"
                    rm -f "$HOME/.senlib.json"
                  fi
                else
                  echo "WARNING: AIU_AUTOGEN_SENLIB_CONFIG_FILE is not set or not readable, skipping SENLIB config generation"
                fi
                if [ -f /opt/rh/gcc-toolset-14/enable ]; then
                  . /opt/rh/gcc-toolset-14/enable
                  export PATH
                fi
                ;;
              x86_64)
                export SENDNN_INFERENCE_REQUIRE_PRECOMPILED_DECODERS=0
                ;;
            esac
          fi

          if [ -f /etc/profile.d/ibm-aiu-setup.sh ]; then
            source /etc/profile.d/ibm-aiu-setup.sh
          fi

          if [ "$KSERVE_INFER_ROCE" = "true" ]; then
            echo "Trying to infer RoCE configs ... "
            grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
            grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null

            cat /proc/driver/nvidia/params

            KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}

            echo "[Infer RoCE] Discovering active HCAs ..."
            active_hcas=()
            # Loop through all mlx5 devices found in sysfs
            for hca_dir in /sys/class/infiniband/mlx5_*; do
                # Ensure it's a directory before proceeding
                if [ -d "$hca_dir" ]; then
                    hca_name=$(basename "$hca_dir")
                    port_state_file="$hca_dir/ports/1/state" # Assume port 1
                    type_file="$hca_dir/ports/1/gid_attrs/types/*"

                    echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
                    if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
                        echo "[Infer RoCE] Found active HCA: $hca_name"
                        active_hcas+=("$hca_name")
                    else
                        echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
                    fi
                fi
            done

            # Check if we found any active HCAs
            if [ ${#active_hcas[@]} -gt 0 ]; then
                # Join the array elements with a comma
                hca_port_pairs=()
                for hca in "${active_hcas[@]}"; do
                  hca_port_pairs+=("${hca}:1")
                done

                active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
                hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
                echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
                export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
                export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
                export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}

                echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
                echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
                echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
            else
                echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
            fi

            if [ ${#active_hcas[@]} -gt 0 ]; then
                echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."

                # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
                declare -A gid_index_count
                declare -A hca_gid_index

                for hca_name in "${active_hcas[@]}"; do
                    echo "[Infer RoCE] Processing HCA: ${hca_name}"

                    # Find all RoCE v2 IPv4 GIDs for this HCA and count by index
                    for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
                        if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
                            idx=$(basename "$tpath")
                            gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
                            # Check for IPv4 GID (contains ffff:)
                            if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
                                gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
                                echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
                                hca_gid_index["${hca_name}"]="${idx}"
                                gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
                                break  # Use first found IPv4 GID per HCA
                            fi
                        fi
                    done
                done

                # Find the most common GID index (most likely to be consistent across nodes)
                best_gid_index=""
                max_count=0
                for idx in "${!gid_index_count[@]}"; do
                    count=${gid_index_count["${idx}"]}
                    echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
                    if [ $count -gt $max_count ]; then
                        max_count=$count
                        best_gid_index="$idx"
                    fi
                done

                # Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
                if [ ${#gid_index_count[@]} -gt 1 ]; then
                    echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
                    # If there's a tie, prefer index 3 as it's most common in SR-IOV setups
                    if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
                        best_gid_index="3"
                        echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
                    fi
                fi

                # Check if GID_INDEX is already set via environment variables
                if [ -n "${NCCL_IB_GID_INDEX}" ]; then
                    echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
                    export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                    export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                    echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
                elif [ -n "$best_gid_index" ]; then
                    echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"

                    export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
                    export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
                    export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}

                    echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
                else
                    echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
                fi
            else
                echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
            fi
          fi

          # --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
          # Older versions still need the blanket --disable-uvicorn-access-log.
          ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
          VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
          echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
          if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
            ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
          fi
          echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"

          # --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
          SHUTDOWN_TIMEOUT_ARGS=""
          if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
            SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout {{ if .Spec.Prefill }}{{ shutdownTimeout .Spec.Prefill.Template 15 }}{{ else }}{{ shutdownTimeout nil 15 }}{{ end }}"
          fi

          # A user-supplied --kv-transfer-config always wins; KServe only fills the flag when it is unset.
          KV_TRANSFER_ARGS=""
          if [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv-transfer-config"* ]] && [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv_transfer_config"* ]] && [[ "$*" != *"--kv-transfer-config"* ]] && [[ "$*" != *"--kv_transfer_config"* ]]; then
            # KSERVE_KV_TRANSFER_ARGS is set as a container env var by the config template;
            # it is empty (no-op) when KV cache offloading is not configured.
            if [ -n "${KSERVE_KV_TRANSFER_ARGS:-}" ]; then
              # --kv-transfer-config with OffloadingConnector requires vLLM 0.22.0+ (vllm-project/vllm#40020).
              if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.22.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.22.0" ]; then
                KV_TRANSFER_ARGS="${KSERVE_KV_TRANSFER_ARGS}"
              fi
            fi
            # This template is only composed for a disaggregated P/D topology (spec.prefill set).
            # Prefill is the KV producer; without a connector here decode has nothing to fetch.
            if [ -z "${KV_TRANSFER_ARGS}" ]; then
              # Only inject when NIXL is importable. CPU and other non-NIXL engine images
              # raise RuntimeError("NIXL is not available") and never finish starting.
              NIXL_PY=$(command -v python3 || command -v python || true)
              if [ -n "${NIXL_PY}" ] && "${NIXL_PY}" -c "import nixl" >/dev/null 2>&1; then
                KV_TRANSFER_ARGS="--kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_producer\"}'"
                echo "[kv-transfer] NIXL available, enabling NixlConnector (kv_producer)"
              else
                echo "[kv-transfer] NIXL not available, P/D KV transfer stays disabled"
              fi
            fi
          fi

          eval "exec vllm serve /mnt/models \
            --served-model-name "{{ .Spec.Model.Name }}" \
            --port 8000 \
            --root-path /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }} \
            ${ACCESS_LOG_ARGS} \
            ${SHUTDOWN_TIMEOUT_ARGS} \
            ${KV_TRANSFER_ARGS} \
            {{- if and .Spec.Prefill .Spec.Prefill.Parallelism .Spec.Prefill.Parallelism.Tensor }} --tensor-parallel-size {{ .Spec.Prefill.Parallelism.Tensor }}{{- end }} \
            {{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
            {{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
            {{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
            ${VLLM_ADDITIONAL_ARGS} \
            $@"
        - --
        env:
        - name: KSERVE_KV_TRANSFER_ARGS
          value: ""
        - name: HOME
          value: /home
        - name: VLLM_LOGGING_LEVEL
          value: INFO
        - name: HF_HUB_CACHE
          value: /models
        - name: VLLM_NIXL_SIDE_CHANNEL_HOST
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        image: registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:fc68d623d1bfc36c8cb2fe4a71f19c8578cfb420ce8ce07b20a02c1ee0be0cf3
        imagePullPolicy: IfNotPresent
        lifecycle:
          preStop:
            exec:
              command:
              - /bin/sleep
              - "15"
        livenessProbe:
          failureThreshold: 10
          httpGet:
            path: /health
            port: 8000
            scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
          periodSeconds: 10
          timeoutSeconds: 1
        name: main
        ports:
        - containerPort: 8000
          protocol: TCP
        readinessProbe:
          failureThreshold: 2
          httpGet:
            path: /health
            port: 8000
            scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
          periodSeconds: 1
          timeoutSeconds: 1
        resources: {}
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          readOnlyRootFilesystem: false
          runAsNonRoot: true
          seccompProfile:
            type: RuntimeDefault
        startupProbe:
          failureThreshold: 60
          httpGet:
            path: /health
            port: 8000
            scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
          periodSeconds: 10
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /home
          name: home
        - mountPath: /tmp
          name: tmp-dir
        - mountPath: /dev/shm
          name: dshm
        - mountPath: /models
          name: model-cache
        - mountPath: /var/run/kserve/tls
          name: tls-certs
          readOnly: true
      terminationGracePeriodSeconds: 60
      volumes:
      - emptyDir: {}
        name: home
      - emptyDir: {}
        name: tmp-dir
      - emptyDir:
          medium: Memory
          sizeLimit: 1Gi
        name: dshm
      - emptyDir: {}
        name: model-cache
      - name: tls-certs
        secret:
          secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
status:
  conditions:
  - lastTransitionTime: "2026-10-29T22:16:43Z"
    message: referenced by 8 LLMInferenceService(s)
    reason: InUse
    severity: Info
    status: "True"
    type: ConfigInUse
  - lastTransitionTime: "2026-10-29T21:52:14Z"
    status: "True"
    type: Ready
  observedGeneration: 3
  referencedBy:
  - name: e2e-distinct-2-simulated
    namespace: llm
  - name: e2e-distinct-simulated
    namespace: llm
  - name: e2e-embedding-simulated
    namespace: llm
  - name: e2e-trlp-test-simulated
    namespace: llm
  - name: e2e-unconfigured-facebook-opt-125m-simulated
    namespace: llm
  - name: echo-simulator
    namespace: llm
  - name: facebook-opt-125m-simulated
    namespace: llm
  - name: premium-simulated-simulated-premium
    namespace: llm