[2026-04-17T07:42:01,974888033+00:00] Upload SBOM INFO: Using mounted CA bundle: /mnt/trusted-ca/ca-bundle.crt '/mnt/trusted-ca/ca-bundle.crt' -> '/etc/pki/ca-trust/source/anchors/ca-bundle.crt' Using token for quay.io/opendatahub/custom-model-grpc Pushing sbom to registry Executing: cosign attach sbom --sbom sbom.json --type spdx quay.io/opendatahub/custom-model-grpc:odh-pr-1374@sha256:c8fcacc6c36249d1e4bb251624ab9c0b641c182077b8bd3c9ebac2eb4e09343f quay.io/opendatahub/custom-model-grpc@sha256:fdea8b230758b7892cc4e87ebc0d683563804a1eb208b8ec3138d2f2107a276e [2026-04-17T07:42:05,451389955+00:00] End upload-sbom 2026/04/17 07:37:24 Entrypoint initialization 2026/04/17 07:37:25 Decoded script /tekton/scripts/script-1-t4gd5 2026/04/17 07:37:25 Decoded script /tekton/scripts/script-2-llvjm 2026/04/17 07:37:25 Decoded script /tekton/scripts/script-3-8tn6m 2026/04/17 07:37:25 Decoded script /tekton/scripts/script-4-w6v52 2026/04/17 07:37:25 Decoded script /tekton/scripts/script-5-sczln Using token for quay.io/opendatahub/kserve Executing: oras blob fetch --registry-config /tmp/use-oci.sh.UWe8Jq/auth-7T36nI.json quay.io/opendatahub/kserve@sha256:9d846f3b7cb9ab52f4ac1408cde527cbb6fe36b371886fd68e530487f4215be8 --output - Restored artifact quay.io/opendatahub/kserve@sha256:9d846f3b7cb9ab52f4ac1408cde527cbb6fe36b371886fd68e530487f4215be8 to /var/workdir/source WARN: artifact URI not provided, (given: =/var/workdir/cachi2) echo "[$(date --utc -Ins)] Prepare connection" [2026-04-17T07:38:05,609680475+00:00] Prepare connection mkdir -p ~/.ssh if [ -e "/ssh/error" ]; then #no server could be provisioned cat /ssh/error exit 1 fi export SSH_HOST=$(cat /ssh/host) if [ "$SSH_HOST" == "localhost" ] ; then IS_LOCALHOST=true echo "Localhost detected; running build in cluster" elif [ -e "/ssh/otp" ]; then curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa Localhost detected; running build in cluster echo "" >> ~/.ssh/id_rsa else cp /ssh/id_rsa ~/.ssh fi mkdir -p scripts if ! [[ $IS_LOCALHOST ]]; then echo "[$(date --utc -Ins)] Setup VM" if [[ "$BUILDAH_HTTP_PROXY" =~ .+\.cluster\.local ]]; then echo "[$(date --utc -Ins)] Ignoring cluster local proxy for remote build" unset BUILDAH_HTTP_PROXY BUILDAH_NO_PROXY fi chmod 0400 ~/.ssh/id_rsa export BUILD_DIR=$(cat /ssh/user-dir) export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10" echo "$BUILD_DIR" # shellcheck disable=SC2086 ssh $SSH_ARGS "$SSH_HOST" mkdir -p "${BUILD_DIR@Q}/workspaces" "${BUILD_DIR@Q}/scripts" "${BUILD_DIR@Q}/volumes" PORT_FORWARD="" PODMAN_PORT_FORWARD="" if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80" PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost" fi echo "[$(date --utc -Ins)] Rsync data" rsync -razW /shared/ "$SSH_HOST:$BUILD_DIR/volumes/shared/" rsync -razW /var/workdir/ "$SSH_HOST:$BUILD_DIR/volumes/workdir/" rsync -razW /entitlement/ "$SSH_HOST:$BUILD_DIR/volumes/etc-pki-entitlement/" rsync -razW /activation-key/ "$SSH_HOST:$BUILD_DIR/volumes/activation-key/" rsync -razW /additional-secret/ "$SSH_HOST:$BUILD_DIR/volumes/additional-secret/" rsync -razW /mnt/trusted-ca/ "$SSH_HOST:$BUILD_DIR/volumes/trusted-ca/" rsync -razW /mnt/proxy-ca-bundle/ "$SSH_HOST:$BUILD_DIR/volumes/proxy-ca-bundle/" rsync -razW "$HOME/.docker/" "$SSH_HOST:$BUILD_DIR/.docker/" rsync -razW --mkpath "/usr/bin/retry" "$SSH_HOST:$BUILD_DIR/usr/bin/retry" rsync -razW "/tekton/results/" "$SSH_HOST:$BUILD_DIR/results/" fi if [ "${IMAGE_APPEND_PLATFORM}" == "true" ]; then IMAGE="${IMAGE}-${PLATFORM//[^a-zA-Z0-9]/-}" export IMAGE fi cat >scripts/script-build.sh <<'REMOTESSHEOF' #!/bin/bash set -euo pipefail cd /var/workdir function set_proxy { if [ -n "${BUILDAH_HTTP_PROXY}" ]; then echo "[$(date --utc -Ins)] Setting proxy to ${BUILDAH_HTTP_PROXY}" export HTTP_PROXY="${BUILDAH_HTTP_PROXY}" export HTTPS_PROXY="${BUILDAH_HTTP_PROXY}" export ALL_PROXY="${BUILDAH_HTTP_PROXY}" if [ -n "${BUILDAH_NO_PROXY}" ]; then echo "[$(date --utc -Ins)] Bypassing proxy for ${BUILDAH_NO_PROXY}" export NO_PROXY="${BUILDAH_NO_PROXY}" fi fi } function unset_proxy { echo "[$(date --utc -Ins)] Unsetting proxy" unset HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY } echo "[$(date --utc -Ins)] Validate context path" if [ -z "$CONTEXT" ]; then echo "WARNING: CONTEXT is empty. Defaulting to '.' (the source directory)." >&2 CONTEXT="." fi source_dir_path=$(realpath "$SOURCE_CODE_DIR") context_dir_path=$(realpath "$SOURCE_CODE_DIR/$CONTEXT") case "$context_dir_path" in "$source_dir_path" | "$source_dir_path/"*) # path is valid, do nothing ;; *) echo "ERROR: The CONTEXT parameter ('$CONTEXT') is invalid because it escapes the source directory." >&2 echo "Source path: $source_dir_path" >&2 echo "Resolved path: $context_dir_path" >&2 exit 1 ;; esac echo "[$(date --utc -Ins)] Update CA trust" ca_bundle=/mnt/trusted-ca/ca-bundle.crt proxy_ca_bundle=/mnt/proxy-ca-bundle/ca-bundle.crt update_ca_trust=false if [ -f "$ca_bundle" ]; then echo "[$(date --utc -Ins)] Using mounted CA bundle: $ca_bundle" cp -vf $ca_bundle /etc/pki/ca-trust/source/anchors/ca-bundle.crt update_ca_trust=true fi if [ -f "$proxy_ca_bundle" ] && [ -n "${BUILDAH_HTTP_PROXY}" ]; then echo "[$(date --utc -Ins)] Using mounted proxy CA bundle: $proxy_ca_bundle" cp -vf $proxy_ca_bundle /etc/pki/ca-trust/source/anchors/proxy-ca-bundle.crt update_ca_trust=true fi if [ "$update_ca_trust" = "true" ]; then update-ca-trust fi echo "[$(date --utc -Ins)] Prepare Dockerfile" if [ -e "$SOURCE_CODE_DIR/$CONTEXT/$DOCKERFILE" ]; then dockerfile_path="$(pwd)/$SOURCE_CODE_DIR/$CONTEXT/$DOCKERFILE" elif [ -e "$SOURCE_CODE_DIR/$DOCKERFILE" ]; then dockerfile_path="$(pwd)/$SOURCE_CODE_DIR/$DOCKERFILE" elif [ -e "$DOCKERFILE" ]; then # Instrumented builds (SAST) use this custom dockerfile step as their base dockerfile_path="$DOCKERFILE" elif echo "$DOCKERFILE" | grep -q "^https\?://"; then echo "Fetch Dockerfile from $DOCKERFILE" dockerfile_path=$(mktemp --suffix=-Dockerfile) http_code=$(curl -s -S -L -w "%{http_code}" --output "$dockerfile_path" "$DOCKERFILE") if [ "$http_code" != 200 ]; then echo "No Dockerfile is fetched. Server responds $http_code" exit 1 fi http_code=$(curl -s -S -L -w "%{http_code}" --output "$dockerfile_path.dockerignore.tmp" "$DOCKERFILE.dockerignore") if [ "$http_code" = 200 ]; then echo "Fetched .dockerignore from $DOCKERFILE.dockerignore" mv "$dockerfile_path.dockerignore.tmp" "$SOURCE_CODE_DIR/$CONTEXT/.dockerignore" fi else echo "Cannot find Dockerfile $DOCKERFILE" exit 1 fi dockerfile_copy=$(mktemp --tmpdir "$(basename "$dockerfile_path").XXXXXX") cp "$dockerfile_path" "$dockerfile_copy" # Inject the image content manifest into the container we are producing. # This will generate the content-sets.json file and copy it by appending a COPY # instruction to the Containerfile. icm_opts=() if [ "${ICM_KEEP_COMPAT_LOCATION}" = "true" ]; then icm_opts+=(-c) fi if [ "${SKIP_INJECTIONS}" = "false" ]; then inject-icm-to-containerfile "${icm_opts[@]}" "$dockerfile_copy" "/var/workdir/cachi2/output/bom.json" "$SOURCE_CODE_DIR/$CONTEXT" fi echo "[$(date --utc -Ins)] Prepare system (architecture: $(uname -m))" # Fixing group permission on /var/lib/containers chown root:root /var/lib/containers sed -i 's/^\s*short-name-mode\s*=\s*.*/short-name-mode = "disabled"/' /etc/containers/registries.conf # Setting new namespace to run buildah - 2^32-2 echo 'root:1:4294967294' | tee -a /etc/subuid >>/etc/subgid build_args=() if [ -n "${BUILD_ARGS_FILE}" ]; then # Parse BUILD_ARGS_FILE ourselves because dockerfile-json doesn't support it echo "Parsing ARGs from $BUILD_ARGS_FILE" mapfile -t build_args < <( # https://www.mankier.com/1/buildah-build#--build-arg-file # delete lines that start with # # delete blank lines sed -e '/^#/d' -e '/^\s*$/d' "${SOURCE_CODE_DIR}/${BUILD_ARGS_FILE}" ) fi LABELS=() ANNOTATIONS=() # Append any annotations from the specified file if [ -n "${ANNOTATIONS_FILE}" ] && [ -f "${SOURCE_CODE_DIR}/${ANNOTATIONS_FILE}" ]; then echo "Reading annotations from file: ${SOURCE_CODE_DIR}/${ANNOTATIONS_FILE}" while IFS= read -r line || [[ -n "$line" ]]; do # Skip empty lines and comments if [[ -n "$line" && ! "$line" =~ ^[[:space:]]*# ]]; then ANNOTATIONS+=("--annotation" "$line") fi done <"${SOURCE_CODE_DIR}/${ANNOTATIONS_FILE}" fi # Split `args` into two sets of arguments. while [[ $# -gt 0 ]]; do case $1 in --build-args) shift # Note: this may result in multiple --build-arg=KEY=value flags with the same KEY being # passed to buildah. In that case, the *last* occurrence takes precedence. This is why # we append BUILD_ARGS after the content of the BUILD_ARGS_FILE while [[ $# -gt 0 && $1 != --* ]]; do build_args+=("$1") shift done ;; --labels) shift while [[ $# -gt 0 && $1 != --* ]]; do LABELS+=("--label" "$1") shift done ;; --annotations) shift while [[ $# -gt 0 && $1 != --* ]]; do ANNOTATIONS+=("--annotation" "$1") shift done ;; *) echo "unexpected argument: $1" >&2 exit 2 ;; esac done BUILD_ARG_FLAGS=() for build_arg in "${build_args[@]}"; do BUILD_ARG_FLAGS+=("--build-arg=$build_arg") done # Dockerfile-json cannot parse Buildah's host variables, we have to pass them manually BUILDAH_INFO=$(buildah info) BUILDAH_OS=$(jq -r '.host.os' <<<"$BUILDAH_INFO") BUILDAH_ARCH=$(jq -r '.host.arch' <<<"$BUILDAH_INFO") BUILDAH_VARIANT=$(jq -r '.host.variant' <<<"$BUILDAH_INFO") BUILDAH_PLATFORM="${BUILDAH_OS}/${BUILDAH_ARCH}" DOCKERFILE_ARG_FLAGS=() # Reference for variables: # https://docs.docker.com/build/building/variables/#pre-defined-build-arguments PREFIXES=('BUILD' 'TARGET') for PREFIX in "${PREFIXES[@]}"; do DOCKERFILE_ARG_FLAGS+=("--build-arg=${PREFIX}PLATFORM=${BUILDAH_PLATFORM}") DOCKERFILE_ARG_FLAGS+=("--build-arg=${PREFIX}OS=${BUILDAH_OS}") DOCKERFILE_ARG_FLAGS+=("--build-arg=${PREFIX}ARCH=${BUILDAH_ARCH}") DOCKERFILE_ARG_FLAGS+=("--build-arg=${PREFIX}VARIANT=${BUILDAH_VARIANT}") done DOCKERFILE_ARG_FLAGS+=("${BUILD_ARG_FLAGS[@]}") dockerfile-json "${DOCKERFILE_ARG_FLAGS[@]}" "$dockerfile_copy" >/shared/parsed_dockerfile.json BASE_IMAGES=$( jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' /shared/parsed_dockerfile.json | tr -d '"' | tr -d "'" ) BUILDAH_ARGS=() UNSHARE_ARGS=() if [ "${HERMETIC}" == "true" ]; then BUILDAH_ARGS+=("--pull=never") UNSHARE_ARGS+=("--net") buildah_retries=3 set_proxy for image in $BASE_IMAGES; do if ! retry unshare -Ufp --keep-caps -r --map-users 1,1,65536 --map-groups 1,1,65536 --mount -- buildah pull --retry "$buildah_retries" "$image"; then echo "Failed to pull base image ${image}" exit 1 fi done unset_proxy echo "Build will be executed with network isolation" fi if [ -n "${TARGET_STAGE}" ]; then BUILDAH_ARGS+=("--target=${TARGET_STAGE}") fi BUILDAH_ARGS+=("${BUILD_ARG_FLAGS[@]}") # Necessary for newer version of buildah if the host system does not contain up to date version of container-selinux # TODO remove the option once all hosts were updated BUILDAH_ARGS+=("--security-opt=unmask=/proc/interrupts") if [ "${PRIVILEGED_NESTED}" == "true" ]; then BUILDAH_ARGS+=("--security-opt=label=disable") BUILDAH_ARGS+=("--cap-add=all") BUILDAH_ARGS+=("--device=/dev/fuse") fi if [ -n "${ADD_CAPABILITIES}" ]; then BUILDAH_ARGS+=("--cap-add=${ADD_CAPABILITIES}") fi if [ "${SQUASH}" == "true" ]; then BUILDAH_ARGS+=("--squash") fi if [ "${SKIP_UNUSED_STAGES}" != "true" ]; then BUILDAH_ARGS+=("--skip-unused-stages=false") fi if [ "${INHERIT_BASE_IMAGE_LABELS}" != "true" ]; then BUILDAH_ARGS+=("--inherit-labels=false") fi if [ -n "${BUILDAH_SOURCE_DATE_EPOCH}" ]; then BUILDAH_ARGS+=("--source-date-epoch=${BUILDAH_SOURCE_DATE_EPOCH}") if [ "${BUILDAH_REWRITE_TIMESTAMP}" = "true" ]; then BUILDAH_ARGS+=("--rewrite-timestamp") fi if [ -n "$BUILD_TIMESTAMP" ]; then echo "ERROR: cannot use both BUILD_TIMESTAMP and SOURCE_DATE_EPOCH" exit 1 fi # but do set it so that we get all the labels/annotations associated with it BUILD_TIMESTAMP="$BUILDAH_SOURCE_DATE_EPOCH" fi if [ "${BUILDAH_OMIT_HISTORY}" == "true" ]; then BUILDAH_ARGS+=("--omit-history") fi VOLUME_MOUNTS=() echo "[$(date --utc -Ins)] Setup prefetched" if [ -f "/var/workdir/cachi2/cachi2.env" ]; then cp -r "/var/workdir/cachi2" /tmp/ chmod -R go+rwX /tmp/cachi2 VOLUME_MOUNTS+=(--volume /tmp/cachi2:/cachi2) # Read in the whole file (https://unix.stackexchange.com/questions/533277), then # for each RUN ... line insert the cachi2.env command *after* any options like --mount sed -E -i \ -e 'H;1h;$!d;x' \ -e 's@^\s*(run((\s|\\\n)+-\S+)*(\s|\\\n)+)@\1. /cachi2/cachi2.env \&\& \\\n @igM' \ "$dockerfile_copy" echo "Prefetched content will be made available" prefetched_repo_for_my_arch="/tmp/cachi2/output/deps/rpm/$(uname -m)/repos.d/cachi2.repo" if [ -f "$prefetched_repo_for_my_arch" ]; then echo "Adding $prefetched_repo_for_my_arch to $YUM_REPOS_D_FETCHED" mkdir -p "$YUM_REPOS_D_FETCHED" if [ ! -f "${YUM_REPOS_D_FETCHED}/cachi2.repo" ]; then cp "$prefetched_repo_for_my_arch" "$YUM_REPOS_D_FETCHED" fi fi fi # if yum repofiles stored in git, copy them to mount point outside the source dir if [ -d "${SOURCE_CODE_DIR}/${YUM_REPOS_D_SRC}" ]; then mkdir -p "${YUM_REPOS_D_FETCHED}" cp -r "${SOURCE_CODE_DIR}/${YUM_REPOS_D_SRC}"/* "${YUM_REPOS_D_FETCHED}" fi # if anything in the repofiles mount point (either fetched or from git), mount it if [ -d "${YUM_REPOS_D_FETCHED}" ]; then chmod -R go+rwX "${YUM_REPOS_D_FETCHED}" mount_point=$(realpath "${YUM_REPOS_D_FETCHED}") VOLUME_MOUNTS+=(--volume "${mount_point}:${YUM_REPOS_D_TARGET}") fi DEFAULT_LABELS=( "--label" "architecture=$(uname -m)" "--label" "vcs-type=git" ) if [ -n "$COMMIT_SHA" ]; then DEFAULT_LABELS+=("--label" "vcs-ref=${COMMIT_SHA}" "--label" "org.opencontainers.image.revision=${COMMIT_SHA}") ANNOTATIONS+=("--annotation" "org.opencontainers.image.revision=${COMMIT_SHA}") fi if [ -n "$SOURCE_URL" ]; then DEFAULT_LABELS+=("--label" "org.opencontainers.image.source=${SOURCE_URL}") ANNOTATIONS+=("--annotation" "org.opencontainers.image.source=${SOURCE_URL}") fi [ -n "$IMAGE_EXPIRES_AFTER" ] && DEFAULT_LABELS+=("--label" "quay.expires-after=$IMAGE_EXPIRES_AFTER") BUILD_TIMESTAMP_RFC3339="" if [ -n "$BUILD_TIMESTAMP" ]; then BUILD_TIMESTAMP_RFC3339=$(date -u -d "@$BUILD_TIMESTAMP" +'%Y-%m-%dT%H:%M:%SZ') else BUILD_TIMESTAMP_RFC3339=$(date -u +'%Y-%m-%dT%H:%M:%SZ') fi DEFAULT_LABELS+=("--label" "build-date=${BUILD_TIMESTAMP_RFC3339}") DEFAULT_LABELS+=("--label" "org.opencontainers.image.created=${BUILD_TIMESTAMP_RFC3339}") ANNOTATIONS+=("--annotation" "org.opencontainers.image.created=${BUILD_TIMESTAMP_RFC3339}") label_pairs=() # If INHERIT_BASE_IMAGE_LABELS is true, get the labels from the final base image only touch base_images_labels.json if [[ "$INHERIT_BASE_IMAGE_LABELS" == "true" ]] && [[ -n "$BASE_IMAGES" ]]; then FINAL_BASE_IMAGE=$( # Get the base image of the final stage # The final stage can refer to a previous `FROM xxx AS yyy` stage, for example 'FROM bar AS foo; ... ; FROM foo; ...' # Define a function that keeps nesting recursively into the parent stages until it finds the original base image # Run the find_root_stage() function on the final stage # If the final stage is scratch or oci-archive, return empty jq -r '.Stages as $all_stages | def find_root_stage($stage): if $stage.From.Stage then find_root_stage($all_stages[$stage.From.Stage.Index]) else $stage end; find_root_stage(.Stages[-1]) | if .From.Scratch or (.BaseName | test("^oci-archive:")) then empty else .BaseName end' /shared/parsed_dockerfile.json | tr -d '"' | tr -d "'" ) if [[ -n "$FINAL_BASE_IMAGE" ]]; then set_proxy buildah pull "$FINAL_BASE_IMAGE" >/dev/null$() unset_proxy buildah inspect "$FINAL_BASE_IMAGE" | jq '.OCIv1.config.Labels' >"base_images_labels.json" fi fi # Concatenate defaults and explicit labels. If a label appears twice, the last one wins. LABELS=("${DEFAULT_LABELS[@]}" "${LABELS[@]}") # Get all the default and explicit labels so that they can be written into labels.json for label in "${LABELS[@]}"; do if [[ "$label" != "--label" ]]; then label_pairs+=("$label") fi done # Labels that we explicitly add to the image label_pairs+=("org.opencontainers.image.created=${BUILD_TIMESTAMP_RFC3339}") label_pairs+=("io.buildah.version=$(buildah version --json | jq -r '.version')") while IFS= read -r label; do label_pairs+=("$label") done < <(jq -r '.Stages[].Commands[] | select(.Name == "LABEL") | .Labels[] | "\(.Key)=\(.Value)"' /shared/parsed_dockerfile.json | sed 's/"//g') printf '%s\n' "${label_pairs[@]}" | jq -Rn ' [ inputs | select(length>0) ] | map( split("=") | {(.[0]): (.[1] // "")} ) | add' >"image_labels.json" jq -s '(.[0] // {}) * (.[1] // {})' "base_images_labels.json" "image_labels.json" >"$SOURCE_CODE_DIR/$CONTEXT/labels.json" jq '.' "$SOURCE_CODE_DIR/$CONTEXT/labels.json" if [ "${SKIP_INJECTIONS}" = "false" ]; then echo "" >>"$dockerfile_copy" # Always write labels.json to the new standard location echo 'COPY labels.json /usr/share/buildinfo/labels.json' >>"$dockerfile_copy" # Conditionally write to the old location for backward compatibility if [ "${ICM_KEEP_COMPAT_LOCATION}" = "true" ]; then echo 'COPY labels.json /root/buildinfo/labels.json' >>"$dockerfile_copy" fi fi # Make sure our labels.json file isn't filtered out containerignore="" if [ -f "$SOURCE_CODE_DIR/$CONTEXT/.containerignore" ]; then containerignore="$SOURCE_CODE_DIR/$CONTEXT/.containerignore" elif [ -f "$SOURCE_CODE_DIR/$CONTEXT/.dockerignore" ]; then containerignore="$SOURCE_CODE_DIR/$CONTEXT/.dockerignore" fi if [ -n "$containerignore" ]; then ignorefile_copy=$(mktemp --tmpdir "$(basename "$containerignore").XXXXXX") cp "$containerignore" "$ignorefile_copy" { echo "" echo "!/labels.json" echo "!/content-sets.json" } >>"$ignorefile_copy" BUILDAH_ARGS+=(--ignorefile "$ignorefile_copy") fi echo "[$(date --utc -Ins)] Register sub-man" ACTIVATION_KEY_PATH="/activation-key" ENTITLEMENT_PATH="/entitlement" # 0. if hermetic=true, skip all subscription related stuff # 1. do not enable activation key and entitlement at same time. If both vars are provided, prefer activation key. # 2. Activation-keys will be used when the key 'org' exists in the activation key secret. # 3. try to pre-register and mount files to the correct location so that users do no need to modify Dockerfiles. # 3. If the Dockerfile contains the string "subcription-manager register", add the activation-keys volume # to buildah but don't pre-register for backwards compatibility. Mount an empty directory on # shared emptydir volume to "/etc/pki/entitlement" to prevent certificates from being included if [ "${HERMETIC}" != "true" ] && [ -e /activation-key/org ]; then cp -r --preserve=mode "$ACTIVATION_KEY_PATH" /tmp/activation-key mkdir -p /shared/rhsm/etc/pki/entitlement mkdir -p /shared/rhsm/etc/pki/consumer VOLUME_MOUNTS+=(-v /tmp/activation-key:/activation-key -v /shared/rhsm/etc/pki/entitlement:/etc/pki/entitlement:Z -v /shared/rhsm/etc/pki/consumer:/etc/pki/consumer:Z) echo "Adding activation key to the build" if ! grep -E "^[^#]*subscription-manager.[^#]*register" "$dockerfile_path"; then # user is not running registration in the Containerfile: pre-register. echo "Pre-registering with subscription manager." export RETRY_MAX_TRIES=6 if ! retry subscription-manager register --org "$(cat /tmp/activation-key/org)" --activationkey "$(cat /tmp/activation-key/activationkey)"; then echo "Subscription-manager register failed" exit 1 fi unset RETRY_MAX_TRIES trap 'subscription-manager unregister || true' EXIT # copy generated certificates to /shared volume cp /etc/pki/entitlement/*.pem /shared/rhsm/etc/pki/entitlement cp /etc/pki/consumer/*.pem /shared/rhsm/etc/pki/consumer # and then mount get /etc/rhsm/ca/redhat-uep.pem into /run/secrets/rhsm/ca VOLUME_MOUNTS+=(--volume /etc/rhsm/ca/redhat-uep.pem:/etc/rhsm/ca/redhat-uep.pem:Z) fi elif [ "${HERMETIC}" != "true" ] && find /entitlement -name "*.pem" >>null; then cp -r --preserve=mode "$ENTITLEMENT_PATH" /tmp/entitlement VOLUME_MOUNTS+=(--volume /tmp/entitlement:/etc/pki/entitlement) echo "Adding the entitlement to the build" fi if [ -n "$WORKINGDIR_MOUNT" ]; then if [[ "$WORKINGDIR_MOUNT" == *:* ]]; then echo "WORKINGDIR_MOUNT contains ':'" >&2 echo "Refusing to proceed in case this is an attempt to set unexpected mount options." >&2 exit 1 fi # ${SOURCE_CODE_DIR}/${CONTEXT} will be the $PWD when we call 'buildah build' # (we set the workdir using 'unshare -w') context_dir=$(realpath "${SOURCE_CODE_DIR}/${CONTEXT}") VOLUME_MOUNTS+=(--volume "$context_dir:${WORKINGDIR_MOUNT}") fi if [ -n "${ADDITIONAL_VOLUME_MOUNTS-}" ]; then # ADDITIONAL_VOLUME_MOUNTS allows to specify more volumes for the build. # Instrumented builds (SAST) use this step as their base and add some other tools. while read -r volume_mount; do VOLUME_MOUNTS+=("--volume=$volume_mount") done <<<"$ADDITIONAL_VOLUME_MOUNTS" fi echo "[$(date --utc -Ins)] Add secrets" ADDITIONAL_SECRET_PATH="/additional-secret" ADDITIONAL_SECRET_TMP="/tmp/additional-secret" if [ -d "$ADDITIONAL_SECRET_PATH" ]; then cp -r --preserve=mode -L "$ADDITIONAL_SECRET_PATH" $ADDITIONAL_SECRET_TMP while read -r filename; do echo "Adding the secret ${ADDITIONAL_SECRET}/${filename} to the build, available at /run/secrets/${ADDITIONAL_SECRET}/${filename}" BUILDAH_ARGS+=("--secret=id=${ADDITIONAL_SECRET}/${filename},src=$ADDITIONAL_SECRET_TMP/${filename}") done < <(find $ADDITIONAL_SECRET_TMP -maxdepth 1 -type f -exec basename {} \;) fi # Prevent ShellCheck from giving a warning because 'image' is defined and 'IMAGE' is not. declare IMAGE buildah_cmd_array=( buildah build "${VOLUME_MOUNTS[@]}" "${BUILDAH_ARGS[@]}" "${LABELS[@]}" "${ANNOTATIONS[@]}" --tls-verify="$TLSVERIFY" --no-cache --ulimit nofile=4096:4096 --http-proxy=false -f "$dockerfile_copy" -t "$IMAGE" . ) buildah_cmd=$(printf "%q " "${buildah_cmd_array[@]}") if [ "${HERMETIC}" == "true" ]; then # enabling loopback adapter enables Bazel builds to work in hermetic mode. command="ip link set lo up && $buildah_cmd" else command="$buildah_cmd" fi # disable host subcription manager integration find /usr/share/rhel/secrets -type l -exec unlink {} \; set_proxy echo "[$(date --utc -Ins)] Run buildah build" echo "[$(date --utc -Ins)] ${command}" unshare -Uf "${UNSHARE_ARGS[@]}" --keep-caps -r --map-users 1,1,65536 --map-groups 1,1,65536 -w "${SOURCE_CODE_DIR}/$CONTEXT" --mount -- sh -c "$command" unset_proxy echo "[$(date --utc -Ins)] Add metadata" # Save the SBOM produced in prefetch so it can be merged into the final SBOM later if [ -f "/tmp/cachi2/output/bom.json" ]; then echo "Making copy of sbom-prefetch.json" cp /tmp/cachi2/output/bom.json ./sbom-prefetch.json fi touch /shared/base_images_digests echo "Recording base image digests used" for image in $BASE_IMAGES; do # Get the image pullspec and filter out a tag if it is not set # Use head -n 1 to ensure we only get one result even if multiple images match the filter base_image_digest=$(buildah images --format '{{ .Name }}{{ if ne .Tag "" }}:{{ .Tag }}{{ end }}@{{ .Digest }}' --filter reference="$image" | head -n 1) # In some cases, there might be BASE_IMAGES, but not any associated digest. This happens # if buildah did not use that particular image during build because it was skipped if [ -n "$base_image_digest" ]; then echo "$image $base_image_digest" | tee -a /shared/base_images_digests fi done image_name=$(echo "${IMAGE##*/}" | tr ':' '-') buildah push "$IMAGE" oci:"/shared/$image_name.oci" echo "/shared/$image_name.oci" >/shared/container_path echo "[$(date --utc -Ins)] End build" buildah push "$IMAGE" "oci:konflux-final-image:$IMAGE" echo "[$(date --utc -Ins)] End push remote" REMOTESSHEOF chmod +x scripts/script-build.sh PODMAN_NVIDIA_ARGS=() if [[ "$PLATFORM" == "linux-g"* ]]; then PODMAN_NVIDIA_ARGS+=("--device=nvidia.com/gpu=all" "--security-opt=label=disable") fi if ! [[ $IS_LOCALHOST ]]; then PRIVILEGED_NESTED_FLAGS=() if [[ "${PRIVILEGED_NESTED}" == "true" ]]; then # This is a workaround for building bootc images because the cache filesystem (/var/tmp/ on the host) must be a real filesystem that supports setting SELinux security attributes. # https://github.com/coreos/rpm-ostree/discussions/4648 # shellcheck disable=SC2086 ssh $SSH_ARGS "$SSH_HOST" mkdir -p "${BUILD_DIR@Q}/var/tmp" PRIVILEGED_NESTED_FLAGS=(--privileged --mount "type=bind,source=$BUILD_DIR/var/tmp,target=/var/tmp,relabel=shared") fi rsync -ra scripts "$SSH_HOST:$BUILD_DIR" echo "[$(date --utc -Ins)] Build via ssh" # shellcheck disable=SC2086 # Please note: all variables below the first ssh line must be quoted with ${var@Q}! # See https://stackoverflow.com/questions/6592376/prevent-ssh-from-breaking-up-shell-script-parameters ssh $SSH_ARGS "$SSH_HOST" $PORT_FORWARD podman run $PODMAN_PORT_FORWARD \ --tmpfs /run/secrets \ -e ACTIVATION_KEY="${ACTIVATION_KEY@Q}" \ -e ADDITIONAL_SECRET="${ADDITIONAL_SECRET@Q}" \ -e ADD_CAPABILITIES="${ADD_CAPABILITIES@Q}" \ -e ANNOTATIONS_FILE="${ANNOTATIONS_FILE@Q}" \ -e BUILD_ARGS_FILE="${BUILD_ARGS_FILE@Q}" \ -e BUILD_TIMESTAMP="${BUILD_TIMESTAMP@Q}" \ -e CONTEXT="${CONTEXT@Q}" \ -e CONTEXTUALIZE_SBOM="${CONTEXTUALIZE_SBOM@Q}" \ -e ENTITLEMENT_SECRET="${ENTITLEMENT_SECRET@Q}" \ -e HERMETIC="${HERMETIC@Q}" \ -e IMAGE="${IMAGE@Q}" \ -e IMAGE_EXPIRES_AFTER="${IMAGE_EXPIRES_AFTER@Q}" \ -e INHERIT_BASE_IMAGE_LABELS="${INHERIT_BASE_IMAGE_LABELS@Q}" \ -e PRIVILEGED_NESTED="${PRIVILEGED_NESTED@Q}" \ -e SBOM_SKIP_VALIDATION="${SBOM_SKIP_VALIDATION@Q}" \ -e SBOM_SOURCE_SCAN_ENABLED="${SBOM_SOURCE_SCAN_ENABLED@Q}" \ -e SBOM_SYFT_SELECT_CATALOGERS="${SBOM_SYFT_SELECT_CATALOGERS@Q}" \ -e SBOM_TYPE="${SBOM_TYPE@Q}" \ -e SKIP_INJECTIONS="${SKIP_INJECTIONS@Q}" \ -e SKIP_SBOM_GENERATION="${SKIP_SBOM_GENERATION@Q}" \ -e SKIP_UNUSED_STAGES="${SKIP_UNUSED_STAGES@Q}" \ -e SOURCE_CODE_DIR="${SOURCE_CODE_DIR@Q}" \ -e SQUASH="${SQUASH@Q}" \ -e STORAGE_DRIVER="${STORAGE_DRIVER@Q}" \ -e TARGET_STAGE="${TARGET_STAGE@Q}" \ -e TLSVERIFY="${TLSVERIFY@Q}" \ -e WORKINGDIR_MOUNT="${WORKINGDIR_MOUNT@Q}" \ -e YUM_REPOS_D_FETCHED="${YUM_REPOS_D_FETCHED@Q}" \ -e YUM_REPOS_D_SRC="${YUM_REPOS_D_SRC@Q}" \ -e YUM_REPOS_D_TARGET="${YUM_REPOS_D_TARGET@Q}" \ -e COMMIT_SHA="${COMMIT_SHA@Q}" \ -e SOURCE_URL="${SOURCE_URL@Q}" \ -e DOCKERFILE="${DOCKERFILE@Q}" \ -e BUILDAH_HTTP_PROXY="${BUILDAH_HTTP_PROXY@Q}" \ -e BUILDAH_NO_PROXY="${BUILDAH_NO_PROXY@Q}" \ -e ICM_KEEP_COMPAT_LOCATION="${ICM_KEEP_COMPAT_LOCATION@Q}" \ -e BUILDAH_OMIT_HISTORY="${BUILDAH_OMIT_HISTORY@Q}" \ -e BUILDAH_SOURCE_DATE_EPOCH="${BUILDAH_SOURCE_DATE_EPOCH@Q}" \ -e BUILDAH_REWRITE_TIMESTAMP="${BUILDAH_REWRITE_TIMESTAMP@Q}" \ -v "${BUILD_DIR@Q}/volumes/shared:/shared:Z" \ -v "${BUILD_DIR@Q}/volumes/workdir:/var/workdir:Z" \ -v "${BUILD_DIR@Q}/volumes/etc-pki-entitlement:/entitlement:Z" \ -v "${BUILD_DIR@Q}/volumes/activation-key:/activation-key:Z" \ -v "${BUILD_DIR@Q}/volumes/additional-secret:/additional-secret:Z" \ -v "${BUILD_DIR@Q}/volumes/trusted-ca:/mnt/trusted-ca:Z" \ -v "${BUILD_DIR@Q}/volumes/proxy-ca-bundle:/mnt/proxy-ca-bundle:Z" \ -v "${BUILD_DIR@Q}/.docker/:/root/.docker:Z" \ -v "${BUILD_DIR@Q}/usr/bin/retry:/usr/bin/retry:Z" \ -v "${BUILD_DIR@Q}/results/:/tekton/results:Z" \ -v "${BUILD_DIR@Q}/scripts:/scripts:Z" \ "${PRIVILEGED_NESTED_FLAGS[@]@Q}" \ --user=0 "${PODMAN_NVIDIA_ARGS[@]@Q}" --rm "${BUILDER_IMAGE@Q}" /scripts/script-build.sh "${@@Q}" echo "[$(date --utc -Ins)] Rsync back" rsync -razW --stats "$SSH_HOST:$BUILD_DIR/volumes/shared/" /shared/ rsync -razW --stats "$SSH_HOST:$BUILD_DIR/volumes/workdir/" /var/workdir/ rsync -razW --stats "$SSH_HOST:$BUILD_DIR/results/" "/tekton/results/" echo "[$(date --utc -Ins)] Buildah pull" buildah pull "oci:konflux-final-image:$IMAGE" else bash scripts/script-build.sh "$@" fi [2026-04-17T07:38:05,620927309+00:00] Validate context path [2026-04-17T07:38:05,625480120+00:00] Update CA trust [2026-04-17T07:38:05,626944769+00:00] Using mounted CA bundle: /mnt/trusted-ca/ca-bundle.crt '/mnt/trusted-ca/ca-bundle.crt' -> '/etc/pki/ca-trust/source/anchors/ca-bundle.crt' [2026-04-17T07:38:08,292372266+00:00] Prepare Dockerfile Checking if /var/workdir/cachi2/output/bom.json exists. Could not find prefetched sbom. No content_sets found for ICM [2026-04-17T07:38:08,300746718+00:00] Prepare system (architecture: x86_64) [2026-04-17T07:38:34,372683768+00:00] Setup prefetched Trying to pull registry.access.redhat.com/ubi9/python-311:9.7... Getting image source signatures Checking if image destination supports signatures Copying blob sha256:ca20ddaf1ecab04f973ee634ced782ccd9ab321c5e2b55dedb16ea1f28163ab1 Copying blob sha256:ba373b90b6f2d2618a2bc1620c38d599a9c0be5539a04db9669652030ee174a1 Copying blob sha256:7ecb09432dbc631c807208909dc90538c274d6955dc77d3bc71d6065c528ae1c Copying blob sha256:38620aa9eda73695e49cbc29288c6eba6860116bd3fc155c2ed32d0a773cd6d7 Copying config sha256:724f655e28b8963071ad7d290ea8b99159075ca4dc572232a2ccc3d5620b1c17 Writing manifest to image destination Storing signatures [2026-04-17T07:38:43,504914053+00:00] Unsetting proxy { "architecture": "x86_64", "build-date": "2026-04-17T07:38:34Z", "com.redhat.component": "python-311-container", "com.redhat.license_terms": "https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI", "cpe": "cpe:/a:redhat:enterprise_linux:9::appstream", "description": "Python 3.11 available as container is a base platform for building and running various Python 3.11 applications and frameworks. Python is an easy to learn, powerful programming language. It has efficient high-level data structures and a simple but effective approach to object-oriented programming. Python's elegant syntax and dynamic typing, together with its interpreted nature, make it an ideal language for scripting and rapid application development in many areas on most platforms.", "distribution-scope": "public", "io.buildah.version": "1.41.4", "io.buildpacks.stack.id": "com.redhat.stacks.ubi9-python-311", "io.k8s.description": "Python 3.11 available as container is a base platform for building and running various Python 3.11 applications and frameworks. Python is an easy to learn, powerful programming language. It has efficient high-level data structures and a simple but effective approach to object-oriented programming. Python's elegant syntax and dynamic typing, together with its interpreted nature, make it an ideal language for scripting and rapid application development in many areas on most platforms.", "io.k8s.display-name": "Python 3.11", "io.openshift.expose-services": "8080:http", "io.openshift.s2i.scripts-url": "image:///usr/libexec/s2i", "io.openshift.tags": "builder,python,python311,python-311,rh-python311", "io.s2i.scripts-url": "image:///usr/libexec/s2i", "maintainer": "SoftwareCollections.org ", "name": "ubi9/python-311", "org.opencontainers.image.created": "2026-04-17T07:38:34Z", "org.opencontainers.image.revision": "41eaf0be8e98885563de45d52c20f0df0f4c563d", "release": "1776370977", "summary": "Platform for building and running Python 3.11 applications", "url": "https://catalog.redhat.com/en/search?searchType=containers", "usage": "s2i build https://github.com/sclorg/s2i-python-container.git --context-dir=3.11/test/setup-test-app/ ubi9/python-311 python-sample-app", "vcs-ref": "41eaf0be8e98885563de45d52c20f0df0f4c563d", "vcs-type": "git", "vendor": "Red Hat, Inc.", "version": "1", "git.url": "https://github.com/bartoszmajsak/kserve", "git.commit": "41eaf0be8e98885563de45d52c20f0df0f4c563d" } [2026-04-17T07:38:43,568588128+00:00] Register sub-man Adding activation key to the build Pre-registering with subscription manager. Executing: subscription-manager register --org 11009103 --activationkey b5fd5ca4-06d5-4b45-86e6-e78776aaa8c1 The system has been registered with ID: e20a8db2-6ffe-49b9-b7fa-ff80fbe938e5 The registered system name is: kserve-group-test-g6mk5-build-custom-model-grpc-pod [2026-04-17T07:38:46,944564941+00:00] Add secrets [2026-04-17T07:38:46,955523142+00:00] Run buildah build [2026-04-17T07:38:46,957278887+00:00] buildah build -v /tmp/activation-key:/activation-key -v /shared/rhsm/etc/pki/entitlement:/etc/pki/entitlement:Z -v /shared/rhsm/etc/pki/consumer:/etc/pki/consumer:Z --volume /etc/rhsm/ca/redhat-uep.pem:/etc/rhsm/ca/redhat-uep.pem:Z --security-opt=unmask=/proc/interrupts --ignorefile /tmp/.dockerignore.ogHXHi --label architecture=x86_64 --label vcs-type=git --label vcs-ref=41eaf0be8e98885563de45d52c20f0df0f4c563d --label org.opencontainers.image.revision=41eaf0be8e98885563de45d52c20f0df0f4c563d --label build-date=2026-04-17T07:38:34Z --label org.opencontainers.image.created=2026-04-17T07:38:34Z --label git.url=https://github.com/bartoszmajsak/kserve --label git.commit=41eaf0be8e98885563de45d52c20f0df0f4c563d --annotation org.opencontainers.image.revision=41eaf0be8e98885563de45d52c20f0df0f4c563d --annotation org.opencontainers.image.created=2026-04-17T07:38:34Z --tls-verify=true --no-cache --ulimit nofile=4096:4096 --http-proxy=false -f /tmp/custom_model_grpc.Dockerfile.jlfSZ6 -t quay.io/opendatahub/custom-model-grpc:odh-pr-1374 . [1/2] STEP 1/22: FROM registry.access.redhat.com/ubi9/python-311:9.7 AS builder [1/2] STEP 2/22: WORKDIR / [1/2] STEP 3/22: USER 0 [1/2] STEP 4/22: RUN dnf install -y gcc gcc-c++ make python3.11-devel && dnf clean all Updating Subscription Management repositories. This system has release set to 9.6 and it receives updates only for this release. Red Hat Enterprise Linux 9 for x86_64 - BaseOS 74 MB/s | 87 MB 00:01 Red Hat Enterprise Linux 9 for x86_64 - AppStre 76 MB/s | 73 MB 00:00 Red Hat Universal Base Image 9 (RPMs) - BaseOS 5.8 MB/s | 533 kB 00:00 Red Hat Universal Base Image 9 (RPMs) - AppStre 28 MB/s | 2.6 MB 00:00 Red Hat Universal Base Image 9 (RPMs) - CodeRea 3.7 MB/s | 288 kB 00:00 Package gcc-11.5.0-11.el9.x86_64 is already installed. Package gcc-c++-11.5.0-11.el9.x86_64 is already installed. Package make-1:4.3-8.el9.x86_64 is already installed. Package python3.11-devel-3.11.13-5.2.el9_7.x86_64 is already installed. Dependencies resolved. Nothing to do. Complete! Updating Subscription Management repositories. This system has release set to 9.6 and it receives updates only for this release. 43 files removed [1/2] STEP 5/22: RUN curl -LsSf https://astral.sh/uv/install.sh | sh && ln -s /root/.local/bin/uv /usr/local/bin/uv downloading uv 0.11.7 x86_64-unknown-linux-gnu installing to /opt/app-root/src/.local/bin uv uvx everything's installed! To add $HOME/.local/bin to your PATH, either restart your shell or run: source $HOME/.local/bin/env (sh, bash, zsh) source $HOME/.local/bin/env.fish (fish) [1/2] STEP 6/22: ARG VENV_PATH [1/2] STEP 7/22: ENV VIRTUAL_ENV=${VENV_PATH} [1/2] STEP 8/22: RUN uv venv $VIRTUAL_ENV Using CPython 3.9.25 interpreter at: /usr/bin/python3 Creating virtual environment at: /prod_venv [1/2] STEP 9/22: ENV PATH="$VIRTUAL_ENV/bin:$PATH" [1/2] STEP 10/22: COPY storage/pyproject.toml storage/uv.lock storage/ [1/2] STEP 11/22: COPY kserve/pyproject.toml kserve/uv.lock kserve/ [1/2] STEP 12/22: RUN cd kserve && uv sync --active --no-cache Using CPython 3.11.13 interpreter at: /usr/bin/python3.11 Removed virtual environment at: /prod_venv Creating virtual environment at: /prod_venv Resolved 263 packages in 1ms Building kserve @ file:///kserve Downloading kubernetes (1.9MiB) Downloading pydantic-core (2.0MiB) Downloading cryptography (4.3MiB) Downloading pandas (12.5MiB) Downloading aiohttp (1.7MiB) Downloading grpcio (6.4MiB) Downloading numpy (15.7MiB) Downloading uvloop (3.8MiB) Downloading black (1.6MiB) Downloading grpcio-tools (2.5MiB) Downloading setuptools (1.2MiB) Downloaded aiohttp Downloaded black Downloaded pydantic-core Downloaded grpcio-tools Downloaded setuptools Downloaded uvloop Downloaded grpcio Downloaded cryptography Downloaded kubernetes Downloaded numpy Downloaded pandas Built kserve @ file:///kserve Prepared 74 packages in 1.60s Installed 74 packages in 1.61s + aiohappyeyeballs==2.6.1 + aiohttp==3.13.3 + aiosignal==1.4.0 + annotated-doc==0.0.4 + annotated-types==0.7.0 + anyio==4.9.0 + attrs==25.3.0 + black==24.3.0 + cachetools==5.5.2 + certifi==2025.1.31 + cffi==2.0.0 + charset-normalizer==3.4.1 + click==8.1.8 + cloudevents==1.11.0 + colorama==0.4.6 + cryptography==46.0.5 + deprecation==2.1.0 + durationpy==0.9 + fastapi==0.121.3 + frozenlist==1.5.0 + google-auth==2.39.0 + grpc-interceptor==0.15.4 + grpcio==1.78.1 + grpcio-tools==1.78.1 + h11==0.16.0 + httpcore==1.0.9 + httptools==0.6.4 + httpx==0.27.2 + idna==3.10 + kserve==0.17.0 (from file:///kserve) + kubernetes==32.0.1 + multidict==6.4.3 + mypy-extensions==1.0.0 + numpy==2.2.4 + oauthlib==3.2.2 + orjson==3.10.16 + packaging==24.2 + pandas==2.2.3 + pathspec==0.12.1 + platformdirs==4.3.7 + prometheus-client==0.21.1 + propcache==0.3.1 + protobuf==6.33.5 + psutil==5.9.8 + pyasn1==0.6.1 + pyasn1-modules==0.4.2 + pycparser==2.22 + pydantic==2.12.4 + pydantic-core==2.41.5 + pyjwt==2.12.1 + python-dateutil==2.9.0.post0 + python-dotenv==1.1.0 + python-multipart==0.0.22 + pytz==2025.2 + pyyaml==6.0.2 + requests==2.32.3 + requests-oauthlib==2.0.0 + rsa==4.9.1 + setuptools==78.1.0 + six==1.17.0 + sniffio==1.3.1 + starlette==0.49.1 + tabulate==0.9.0 + timing-asgi==0.3.1 + typing-extensions==4.15.0 + typing-inspection==0.4.2 + tzdata==2025.2 + urllib3==2.6.2 + uvicorn==0.34.1 + uvloop==0.21.0 + watchfiles==1.0.5 + websocket-client==1.8.0 + websockets==15.0.1 + yarl==1.20.0 [1/2] STEP 13/22: COPY kserve kserve [1/2] STEP 14/22: RUN cd kserve && uv sync --active --no-cache Resolved 263 packages in 1ms Building kserve @ file:///kserve Built kserve @ file:///kserve Prepared 1 package in 1.28s Uninstalled 1 package in 0.41ms Installed 1 package in 7ms ~ kserve==0.17.0 (from file:///kserve) [1/2] STEP 15/22: COPY custom_model/pyproject.toml custom_model/uv.lock custom_model/ [1/2] STEP 16/22: RUN cd custom_model && uv sync --active --no-cache Resolved 113 packages in 3.33s Building kserve @ file:///kserve Downloading networkx (2.0MiB) Downloading uvloop (3.6MiB) Downloading aiohttp (1.7MiB) Downloading pillow (4.3MiB) Downloading cryptography (4.3MiB) Downloading sympy (6.0MiB) Downloading pandas (12.2MiB) Downloading numpy (16.1MiB) Downloading grpcio (6.5MiB) Downloading kubernetes (1.9MiB) Downloading py-spy (2.6MiB) Downloading virtualenv (5.6MiB) Downloading ray (69.3MiB) Downloading grpcio-tools (2.5MiB) Downloaded aiohttp Downloaded py-spy Downloaded grpcio-tools Downloaded uvloop Downloaded pillow Downloaded cryptography Downloaded virtualenv Downloaded grpcio Downloaded networkx Downloaded kubernetes Downloaded numpy Downloaded sympy Downloaded pandas Downloaded ray Built kserve @ file:///kserve Prepared 84 packages in 2.00s Uninstalled 53 packages in 293ms Installed 86 packages in 18.06s - aiohttp==3.13.3 + aiohttp==3.13.5 + aiohttp-cors==0.8.1 - anyio==4.9.0 + anyio==4.13.0 - attrs==25.3.0 + attrs==26.1.0 - cachetools==5.5.2 - certifi==2025.1.31 + certifi==2026.2.25 - charset-normalizer==3.4.1 + charset-normalizer==3.4.7 - click==8.1.8 + click==8.3.2 - cloudevents==1.11.0 + cloudevents==1.12.1 + colorful==0.5.8 - cryptography==46.0.5 + cryptography==46.0.7 + distlib==0.4.0 - durationpy==0.9 + durationpy==0.10 - fastapi==0.121.3 + fastapi==0.135.3 + filelock==3.25.2 - frozenlist==1.5.0 + frozenlist==1.8.0 + fsspec==2026.3.0 + google-api-core==2.30.2 - google-auth==2.39.0 + google-auth==2.49.1 + googleapis-common-protos==1.74.0 - grpcio==1.78.1 + grpcio==1.80.0 - grpcio-tools==1.78.1 + grpcio-tools==1.80.0 - httptools==0.6.4 + httptools==0.7.1 - httpx==0.27.2 + httpx==0.28.1 - idna==3.10 + idna==3.11 + importlib-metadata==8.7.1 + jinja2==3.1.6 + jsonschema==4.26.0 + jsonschema-specifications==2025.9.1 ~ kserve==0.17.0 (from file:///kserve) - kubernetes==32.0.1 + kubernetes==35.0.0 + markupsafe==3.0.3 + mpmath==1.3.0 + msgpack==1.1.2 - multidict==6.4.3 + multidict==6.7.1 - mypy-extensions==1.0.0 + mypy-extensions==1.1.0 + networkx==3.6.1 - numpy==2.2.4 + numpy==2.4.4 - oauthlib==3.2.2 + oauthlib==3.3.1 + opencensus==0.11.4 + opencensus-context==0.1.3 + opentelemetry-api==1.40.0 + opentelemetry-exporter-prometheus==0.61b0 + opentelemetry-proto==1.40.0 + opentelemetry-sdk==1.40.0 + opentelemetry-semantic-conventions==0.61b0 - orjson==3.10.16 + orjson==3.11.8 - packaging==24.2 + packaging==26.0 - pandas==2.2.3 + pandas==2.3.3 - pathspec==0.12.1 + pathspec==1.0.4 + pillow==10.4.0 - platformdirs==4.3.7 + platformdirs==4.9.4 - prometheus-client==0.21.1 + prometheus-client==0.24.1 - propcache==0.3.1 + propcache==0.4.1 + proto-plus==1.27.2 - protobuf==6.33.5 + protobuf==6.33.6 + py-spy==0.4.1 - pyasn1==0.6.1 + pyasn1==0.6.3 - pycparser==2.22 + pycparser==3.0 - pydantic==2.12.4 + pydantic==2.12.5 + python-discovery==1.2.2 - python-dotenv==1.1.0 + python-dotenv==1.2.2 - python-multipart==0.0.22 + python-multipart==0.0.24 - pytz==2025.2 + pytz==2026.1.post1 - pyyaml==6.0.2 + pyyaml==6.0.3 + ray==2.54.1 + referencing==0.37.0 - requests==2.32.3 + requests==2.33.1 + rpds-py==0.30.0 - rsa==4.9.1 - setuptools==78.1.0 + setuptools==82.0.1 + smart-open==7.5.1 - sniffio==1.3.1 + sympy==1.14.0 - tabulate==0.9.0 + tabulate==0.10.0 - timing-asgi==0.3.1 + timing-asgi==0.3.2 + torch==2.9.0+cpu (from https://download.pytorch.org/whl/cpu/torch-2.9.0%2Bcpu-cp311-cp311-manylinux_2_28_x86_64.whl) + torchvision==0.24.0+cpu (from https://download.pytorch.org/whl/cpu/torchvision-0.24.0%2Bcpu-cp311-cp311-manylinux_2_28_x86_64.whl) - tzdata==2025.2 + tzdata==2026.1 - urllib3==2.6.2 + urllib3==2.6.3 - uvicorn==0.34.1 + uvicorn==0.44.0 - uvloop==0.21.0 + uvloop==0.22.1 + virtualenv==21.2.0 - watchfiles==1.0.5 + watchfiles==1.1.1 - websocket-client==1.8.0 + websocket-client==1.9.0 - websockets==15.0.1 + websockets==16.0 + wrapt==2.1.2 - yarl==1.20.0 + yarl==1.23.0 + zipp==3.23.0 [1/2] STEP 17/22: COPY custom_model custom_model [1/2] STEP 18/22: RUN cd custom_model && uv sync --active --no-cache Resolved 113 packages in 3.14s Checked 107 packages in 33ms [1/2] STEP 19/22: COPY pyproject.toml pyproject.toml [1/2] STEP 20/22: COPY third_party/pip-licenses.py pip-licenses.py [1/2] STEP 21/22: RUN pip install --no-cache-dir tomli Collecting tomli Downloading tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (10 kB) Downloading tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (243 kB) Installing collected packages: tomli Successfully installed tomli-2.4.1 [notice] A new release of pip is available: 24.2 -> 26.0.1 [notice] To update, run: pip install --upgrade pip [1/2] STEP 22/22: RUN mkdir -p third_party/library && python3 pip-licenses.py 2026-04-17 07:39:54,803 [INFO] {'from': 'mixed', 'format': 'plain-vertical', 'with_license_file': True, 'with_notice_file': True, 'with_urls': True, 'output_path': 'third_party/library', 'ignore_packages': ['nvidia-cublas-cu12', 'nvidia-cuda-cupti-cu12', 'nvidia-cuda-nvrtc-cu12', 'nvidia-cuda-runtime-cu12', 'nvidia-cudnn-cu12', 'nvidia-cufft-cu12', 'nvidia-cufile-cu12', 'nvidia-curand-cu12', 'nvidia-cusolver-cu12', 'nvidia-cusparse-cu12', 'nvidia-nccl-cu12', 'nvidia-nvjitlink-cu12', 'nvidia-nvtx-cu12', 'nvidia-cutlass-dsl', 'nvidia-cutlass-dsl-libs-base', 'nvidia-cusparselt-cu12', 'nvidia-cudnn-frontend', 'nvidia-nvshmem-cu12', 'intel-openmp', 'intel-cmplr-lib-ur', 'umf', 'tcmlib'], 'allow_only': 'Apache Software License;MIT License;BSD License;Mozilla Public License 2.0 (MPL 2.0);Python Software Foundation License;CMU License (MIT-CMU);The Unlicense (Unlicense);ISC License (ISCL);GNU Lesser General Public License v2 (LGPLv2);GNU General Public License (GPL);Historical Permission Notice and Disclaimer (HPND);'} [2/2] STEP 1/17: FROM registry.access.redhat.com/ubi9/python-311:9.7 AS prod [2/2] STEP 2/17: WORKDIR / [2/2] STEP 3/17: ARG VENV_PATH [2/2] STEP 4/17: ENV VIRTUAL_ENV=${VENV_PATH} [2/2] STEP 5/17: ENV PATH="$VIRTUAL_ENV/bin:$PATH" [2/2] STEP 6/17: USER 0 [2/2] STEP 7/17: RUN useradd kserve -m -u 1000 -d /home/kserve [2/2] STEP 8/17: COPY --from=builder --chown=kserve:kserve third_party third_party [2/2] STEP 9/17: COPY --from=builder --chown=kserve:kserve $VIRTUAL_ENV $VIRTUAL_ENV [2/2] STEP 10/17: COPY --from=builder kserve kserve [2/2] STEP 11/17: COPY --from=builder custom_model custom_model [2/2] STEP 12/17: USER 1000 [2/2] STEP 13/17: ENV PYTHONPATH=/custom_model [2/2] STEP 14/17: ENTRYPOINT ["python", "-m", "custom_model.model_grpc"] [2/2] STEP 15/17: COPY labels.json /usr/share/buildinfo/labels.json [2/2] STEP 16/17: COPY labels.json /root/buildinfo/labels.json [2/2] STEP 17/17: LABEL "architecture"="x86_64" "vcs-type"="git" "vcs-ref"="41eaf0be8e98885563de45d52c20f0df0f4c563d" "org.opencontainers.image.revision"="41eaf0be8e98885563de45d52c20f0df0f4c563d" "build-date"="2026-04-17T07:38:34Z" "org.opencontainers.image.created"="2026-04-17T07:38:34Z" "git.url"="https://github.com/bartoszmajsak/kserve" "git.commit"="41eaf0be8e98885563de45d52c20f0df0f4c563d" [2/2] COMMIT quay.io/opendatahub/custom-model-grpc:odh-pr-1374 --> c24ff5eeab2b Successfully tagged quay.io/opendatahub/custom-model-grpc:odh-pr-1374 c24ff5eeab2ba9fb8d9fee564de07f6a0b29558b5522f0ab01a75b2dbc91652e [2026-04-17T07:40:34,566029019+00:00] Unsetting proxy [2026-04-17T07:40:34,567750203+00:00] Add metadata Recording base image digests used registry.access.redhat.com/ubi9/python-311:9.7 registry.access.redhat.com/ubi9/python-311:9.7@sha256:1ee92c29dcc1ce55af43f225a4fc7a025ae6d18ce91522b53e58eb971af95742 registry.access.redhat.com/ubi9/python-311:9.7 registry.access.redhat.com/ubi9/python-311:9.7@sha256:1ee92c29dcc1ce55af43f225a4fc7a025ae6d18ce91522b53e58eb971af95742 Getting image source signatures Copying blob sha256:2861c8d2aef8921470ecd55ecba293b2053f5d79fb517e29236e42c8f0a975b7 Copying blob sha256:8121c751aa96cd9a453045d9d41484ee56bf8dae3575103bcc1806fd92f1db25 Copying blob sha256:2fbe88a1c9840ea6039bffa0fc8f9f89487a3bc18e0e46fba81435e6cfa4ed1a Copying blob sha256:eb14e5ae93ae3f9d30fa532f71a0026f81c4d11fce14d716aa64297272576329 Copying blob sha256:65910b1492eb2097aecfc9a5a677ac6513448e46b46bbdf2e99be528513e9c7c Copying config sha256:c24ff5eeab2ba9fb8d9fee564de07f6a0b29558b5522f0ab01a75b2dbc91652e Writing manifest to image destination [2026-04-17T07:40:43,641008973+00:00] End build Getting image source signatures Copying blob sha256:2861c8d2aef8921470ecd55ecba293b2053f5d79fb517e29236e42c8f0a975b7 Copying blob sha256:eb14e5ae93ae3f9d30fa532f71a0026f81c4d11fce14d716aa64297272576329 Copying blob sha256:2fbe88a1c9840ea6039bffa0fc8f9f89487a3bc18e0e46fba81435e6cfa4ed1a Copying blob sha256:8121c751aa96cd9a453045d9d41484ee56bf8dae3575103bcc1806fd92f1db25 Copying blob sha256:65910b1492eb2097aecfc9a5a677ac6513448e46b46bbdf2e99be528513e9c7c Copying config sha256:c24ff5eeab2ba9fb8d9fee564de07f6a0b29558b5522f0ab01a75b2dbc91652e Writing manifest to image destination [2026-04-17T07:40:52,717135951+00:00] End push remote Unregistering from: subscription.rhsm.redhat.com:443/subscription System has been unregistered. echo "Build on remote host $SSH_HOST finished" Build on remote host localhost finished echo "[$(date --utc -Ins)] Final touches" [2026-04-17T07:40:53,197615492+00:00] Final touches buildah images REPOSITORY TAG IMAGE ID CREATED SIZE quay.io/opendatahub/custom-model-grpc odh-pr-1374 c24ff5eeab2b 35 seconds ago 2.3 GB registry.access.redhat.com/ubi9/python-311 9.7 724f655e28b8 11 hours ago 1.1 GB echo "[$(date --utc -Ins)] End remote" [2026-04-17T07:40:53,226291062+00:00] End remote [2026-04-17T07:40:53,766624780+00:00] Update CA trust INFO: Using mounted CA bundle: /mnt/trusted-ca/ca-bundle.crt '/mnt/trusted-ca/ca-bundle.crt' -> '/etc/pki/ca-trust/source/anchors/ca-bundle.crt' [2026-04-17T07:40:56,446577856+00:00] Convert image [2026-04-17T07:40:56,448020187+00:00] Push image with unique tag Pushing to quay.io/opendatahub/custom-model-grpc:kserve-group-test-g6mk5-build-custom-model-grpc Executing: buildah push --format=docker --retry 3 --tls-verify=true quay.io/opendatahub/custom-model-grpc:odh-pr-1374 docker://quay.io/opendatahub/custom-model-grpc:kserve-group-test-g6mk5-build-custom-model-grpc [2026-04-17T07:41:25,257629018+00:00] Push image with git revision Pushing to quay.io/opendatahub/custom-model-grpc:odh-pr-1374 Executing: buildah push --format=docker --retry 3 --tls-verify=true --digestfile /var/workdir/image-digest quay.io/opendatahub/custom-model-grpc:odh-pr-1374 docker://quay.io/opendatahub/custom-model-grpc:odh-pr-1374 sha256:c8fcacc6c36249d1e4bb251624ab9c0b641c182077b8bd3c9ebac2eb4e09343fquay.io/opendatahub/custom-model-grpc:odh-pr-1374 [2026-04-17T07:41:25,995247870+00:00] End push [2026-04-17T07:41:26,174675347+00:00] Generate SBOM Running syft on the image Running syft on the source code [0000] WARN no explicit name and version provided for directory source, deriving artifact ID from the given path (which is not ideal) [2026-04-17T07:41:57,561862119+00:00] End sbom-syft-generate [2026-04-17T07:41:58,310045140+00:00] Prepare SBOM [2026-04-17T07:41:58,315667867+00:00] Generate SBOM with mobster Skipping SBOM validation 2026-04-17 07:41:59,368 [INFO] mobster.log: Logging level set to 20 2026-04-17 07:42:00,190 [INFO] mobster.oci: Fetching manifest for registry.access.redhat.com/ubi9/python-311@sha256:1ee92c29dcc1ce55af43f225a4fc7a025ae6d18ce91522b53e58eb971af95742 2026-04-17 07:42:00,628 [INFO] mobster.cmd.generate.oci_image.contextual_sbom.contextualize: Contextual workflow will be used. Parent SBOM used for contextualization: https://konflux-ci.dev/spdxdocs/quay.io/redhat-user-workloads/osci-rhel-containers-tenant/rhel-9-7/python-311-9-7@sha256:4f2804444253491092e86e574d2e4fdb944b1f1110b97d9c5501608f64c6cc3e-386f9b38-5fc7-4206-8e2c-6ab54fe1a9ad 2026-04-17 07:42:00,832 [INFO] mobster.cmd.generate.oci_image.contextual_sbom.logging: {"event_type": "contextual_sbom_matching_statistics", "parent_sbom_reference": "https://konflux-ci.dev/spdxdocs/quay.io/redhat-user-workloads/osci-rhel-containers-tenant/rhel-9-7/python-311-9-7@sha256:4f2804444253491092e86e574d2e4fdb944b1f1110b97d9c5501608f64c6cc3e-386f9b38-5fc7-4206-8e2c-6ab54fe1a9ad", "component_sbom_reference": "https://konflux-ci.dev/spdxdocs/quay.io/opendatahub/custom-model-grpc@sha256:c8fcacc6c36249d1e4bb251624ab9c0b641c182077b8bd3c9ebac2eb4e09343f-8c290c2a-9410-46b9-ac60-f13c6588cdd2", "component_packages": {"total": 1322, "matched": 681, "unmatched_all": 641, "unmatched_component_only": 632, "unmatched_without_unique_id": 9}, "parent_packages": {"total": 755, "matched": 750, "unmatched_all": 5, "unmatched_removed_at_build": 5, "unmatched_without_unique_id": 0}, "match_methods": {"by_checksum": 0, "by_verification_code": 461, "by_purl": 220, "total": 681}, "match_origins": {"syft_to_syft": 681, "syft_to_hermeto": 0, "hermeto_to_syft": 0, "hermeto_to_hermeto": 0}, "duplicate_identifiers": {"checksums": {"count": 0, "details": []}, "verification_codes": {"count": 0, "details": []}, "purls": {"count": 0, "details": []}}} 2026-04-17 07:42:00,834 [INFO] mobster.cmd.generate.oci_image: Contextual SBOM workflow finished successfully. 2026-04-17 07:42:00,834 [INFO] mobster.log: Contextual workflow completed in 0.92s 2026-04-17 07:42:01,237 [INFO] mobster.main: Exiting with code 0. [2026-04-17T07:42:01,322440236+00:00] End prepare-sboms