Preparing the Training Scripts

The parameters in the sh script are as follows:

• ${output_dir}: output path of training or format conversion. Configure the OBS path for the output_dir environment variable. For example, /home/ma-user/modelarts/outputs/output_dir_0.
• ${model_path}: path of the training model. Set TOKENIZER_PATH based on this parameter.
• ${dataset_path}: path of the dataset to be processed.

The environment variables are set in the run_distributed_task.sh script for starting the training job.

# Change the ascend-toolkit path.
export CUDA_DEVICE_MAX_CONNECTIONS=1
source /usr/local/Ascend/ascend-toolkit/set_env.sh
python /home/ma-user/MA_Turbo/src/open_source/MindSpeed-LLM/convert_ckpt.py \ # Location of the MindSpeed-LLM convert code pre-installed in the image. No modification is required.
    --use-mcore-models \
    --model-type GPT \
    --load-model-type hf \
    --save-model-type mg \
    --target-tensor-parallel-size 8 \
    --target-pipeline-parallel-size 2 \
    --spec mindspeed_llm.tasks.models.spec.qwen3_spec layer_spec \
    --load-dir "$TOKENIZER_PATH" \                                   # Model location, which is set in run_distributed_task.sh.
    --save-dir "$CKPT_LOAD_DIR" \                   # Location for saving the converted weights, which is set in run_distributed_task.sh.
    --tokenizer-model "$TOKENIZER_PATH/tokenizer.json"  \             # Location of tokenizer.json. The variable part is set in run_distributed_task.sh.
    --params-dtype bf16 \
    --model-type-hf qwen3

The environment variables are set in the run_distributed_task.sh script for starting the training job.

# Change the set_env.sh path as needed.
source /usr/local/Ascend/ascend-toolkit/set_env.sh
mkdir -p "${output_dir}/finetune_dataset" # Retain the value.
python /home/ma-user/MA_Turbo/src/open_source/MindSpeed-LLM/preprocess_data.py \ # Location of the MindSpeed-LLM convert code pre-installed in the image. No modification is required.
    --input ${dataset_path}/alpaca_gpt4_data.json \                              # Path of the dataset to be processed.
    --tokenizer-name-or-path "$TOKENIZER_PATH" \                          # Path of the model used to process the dataset, which is set in run_distributed_task.sh.
    --output-prefix "$DATA_PATH" \                      # Path and format of the output dataset, which is set in run_distributed_task.sh.
    --handler-name AlpacaStyleInstructionHandler \    # Modify this if the format is changed.
    --tokenizer-type PretrainedFromHF \
    --workers 4 \
    --log-interval 1000 \
    --enable-thinking true \
    --prompt-type qwen3

The content of the tune_qwen3_32b_4K_full_ptd.sh script is as follows. For details about the parameters to be modified in the script, see the comments in the script.

#!/bin/bash
export HCCL_CONNECT_TIMEOUT=7200   # Maximum timeout interval. You can change the value as needed.
export HCCL_EXEC_TIMEOUT=7200
export CUDA_DEVICE_MAX_CONNECTIONS=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
# =========================================================
# 1. Receive external environment variables. (If no external environment variables are transferred, the default values after the colons are used.)
You are advised to set the environment variables in run_distributed_task.sh.
# =========================================================
NPUS_PER_NODE=${NPUS_PER_NODE:-8}
MASTER_ADDR=${MASTER_ADDR:-"localhost"}
MASTER_PORT=${MASTER_PORT:-6000}
NNODES=${NNODES:-1}
NODE_RANK=${NODE_RANK:-0}
WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
# The path configuration is also changed to receive environment variables. You are advised to set the environment variables in run_distributed_task.sh.
CKPT_LOAD_DIR=${CKPT_LOAD_DIR:-"${output_dir}/model_weights/qwen3_mcore/"}    # Default reference value, which is overwritten in run_distributed_task.sh
CKPT_SAVE_DIR=${CKPT_SAVE_DIR:-"${output_dir}/ckpt/qwen3-32b/"}               # Default reference value, which is overwritten in run_distributed_task.sh
DATA_PATH=${DATA_PATH:-"${output_dir}/finetune_dataset/alpaca"}               # Default reference value, which is overwritten in run_distributed_task.sh
TOKENIZER_PATH=${TOKENIZER_PATH:-"${model_path}/Qwen3-32B/"}                  # Default reference value, which is overwritten in run_distributed_task.sh)
# =========================================================
# 2. Training parameters
# =========================================================
TP=8     # The value must be the same as that in ckpt_convert_qwen3_hf2mcore.sh.
PP=2     # The value must be the same as that in ckpt_convert_qwen3_hf2mcore.sh.
MBS=1
GBS=32
TRAIN_ITERS=50
SEQ_LENGTH=4096
DISTRIBUTED_ARGS="
    --nproc_per_node $NPUS_PER_NODE \
    --nnodes $NNODES \
    --node_rank $NODE_RANK \
    --master_addr $MASTER_ADDR \
    --master_port $MASTER_PORT
"
OPTIMIZE_ARGS="
    --use-flash-attn \
    --use-fused-rotary-pos-emb \
    --use-rotary-position-embeddings \
    --use-fused-swiglu \
    --use-fused-rmsnorm \
    --no-masked-softmax-fusion \
    --use-distributed-optimizer
"
TRAIN_ARGS="
    --micro-batch-size 1 \
    --global-batch-size 16 \
    --lr 1.25e-6 \
    --lr-decay-style cosine \
    --min-lr 1.25e-7 \
    --weight-decay 1e-1 \
    --lr-warmup-fraction 0.01 \
    --attention-dropout 0.0 \
    --init-method-std 0.01 \
    --hidden-dropout 0.0 \
    --clip-grad 1.0 \
    --adam-beta1 0.9 \
    --adam-beta2 0.95 \
    --initial-loss-scale 4096 \
    --seed 42 \
    --bf16 \
    --train-iters ${TRAIN_ITERS} \
    --seq-length ${SEQ_LENGTH} \
    --no-shared-storage
"
MODEL_PARALLEL_ARGS="
    --tensor-model-parallel-size ${TP} \
    --pipeline-model-parallel-size ${PP}
"
GPT_ARGS="
    --use-mcore-models \
    --spec mindspeed_llm.tasks.models.spec.qwen3_spec layer_spec \
    --kv-channels 128 \
    --qk-layernorm \
    --tokenizer-name-or-path ${TOKENIZER_PATH} \
    --max-position-embeddings ${SEQ_LENGTH} \
    --num-layers 64 \
    --hidden-size 5120 \
    --ffn-hidden-size 25600 \
    --num-attention-heads 64 \
    --tokenizer-type PretrainedFromHF \
    --make-vocab-size-divisible-by 1 \
    --padded-vocab-size 151936 \
    --rotary-base 1000000 \
    --untie-embeddings-and-output-weights \
    --disable-bias-linear \
    --position-embedding-type rope \
    --normalization RMSNorm \
    --swiglu \
    --attention-softmax-in-fp32 \
    --no-gradient-accumulation-fusion \
    --group-query-attention \
    --num-query-groups 8
"
DATA_ARGS="
    --data-path $DATA_PATH \
    --split 100,0,0
"
OUTPUT_ARGS="
    --load ${CKPT_LOAD_DIR} \
    --save ${CKPT_SAVE_DIR} \
    --log-interval 1 \
    --save-interval ${TRAIN_ITERS} \
    --eval-interval ${TRAIN_ITERS} \
    --eval-iters 0 \
    --no-load-optim \
    --no-load-rng
"
TUNE_ARGS="
    --finetune \
    --stage sft \
    --is-instruction-dataset \
    --tokenizer-not-use-fast \
    --prompt-type qwen3 \
    --variable-seq-lengths
"
# Run torchrun. Note that NODE_RANK is added to the log file name to prevent conflicts.
mkdir -p "$CHECK_LOG_DIR" # Verification log, which does not need to be modified
set -o pipefail             # Enable pipefail.
torchrun $DISTRIBUTED_ARGS /home/ma-user/MA_Turbo/src/open_source/MindSpeed-LLM/posttrain_gpt.py \  # Location of the MindSpeed-LLM startup code pre-installed in the image. No modification is required.
    $GPT_ARGS \
    $DATA_ARGS \
    $OUTPUT_ARGS \
    $OPTIMIZE_ARGS \
    $TRAIN_ARGS \
    $TUNE_ARGS \
    $MODEL_PARALLEL_ARGS \
    --distributed-backend nccl \
    2>&1 | tee "$CHECK_LOG_PATH" && \
    echo "Training completes" >> "$CHECK_LOG_PATH"  # Append a completion flag to the log. This environment variable is centrally configured in run_distributed_task.sh and does not require manual modification.

sleep 10s
tail -n 500 "$CHECK_LOG_PATH" | grep -q -E "Training completes" || exit 1 # Verifies the abnormal status.

The content of the run_distributed_task.sh script is as follows. For details about the parameters to be modified in the script, see the comments in the script.

#!/bin/bash
set -e
# =========================
# 1. Basic environment and network configuration
# =========================
export HCCL_CONNECT_TIMEOUT=7200   # Maximum timeout interval. You can change the value as needed.
export HCCL_EXEC_TIMEOUT=7200
export HCCL_IF_BASE_PORT=61000
export HCCL_NPU_SOCKET_PORT_RANGE="61000-61050"
unset https_proxy http_proxy proxy ASCEND_RT_VISIBLE_DEVICES
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
# Automatically obtain the local IP address.
get_current_ip() {
    local ip
    ip=$(ip -4 addr | grep -v '127.0.0.1' | grep -oP '(?<=inet\s)\d+(\.\d+){3}' | head -n1 2>/dev/null)
    if [[ -z "$ip" ]]; then
        ip=$(ip -4 addr | grep -v '127.0.0.1' | awk '/inet/ {gsub(/\/.*/,""); print $2}' | head -n1 2>/dev/null)
    fi
    echo "${ip:-}"
}
CURRENT_IP=$(get_current_ip)
# Parse ModelArts environment variables to automatically obtain the number of NPUs and nodes.
export NNODES=${VC_WORKER_NUM:-1}
export NPUS_PER_NODE=${MA_NUM_GPUS:-8}
export MASTER_PORT=6000
# Automatically calculate NODE_RANK and MASTER_ADDR.
if [ "$NNODES" -gt 1 ]; then
    MASTER_ADDR_RAW="${VC_WORKER_HOSTS%%,*}"
    export MASTER_ADDR=$(python3 -c "import socket; print(socket.gethostbyname('${MASTER_ADDR_RAW}'))")
    export NODE_RANK=$(python3 -c "import os, socket; hosts = os.environ.get('VC_WORKER_HOSTS', '').split(','); cur='${CURRENT_IP}'; print(next((i for i, h in enumerate(hosts) if socket.gethostbyname(h.strip()) == cur), 0))")
    echo "[INFO] Multi-Node: Master=$MASTER_ADDR, Rank=$NODE_RANK, Total=$NNODES"
else
    export MASTER_ADDR="localhost"
    export NODE_RANK=0
    echo "[INFO] Single-Node Mode"
fi
# =========================
# 2. Define the global path (for subscripts, which will overwrite the corresponding parameters in the fine-tuning script).
# =========================
export CKPT_LOAD_DIR="${output_dir}/model_weights/qwen3_mcore/" # Path of the converted mg format
export CKPT_SAVE_DIR="${output_dir}/ckpt/qwen3-32b/"              # CKPT save path
export DATA_PATH="${output_dir}/finetune_dataset/alpaca"         # Path and format of the processed fine-tuning dataset (alpaca in this example). The path does not end with a slash (/).
export TOKENIZER_PATH="${model_path}/Qwen3-32B/"                 # Model path
export TRAINED_HF_MODELS="${output_dir}/trained_hf_models/"      # Path of the model converted from the mg format to the hf format after training is complete.

export CHECK_LOG_DIR="/home/ma-user/logs"                        # Folder where the verification log is stored in the container. The path in the container is not the mount path and does not need to be modified. The path does not end with a slash (/).
export CHECK_LOG_FILE="tune_qwen3_32b_full_ptd.log"              # Validation log name in the container. You can set the name based on the model and task.
export CHECK_LOG_PATH="${CHECK_LOG_DIR}/${CHECK_LOG_FILE}"       # Full path to the validation log within the container. This is an internal container file and does not require manual modification.

# Ensure that the directory exists.
mkdir -p "$CKPT_SAVE_DIR"
mkdir -p "$CHECK_LOG_DIR" # Validation log.
# =========================
# 3. Weight conversion (hf2mg) and data processing
# =========================
echo " Starting Checkpoint & Data Conversion..."
# Perform weight conversion.
echo " Running ckpt_convert_qwen3_hf2mcore.sh..."
bash ckpt_convert_qwen3_hf2mcore.sh                   # Weight conversion script of the model.
# Perform data processing.
echo " Running data_convert_qwen3_instruction.sh..."
bash data_convert_qwen3_instruction.sh                # Data processing script of the model.
echo " Preparation Done. Creating flag file."


# =========================
# 4. Start distributed training (on all nodes).
# =========================

# Check for the existence of r+ mode (required for OBS mounting)
FILE_PATH="/home/ma-user/MA_Turbo/src/open_source/MindSpeed-LLM/mindspeed_llm/tasks/preprocess/decoder_packed_mtf_dataset.py"
if grep -q "mmap_mode='r+'" "$FILE_PATH"; then
    # Precisely replace only the lines that contain mmap_mode='r+'.
    sed -i.bak "s/\(mmap_mode=['\"]\)\(r+\)\(['\"]\)/\1r\3/g" "$FILE_PATH"
else
    echo "mmap_mode='r+' not detected; operation is skipped."
fi

echo " Starting Training Script on Rank $NODE_RANK..."
# Run the modified training script.
bash tune_qwen3_32b_4K_full_ptd.sh                # Fine-tuning script of the model


# =========================
# 5. Weight conversion (mg2hf) and saving the model
# =========================

# Execute weight conversion (mg2hf) on the master node only.
if [ "$NODE_RANK" -eq 0 ]; then
    echo "[Master] Master. Doing Convert ckpt from mcore to hf models"
    mkdir -p "$TRAINED_HF_MODELS"
    find "$TOKENIZER_PATH" -type f ! -name "*.safetensors" -exec cp {} "$TRAINED_HF_MODELS" \;  # Copies the original model configuration files (e.g., config.json) and tokenizer files (e.g., tokenizer.model, vocab.json)
    bash ckpt_convert_qwen3_mcore2hf.sh                                                         # Executes the mg2hf weight conversion script.
    mv "$TOKENIZER_PATH/mg2hf/"* "$TRAINED_HF_MODELS/"                                          # Moves the converted weights to the corresponding directory in OBS.
    echo "All steps completes" >> "$CHECK_LOG_PATH"
else
    echo "[Worker $NODE_RANK] Task finished. Waiting for Master to complete conversion..."
    echo "All steps completes" >> "$CHECK_LOG_PATH"
fi

sleep 10s
tail -n 500 "$CHECK_LOG_PATH" | grep -q -E "All steps completes"  && exit 0 || exit 1 # Reports abnormal status back to the platform; no modifications to environment variables are needed.