准备训练脚本

train.sh脚本内容如下，脚本中需要修改的参数请参见脚本中的注释。准备好的train.sh脚本放在自己的OBS桶内的相应位置。

#!/bin/bash
set -x
set -o pipefail
export HCCL_IF_BASE_PORT=61000
export HCCL_NPU_SOCKET_PORT_RANGE="61000-61050"
export HCCL_CONNECT_TIMEOUT=7200   # 最大超时时间，可以根据实际情况调整
export HCCL_EXEC_TIMEOUT=7200
unset https_proxy http_proxy proxy ASCEND_RT_VISIBLE_DEVICES
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export TOKENIZERS_PARALLELISM=false
export TORCH_NCCL_AVOID_RECORD_STREAMS=1
# 获取本机 IP
get_current_ip() {
    local ip
    ip=$(ip -4 addr | grep -v '127.0.0.1' | grep -oP '(?<=inet\s)\d+(\.\d+){3}' | head -n1 2>/dev/null)
    if [[ -z "$ip" ]]; then
        ip=$(ip -4 addr | grep -v '127.0.0.1' | awk '/inet/ {gsub(/\/.*/,""); print $2}' | head -n1 2>/dev/null)
    fi
    echo "${ip:-}"
}
CURRENT_IP=$(get_current_ip)
export NNODES=${VC_WORKER_NUM:-1}
export MASTER_PORT=${MASTER_PORT:=12345}
export CHECK_LOG_DIR="/home/ma-user/logs"
export CHECK_LOG_PATH="${CHECK_LOG_DIR}/veomni.log" #检查日志路径，无需修改
mkdir -p "$CHECK_LOG_DIR"
# 自动计算 NODE_RANK 和 MASTER_ADDR
if [ "$NNODES" -gt 1 ]; then
    MASTER_ADDR_RAW="${VC_WORKER_HOSTS%%,*}"
    export MASTER_ADDR=$(python3 -c "import socket; print(socket.gethostbyname('${MASTER_ADDR_RAW}'))")
    export NODE_RANK=$(python3 -c "import os, socket; hosts = os.environ.get('VC_WORKER_HOSTS', '').split(','); cur='${CURRENT_IP}'; print(next((i for i, h in enumerate(hosts) if socket.gethostbyname(h.strip()) == cur), 0))")
    echo "[INFO] Multi-Node: Master=$MASTER_ADDR, Rank=$NODE_RANK, Total=$NNODES"
    additional_args="--rdzv_endpoint=${MASTER_ADDR}:${MASTER_PORT}"
else
    export MASTER_ADDR="localhost"
    export NODE_RANK=0
    echo "[INFO] Single-Node Mode"
    additional_args="$additional_args --standalone"
fi
if command -v nvidia-smi &> /dev/null && nvidia-smi --list-gpus &> /dev/null; then
  # GPU
  if [[ -n "${CUDA_VISIBLE_DEVICES}" ]]; then
    NPROC_PER_NODE=${NPROC_PER_NODE:=$(echo "${CUDA_VISIBLE_DEVICES}" | tr ',' '\n' | wc -l)}
  else
    NPROC_PER_NODE=${NPROC_PER_NODE:=$(nvidia-smi --list-gpus | wc -l)}
  fi
  export NCCL_DEBUG=WARN
else
  # NPU
  if [[ -n "${ASCEND_RT_VISIBLE_DEVICES}" ]]; then
    NPROC_PER_NODE=${NPROC_PER_NODE:=$(echo "${ASCEND_RT_VISIBLE_DEVICES}" | tr ',' '\n' | wc -l)}
  else
    NPROC_PER_NODE=${NPROC_PER_NODE:=$(ls -l /dev/davinci* | grep -v "davinci_manager" | wc -l)}
  fi
  # NPU env that may optimize performance
  export PYTORCH_NPU_ALLOC_CONF=${PYTORCH_NPU_ALLOC_CONF:='expandable_segments:True'}
fi
torchrun \
  --nnodes=$NNODES \
  --nproc-per-node=$NPROC_PER_NODE \
  --node-rank=$NODE_RANK \
  $additional_args $@ 2>&1 | tee "$CHECK_LOG_PATH" && \
  echo "Training completes" >> "$CHECK_LOG_PATH"
sleep 60s
tail -n 500 "$CHECK_LOG_PATH" | grep -q -E "Training completes"  || exit 1 #异常状态校验，返回状态码给平台

# 主节点进行hf格式转化
if [ "$NODE_RANK" -eq 0 ]; then
    echo "[Master] Master. Doing Convert ckpt to hf models"
    #此处与平台启动命令的$output_dir/Qwen3-32B-Base-sft-tulu保持一致
    CHECKPOINT_ROOT="$output_dir/Qwen3-32B-Base-sft-tulu/checkpoints"
    #获取global_step最大的一次保存的ckpt结果转化为hf格式
    LATEST_STEP_DIR=$(ls -v "$CHECKPOINT_ROOT" | grep "global_step_" | tail -n 1)
    FULL_LOAD_PATH="${CHECKPOINT_ROOT}/${LATEST_STEP_DIR}"
    echo "[INFO] Detected latest checkpoint: $FULL_LOAD_PATH"
    # 执行ckpt转化为hf格式，model-assets-dir是模型路径，load-dir是训练完保存的dcp路径，save-dir是最后保存的hf路径
    if python /home/ma-user/VeOmni/scripts/merge_dcp_to_hf.py \
        --model-assets-dir "$model_dir/Qwen3-32B" \
        --save-dir "$output_dir/qwen3-32b-saved-hf_ckpt" \
        --load-dir "$FULL_LOAD_PATH"; then
        echo "转换成功"
        echo "All steps completes" >> "$CHECK_LOG_PATH"
    else
        echo "[ERROR] Master. Convert ckpt to hf models FAILED!" >> "$CHECK_LOG_PATH"
        exit 1  # 失败则显式退出
    fi
else
    echo "[Worker $NODE_RANK] Task finished. Waiting for Master to complete conversion..."
    echo "All steps completes" >> "$CHECK_LOG_PATH"
fi
sleep 10s
tail -n 500 "$CHECK_LOG_PATH" | grep -q -E "All steps completes"  && exit 0 || exit 1 #异常状态校验，返回状态码给平台

train.sh脚本内容如下，脚本中需要修改的参数请参见脚本中的注释。其中脚本中的注释仅为说明参数含义，实际使用时请去除，在每行的\后不要留有空格。

#!/bin/bash
set -x
set -o pipefail
export HCCL_IF_BASE_PORT=61000
export HCCL_NPU_SOCKET_PORT_RANGE="61000-61050"
export HCCL_CONNECT_TIMEOUT=7200   # 最大超时时间，可以根据实际情况调整
export HCCL_EXEC_TIMEOUT=7200
unset https_proxy http_proxy proxy ASCEND_RT_VISIBLE_DEVICES
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export TOKENIZERS_PARALLELISM=false
export TORCH_NCCL_AVOID_RECORD_STREAMS=1
# 获取本机 IP
get_current_ip() {
    local ip
    ip=$(ip -4 addr | grep -v '127.0.0.1' | grep -oP '(?<=inet\s)\d+(\.\d+){3}' | head -n1 2>/dev/null)
    if [[ -z "$ip" ]]; then
        ip=$(ip -4 addr | grep -v '127.0.0.1' | awk '/inet/ {gsub(/\/.*/,""); print $2}' | head -n1 2>/dev/null)
    fi
    echo "${ip:-}"
}
CURRENT_IP=$(get_current_ip)
export NNODES=${VC_WORKER_NUM:-1}
export MASTER_PORT=${MASTER_PORT:=12345}
export CHECK_LOG_DIR="/home/ma-user/logs"
export CHECK_LOG_PATH="${CHECK_LOG_DIR}/veomni.log" #检查日志路径，无需修改
mkdir -p "$CHECK_LOG_DIR"
# 自动计算 NODE_RANK 和 MASTER_ADDR
if [ "$NNODES" -gt 1 ]; then
    MASTER_ADDR_RAW="${VC_WORKER_HOSTS%%,*}"
    export MASTER_ADDR=$(python3 -c "import socket; print(socket.gethostbyname('${MASTER_ADDR_RAW}'))")
    export NODE_RANK=$(python3 -c "import os, socket; hosts = os.environ.get('VC_WORKER_HOSTS', '').split(','); cur='${CURRENT_IP}'; print(next((i for i, h in enumerate(hosts) if socket.gethostbyname(h.strip()) == cur), 0))")
    echo "[INFO] Multi-Node: Master=$MASTER_ADDR, Rank=$NODE_RANK, Total=$NNODES"
    additional_args="--rdzv_endpoint=${MASTER_ADDR}:${MASTER_PORT}"
else
    export MASTER_ADDR="localhost"
    export NODE_RANK=0
    echo "[INFO] Single-Node Mode"
    additional_args="$additional_args --standalone"
fi
if command -v nvidia-smi &> /dev/null && nvidia-smi --list-gpus &> /dev/null; then
  # GPU
  if [[ -n "${CUDA_VISIBLE_DEVICES}" ]]; then
    NPROC_PER_NODE=${NPROC_PER_NODE:=$(echo "${CUDA_VISIBLE_DEVICES}" | tr ',' '\n' | wc -l)}
  else
    NPROC_PER_NODE=${NPROC_PER_NODE:=$(nvidia-smi --list-gpus | wc -l)}
  fi
  export NCCL_DEBUG=WARN
else
  # NPU
  if [[ -n "${ASCEND_RT_VISIBLE_DEVICES}" ]]; then
    NPROC_PER_NODE=${NPROC_PER_NODE:=$(echo "${ASCEND_RT_VISIBLE_DEVICES}" | tr ',' '\n' | wc -l)}
  else
    NPROC_PER_NODE=${NPROC_PER_NODE:=$(ls -l /dev/davinci* | grep -v "davinci_manager" | wc -l)}
  fi
  # NPU env that may optimize performance
  export PYTORCH_NPU_ALLOC_CONF=${PYTORCH_NPU_ALLOC_CONF:='expandable_segments:True'}
fi
torchrun \
  --nnodes=$NNODES \
  --nproc-per-node=$NPROC_PER_NODE \
  --node-rank=$NODE_RANK \
  $additional_args $@ 2>&1 | tee "$CHECK_LOG_PATH" && \
  echo "Training completes" >> "$CHECK_LOG_PATH"
sleep 30s
tail -n 500 "$CHECK_LOG_PATH" | grep -q -E "Training completes"  || exit 1 #异常状态校验，返回状态码给平台
# 主节点进行hf格式转化
if [ "$NODE_RANK" -eq 0 ]; then
    echo "[Master] Master. Doing Convert ckpt to hf models"
    # 与平台启动命令的$output_dir/Qwen3-30B-A3B-Base-sft-tulu保持一致
    CHECKPOINT_ROOT="$output_dir/Qwen3-30B-A3B-Base-sft-tulu/checkpoints"
    # 获取global_step最大的一次保存的ckpt结果转化为hf格式
    LATEST_STEP_DIR=$(ls -v "$CHECKPOINT_ROOT" | grep "global_step_" | tail -n 1)
    FULL_LOAD_PATH="${CHECKPOINT_ROOT}/${LATEST_STEP_DIR}"
    echo "[INFO] Detected latest checkpoint: $FULL_LOAD_PATH"
    # 执行ckpt转化为hf格式，其中save-dir为hf格式权重的保存路径，包含模型的其他配置文件
    python /home/ma-user/VeOmni/scripts/merge_dcp_to_hf.py \
    --model-assets-dir "$model_dir/Qwen3-30B-A3B" \
    --save-dir "$output_dir/qwen3-30b-A3B-saved-hf_ckpt" \
    --load-dir "$FULL_LOAD_PATH" 2>&1 | tee "$CHECK_LOG_PATH" && \
    echo "Converting ckpt to hf completes" >> "$CHECK_LOG_PATH"
    sleep 5s
    tail -n 500 "$CHECK_LOG_PATH" | grep -q -E "Converting ckpt to hf completes"  || exit 1 #异常状态校验，返回状态码给平台    
    # 对得到的hf权重拆分MoE专家
    sleep 10s
    python /home/ma-user/VeOmni/scripts/moe_ckpt_merge/moe_spilt.py \
    --merge_hf_path "$output_dir/qwen3-30b-A3B-saved-hf_ckpt" \
    --split_hf_path "$output_dir/qwen3-30b-saved-hf_ckpt_split" 2>&1 | tee "$CHECK_LOG_PATH" && \
    echo "All steps completes" >> "$CHECK_LOG_PATH"
else
    echo "[Worker $NODE_RANK] Task finished. Waiting for Master to complete conversion..."
    echo "All steps completes" >> "$CHECK_LOG_PATH"
fi
sleep 30s
tail -n 500 "$CHECK_LOG_PATH" | grep -q -E "All steps completes"  && exit 0 || exit 1 #异常状态校验，返回状态码给平台

train.sh脚本内容如下，脚本中需要修改的参数请参见脚本中的注释。

#!/bin/bash
set -x
set -o pipefail
export HCCL_IF_BASE_PORT=61000
export HCCL_NPU_SOCKET_PORT_RANGE="61000-61050"
export HCCL_CONNECT_TIMEOUT=7200   # 最大超时时间，可以根据实际情况调整
export HCCL_EXEC_TIMEOUT=7200
unset https_proxy http_proxy proxy ASCEND_RT_VISIBLE_DEVICES
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export TOKENIZERS_PARALLELISM=false
export TORCH_NCCL_AVOID_RECORD_STREAMS=1
# 获取本机 IP
get_current_ip() {
    local ip
    ip=$(ip -4 addr | grep -v '127.0.0.1' | grep -oP '(?<=inet\s)\d+(\.\d+){3}' | head -n1 2>/dev/null)
    if [[ -z "$ip" ]]; then
        ip=$(ip -4 addr | grep -v '127.0.0.1' | awk '/inet/ {gsub(/\/.*/,""); print $2}' | head -n1 2>/dev/null)
    fi
    echo "${ip:-}"
}
CURRENT_IP=$(get_current_ip)
export NNODES=${VC_WORKER_NUM:-1}
export MASTER_PORT=${MASTER_PORT:=12345}
export CHECK_LOG_DIR="/home/ma-user/logs"
export CHECK_LOG_PATH="${CHECK_LOG_DIR}/veomni.log" # 检查日志路径，无需修改
mkdir -p "$CHECK_LOG_DIR"
# 自动计算 NODE_RANK 和 MASTER_ADDR
if [ "$NNODES" -gt 1 ]; then
    MASTER_ADDR_RAW="${VC_WORKER_HOSTS%%,*}"
    export MASTER_ADDR=$(python3 -c "import socket; print(socket.gethostbyname('${MASTER_ADDR_RAW}'))")
    export NODE_RANK=$(python3 -c "import os, socket; hosts = os.environ.get('VC_WORKER_HOSTS', '').split(','); cur='${CURRENT_IP}'; print(next((i for i, h in enumerate(hosts) if socket.gethostbyname(h.strip()) == cur), 0))")
    echo "[INFO] Multi-Node: Master=$MASTER_ADDR, Rank=$NODE_RANK, Total=$NNODES"
    additional_args="--rdzv_endpoint=${MASTER_ADDR}:${MASTER_PORT}"
else
    export MASTER_ADDR="localhost"
    export NODE_RANK=0
    echo "[INFO] Single-Node Mode"
    additional_args="$additional_args --standalone"
fi
if command -v nvidia-smi &> /dev/null && nvidia-smi --list-gpus &> /dev/null; then
  # GPU
  if [[ -n "${CUDA_VISIBLE_DEVICES}" ]]; then
    NPROC_PER_NODE=${NPROC_PER_NODE:=$(echo "${CUDA_VISIBLE_DEVICES}" | tr ',' '\n' | wc -l)}
  else
    NPROC_PER_NODE=${NPROC_PER_NODE:=$(nvidia-smi --list-gpus | wc -l)}
  fi
  export NCCL_DEBUG=WARN
else
  # NPU
  if [[ -n "${ASCEND_RT_VISIBLE_DEVICES}" ]]; then
    NPROC_PER_NODE=${NPROC_PER_NODE:=$(echo "${ASCEND_RT_VISIBLE_DEVICES}" | tr ',' '\n' | wc -l)}
  else
    NPROC_PER_NODE=${NPROC_PER_NODE:=$(ls -l /dev/davinci* | grep -v "davinci_manager" | wc -l)}
  fi
  # NPU env that may optimize performance
  export PYTORCH_NPU_ALLOC_CONF=${PYTORCH_NPU_ALLOC_CONF:='expandable_segments:True'}
fi
torchrun \
  --nnodes=$NNODES \
  --nproc-per-node=$NPROC_PER_NODE \
  --node-rank=$NODE_RANK \
  $additional_args $@ 2>&1 | tee "$CHECK_LOG_PATH" && \
  echo "Training completes" >> "$CHECK_LOG_PATH"
sleep 20s
tail -n 500 "$CHECK_LOG_PATH" | grep -q -E "Training completes"  || exit 1 # 异常状态校验，返回状态码给平台
# 主节点进行hf格式转化
if [ "$NODE_RANK" -eq 0 ]; then
    echo "[Master] Master. Doing Convert ckpt to hf models"
    CHECKPOINT_ROOT="$output_dir/qwen3_vl_dense_sft/checkpoints" # 与平台启动命令的$output_dir/qwen3_vl_dense_sft保持一致
    LATEST_STEP_DIR=$(ls -v "$CHECKPOINT_ROOT" | grep "global_step_" | tail -n 1) # 获取global_step最大的一次保存的ckpt结果转化为hf格式
    FULL_LOAD_PATH="${CHECKPOINT_ROOT}/${LATEST_STEP_DIR}"
    echo "[INFO] Detected latest checkpoint: $FULL_LOAD_PATH"
    # 执行ckpt转化为hf格式，model-assets-dir是config的路径，load-dir是训练完保存的dcp路径，save-dir是最后保存的hf路径
    python /home/ma-user/VeOmni/scripts/merge_dcp_to_hf.py \
    --model-assets-dir "$model_dir/Qwen3-VL-8B" \
    --save-dir "$output_dir/qwen3-vl-8b-saved-hf_ckpt" \
    --load-dir "$FULL_LOAD_PATH" 2>&1 | tee "$CHECK_LOG_PATH" && \
    echo "All steps completes" >> "$CHECK_LOG_PATH"
else
    echo "[Worker $NODE_RANK] Task finished. Waiting for Master to complete conversion..."
    echo "All steps completes" >> "$CHECK_LOG_PATH"
fi
sleep 10s
tail -n 500 "$CHECK_LOG_PATH" | grep -q -E "All steps completes"  && exit 0 || exit 1 # 异常状态校验，返回状态码给平台