Java

Sample code of multipart upload using Java:

/*
 * Copyright (c) Huawei Technologies Co., Ltd. 2019-2022. All rights reserved.
 */
package com.huaweicloud.vod;

import com.huaweicloud.sdk.core.auth.BasicCredentials;
import com.huaweicloud.sdk.core.auth.ICredential;
import com.huaweicloud.sdk.core.exception.ConnectionException;
import com.huaweicloud.sdk.core.exception.RequestTimeoutException;
import com.huaweicloud.sdk.core.exception.ServiceResponseException;
import com.huaweicloud.sdk.vod.v1.VodClient;
import com.huaweicloud.sdk.vod.v1.model.ConfirmAssetUploadReq;
import com.huaweicloud.sdk.vod.v1.model.ConfirmAssetUploadRequest;
import com.huaweicloud.sdk.vod.v1.model.ConfirmAssetUploadResponse;
import com.huaweicloud.sdk.vod.v1.model.CreateAssetByFileUploadReq;
import com.huaweicloud.sdk.vod.v1.model.CreateAssetByFileUploadRequest;
import com.huaweicloud.sdk.vod.v1.model.CreateAssetByFileUploadResponse;
import com.huaweicloud.sdk.vod.v1.model.ShowAssetTempAuthorityRequest;
import com.huaweicloud.sdk.vod.v1.model.ShowAssetTempAuthorityResponse;
import com.huaweicloud.sdk.vod.v1.region.VodRegion;

import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;

import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils;
import org.dom4j.Document;
import org.dom4j.DocumentException;
import org.dom4j.DocumentHelper;
import org.dom4j.Element;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.security.MessageDigest;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Arrays;
import java.util.List;
import java.util.TimeZone;

/**
 * Example of multipart upload
 */
public class PartUploadDemo {

    // Set the buffer size as needed, that is, the size of the file part read each time.
    public static final int bufferSize = 1024 * 1024; // 1MB

    // Region
    public static final String REGION_NORTH4 = "cn-north-4";
    public static final String REGION_NORTH1 = "cn-north-1";
    public static final String REGION_EAST2 = "cn-east-2";

    // ak/sk
    private static final String AK = "";
    private static final String SK = "";

    public static void main(String[] args) {
        // Path of the local media asset to be uploaded
        String filePath = "";

        PartUploadDemo uploadDemo = new PartUploadDemo();
        // Upload the media asset.
        uploadDemo.uploadPartFile(filePath);
    }

    /**
     * Multipart upload
     * @param filePath: local path where the file to be uploaded is
     */
    public void uploadPartFile(String filePath) {
        // Verify the file and its path.
        File file = validFile(filePath);
        try {
            String fileName = file.getName();

            // An MP4 file is used as an example. For details about other formats, see the official website.
            String fileType = "MP4";
            String fileContentType = "video/mp4";

            // FileInfo fileInfo = new FileInfo(filePath, file);
            // 1. Initialize authentication and obtain vodClient.
            VodClient vodClient = this.createVodClient();
            System.out.println("Start creating the media asset:" + file.getName());
            // 2. Create a VOD media asset.
            CreateAssetByFileUploadReq reqbody = this.buildFileUploadReq(fileName, fileType, null, null);
            CreateAssetByFileUploadResponse assetResponse = this.createAssetByFileUpload(vodClient, reqbody);
            // 3. Obtain authorization for initializing an upload task.
            ShowAssetTempAuthorityResponse initAuthResponse = this.initPartUploadAuthority(vodClient, assetResponse, fileContentType);
            // 4. Initialize the upload task.
            String uploadId = this.initPartUpload(initAuthResponse.getSignStr(), fileContentType);

            // Count the number of file parts.
            int partNumber = 1;
            // Buffer
            byte[] fileByte = new byte[bufferSize];
            // Record the read length.
            int readLength = 0;

            // 7. Read the file content and repeat steps 5 and 6 to upload all parts.
            FileInputStream fis = new FileInputStream(file);
            // MD5
            MessageDigest md = MessageDigest.getInstance("MD5");
            while ((readLength = fis.read(fileByte)) > 0) {
                // If the read length is less than the buffer length, copy the data within the valid length and apply the data to the last part.
                if(readLength < bufferSize) {
                    fileByte = Arrays.copyOf(fileByte, readLength);
                }
                // Perform MD5 encoding and then Base64 encoding.
                byte[] digest = md.digest(fileByte);
                String contentMd5 = new String(Base64.encodeBase64(digest));
                System.out.println("The MD5 value of the "+(partNumber)+" part in the file is " + contentMd5);

                // 5. Obtain authorization for multipart upload.
                ShowAssetTempAuthorityResponse partUploadAuthorityResponse = this.getPartUploadAuthority(vodClient, fileContentType, assetResponse, contentMd5, uploadId, partNumber);

                // 6. Upload parts.
                this.uploadPartFile(partUploadAuthorityResponse.getSignStr(), fileByte, contentMd5);

                // The part number automatically increments by one.
                partNumber++;
            }

            fis.close();

            // 8. Obtain authorization for obtaining uploaded parts.
            ShowAssetTempAuthorityResponse listPartUploadAuthorityResponse = this.listUploadedPartAuthority(vodClient, assetResponse, uploadId);
            // 9. Obtain uploaded parts.
            String partInfo = this.listUploadedPart(listPartUploadAuthorityResponse.getSignStr());
            // 10. Obtain authorization for merging parts.
            ShowAssetTempAuthorityResponse mergePartUploadAuthorityResponse = this.mergeUploadedPartAuthority(vodClient, assetResponse, uploadId);
            // 11. Merge uploaded parts.
            this.mergeUploadedPart(mergePartUploadAuthorityResponse.getSignStr(), partInfo);
            // 12. Confirm media asset upload.
            this.confirmUploaded(vodClient, assetResponse);
            System.out.println("Media asset created. assetId:" + assetResponse.getAssetId());
        } catch (ServiceResponseException e) {
            e.printStackTrace();
            System.out.println(String.valueOf(e.getHttpStatusCode()));
            System.out.println(e.getRequestId());
            System.out.println(e.getErrorCode());
            System.out.println(e.getErrorMsg());
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * Verify the file and its path.
     * @param filePath
     */
    private File validFile(String filePath){
        if (StringUtils.isEmpty(filePath)) {
            throw new RuntimeException("The file path is null.");
        }
        File file = new File(filePath);
        if (!file.exists()) {
            throw new RuntimeException("The file does not exist.");
        } else if (file.isDirectory()) {
            try {
                throw new RuntimeException(file.getCanonicalPath() + "This is a directory.");
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
        return file;
    }


    /**
     * 1. Construct authentication.
     * @return
     */
    public VodClient createVodClient(){
        ICredential auth = new BasicCredentials()
            .withAk(AK)
            .withSk(SK)
            ;
        return VodClient.newBuilder()
            .withCredential(auth)
            .withRegion(VodRegion.valueOf(REGION_NORTH4))
            .build();
    }

    /**
     * 2. Create a media asset.
     * @param client
     * @param reqbody
     * @return
     */
    public CreateAssetByFileUploadResponse createAssetByFileUpload(VodClient client, CreateAssetByFileUploadReq reqbody){
        System.out.println("createAssetByFileUpload start");
        CreateAssetByFileUploadRequest request = new CreateAssetByFileUploadRequest();
        // Configure the X-Sdk-Date parameter, which is mandatory for AK/SK authentication.
        request.setXSdkDate(getXSdkDate());
        // Configure upload parameters.
        request.withBody(reqbody);

        // Call the created media asset.
        CreateAssetByFileUploadResponse response = client.createAssetByFileUpload(request);
        System.out.println("createAssetByFileUpload end; createAssetResponse:" + response.toString());
        return response;
    }

    /**
     * Construct request parameters for media asset creation.
     * @param fileName
     * @param videoName
     * @param title
     * @return
     */
    public CreateAssetByFileUploadReq buildFileUploadReq(String fileName, String videoType, String videoName, String title){
        CreateAssetByFileUploadReq req = new CreateAssetByFileUploadReq();
        req.withVideoName(StringUtils.isNotEmpty(videoName) ? videoName : fileName);
        req.withTitle(StringUtils.isNotEmpty(title) ? title : fileName);
        // Set the media asset type.
        req.withVideoType(videoType);
        return req;
    }

    /**
     * The X-Sdk-Date parameter indicates the current UTC time (format: yyyyHHddTHHmmssZ), for example, 20240312T092514Z.
     * @return
     */
    public String getXSdkDate(){
        TimeZone zone = TimeZone.getTimeZone("UTC");
        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'");
        return LocalDateTime.now(zone.toZoneId()).format(formatter);
    }

    /**
     * 3. Obtain authorization for initializing an upload task.
     * @param client
     * @param assetResponse
     */
    public ShowAssetTempAuthorityResponse initPartUploadAuthority(VodClient client, CreateAssetByFileUploadResponse assetResponse, String fileContentType){
        System.out.println("Obtain authorization for initializing an upload task. initPartUploadAuthority start");
        ShowAssetTempAuthorityRequest request = new ShowAssetTempAuthorityRequest();
        request.setXSdkDate(getXSdkDate());
        // Configure parameters.
        request.withHttpVerb("POST");
        request.withBucket(assetResponse.getTarget().getBucket());
        request.withObjectKey(assetResponse.getTarget().getObject());
        request.withContentType(fileContentType);
        // Send an initialization request.
        ShowAssetTempAuthorityResponse response = client.showAssetTempAuthority(request);
        System.out.println("Obtain authorization for initializing an upload task. initPartUploadAuthority end; response: " + response.toString());
        return response;
    }

    /**
     * 4. Initialize multipart upload.
     * @param signStr
     * @param contentType
     * @return
     */
    public String initPartUpload(String signStr, String contentType) throws DocumentException {
        System.out.println("Initialize multipart upload. initPartUpload start");
        HttpResponse response = HttpRequest.post(signStr).header("Content-type",contentType).execute();
        System.out.println(response.body());
        Document document = DocumentHelper.parseText(response.body());
        Element root = document.getRootElement();
        Element u = root.element("UploadId");
        System.out.println("Initialize multipart upload. initPartUpload end; UploadId:" + u.getText());
        return u.getText();
    }

    /**
     * 5. Obtain authorization for multipart upload.
     * @param client
     * @param fileContentType
     * @param assetResponse
     * @param contentMd5
     * @param uploadId
     * @param partNumber
     * @return
     */
    public ShowAssetTempAuthorityResponse getPartUploadAuthority(VodClient client, String fileContentType, CreateAssetByFileUploadResponse assetResponse, String contentMd5, String uploadId, int partNumber) {
        System.out.println("Obtain authorization for multipart upload. getPartUploadAuthority start; partNumber: " + partNumber);
        ShowAssetTempAuthorityRequest request = new ShowAssetTempAuthorityRequest();
        request.setXSdkDate(getXSdkDate());
        request.withHttpVerb("PUT");
        request.withBucket(assetResponse.getTarget().getBucket());
        request.withObjectKey(assetResponse.getTarget().getObject());
        request.withContentType(fileContentType);
        request.withContentMd5(contentMd5);
        request.withUploadId(uploadId);
        request.withPartNumber(partNumber);
        ShowAssetTempAuthorityResponse response = client.showAssetTempAuthority(request);
        System.out.println("Obtain authorization for multipart upload. getPartUploadAuthority end; partNumber: " + partNumber + "; response" + response.toString());
        return response;
    }

    /**
     * 6. Upload parts.
     * @param signStr
     * @param fileByte
     * @param contentMd5
     */
    public void uploadPartFile(String signStr, byte[] fileByte, String contentMd5) {
        System.out.println("Upload parts. uploadPartFile start");
        HttpResponse response = HttpRequest.put(signStr)
            // The contentMd5 does not need to be escaped.
            .header("Content-MD5", contentMd5)
            .header("Content-Type", "application/octet-stream")
            .body(fileByte).execute();
        System.out.println(response.toString());
        if (response.getStatus() != 200) {
            throw new RuntimeException("Upload parts. uploadPartFile end; upload failed.");
        }
        System.out.println("Upload parts. uploadPartFile end");
    }

    /**
     * 8. Obtain authorization for listing uploaded parts.
     * @param client
     * @param assetResponse
     * @param uploadId
     * @return
     */
    public ShowAssetTempAuthorityResponse listUploadedPartAuthority(VodClient client, CreateAssetByFileUploadResponse assetResponse, String uploadId){
        System.out.println("Obtain authorization for listing uploaded parts. listUploadedPartAuthority start");
        ShowAssetTempAuthorityRequest request = new ShowAssetTempAuthorityRequest();
        request.setXSdkDate(getXSdkDate());
        request.withHttpVerb("GET");
        request.withBucket(assetResponse.getTarget().getBucket());
        request.withObjectKey(assetResponse.getTarget().getObject());
        request.withUploadId(uploadId);
        ShowAssetTempAuthorityResponse response = client.showAssetTempAuthority(request);
        System.out.println("Obtain authorization for listing uploaded parts. listUploadedPartAuthority end; response: " + response.toString());
        return response;
    }

    /**
     * 9. Query uploaded parts.
     * @param signStr
     * @return
     */
    public String listUploadedPart(String signStr) throws DocumentException {
        System.out.println("Query uploaded parts. listUploadedPart start");
        int partNumberMarker = 0;
        Element mergerRoot = DocumentHelper.createElement("CompleteMultipartUpload");
        while(true) {
            // List parts.
            HttpResponse response = HttpRequest.get(signStr + "&part-number-marker=" + partNumberMarker).execute();
            System.out.println("listUploadedPartResponse:" + response.body());
            Document responseDocument = DocumentHelper.parseText(response.body());
            Element rootResponse = responseDocument.getRootElement();
            List<Element> elementsResponse = rootResponse.elements("Part");
            Element partNumberMarkerElement = rootResponse.element("NextPartNumberMarker");
            for (Element e : elementsResponse) {
                Element te = DocumentHelper.createElement("Part");
                te.add(e.element("PartNumber").createCopy());
                te.add(e.element("ETag").createCopy());
                mergerRoot.add(te);
            }
            partNumberMarker = Integer.valueOf(partNumberMarkerElement.getText());
            if(partNumberMarker % 1000 != 0) {
                break;
            }
        }
        System.out.println(mergerRoot.asXML());
        System.out.println("Query uploaded parts. listUploadedPart end");
        return mergerRoot.asXML();
    }

    /**
     * 10. Obtain authorization for merging parts.
     * @param client
     * @param assetResponse
     * @param uploadId
     * @return
     */
    public ShowAssetTempAuthorityResponse mergeUploadedPartAuthority(VodClient client, CreateAssetByFileUploadResponse assetResponse, String uploadId) {
        System.out.println("Obtain authorization for merging parts. mergeUploadedPartAuthority start");
        ShowAssetTempAuthorityRequest request = new ShowAssetTempAuthorityRequest();
        request.setXSdkDate(getXSdkDate());
        request.withHttpVerb("POST");
        request.withBucket(assetResponse.getTarget().getBucket());
        request.withObjectKey(assetResponse.getTarget().getObject());
        request.withUploadId(uploadId);
        ShowAssetTempAuthorityResponse response = client.showAssetTempAuthority(request);
        System.out.println("Obtain authorization for merging parts. mergeUploadedPartAuthority end; response: " + response.toString());
        return response;
    }

    /**
     * 11. Merge parts.
     */
    public void mergeUploadedPart(String signStr, String partInfo) {
        System.out.println("Merge parts. mergeUploadedPart start");
        // Add Content-Type to the request header and set the value to application/xml.
        HttpResponse response = HttpRequest.post(signStr)
            .header("Content-Type","application/xml")
            .body(partInfo)
            .execute();
        System.out.println(response.toString());
        if (response.getStatus() != 200) {
            throw new RuntimeException("Merge parts. mergeUploadedPart end; part merging failed.");
        }
        System.out.println("Merge parts. mergeUploadedPart end");
    }

    /**
     * 12. Confirm the upload completion.
     */
    public void confirmUploaded(VodClient client, CreateAssetByFileUploadResponse assetResponse) {
        System.out.println("Confirm the upload completion. confirmUploaded start");
        ConfirmAssetUploadRequest request = new ConfirmAssetUploadRequest();
        request.setXSdkDate(getXSdkDate());
        ConfirmAssetUploadReq body = new ConfirmAssetUploadReq();
        body.withStatus(ConfirmAssetUploadReq.StatusEnum.fromValue("CREATED"));
        body.withAssetId(assetResponse.getAssetId());
        request.withBody(body);
        ConfirmAssetUploadResponse response = client.confirmAssetUpload(request);
        System.out.println("Uploaded, assetId:" + response.toString());
    }

}