Performing a Multipart Upload

Initiating a Multipart Upload

Before using a multipart upload, you need to first let OBS initiate it. This operation will return a globally unique identifier (upload_id) created by the OBS server to identify the multipart upload. You can use this upload ID to initiate related operations, such as aborting a multipart upload, listing multipart uploads, and listing uploaded parts.

Note that in a multipart upload, the object properties (such as the ACL and expiration time) can only be configured in the initiation phase.

You can use initiate_multi_part_upload to initiate a multipart upload. The following table describes related parameters.

static void test_initiate_multi_part_upload()
{
    obs_status ret_status = OBS_STATUS_BUTT;
    // Create and initialize option.
    obs_options option;
    init_obs_options(&option);
    option.bucket_options.host_name = "<your-endpoint>";
    option.bucket_options.bucket_name = "<Your bucketname>";

    // Hard-coded or plaintext AK/SK are risky. For security purposes, encrypt your AK/SK and store them in the configuration file or environment variables. In this example, the AK/SK are stored in environment variables for identity authentication. Before running this example, configure environment variables ACCESS_KEY_ID and SECRET_ACCESS_KEY.
    // Obtain an AK/SK pair on the management console. For details, see https://support.huaweicloud.com/intl/en-us/usermanual-ca/ca_01_0003.html.
    option.bucket_options.access_key = getenv("ACCESS_KEY_ID");
    option.bucket_options.secret_access_key = getenv("SECRET_ACCESS_KEY");
    // Define the cache and size of multipart upload IDs.
    char upload_id[OBS_COMMON_LEN_256] = {0};
    int upload_id_size = OBS_COMMON_LEN_256;
    // Set response callback function.
    obs_response_handler handler =
    { 
        &response_properties_callback,
        &response_complete_callback 
    };
    // Initiate a multipart upload. Set upload_id below to the value of upload_id_return listed in the parameter table above.
    initiate_multi_part_upload(&option, "<object key>", upload_id_size, upload_id, NULL, NULL, &handler, &ret_status);
    if (OBS_STATUS_OK == ret_status)
    {
        printf("test init upload part successfully. uploadId= %s\n", upload_id);
    }
    else
    {
        printf("test init upload part faied(%s).\n", obs_get_status_name(ret_status));
    }
}

• In obs_put_properties structure, you can specify the MIME type and customized metadata for the object.
• upload_id of the multipart upload returned by initiate_multi_part_upload will be used in follow-up operations.

Uploading Parts

After initializing a multipart upload, you can specify the object name and upload ID to upload a part. Each part has a part number (ranging from 1 to 10000). For parts with the same upload ID, their part numbers are unique and identify their comparative locations in the object. If you use the same part number to upload two parts, the latter one being uploaded will overwrite the former. Except for the part last uploaded whose size ranges from 0 to 5 GB, sizes of the other parts range from 100 KB to 5 GB. Parts are uploaded in random order and can be uploaded through different processes or machines. OBS will combine them into the object based on their part numbers.

You can upload a part by using upload_part. The following table describes the parameters.

Sample code:

static void test_upload_part()
{
    // Create and initialize option.
    obs_options option;
    init_obs_options(&option);
    option.bucket_options.host_name = "<your-endpoint>";
    option.bucket_options.bucket_name = "<Your bucketname>";

// Hard-coded or plaintext AK/SK are risky. For security purposes, encrypt your AK/SK and store them in the configuration file or environment variables. In this example, the AK/SK are stored in environment variables for identity authentication. Before running this example, configure environment variables ACCESS_KEY_ID and SECRET_ACCESS_KEY.
    // Obtain an AK/SK pair on the management console. For details, see https://support.huaweicloud.com/intl/en-us/usermanual-ca/ca_01_0003.html.
    option.bucket_options.access_key = getenv("ACCESS_KEY_ID");
    option.bucket_options.secret_access_key = getenv("SECRET_ACCESS_KEY");
    // Define the slice size by 5 MB.
    uint64_t uploadSliceSize =5L * 1024 * 1024; 
    // Define and initialize the size of the uploading part.
    uint64_t uploadSize = uploadSliceSize; 
    // Define and Initialize the length variable of the uploading part.
    uint64_t filesize = 0;                                          
    // Initialize put_properties.
    obs_put_properties put_properties;
    init_put_properties(&put_properties);
    // Callback function
    obs_upload_handler Handler =
    { 
        {&response_properties_callback, &response_complete_callback},
        &test_upload_file_data_callback
    };   
    // Initialize callback data.
    test_upload_file_callback_data data;
    memset(&data, 0, sizeof(test_upload_file_callback_data));
    filesize = get_file_info(filename,&data);
    data.noStatus = 1;
    data.part_size = uploadSize;
    data.part_num = (filesize % uploadSize == 0) ? (filesize / uploadSize) : (filesize / uploadSize +1);
 
    // Upload the first part.
    uploadPartInfo.part_number=1;
    uploadPartInfo.upload_id = "<upload id>";
    data.start_byte  = 0;
    upload_part(&option,key,&uploadPartInfo,uploadSize,
                                      &put_properties,0,&Handler,&data);
    if (OBS_STATUS_OK == data.ret_status) {
        printf("test upload part 1 successfully. \n");
    }
    else
    {
        printf("test upload part 1 faied(%s).\n", obs_get_status_name(data.ret_status));
    }
    // Upload the second part.
    uploadPartInfo.part_number=2;
    uploadPartInfo.upload_id = "<upload id>";
    filesize = get_file_info(filename,&data);
    uploadSize =filesize - uploadSize;
    data.part_size = uploadSize;
    data.start_byte = uploadSliceSize;
    fseek(data.infile, data.start_byte, SEEK_SET);
    upload_part(&option,key,&uploadPartInfo,uploadSize, &put_properties,0,&Handler,&data);
    if (OBS_STATUS_OK == data.ret_status) {
        printf("test upload part 2 successfully. \n");
    }
    else
    {
        printf("test upload part 2 faied(%s).\n", obs_get_status_name(data.ret_status));
    }

}

• Except the part last uploaded, other parts must be larger than 100 KB. Part sizes will not be verified during upload because which one is last uploaded is not identified until parts are combined.
• OBS will return ETags (MD5 values) of the received parts to users.
• To ensure data integrity, set the value of MD5 and add the MD5 value to the Content-MD5 request header. The OBS server will compare the MD5 value contained by each part and that calculated by SDK to verify the data integrity.
• You can call put_properties.md5 to set the MD5 value of the uploaded data directly.
• Part numbers range from 1 to 10000. If a part number exceeds this range, OBS will return error 400 Bad Request.
• The minimum part size supported by an OBS 3.0 bucket is 100 KB, and the minimum part size supported by an OBS 2.0 bucket is 5 MB. You are advised to perform multipart upload to OBS 3.0 buckets.

Complete Multipart Upload

After all parts are uploaded, call the API for combining parts to form the object. Before this operation, valid part numbers and ETags of all parts must be sent to OBS. After receiving this information, OBS verifies the validity of each part one by one. After all parts pass the verification, OBS combines these parts to form the final object.

You can combine parts by using complete_multi_part_upload. The following table describes the parameters.

Sample code:

static void test_complete_upload(char *filename, char *key)
{
    obs_status ret_status = OBS_STATUS_BUTT;

    // Create and initialize option.
    obs_options option;
    init_obs_options(&option);
    option.bucket_options.host_name = "<your-endpoint>";
    option.bucket_options.bucket_name = "<Your bucketname>";

    //Read the AK/SK from environment variables.
    option.bucket_options.access_key = getenv("ACCESS_KEY_ID");
    option.bucket_options.secret_access_key = getenv("SECRET_ACCESS_KEY");

    // Initialize the put_properties structure.
    obs_put_properties put_properties;
    init_put_properties(&put_properties);
    // Set part information.
    char *uploadId = "<upload id>";
    obs_complete_upload_Info info[2];
    info[0].part_number="1";
    info[0].etag="65fe0e161b35c8deead213871033f7fa";
    info[1].part_number="2";
    info[1].etag="0433d5ffc28450be3b6cf25ab8955267";
     // Set response callback function.
    obs_complete_multi_part_upload_handler Handler =
    { 
        {&response_properties_callback,
         &response_complete_callback},
        &CompleteMultipartUploadCallback
    };
    // Combine parts.
    complete_multi_part_upload(&option,key,uploadId,number,info,&putProperties,
                &Handler, &ret_status);
    if (OBS_STATUS_OK == ret_status) {
        printf("test complete upload successfully. \n");
    }
    else
    {
        printf("test complete upload faied(%s).\n", obs_get_status_name(ret_status));
    }
}

• In the preceding code, the info structure array indicates the list of part numbers and ETags of uploaded parts.
• Part numbers can be inconsecutive.

Concurrently Uploading Parts

Multipart upload is mainly used for large file upload or when the network condition is poor. The following sample code shows how to concurrently upload parts involved in a multipart upload:

static void test_concurrent_upload_part(char *filename, char *key, uint64_t uploadSliceSize)
{
    obs_status ret_status = OBS_STATUS_BUTT;
    // Create and initialize option.
    obs_options option;
    init_obs_options(&option);
    option.bucket_options.host_name = "<your-endpoint>";
    option.bucket_options.bucket_name = "<Your bucketname>";

    //Read the AK/SK from environment variables.
    option.bucket_options.access_key = getenv("ACCESS_KEY_ID");
    option.bucket_options.secret_access_key = getenv("SECRET_ACCESS_KEY");
    char *concurrent_upload_id;
    uint64_t uploadSize = uploadSliceSize;                             
    uint64_t filesize =0;                                          
    // Initialize the put_properties structure.
    obs_put_properties put_properties;
    init_put_properties(&put_properties);
    // Large file information: file pointer, file size, number of parts determined by part size
    test_upload_file_callback_data data;
    memset(&data, 0, sizeof(test_upload_file_callback_data));
    filesize = get_file_info(filename,&data);
    data.noStatus = 1;
    data.part_size = uploadSize;
    data.part_num = (filesize % uploadSize == 0) ? (filesize / uploadSize) : (filesize / uploadSize +1);
    // Initialize the callback function of the uploading part.
    obs_response_handler Handler =
    { 
         &response_properties_callback, &response_complete_callback 
    };
    // Callback function of the combined parts.
    obs_complete_multi_part_upload_handler complete_multi_handler =
    { 
        {&response_properties_callback,
         &response_complete_callback},
        &CompleteMultipartUploadCallback
    };
    // Initialization of the uploading part returns uploadId: uploadIdReturn.
    char uploadIdReturn[256] = {0};
    int upload_id_return_size = 255;
    initiate_multi_part_upload(&option,key,upload_id_return_size,uploadIdReturn, &putProperties,
            0,&Handler, &ret_status);
    if (OBS_STATUS_OK == ret_status) {
        printf("test init upload part return uploadIdReturn(%s). \n", uploadIdReturn);
        strcpy(concurrent_upload_id,uploadIdReturn);
    }
    else
    {
        printf("test init upload part faied(%s).\n", obs_get_status_name(ret_status));
    }
    // Concurrent upload
    test_concurrent_upload_file_callback_data *concurrent_upload_file;
    concurrent_upload_file =(test_concurrent_upload_file_callback_data *)malloc(
                    sizeof(test_concurrent_upload_file_callback_data)*(data.part_num+1));
    if(concurrent_upload_file == NULL)
    {
         printf("malloc test_concurrent_upload_file_callback_data failed!!!");
         return ;
    }
    test_concurrent_upload_file_callback_data *concurrent_upload_file_complete =   
                              concurrent_upload_file;
    start_upload_threads(data, concurrent_upload_id,filesize, key, option, concurrent_upload_file_complete);
     // Combine parts.
    obs_complete_upload_Info *upload_Info = (obs_complete_upload_Info *)malloc(
                sizeof(obs_complete_upload_Info)*data.part_num);
    for(i=0; i<data.part_num; i++)
    {
        upload_Info[i].part_number = concurrent_upload_file_complete[i].part_num;
        upload_Info[i].etag = concurrent_upload_file_complete[i].etag;
    }
    complete_multi_part_upload(&option, key, uploadIdReturn, data.part_num,upload_Info,&putProperties,&complete_multi_handler,&ret_status);
    if (ret_status == OBS_STATUS_OK) {
        printf("test complete upload successfully. \n");
    }
    else
    {
        printf("test complete upload faied(%s).\n", obs_get_status_name(ret_status));
    }
    if(concurrent_upload_file)
    {
        free(concurrent_upload_file);
        concurrent_upload_file = NULL;
    }
    if(upload_Info)
    {
        free(upload_Info);
        upload_Info = NULL;
    }
}