Compute
Elastic Cloud Server
Huawei Cloud Flexus
Bare Metal Server
Auto Scaling
Image Management Service
Dedicated Host
FunctionGraph
Cloud Phone Host
Huawei Cloud EulerOS
Networking
Virtual Private Cloud
Elastic IP
Elastic Load Balance
NAT Gateway
Direct Connect
Virtual Private Network
VPC Endpoint
Cloud Connect
Enterprise Router
Enterprise Switch
Global Accelerator
Management & Governance
Cloud Eye
Identity and Access Management
Cloud Trace Service
Resource Formation Service
Tag Management Service
Log Tank Service
Config
OneAccess
Resource Access Manager
Simple Message Notification
Application Performance Management
Application Operations Management
Organizations
Optimization Advisor
IAM Identity Center
Cloud Operations Center
Resource Governance Center
Migration
Server Migration Service
Object Storage Migration Service
Cloud Data Migration
Migration Center
Cloud Ecosystem
KooGallery
Partner Center
User Support
My Account
Billing Center
Cost Center
Resource Center
Enterprise Management
Service Tickets
HUAWEI CLOUD (International) FAQs
ICP Filing
Support Plans
My Credentials
Customer Operation Capabilities
Partner Support Plans
Professional Services
Analytics
MapReduce Service
Data Lake Insight
CloudTable Service
Cloud Search Service
Data Lake Visualization
Data Ingestion Service
GaussDB(DWS)
DataArts Studio
Data Lake Factory
DataArts Lake Formation
IoT
IoT Device Access
Others
Product Pricing Details
System Permissions
Console Quick Start
Common FAQs
Instructions for Associating with a HUAWEI CLOUD Partner
Message Center
Security & Compliance
Security Technologies and Applications
Web Application Firewall
Host Security Service
Cloud Firewall
SecMaster
Anti-DDoS Service
Data Encryption Workshop
Database Security Service
Cloud Bastion Host
Data Security Center
Cloud Certificate Manager
Edge Security
Managed Threat Detection
Blockchain
Blockchain Service
Web3 Node Engine Service
Media Services
Media Processing Center
Video On Demand
Live
SparkRTC
MetaStudio
Storage
Object Storage Service
Elastic Volume Service
Cloud Backup and Recovery
Storage Disaster Recovery Service
Scalable File Service Turbo
Scalable File Service
Volume Backup Service
Cloud Server Backup Service
Data Express Service
Dedicated Distributed Storage Service
Containers
Cloud Container Engine
SoftWare Repository for Container
Application Service Mesh
Ubiquitous Cloud Native Service
Cloud Container Instance
Databases
Relational Database Service
Document Database Service
Data Admin Service
Data Replication Service
GeminiDB
GaussDB
Distributed Database Middleware
Database and Application Migration UGO
TaurusDB
Middleware
Distributed Cache Service
API Gateway
Distributed Message Service for Kafka
Distributed Message Service for RabbitMQ
Distributed Message Service for RocketMQ
Cloud Service Engine
Multi-Site High Availability Service
EventGrid
Dedicated Cloud
Dedicated Computing Cluster
Business Applications
Workspace
ROMA Connect
Message & SMS
Domain Name Service
Edge Data Center Management
Meeting
AI
Face Recognition Service
Graph Engine Service
Content Moderation
Image Recognition
Optical Character Recognition
ModelArts
ImageSearch
Conversational Bot Service
Speech Interaction Service
Huawei HiLens
Video Intelligent Analysis Service
Developer Tools
SDK Developer Guide
API Request Signing Guide
Terraform
Koo Command Line Interface
Content Delivery & Edge Computing
Content Delivery Network
Intelligent EdgeFabric
CloudPond
Intelligent EdgeCloud
Solutions
SAP Cloud
High Performance Computing
Developer Services
ServiceStage
CodeArts
CodeArts PerfTest
CodeArts Req
CodeArts Pipeline
CodeArts Build
CodeArts Deploy
CodeArts Artifact
CodeArts TestPlan
CodeArts Check
CodeArts Repo
Cloud Application Engine
MacroVerse aPaaS
KooMessage
KooPhone
KooDrive
On this page

Show all

Go

Updated on 2024-12-04 GMT+08:00

Sample code of multipart upload using Go:

package main

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/xml"
	"errors"
	"fmt"
	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/basic"
	vod "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/vod/v1"
	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/vod/v1/model"
	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/vod/v1/region"
	"io"
	"net/http"
	"os"
)

// Region
const regionNorth4 = "cn-north-4"
const regionNorth1 = "cn-north-1"
const regionEast2 = "cn-east-2"

// ak/sk
const ak = ""
const sk = ""

// Set the buffer size as needed, that is, the size of the file part read each time.
// 1M
const bufferSize = 1024 * 1024

/*
Example of multipart upload
*/
func main() {
	// Path of the local media asset to be uploaded
	filePath := ""

	// Upload the media asset.
	partUpload(filePath)
}

/*
Multipart upload
filePath: local path where the file to be uploaded is
*/
func partUpload(filePath string) {
	// Verify the file and its path.
	fileInfo := validFile(filePath)
	fileName := fileInfo.Name()
	fmt.Println(fileName)

	file, err := os.Open(filePath)
	if err != nil {
		fmt.Println("Error:", err)
		return
	}
	defer func(file *os.File) {
		err := file.Close()
		if err != nil {
			panic(err)
		}
	}(file)

	// An MP4 file is used as an example. For details about other formats, see the official website.
	fileType := "MP4"
	fileContentType := "video/mp4"

	// 1. Initialize authentication and obtain vodClient.
	client := createVodClient()
	// 2. Create a VOD media asset.
	fmt.Println("Start creating a media file: " + fileName)
	assetResponse := createAsset(client, fileName, fileName, fileType)

	// 3. Obtain authorization for initializing an upload task.
	initAuthResponse := initPartUploadAuthority(client, assetResponse, fileContentType)

	// 4. Initialize the upload task.
	uploadId := initPartUpload(initAuthResponse.SignStr, fileContentType)

	// Count the number of file parts.
	partNumber := 1

	// 7. Read the file content and repeat steps 5 and 6 to upload all parts.
	for {
		buf := make([]byte, bufferSize)
		n, err := file.Read(buf)
		if n == 0 || err != nil || err == io.EOF {
			break
		}
		// Perform MD5 encoding and then Base64 encoding.
		h := md5.New()
		h.Write(buf[:n])
		data := h.Sum(nil)
		contentMd5 := base64.StdEncoding.EncodeToString(data)
		fmt.Println("The MD5 value of the ", partNumber, " part of the file is :", contentMd5)

		// 5. Obtain authorization for multipart upload.
		partUploadAuthorityResponse := getPartUploadAuthority(client, fileContentType, assetResponse,
			contentMd5, uploadId, partNumber)

		// 6. Upload parts.
		uploadPartFile(partUploadAuthorityResponse.SignStr, buf[:n], contentMd5)

		// The part number automatically increments by one.
		partNumber++
	}

	// 8. Obtain authorization for obtaining uploaded parts.
	listPartUploadAuthorityResponse := listUploadedPartAuthority(client, assetResponse, uploadId)

	// 9. Obtain uploaded parts.
	partInfo := listUploadedPart(listPartUploadAuthorityResponse.SignStr)

	// 10. Obtain authorization for merging parts.
	mergePartUploadAuthorityResponse := mergeUploadedPartAuthority(client, assetResponse, uploadId)

	// 11. Merge uploaded parts.
	mergeUploadedPart(mergePartUploadAuthorityResponse.SignStr, partInfo)

	// 12. Confirm media asset upload.
	confirmUploaded(client, assetResponse)

	fmt.Println("Media asset created. assetId:", *assetResponse.AssetId)
}

/*
Verify the file and its path.
filePath: file path
*/
func validFile(filePath string) os.FileInfo {
	// Check whether the file exists.
	fileInfo, err := os.Stat(filePath)
	if os.IsNotExist(err) {
		fmt.Println("The file does not exist.")
		// Throw an error.
		panic(errors.New("The file does not exist."))
	}
	if err != nil {
		fmt.Println("Error:", err)
		panic(err)
	}
	return fileInfo
}

/*
1. Construct authentication.
*/
func createVodClient() *vod.VodClient {
	auth, _ := basic.NewCredentialsBuilder().
		WithAk(ak).
		WithSk(sk).
		SafeBuild()
	reg, _ := region.SafeValueOf(regionNorth4)

	build, _ := vod.VodClientBuilder().
		WithRegion(reg).
		WithCredential(auth).
		SafeBuild()

	client := vod.NewVodClient(build)
	return client
}

/*
2. Create a media asset.
videoName: name
title: title
videoType: file type
*/
func createAsset(client *vod.VodClient, videoName string, title string,
	videoType string) *model.CreateAssetByFileUploadResponse {
	request := &model.CreateAssetByFileUploadRequest{}
	request.Body = &model.CreateAssetByFileUploadReq{
		VideoName: videoName,
		Title:     title,
		VideoType: videoType,
	}
	response, err := client.CreateAssetByFileUpload(request)
	if err == nil {
		fmt.Println(response)
	} else {
		fmt.Println(err)
		panic(err)
	}
	return response
}

/*
3. Obtain authorization for initializing an upload task.
client
assetResponse
*/
func initPartUploadAuthority(client *vod.VodClient, assetResponse *model.CreateAssetByFileUploadResponse,
	fileContentType string) *model.ShowAssetTempAuthorityResponse {
	fmt.Println("Obtain authorization for initializing an upload task. initPartUploadAuthority start")
	// Configure parameters.
	request := &model.ShowAssetTempAuthorityRequest{}
	request.HttpVerb = "POST"
	request.Bucket = assetResponse.Target.Bucket
	request.ObjectKey = assetResponse.Target.Object
	request.ContentType = &fileContentType
	// Send an initialization request.
	response, err := client.ShowAssetTempAuthority(request)
	if err == nil {
		fmt.Print(response)
	} else {
		fmt.Println(err)
		panic(err)
	}
	fmt.Println("Obtain authorization for initializing an upload task. initPartUploadAuthority end\n")
	return response
}

/*
4. Initialize multipart upload.
signStr: signed URL for initialization returned in step 3
contentType: media file format
return uploadId
*/
func initPartUpload(signStr *string, contentType string) string {
	fmt.Println("Initialize multipart upload. initPartUpload start")
	// Create an HTTP client.
	client := &http.Client{}
	// Create a request.
	req, err := http.NewRequest("POST", *signStr, nil)

	req.Header.Set("Content-Type", contentType)
	// Send a request.
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("Error sending request:", err)
		panic(err)
	}
	defer func(Body io.ReadCloser) {
		err := Body.Close()
		if err != nil {
			panic(err)
		}
	}(resp.Body)

	// Read the response body.
	body, _ := io.ReadAll(resp.Body)
	// Print the response body.
	fmt.Println(string(body))

	// Parse the response body. The returned result is XML text, which is parsed based on the format.
	var initiateMultipartUploadResult InitiateMultipartUploadResult
	err = xml.Unmarshal(body, &initiateMultipartUploadResult)
	if err != nil {
		panic(err)
	}
	fmt.Printf("Bucket:%s, Key:%s, UploadId:%s\n", initiateMultipartUploadResult.Bucket,
		initiateMultipartUploadResult.Key, initiateMultipartUploadResult.UploadId)
	fmt.Println("Initialize multipart upload. initPartUpload end\n")
	return initiateMultipartUploadResult.UploadId
}

/*
5. Obtain authorization for multipart upload.
client
fileContentType: media file format
assetResponse: response to media asset creation
contentMd5: contentMd5 of the current part
uploadId
partNumber: part number
return: authorization result
*/
func getPartUploadAuthority(client *vod.VodClient, fileContentType string, assetResponse *model.CreateAssetByFileUploadResponse,
	contentMd5 string, uploadId string, partNumber int) *model.ShowAssetTempAuthorityResponse {
	fmt.Println("Obtain authorization for multipart upload. getPartUploadAuthority start; partNumber:", partNumber)
	// Configure parameters.
	request := &model.ShowAssetTempAuthorityRequest{}
	request.HttpVerb = "PUT"
	request.Bucket = assetResponse.Target.Bucket
	request.ObjectKey = assetResponse.Target.Object
	request.ContentType = &fileContentType
	request.ContentMd5 = &contentMd5
	request.UploadId = &uploadId
	partNumberRequest := int32(partNumber)
	request.PartNumber = &partNumberRequest
	// Send a request.
	response, err := client.ShowAssetTempAuthority(request)
	if err == nil {
		fmt.Print(response)
	} else {
		fmt.Println(err)
		panic(err)
	}
	fmt.Println("Obtain authorization for multipart upload. getPartUploadAuthority end; partNumber:", partNumber, "\n")
	return response
}

/*
6. Upload parts.
signStr: signed URL for upload returned in step 5
fileByte: data of the current part
contentMd5: contentMd5 of the current part
*/
func uploadPartFile(signStr *string, fileByte []byte, contentMd5 string) {
	fmt.Print("Upload parts. uploadPartFile start")

	// Create an HTTP client.
	client := &http.Client{}
	// Create a request.
	req, err := http.NewRequest("PUT", *signStr, bytes.NewBuffer(fileByte))
	req.Header.Set("Content-MD5", contentMd5)
	req.Header.Set("Content-Type", "application/octet-stream")

	// Send a request.
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("Error sending request:", err)
		panic(err)
	}
	defer func(Body io.ReadCloser) {
		err := Body.Close()
		if err != nil {
			panic(err)
		}
	}(resp.Body)
	// Read the response body.
	body, _ := io.ReadAll(resp.Body)
	// Print the response body.
	fmt.Println(string(body))
	if resp.StatusCode != 200 {
		panic("File upload failed.")
	}
	fmt.Println("Upload parts. uploadPartFile end \n")
}

/*
8. Obtain authorization for listing uploaded parts.
client
assetResponse: response to media asset creation
uploadId
return
*/
func listUploadedPartAuthority(client *vod.VodClient, assetResponse *model.CreateAssetByFileUploadResponse,
	uploadId string) *model.ShowAssetTempAuthorityResponse {
	fmt.Println("Obtain authorization for listing uploaded parts. listUploadedPartAuthority start")
	// Configure parameters.
	request := &model.ShowAssetTempAuthorityRequest{}
	request.HttpVerb = "GET"
	request.Bucket = assetResponse.Target.Bucket
	request.ObjectKey = assetResponse.Target.Object
	request.UploadId = &uploadId
	// Send a request.
	response, err := client.ShowAssetTempAuthority(request)
	if err == nil {
		fmt.Print(response)
	} else {
		fmt.Println(err)
		panic(err)
	}
	fmt.Println("Obtain authorization for listing uploaded parts. listUploadedPartAuthority end\n")
	return response
}

/*
9. Query uploaded parts.
signStr: signed URL for query returned in step 8
return
*/
func listUploadedPart(signStr *string) string {
	fmt.Println("Query uploaded parts. listUploadedPart start")
	// Query the start number of file parts.
	partNumberMarker := 0
	// Parameter (XML) for assembling merged parts
	result := "<CompleteMultipartUpload>"
	// Create an HTTP client.
	client := &http.Client{}
	for {
		// Create a request.
		url := *signStr + "&part-number-marker=" + fmt.Sprintf("%d", partNumberMarker)
		req, _ := http.NewRequest("GET", url, nil)
		// Send a request.
		resp, _ := client.Do(req)
		// Read the response body.
		body, _ := io.ReadAll(resp.Body)
		// Print the response body.
		fmt.Println(string(body))
		// Parse and convert the response result to XML text.
		var listPartsResult ListPartsResult
		_ = xml.Unmarshal(body, &listPartsResult)
		// parts is not displayed in the response result.
		if len(listPartsResult.Parts) < 1 {
			break
		}
		// Loop parts to assemble Part data.
		for _, part := range listPartsResult.Parts {
			num := part.PartNumber
			tag := part.Etag
			result += "<Part>" +
				"<PartNumber>" +
				fmt.Sprintf("%d", num) +
				"</PartNumber>" +
				"<ETag>" +
				tag +
				"</ETag>" +
				"</Part>"
		}
		// Set the next part number in the response to the start part number and send the request again.
		partNumberMarker = listPartsResult.NextPartNumberMarker
		if partNumberMarker%1000 != 0 {
			break
		}
	}
	result += "</CompleteMultipartUpload>"
	fmt.Println(result)
	fmt.Println("Query uploaded parts. listUploadedPart end\n")
	return result
}

/*
10. Obtain authorization for merging parts.
client
assetResponse
uploadId
return
*/
func mergeUploadedPartAuthority(client *vod.VodClient, assetResponse *model.CreateAssetByFileUploadResponse,
	uploadId string) *model.ShowAssetTempAuthorityResponse {
	fmt.Println("Obtain authorization for merging parts. mergeUploadedPartAuthority start")
	// Configure parameters.
	request := &model.ShowAssetTempAuthorityRequest{}
	request.HttpVerb = "POST"
	request.Bucket = assetResponse.Target.Bucket
	request.ObjectKey = assetResponse.Target.Object
	request.UploadId = &uploadId
	// Send a request.
	response, err := client.ShowAssetTempAuthority(request)
	if err == nil {
		fmt.Print(response)
	} else {
		fmt.Println(err)
		panic(err)
	}
	fmt.Println("Obtain authorization for merging parts. mergeUploadedPartAuthority end\n")
	return response
}

/*
11. Merge parts.
signStr: signed URL for merging parts returned in step 10
partInfo: data of the parts to be merged
*/
func mergeUploadedPart(signStr *string, partInfo string) {
	fmt.Println("Merge parts. mergeUploadedPart start")
	// Create an HTTP client.
	client := &http.Client{}
	// Create a request.
	//req, err := http.NewRequest("POST", *signStr, bytes.NewBuffer([]byte(partInfo)))
	req, err := http.NewRequest("POST", *signStr, bytes.NewBufferString(partInfo))
	// Add "Content-Type":"application/xml" to the request header.
	req.Header.Set("Content-Type", "application/xml")
	// Send a request.
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("Error sending request:", err)
		panic(err)
	}
	defer func(Body io.ReadCloser) {
		err := Body.Close()
		if err != nil {
			panic(err)
		}
	}(resp.Body)
	// Read the response body.
	body, _ := io.ReadAll(resp.Body)
	// Print the response body.
	fmt.Println(string(body))
	fmt.Println("Merge parts. mergeUploadedPart end\n")
}

/*
12. Confirm the upload completion.
*/
func confirmUploaded(client *vod.VodClient, assetResponse *model.CreateAssetByFileUploadResponse) {
	fmt.Println("Confirm the upload completion. confirmUploaded start")
	// Configure request parameters.
	request := &model.ConfirmAssetUploadRequest{}
	request.Body = &model.ConfirmAssetUploadReq{
		Status:  model.GetConfirmAssetUploadReqStatusEnum().CREATED,
		AssetId: *assetResponse.AssetId,
	}
	// Send a request.
	response, err := client.ConfirmAssetUpload(request)
	if err == nil {
		fmt.Print(response)
	} else {
		fmt.Println(err)
		panic(err)
	}
	fmt.Println("Uploaded, assetId:", *response.AssetId)
}

// InitiateMultipartUploadResult: XML structure returned after multipart upload initialization
type InitiateMultipartUploadResult struct {
	Bucket   string `xml:"Bucket"`
	Key      string `xml:"Key"`
	UploadId string `xml:"UploadId"`
}

// ListPartsResult: XML structure returned after querying uploaded parts
type ListPartsResult struct {
	Bucket               string `xml:"Bucket"`
	Key                  string `xml:"Key"`
	UploadId             string `xml:"UploadId"`
	NextPartNumberMarker int    `xml:"NextPartNumberMarker"`
	Parts                []Part `xml:"Part"`
}

// Part: part structure of the returned xml listPartsResult after querying uploaded parts
type Part struct {
	Etag       string `xml:"ETag"`
	PartNumber int    `xml:"PartNumber"`
}

We use cookies to improve our site and your experience. By continuing to browse our site you accept our cookie policy. Find out more

Feedback

Feedback

Feedback

0/500

Selected Content

Submit selected content with the feedback