Uploading Objects in Batches (SDK for Node.js)

// Import the OBS library.
//Use npm for installation.
const ObsClient = require("esdk-obs-nodejs");
// Use the source code for installation.
// var ObsClient = require('./lib/obs');

// Create an ObsClient instance.
const obsClient = new ObsClient({
  // Obtain an AK and SK pair using environment variables (recommended) or import it in other ways. Using hard coding may result in leakage.
  // Obtain an AK/SK pair on the management console. For details, see https://support.huaweicloud.com/intl/en-us/usermanual-ca/ca_01_0003.html.
  access_key_id: process.env.ACCESS_KEY_ID,
  secret_access_key: process.env.SECRET_ACCESS_KEY,
  // (Optional) If you use a temporary AK/SK pair and a security token to access OBS, you are not advised to use hard coding, which may result in information leakage. You can obtain an AK/SK pair using environment variables or import an AK/SK pair in other ways.
  // security_token: process.env.SECURITY_TOKEN,
  // Enter the endpoint corresponding to the bucket. CN-Hong Kong is used here as an example. Replace it with the one currently in use.
  server: "https://obs.ap-southeast-1.myhuaweicloud.com"
});

// Local directory to be uploaded
const localUploadDir = "./uploads";
// Target prefix in OBS
const obsPrefix = "uploaded/";
/**
 * Recursively obtaining all files in the directory
 */
function getAllFiles(dir, fileList = []) {
  const files = fs.readdirSync(dir);
  for (const file of files) {
    const filePath = path.join(dir, file);
    const stat = fs.statSync(filePath);
    if (stat.isDirectory()) {
      getAllFiles(filePath, fileList);
    } else {
      fileList.push(filePath);
    }
  }
  return fileList;
}
/**
 * Uploading objects in batches
 */
async function batchUpload() {
  try {
    const bucketName = "examplebucket";
    // Obtain the list of local files to be uploaded.
    if (!fs.existsSync(localUploadDir)) {
      console.log(`Local directory ${localUploadDir} does not exist`);
      return;
    }
    const filesToUpload = getAllFiles(localUploadDir);
    if (filesToUpload.length === 0) {
      console.log("No files found to upload");
      return;
    }
    console.log(`Found ${filesToUpload.length} files, starting batch upload...`);
    // Use Promise.allSettled for parallel uploads (limit the number of concurrent requests to avoid resource exhaustion).
    const concurrency = 5; // Number of concurrent requests
    let completed = 0;
    let successCount = 0;
    let failCount = 0;
    for (let i = 0; i < filesToUpload.length; i += concurrency) {
      const batch = filesToUpload.slice(i, i + concurrency);
      const uploadPromises = batch.map(async (filePath) => {
        // Calculate the object keys in OBS.
        const relativePath = path.relative(localUploadDir, filePath);
        const objectKey = obsPrefix + relativePath.replace(/\\/g, "/");
        const uploadParams = {
          Bucket: bucketName,
          Key: objectKey,
          SourceFile: filePath,
        };
        const result = await obsClient.putObject(uploadParams);
        if (result.CommonMsg.Status <= 300) {
          console.log(`[${++completed}/${filesToUpload.length}] File [${filePath}] uploaded successfully, ETag: ${result.InterfaceResult.ETag}`);
          return true;
        } else {
          console.log(`[${++completed}/${filesToUpload.length}] File [${filePath}] upload failed: ${result.CommonMsg.Message}`);
          return false;
        }
      });
      const results = await Promise.allSettled(uploadPromises);
      results.forEach((r) => {
        if (r.status === "fulfilled" && r.value) {
          successCount++;
        } else {
          failCount++;
        }
      });
    }
    console.log(`Batch upload complete: ${successCount} succeeded, ${failCount} failed`);
  } catch (error) {
    console.log("Client exception:");;
    console.log(error);
  }
}

batchUpload();