Downloading Objects in Batches (SDK for Node.js)

// Import the OBS library.
//Use npm for installation.
const ObsClient = require("esdk-obs-nodejs");
// Use the source code for installation.
// var ObsClient = require('./lib/obs');

// Create an ObsClient instance.
const obsClient = new ObsClient({
  // Obtain an AK and SK pair using environment variables (recommended) or import it in other ways. Using hard coding may result in leakage.
  // Obtain an AK/SK pair on the management console. For details, see https://support.huaweicloud.com/intl/en-us/usermanual-ca/ca_01_0003.html.
  access_key_id: process.env.ACCESS_KEY_ID,
  secret_access_key: process.env.SECRET_ACCESS_KEY,
  // (Optional) If you use a temporary AK/SK pair and a security token to access OBS, you are not advised to use hard coding, which may result in information leakage. You can obtain an AK/SK pair using environment variables or import an AK/SK pair in other ways.
  // security_token: process.env.SECURITY_TOKEN,
  // Enter the endpoint corresponding to the bucket. CN-Hong Kong is used here as an example. Replace it with the one currently in use.
  server: "https://obs.ap-southeast-1.myhuaweicloud.com"
});

// Local storage directory
const localSaveDir = "./downloads";
/**
 * Downloading objects in batches
 * List objects and then download them one by one.
 */
async function batchDownload() {
  try {
    const bucketName = "examplebucket";
    const prefix = "test/";
    const params = {
      Bucket: bucketName,
      Prefix: prefix,
      EncodingType: "url",
    };
    // Ensure that the local storage directory exists.
    if (!fs.existsSync(localSaveDir)) {
      fs.mkdirSync(localSaveDir, { recursive: true });
    }
    // List the objects to be downloaded.
    let objectsToDownload = [];
    while (true) {
      const listResult = await obsClient.listObjects(params);
      if (listResult.CommonMsg.Status > 300) {
        console.log("List objects failed: Status=%d, Code=%s, Message=%s",
          listResult.CommonMsg.Status, listResult.CommonMsg.Code, listResult.CommonMsg.Message);
        return;
      }
      if (listResult.InterfaceResult.Contents && listResult.InterfaceResult.Contents.length > 0) {
        objectsToDownload = objectsToDownload.concat(listResult.InterfaceResult.Contents);
      }
      if (listResult.InterfaceResult.IsTruncated === "true") {
        params.Marker = listResult.InterfaceResult.NextMarker;
      } else {
        break;
      }
    }
    console.log(`Found ${objectsToDownload.length} objects, starting batch download...`);
    // Use Promise.allSettled for parallel downloads (limit the number of concurrent requests to avoid resource exhaustion).
    const concurrency = 5; // Number of concurrent requests
    let completed = 0;
    let successCount = 0;
    let failCount = 0;
    for (let i = 0; i < objectsToDownload.length; i += concurrency) {
      const batch = objectsToDownload.slice(i, i + concurrency);
      const downloadPromises = batch.map(async (obj) => {
        const downloadParams = {
          Bucket: bucketName,
          Key: obj.Key,
          SavePath: path.join(localSaveDir, decodeURIComponent(obj.Key)),
        };
        const result = await obsClient.getObject(downloadParams);
        if (result.CommonMsg.Status <= 300) {
          console.log(`[${++completed}/${objectsToDownload.length}] Object [${obj.Key}] downloaded successfully`);
          return true;
        } else {
          console.log(`[${++completed}/${objectsToDownload.length}] Object [${obj.Key}] download failed:: ${result.CommonMsg.Message}`);
          return false;
        }
      });
      const results = await Promise.allSettled(downloadPromises);
      results.forEach((r) => {
        if (r.status === "fulfilled" && r.value) {
          successCount++;
        } else {
          failCount++;
        }
      });
    }
    console.log(`Batch download complete: ${successCount} succeeded, ${failCount} failed`);
} catch (error) {
    console.log("Client exception:");
    console.log(error);
  }
}
batchDownload();