Copying Objects in Batches (SDK for Node.js)

// Import the OBS library.
//Use npm for installation.
const ObsClient = require("esdk-obs-nodejs");
// Use the source code for installation.
// var ObsClient = require('./lib/obs');

// Create an ObsClient instance.
const obsClient = new ObsClient({
  // Obtain an AK and SK pair using environment variables (recommended) or import it in other ways. Using hard coding may result in leakage.
  // Obtain an AK/SK pair on the management console. For details, see https://support.huaweicloud.com/intl/en-us/usermanual-ca/ca_01_0003.html.
  access_key_id: process.env.ACCESS_KEY_ID,
  secret_access_key: process.env.SECRET_ACCESS_KEY,
  // (Optional) If you use a temporary AK/SK pair and a security token to access OBS, you are not advised to use hard coding, which may result in information leakage. You can obtain an AK/SK pair using environment variables or import an AK/SK pair in other ways.
  // security_token: process.env.SECURITY_TOKEN,
  // Enter the endpoint corresponding to the bucket. CN-Hong Kong is used here as an example. Replace it with the one currently in use.
  server: "https://obs.ap-southeast-1.myhuaweicloud.com"
});

/**
 * Copying objects in batches
 * It is applicable to scenarios such as cross-bucket replication, intra-bucket backup, and version management.
 */
async function batchCopyObjects() {
  try {
    const sourceBucket = "source-bucket";
    const destBucket = "dest-bucket";
    const sourcePrefix = "source/";
    const destPrefix = "backup/";
    const params = {
      Bucket: sourceBucket,
      Prefix: sourcePrefix,
      EncodingType: "url",
    };
    // List the source objects to be copied.
    let objectsToCopy = [];
    while (true) {
      const listResult = await obsClient.listObjects(params);
      if (listResult.CommonMsg.Status > 300) {
        console.log("List objects failed: Status=%d, Code=%s, Message=%s",
          listResult.CommonMsg.Status, listResult.CommonMsg.Code, listResult.CommonMsg.Message);
        return;
      }
      if (listResult.InterfaceResult.Contents && listResult.InterfaceResult.Contents.length > 0) {
        objectsToCopy = objectsToCopy.concat(listResult.InterfaceResult.Contents);
      }
      if (listResult.InterfaceResult.IsTruncated === "true") {
        params.Marker = listResult.InterfaceResult.NextMarker;
      } else {
        break;
      }
    }
    console.log(`Found ${objectsToCopy.length} objects, starting batch copy...`);
    // Use Promise.allSettled for parallel replication (limit the number of concurrent requests to avoid resource exhaustion).
    const concurrency = 5; // Number of concurrent requests
    let completed = 0;
    let successCount = 0;
    let failCount = 0;
    for (let i = 0; i < objectsToCopy.length; i += concurrency) {
      const batch = objectsToCopy.slice(i, i + concurrency);
      const copyPromises = batch.map(async (obj) => {
        // Destination object keys: Replace the prefixes.
        let destKey = obj.Key.replace(sourcePrefix, destPrefix);
        const copyParams = {
          Bucket: destBucket,
          Key: destKey,
          CopySource: `${sourceBucket}/${obj.Key}`,
        };
        const result = await obsClient.copyObject(copyParams);
        if (result.CommonMsg.Status <= 300) {
          console.log(`[${++completed}/${objectsToCopy.length}] Object [${obj.Key}] -> [${destKey}] copied successfully`);
          return true;
        } else {
          console.log(`[${++completed}/${objectsToCopy.length}] Object [${obj.Key}] copy failed: ${result.CommonMsg.Message}`);
          return false;
        }
      });
      const results = await Promise.allSettled(copyPromises);
      results.forEach((r) => {
        if (r.status === "fulfilled" && r.value) {
          successCount++;
        } else {
          failCount++;
        }
      });
    }
    console.log(`Batch copy complete: ${successCount} succeeded, ${failCount} failed`);
  } catch (error) {
    console.log("Client exception:");
    console.log(error);
  }
}
batchCopyObjects();