Copying Objects in Batches

Sample Code

This example copies objects from an OBS bucket to a destination bucket in batches. You can list objects whose names contain the specified prefix and copy them to the destination bucket in batches based on the specified concurrency. The destination keys are generated by replacing the original prefixes. Cross-bucket replication and intra-bucket backup are supported. This method is suitable for scenarios such as data migration and backup.

<?php
// Import the dependency library.
require 'vendor/autoload.php';
// Import the SDK code library during source code installation.
// require 'obs-autoloader.php';
// Declare the namespace.
use Obs\ObsClient;
// Create an instance of ObsClient.
$obsClient = new ObsClient ( [
      //Obtain an AK/SK pair using environment variables or import the AK/SK pair in other ways. Using hard coding may result in leakage.
      //Obtain an AK/SK pair on the management console. For details, see https://support.huaweicloud.com/intl/en-us/usermanual-ca/ca_01_0003.html.
      'key' => getenv('ACCESS_KEY_ID'),
      'secret' => getenv('SECRET_ACCESS_KEY'),
      // Enter the endpoint corresponding to the bucket. CN-Hong Kong is used here as an example. Replace it with the one currently in use.
       'endpoint' => "obs.ap-southeast-1.myhuaweicloud.com",
      'signature' => 'obs'
] );

$sourceBucket = 'sourcebucketname';
$destBucket = 'destbucketname';
$sourcePrefix = 'data/';
$destPrefix = 'backup/';
$concurrency = 5;

// List objects whose names contain the specified prefix.
$isTruncated = true;
$marker = null;
$objectsToCopy = [];

while ($isTruncated) {
       $resp = $obsClient->listObjects ( [
              'Bucket' => $sourceBucket,
              'Prefix' => $sourcePrefix,
              'Marker' => $marker
       ] );
       if (!empty($resp['Contents'])) {
              foreach ($resp['Contents'] as $content) {
                     // The destination keys are generated by replacing the original prefixes.
                     $destKey = $destPrefix . substr($content['Key'], strlen($sourcePrefix));
                     // Skip the directory objects (destKey is left blank when the value of Key is the same as that of sourcePrefix).
                     if ($destKey === '' || substr($destKey, -1) === '/') {
                            continue;
                     }
                     $objectsToCopy[] = ['SourceKey' => $content['Key'], 'DestKey' => $destKey, 'Size' => $content['Size']];
              }
       }
       $isTruncated = $resp['IsTruncated'];
       $marker = $resp['NextMarker'];
}
printf("Found %d objects to copy\n\n", count($objectsToCopy));

// Copy objects concurrently.
// In the production environment, you are advised to add a retry mechanism to prevent incomplete replication caused by network fluctuation or temporary faults.
// A single copyObject operation can only copy objects with a total size of no more than 5 GB. For objects exceeding 5 GB in total size, use multipart copy (Upload Part Copy).
// The source and destination buckets must be in the same region. For cross-region replication, configure a cross-region replication rule first.
// To copy a specified object version, add the version ID to CopySource in the format of bucket/key?versionId=xxx.
// Before copying an object, ensure that no object with the same name as the destination object exists in the destination bucket. Otherwise, the copyObject operation will overwrite the existing one.

$successCount = 0;
$failCount = 0;

$promiseGenerator = function () use ($obsClient, $sourceBucket, $destBucket, $objectsToCopy, &$successCount, &$failCount) {
       foreach ($objectsToCopy as $object) {
              yield $obsClient->copyObjectAsync ([
                     'Bucket' => $destBucket,
                     'Key' => $object['DestKey'],
                     // If Key in CopySource contains special characters, URL encoding is required.
                     'CopySource' => $sourceBucket . '/' . rawurlencode($object['SourceKey']),
                     // (Optional) Specify the storage class of the destination objects, for example, 'StorageClass' => ObsClient::StorageClassStandard.
              ], function ($exception, $resp) use ($object, &$successCount, &$failCount) {
                     if ($exception === null) {
                            printf ( "CopyTo:%s, RequestId:%s\n", $object['DestKey'], $resp['RequestId'] );
                            $successCount++;
                     } else {
                            printf ( "CopyFailed: Key:%s, Status:%d, Message:%s\n", $object['SourceKey'], $exception->getStatusCode(), $exception->getExceptionMessage() );
                            $failCount++;
                     }
              } );
       }
};

$eachPromise = new EachPromise($promiseGenerator(), [
       'concurrency' => $concurrency
]);

$eachPromise->promise()->wait();

printf("\nResult: success=%d, fail=%d\n", $successCount, $failCount);