Downloading Objects in Batches

This example downloads objects from an OBS bucket to a local directory in batches. You can list objects whose names contain the specified prefix and download them concurrently based on the specified concurrency. The files are saved to the local directory based on the original key structure. This method is suitable for scenarios such as batch export and offline backup.

<?php
// This example downloads objects from an OBS bucket to a local directory in batches. You can list objects whose names contain the specified prefix and download them concurrently based on the specified concurrency. The files are saved to the local directory based on the original key structure. This method is suitable for scenarios such as batch export and offline backup.
// Import the dependency library.
require 'vendor/autoload.php';
// Import the SDK code library during source code installation.
// require 'obs-autoloader.php';
// Declare the namespace.
use Obs\ObsClient;
// Create an instance of ObsClient.
$obsClient = new ObsClient ( [
      // Obtain an AK/SK pair using environment variables or import the AK/SK pair in other ways. Using hard coding may result in leakage.
      //Obtain an AK/SK pair on the management console. For details, see https://support.huaweicloud.com/intl/en-us/usermanual-ca/ca_01_0003.html.
      'key' => getenv('ACCESS_KEY_ID'),
      'secret' => getenv('SECRET_ACCESS_KEY'),
      // Enter the endpoint corresponding to the bucket. CN-Hong Kong is used here as an example. Replace it with the one currently in use.
       'endpoint' => "obs.ap-southeast-1.myhuaweicloud.com",
      'signature' => 'obs'
] );
$bucketName = 'bucketname';
$localDir = '/tmp/download-dir';
$keyPrefix = 'backup/';

$concurrency = 5;
// List objects whose names contain the specified prefix.
$isTruncated = true;
$marker = null;
$objectsToDownload = [];

while ($isTruncated) {
       $resp = $obsClient->listObjects ( [
              'Bucket' => $bucketName,
              'Prefix' => $keyPrefix,
              'Marker' => $marker,
              // MaxKeys specifies the maximum number of objects returned on each page. The default value is 1000. The value ranges from 1 to 1000.
              'MaxKeys' => 1000                                                                         
        ] );
       if (!empty($resp['Contents'])) {
              foreach ($resp['Contents'] as $content) {
                     $objectsToDownload[] = ['Key' => $content['Key']];
              }
       }
       $isTruncated = $resp['IsTruncated'];
       $marker = $resp['NextMarker'];
}
printf("Found %d objects to download\n\n", count($objectsToDownload));

// Download objects concurrently and save them to the local directory based on the original key structure.
// In the production environment, you are advised to add a retry mechanism to prevent data incompleteness caused by network fluctuation or temporary faults.
$successCount = 0;
$failCount = 0;

$promiseGenerator = function () use ($obsClient, $bucketName, $localDir, $objectsToDownload, &$successCount, &$failCount) {
       foreach ($objectsToDownload as $object) {
              $localPath = $localDir . '/' . $object['Key'];
              // Path traversal protection: Check whether the key contains .. to prevent directory traversal.
              if (strpos($object['Key'], '..') !== false) {
                     printf("SkipUnsafeKey: Key:%s contains path traversal\n", $object['Key']);
                     $failCount++;
                     continue;
              }
              $localDirPath = dirname($localPath);
              if (!is_dir($localDirPath)) {
                     mkdir($localDirPath, 0755, true);
              }
              yield $obsClient->getObjectAsync ( [
                     'Bucket' => $bucketName,
                     'Key' => $object['Key'],
                     'SaveAsFilepath' => $localPath
              ], function ($exception, $resp) use ($object, &$successCount, &$failCount) {
                     if ($exception === null) {
                            printf ( "Key:%s, ETag:%s, Size:%s\n", $object['Key'], $resp['ETag'], $resp['ContentLength'] );
                            $successCount++;
                     } else {
                            printf ( "DownloadFailed: Key:%s, Status:%d, Message:%s\n", $object['Key'], $exception->getStatusCode(), $exception->getExceptionMessage() );
                            $failCount++;
                     }
              } );
       }
};

$eachPromise = new EachPromise($promiseGenerator(), [
       'concurrency' => $concurrency
]);

$eachPromise->promise()->wait();

printf("\nResult: success=%d, fail=%d\n", $successCount, $failCount);