Does the SDK Support Uploading, Downloading, or Copying Objects in a Batch? (SDK for Java)
The SDK does not provide dedicated APIs for uploading, downloading, or copying objects in a batch. However, you can combine existing APIs to implement this function. The procedure is as follows:
- List all objects to be uploaded, downloaded, or copied. For details, see Listing Objects (SDK for Java).
- Call the API for uploading, downloading, or copying a single object for the listed objects.
The sample code for uploading objects in a batch is as follows:
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | import com.obs.services.ObsClient; import com.obs.services.exception.ObsException; import com.obs.services.model.ObsObject; import java.io.ByteArrayInputStream; import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; public class BatchUpload001 { private List<File> fileList = new ArrayList<>(); public static void main(String[] args) { // Obtain an AK/SK pair using environment variables or import the AK/SK pair in other ways. Using hard coding may result in leakage. // Obtain an AK/SK pair on the management console. String ak = System.getenv("ACCESS_KEY_ID"); String sk = System.getenv("SECRET_ACCESS_KEY_ID"); // (Optional) If you are using a temporary AK/SK pair and a security token to access OBS, you are not advised to use hard coding, which may result in information leakage. // Obtain an AK/SK pair and a security token using environment variables or import them in other ways. // String securityToken = System.getenv("SECURITY_TOKEN"); // Enter the endpoint corresponding to the bucket. CN-Hong Kong is used here as an example. Replace it with the one currently in use. String endPoint = "https://obs.ap-southeast-1.myhuaweicloud.com"; // Obtain an endpoint using environment variables or import it in other ways. //String endPoint = System.getenv("ENDPOINT"); // Create an ObsClient instance. // Use a permanent AK/SK pair to initialize the client. ObsClient obsClient = new ObsClient(ak, sk, endPoint); // Use a temporary AK/SK pair and security token to initialize the client. // ObsClient obsClient = new ObsClient(ak, sk, securityToken, endPoint); try { String bucketName = "examplebucket"; // Define the prefix of objects in the bucket. String objectPre = "object/"; // Folder to be uploaded String localDirPath = "localfile"; // Scan all files and empty directories in the folder. File localDir = new File(localDirPath); if (!localDir.exists() || !localDir.isDirectory()) { System.out.println("The local folder does not exist: " + localDirPath); return; } BatchUpload001 uploader = new BatchUpload001(); uploader.listFiles(localDir); if (uploader.fileList.isEmpty()) { System.out.println("The file to be uploaded is not found."); return; } System.out.println("Number of objects to be uploaded: " + uploader.fileList.size())); // Initialize the thread pool. int threadCount = 20; ExecutorService executorService = Executors.newFixedThreadPool(threadCount); CountDownLatch latch = new CountDownLatch(uploader.fileList.size()); AtomicInteger successCount = new AtomicInteger(0); AtomicInteger failCount = new AtomicInteger(0); // Concurrently upload files. for (File f : uploader.fileList) { executorService.execute(() -> { try { String relativePath = f.getPath().substring(localDirPath.length() + 1); // Replace the Windows path separator with the separator of the OBS object key. String remoteObjectKey = objectPre + relativePath.replace("\\", "/"); if (f.isDirectory()) { // For empty folders, create empty folder objects in the bucket. remoteObjectKey += "/"; obsClient.putObject(bucketName, remoteObjectKey, new ByteArrayInputStream(new byte[0])); } else { obsClient.putObject(bucketName, remoteObjectKey, f); } successCount.incrementAndGet(); System.out.println("Upload succeeded: " + remoteObjectKey); } catch (ObsException e) { failCount.incrementAndGet(); System.out.println("Upload failed: " + f.getPath()); + ", HTTP Code: " + e.getResponseCode() + ", Error Code: " + e.getErrorCode() + ", Error Message: " + e.getErrorMessage()); } catch (Exception e) { failCount.incrementAndGet(); System.out.println("Upload failed: " + f.getPath() + ", " + e.getMessage()); } finally { latch.countDown(); } }); } // Wait until the upload is complete. executorService.shutdown(); latch.await(30, TimeUnit.MINUTES); System.out.println("Upload completed. Successful: " + successCount.get() + ", Failed: " + failCount.get()); } catch (ObsException e) { System.out.println("Failed to create ObsClient."); System.out.println("HTTP Code: " + e.getResponseCode()); System.out.println("Error Code: " + e.getErrorCode()); System.out.println("Error Message: " + e.getErrorMessage()); } catch (Exception e) { System.out.println("Upload exception: " + e.getMessage()); e.printStackTrace(); } } // Recursively scan all files and empty directories in the folder. private void listFiles(File file) { File[] children = file.listFiles(); if (children == null) { return; } if (children.length == 0) { // An empty folder also needs to be uploaded. Add it to the list. fileList.add(file); return; } for (File f : children) { if (f.isDirectory()) { listFiles(f); } else if (f.isFile()) { fileList.add(f); } } } } |
You can use multiple threads to concurrently upload, download, and copy data to improve efficiency.
What is your overall rating for this page?
Thank you very much for your feedback. We will continue working to improve the documentation.See the reply and handling status in My Cloud VOC.
For any further questions, feel free to contact us through the chatbot.
Chatbot