Downloading Objects in Batches (SDK for C)

Sample Code

This example downloads objects from an OBS bucket to a local directory in batches. You can list objects whose names contain the specified prefix and download them concurrently based on the specified concurrency. The files are saved to the local directory based on the original key structure. This method is suitable for scenarios such as batch export and offline backup.

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <pthread.h>
#include <sys/stat.h>
#include <errno.h>
#include "eSDKOBS.h"
#define MAX_KEY_LEN     1024
#define MAX_PATH_LEN    256
#define MAX_TASK        10000
typedef struct {
    char key[MAX_KEY_LEN];
    char local_path[MAX_PATH_LEN];
} download_task;
typedef struct {
    download_task *task;
    FILE *fp;
} download_context;
typedef struct {
    int total;
    int success;
    int failed;
    char failed_keys[100][MAX_KEY_LEN];
    int failed_count;
    pthread_mutex_t mutex;
} batch_result;
static batch_result g_result = {0, 0, 0, {{0}}, 0, PTHREAD_MUTEX_INITIALIZER};
static download_task *g_tasks = NULL;
static int g_task_count = 0;
static int g_task_index = 0;
static pthread_mutex_t g_task_mutex = PTHREAD_MUTEX_INITIALIZER;
static obs_status get_object_properties_callback(const obs_response_properties *properties, void *callback_data)
{
    (void)properties;
    (void)callback_data;
    return OBS_STATUS_OK;
}
static obs_status get_object_data_callback(int buffer_size, const char *buffer, void *callback_data)
{
    download_context *ctx = (download_context *)callback_data;
    if (!ctx || !ctx->fp) return OBS_STATUS_InternalError;
    size_t written = fwrite(buffer, 1, buffer_size, ctx->fp);
    return (written == (size_t)buffer_size) ? OBS_STATUS_OK : OBS_STATUS_InternalError;
}
static void get_object_complete_callback(obs_status status,
                                          const obs_error_details *error,
                                          void *callback_data)
{
    (void)error;
    download_context *ctx = (download_context *)callback_data;
    char *key = ctx && ctx->task ? ctx->task->key : "unknown";
    pthread_mutex_lock(&g_result.mutex);
    if (status == OBS_STATUS_OK) {
        g_result.success++;
        printf("[SUCCESS] download %s\n", key);
    } else {
        g_result.failed++;
        if (g_result.failed_count < 100) {
            strncpy(g_result.failed_keys[g_result.failed_count], key, MAX_KEY_LEN - 1);
            g_result.failed_keys[g_result.failed_count][MAX_KEY_LEN - 1] = '\0';
            g_result.failed_count++;
        }
        printf("[FAILED] download %s: %s\n", key, obs_get_status_name(status));
    }
    pthread_mutex_unlock(&g_result.mutex);
}
typedef struct {
    download_task *tasks;
    int max_count;
    int current_count;
    const char *target_dir;
    obs_status ret_status;
} list_cb_data;
static obs_status list_objects_callback(int is_truncated, const char *next_marker,
                                         int contents_count,
                                         const obs_list_objects_content *contents,
                                         int common_prefixes_count,
                                         const char **common_prefixes,
                                         void *callback_data)
{
    (void)is_truncated;
    (void)next_marker;
    (void)common_prefixes_count;
    (void)common_prefixes;
    list_cb_data *data = (list_cb_data *)callback_data;
    for (int i = 0; i < contents_count && data->current_count < data->max_count; i++) {
        memset(&data->tasks[data->current_count], 0, sizeof(download_task));
        strncpy(data->tasks[data->current_count].key, contents[i].key, MAX_KEY_LEN - 1);
        snprintf(data->tasks[data->current_count].local_path, MAX_PATH_LEN, "%s/%s",
                 data->target_dir, contents[i].key);
        data->current_count++;
    }
    return OBS_STATUS_OK;
}
static void list_complete_callback(obs_status status,
                                    const obs_error_details *error,
                                    void *callback_data)
{
    (void)error;
    list_cb_data *data = (list_cb_data *)callback_data;
    data->ret_status = status;
}
static void *worker_thread(void *arg)
{
    obs_options *options_ptr = (obs_options *)arg;
    while (1) {
        pthread_mutex_lock(&g_task_mutex);
        if (g_task_index >= g_task_count) {
            pthread_mutex_unlock(&g_task_mutex);
            break;
        }
        int idx = g_task_index++;
        pthread_mutex_unlock(&g_task_mutex);
        download_task *task = &g_tasks[idx];
        char dir_path[MAX_PATH_LEN];
        strncpy(dir_path, task->local_path, MAX_PATH_LEN - 1);
        char *last_slash = strrchr(dir_path, '/');
        if (last_slash) {
            *last_slash = '\0';
            mkdir(dir_path, 0755);
        }
        FILE *fp = fopen(task->local_path, "wb");
        if (!fp) {
            pthread_mutex_lock(&g_result.mutex);
            g_result.failed++;
            if (g_result.failed_count < 100) {
                strncpy(g_result.failed_keys[g_result.failed_count], task->key, MAX_KEY_LEN - 1);
                g_result.failed_keys[g_result.failed_count][MAX_KEY_LEN - 1] = '\0';
                g_result.failed_count++;
            }
            printf("[FAILED] cannot create %s: %s\n", task->local_path, strerror(errno));
            pthread_mutex_unlock(&g_result.mutex);
            continue;
        }
        download_context ctx;
        ctx.task = task;
        ctx.fp = fp;
        obs_get_object_handler handler = {
            { get_object_properties_callback, get_object_complete_callback },
            get_object_data_callback
        };
        obs_get_conditions get_conditions;
        init_get_properties(&get_conditions);
        obs_object_info object_info;
        memset(&object_info, 0, sizeof(object_info));
        object_info.key = task->key;
        get_object(options_ptr, &object_info, &get_conditions, 0, &handler, &ctx);
        fclose(fp);
    }
    return NULL;
}
int main()
{
    const char *BUCKET_NAME ="example-bucket-name";
    const char *TARGET_DIR = "/tmp/download_files";
    const char *PREFIX = "archive/";
    int THREAD_NUM = 8;
    obs_status status = obs_initialize(OBS_INIT_ALL);
    if (status != OBS_STATUS_OK) {
        printf("obs_initialize failed: %s\n", obs_get_status_name(status));
        return 1;
    }
    obs_options options;
    init_obs_options(&options);
    // Enter the endpoint corresponding to the bucket for host_name. CN-Hong Kong is used here as an example. Replace it with the one currently in use.
    options.bucket_options.host_name = "obs.ap-southeast-1.myhuaweicloud.com";
    options.bucket_options.bucket_name = (char *)BUCKET_NAME;
    options.bucket_options.access_key = getenv("ACCESS_KEY_ID");
    options.bucket_options.secret_access_key = getenv("SECRET_ACCESS_KEY");

    printf("\nListing objects with prefix '%s'...\n", PREFIX);
    g_tasks = (download_task *)malloc(sizeof(download_task) * MAX_TASK);
    if (!g_tasks) {
        printf("Failed to allocate memory\n");
        obs_deinitialize();
        return 1;
    }
    list_cb_data cb_data = {g_tasks, MAX_TASK, 0, TARGET_DIR, OBS_STATUS_BUTT};
    obs_list_objects_handler list_handler = {
        { NULL, list_complete_callback },
        list_objects_callback
    };
    list_bucket_objects(&options, PREFIX, NULL, NULL, 1000, &list_handler, &cb_data);
    if (cb_data.ret_status != OBS_STATUS_OK) {
        printf("list_bucket_objects failed: %s\n", obs_get_status_name(cb_data.ret_status));
        free(g_tasks);
        obs_deinitialize();
        return 1;
    }
    g_task_count = cb_data.current_count;
    if (g_task_count == 0) {
        printf("No objects found with prefix '%s'\n", PREFIX);
        free(g_tasks);
        obs_deinitialize();
        return 0;
    }
    printf("Found %d objects to download\n", g_task_count);
    g_result.total = g_task_count;
    g_result.success = 0;
    g_result.failed = 0;
    g_result.failed_count = 0;
    g_task_index = 0;
    pthread_t threads[THREAD_NUM];
    for (int i = 0; i < THREAD_NUM; i++) {
        pthread_create(&threads[i], NULL, worker_thread, &options);
    }
    for (int i = 0; i < THREAD_NUM; i++) {
        pthread_join(threads[i], NULL);
    }
    printf("\n========== Batch Download Summary ==========\n");
    printf("Total: %d\n", g_result.total);
    printf("Success: %d\n", g_result.success);
    printf("Failed: %d\n", g_result.failed);
    if (g_result.failed_count > 0) {
        printf("\nFailed objects:\n");
        for (int i = 0; i < g_result.failed_count; i++) {
            printf("  - %s\n", g_result.failed_keys[i]);
        }
    }
    printf("==========================================\n");
    free(g_tasks);
    obs_deinitialize();
    return 0;
}