批量下载对象(C SDK)

示例代码

本示例用于批量下载OBS桶中的对象到本地目录。您可以通过列举满足前缀条件的对象， 再以指定并发数并发下载，文件按原Key结构保存到本地目录，适用于批量导出、离线备份等场景。

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <pthread.h>
#include <sys/stat.h>
#include <errno.h>
#include "eSDKOBS.h"
#define MAX_KEY_LEN     1024
#define MAX_PATH_LEN    256
#define MAX_TASK        10000
typedef struct {
    char key[MAX_KEY_LEN];
    char local_path[MAX_PATH_LEN];
} download_task;
typedef struct {
    download_task *task;
    FILE *fp;
} download_context;
typedef struct {
    int total;
    int success;
    int failed;
    char failed_keys[100][MAX_KEY_LEN];
    int failed_count;
    pthread_mutex_t mutex;
} batch_result;
static batch_result g_result = {0, 0, 0, {{0}}, 0, PTHREAD_MUTEX_INITIALIZER};
static download_task *g_tasks = NULL;
static int g_task_count = 0;
static int g_task_index = 0;
static pthread_mutex_t g_task_mutex = PTHREAD_MUTEX_INITIALIZER;
static obs_status get_object_properties_callback(const obs_response_properties *properties, void *callback_data)
{
    (void)properties;
    (void)callback_data;
    return OBS_STATUS_OK;
}
static obs_status get_object_data_callback(int buffer_size, const char *buffer, void *callback_data)
{
    download_context *ctx = (download_context *)callback_data;
    if (!ctx || !ctx->fp) return OBS_STATUS_InternalError;
    size_t written = fwrite(buffer, 1, buffer_size, ctx->fp);
    return (written == (size_t)buffer_size) ? OBS_STATUS_OK : OBS_STATUS_InternalError;
}
static void get_object_complete_callback(obs_status status,
                                          const obs_error_details *error,
                                          void *callback_data)
{
    (void)error;
    download_context *ctx = (download_context *)callback_data;
    char *key = ctx && ctx->task ? ctx->task->key : "unknown";
    pthread_mutex_lock(&g_result.mutex);
    if (status == OBS_STATUS_OK) {
        g_result.success++;
        printf("[SUCCESS] download %s\n", key);
    } else {
        g_result.failed++;
        if (g_result.failed_count < 100) {
            strncpy(g_result.failed_keys[g_result.failed_count], key, MAX_KEY_LEN - 1);
            g_result.failed_keys[g_result.failed_count][MAX_KEY_LEN - 1] = '\0';
            g_result.failed_count++;
        }
        printf("[FAILED] download %s: %s\n", key, obs_get_status_name(status));
    }
    pthread_mutex_unlock(&g_result.mutex);
}
typedef struct {
    download_task *tasks;
    int max_count;
    int current_count;
    const char *target_dir;
    obs_status ret_status;
} list_cb_data;
static obs_status list_objects_callback(int is_truncated, const char *next_marker,
                                         int contents_count,
                                         const obs_list_objects_content *contents,
                                         int common_prefixes_count,
                                         const char **common_prefixes,
                                         void *callback_data)
{
    (void)is_truncated;
    (void)next_marker;
    (void)common_prefixes_count;
    (void)common_prefixes;
    list_cb_data *data = (list_cb_data *)callback_data;
    for (int i = 0; i < contents_count && data->current_count < data->max_count; i++) {
        memset(&data->tasks[data->current_count], 0, sizeof(download_task));
        strncpy(data->tasks[data->current_count].key, contents[i].key, MAX_KEY_LEN - 1);
        snprintf(data->tasks[data->current_count].local_path, MAX_PATH_LEN, "%s/%s",
                 data->target_dir, contents[i].key);
        data->current_count++;
    }
    return OBS_STATUS_OK;
}
static void list_complete_callback(obs_status status,
                                    const obs_error_details *error,
                                    void *callback_data)
{
    (void)error;
    list_cb_data *data = (list_cb_data *)callback_data;
    data->ret_status = status;
}
static void *worker_thread(void *arg)
{
    obs_options *options_ptr = (obs_options *)arg;
    while (1) {
        pthread_mutex_lock(&g_task_mutex);
        if (g_task_index >= g_task_count) {
            pthread_mutex_unlock(&g_task_mutex);
            break;
        }
        int idx = g_task_index++;
        pthread_mutex_unlock(&g_task_mutex);
        download_task *task = &g_tasks[idx];
        char dir_path[MAX_PATH_LEN];
        strncpy(dir_path, task->local_path, MAX_PATH_LEN - 1);
        char *last_slash = strrchr(dir_path, '/');
        if (last_slash) {
            *last_slash = '\0';
            mkdir(dir_path, 0755);
        }
        FILE *fp = fopen(task->local_path, "wb");
        if (!fp) {
            pthread_mutex_lock(&g_result.mutex);
            g_result.failed++;
            if (g_result.failed_count < 100) {
                strncpy(g_result.failed_keys[g_result.failed_count], task->key, MAX_KEY_LEN - 1);
                g_result.failed_keys[g_result.failed_count][MAX_KEY_LEN - 1] = '\0';
                g_result.failed_count++;
            }
            printf("[FAILED] cannot create %s: %s\n", task->local_path, strerror(errno));
            pthread_mutex_unlock(&g_result.mutex);
            continue;
        }
        download_context ctx;
        ctx.task = task;
        ctx.fp = fp;
        obs_get_object_handler handler = {
            { get_object_properties_callback, get_object_complete_callback },
            get_object_data_callback
        };
        obs_get_conditions get_conditions;
        init_get_properties(&get_conditions);
        obs_object_info object_info;
        memset(&object_info, 0, sizeof(object_info));
        object_info.key = task->key;
        get_object(options_ptr, &object_info, &get_conditions, 0, &handler, &ctx);
        fclose(fp);
    }
    return NULL;
}
int main()
{
    const char *BUCKET_NAME ="example-bucket-name";
    const char *TARGET_DIR = "/tmp/download_files";
    const char *PREFIX = "archive/";
    int THREAD_NUM = 8;
    obs_status status = obs_initialize(OBS_INIT_ALL);
    if (status != OBS_STATUS_OK) {
        printf("obs_initialize failed: %s\n", obs_get_status_name(status));
        return 1;
    }
    obs_options options;
    init_obs_options(&options);
    // host_name填写桶所在的endpoint, 此处以华北-北京四为例，其他地区请按实际情况填写。
    options.bucket_options.host_name = "obs.cn-north-4.myhuaweicloud.com";
    options.bucket_options.bucket_name = (char *)BUCKET_NAME;
    options.bucket_options.access_key = getenv("ACCESS_KEY_ID");
    options.bucket_options.secret_access_key = getenv("SECRET_ACCESS_KEY");

    printf("\nListing objects with prefix '%s'...\n", PREFIX);
    g_tasks = (download_task *)malloc(sizeof(download_task) * MAX_TASK);
    if (!g_tasks) {
        printf("Failed to allocate memory\n");
        obs_deinitialize();
        return 1;
    }
    list_cb_data cb_data = {g_tasks, MAX_TASK, 0, TARGET_DIR, OBS_STATUS_BUTT};
    obs_list_objects_handler list_handler = {
        { NULL, list_complete_callback },
        list_objects_callback
    };
    list_bucket_objects(&options, PREFIX, NULL, NULL, 1000, &list_handler, &cb_data);
    if (cb_data.ret_status != OBS_STATUS_OK) {
        printf("list_bucket_objects failed: %s\n", obs_get_status_name(cb_data.ret_status));
        free(g_tasks);
        obs_deinitialize();
        return 1;
    }
    g_task_count = cb_data.current_count;
    if (g_task_count == 0) {
        printf("No objects found with prefix '%s'\n", PREFIX);
        free(g_tasks);
        obs_deinitialize();
        return 0;
    }
    printf("Found %d objects to download\n", g_task_count);
    g_result.total = g_task_count;
    g_result.success = 0;
    g_result.failed = 0;
    g_result.failed_count = 0;
    g_task_index = 0;
    pthread_t threads[THREAD_NUM];
    for (int i = 0; i < THREAD_NUM; i++) {
        pthread_create(&threads[i], NULL, worker_thread, &options);
    }
    for (int i = 0; i < THREAD_NUM; i++) {
        pthread_join(threads[i], NULL);
    }
    printf("\n========== Batch Download Summary ==========\n");
    printf("Total: %d\n", g_result.total);
    printf("Success: %d\n", g_result.success);
    printf("Failed: %d\n", g_result.failed);
    if (g_result.failed_count > 0) {
        printf("\nFailed objects:\n");
        for (int i = 0; i < g_result.failed_count; i++) {
            printf("  - %s\n", g_result.failed_keys[i]);
        }
    }
    printf("==========================================\n");
    free(g_tasks);
    obs_deinitialize();
    return 0;
}