Updated on 2022-11-18 GMT+08:00

C API

Function Description

Users can use the C application programming interface (API) to create, read and write, append, and delete files. For details of the C API, see the following official guidelines:

http://hadoop.apache.org/docs/r3.1.1/hadoop-project-dist/hadoop-hdfs/LibHdfs.html

Code Sample

The following code snippets are used as an example. For complete code, see the HDFS C sample code HDFS/hdfs-c-example/hdfs_test.c in the HDFS sample code decompression directory.

  1. Configure the HDFS NameNode parameter and create the link connecting to the HDFS files.
    hdfsFS fs = hdfsConnect("default", 0);
    fprintf(stderr, "hdfsConnect- SUCCESS!\n");
  2. Create the HDFS directory.
    const char* dir = "/tmp/nativeTest";
    int exitCode = hdfsCreateDirectory(fs, dir);
    if( exitCode == -1 ){
         fprintf(stderr, "Failed to create directory %s \n", dir );
         exit(-1);
    }
    fprintf(stderr, "hdfsCreateDirectory- SUCCESS! : %s\n", dir);
  3. Write files.
    const char* file = "/tmp/nativeTest/testfile.txt";
    hdfsFile writeFile = openFile(fs, (char*)file, O_WRONLY |O_CREAT, 0, 0, 0);
    fprintf(stderr, "hdfsOpenFile- SUCCESS! for write : %s\n", file);
    
    if(!hdfsFileIsOpenForWrite(writeFile)){
        fprintf(stderr, "Failed to open %s for writing.\n", file);
        exit(-1);
    }
    
    char* buffer = "Hadoop HDFS Native file write!";
    
    hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer)+1);
    fprintf(stderr, "hdfsWrite- SUCCESS! : %s\n", file);
    
    printf("Flushing file data ....\n");
    if (hdfsFlush(fs, writeFile)) {
         fprintf(stderr, "Failed to 'flush' %s\n", file);
         exit(-1);
    }
    hdfsCloseFile(fs, writeFile);
    fprintf(stderr, "hdfsCloseFile- SUCCESS! : %s\n", file);
  4. Read files.
    hdfsFile readFile = openFile(fs, (char*)file, O_RDONLY, 100, 0, 0);
    fprintf(stderr, "hdfsOpenFile- SUCCESS! for read : %s\n", file);
    
    if(!hdfsFileIsOpenForRead(readFile)){
        fprintf(stderr, "Failed to open %s for reading.\n", file);
        exit(-1);
    }
    
    buffer = (char *) malloc(100);
    tSize num_read = hdfsRead(fs, readFile, (void*)buffer, 100);
    fprintf(stderr, "hdfsRead- SUCCESS!, Byte read : %d, File contant : %s \n", num_read ,buffer);
    hdfsCloseFile(fs, readFile);
  5. From the specified location to read the file.
    buffer = (char *) malloc(100);
    readFile = openFile(fs, file, O_RDONLY, 100, 0, 0);
    if (hdfsSeek(fs, readFile, 10)) {
         fprintf(stderr, "Failed to 'seek' %s\n", file);
         exit(-1);
    }
    num_read = hdfsRead(fs, readFile, (void*)buffer, 100);
    fprintf(stderr, "hdfsSeek- SUCCESS!, Byte read : %d, File seek contant : %s \n", num_read ,buffer);
    hdfsCloseFile(fs, readFile);
  6. Copy the file.
    const char* destfile = "/tmp/nativeTest/testfile1.txt";
    if (hdfsCopy(fs, file, fs, destfile)) {
        fprintf(stderr, "File copy failed, src : %s, des : %s \n", file, destfile);
        exit(-1);
    }
    fprintf(stderr, "hdfsCopy- SUCCESS!, File copied, src : %s, des : %s \n", file, destfile);
  7. Move the file.
    const char* mvfile = "/tmp/nativeTest/testfile2.txt";
    if (hdfsMove(fs, destfile, fs, mvfile )) {
        fprintf(stderr, "File move failed, src : %s, des : %s \n", destfile , mvfile);
        exit(-1);
    }
    fprintf(stderr, "hdfsMove- SUCCESS!, File moved, src : %s, des : %s \n", destfile , mvfile);
  8. Rename the file.
    const char* renamefile = "/tmp/nativeTest/testfile3.txt";
    if (hdfsRename(fs, mvfile, renamefile)) {
         fprintf(stderr, "File rename failed, Old name : %s, New name : %s \n", mvfile, renamefile);
         exit(-1);
    }
    fprintf(stderr, "hdfsRename- SUCCESS!, File renamed, Old name : %s, New name : %s \n", mvfile, renamefile);
  9. Delete Files.
    if (hdfsDelete(fs, renamefile, 0)) {
        fprintf(stderr, "File delete failed : %s \n", renamefile);
        exit(-1);
    }
    fprintf(stderr, "hdfsDelete- SUCCESS!, File deleted : %s\n",renamefile);
  10. Set the number of replications.
    if (hdfsSetReplication(fs, file, 10)) {
        fprintf(stderr, "Failed to set replication : %s \n", file );
        exit(-1);
    }
    fprintf(stderr, "hdfsSetReplication- SUCCESS!, Set replication 10 for %s\n",file);
  11. Set users, user groups.
    if (hdfsChown(fs, file, "root", "root")) {
       fprintf(stderr, "Failed to set chown : %s \n", file );
       exit(-1);
    }
    fprintf(stderr, "hdfsChown- SUCCESS!, Chown success for %s\n",file);
  12. Set permissions.
    if (hdfsChmod(fs, file, S_IRWXU | S_IRWXG | S_IRWXO)) {
       fprintf(stderr, "Failed to set chmod: %s \n", file );
       exit(-1);
    }
    fprintf(stderr, "hdfsChmod- SUCCESS!, Chmod success for %s\n",file);
  13. Set the file time.
    struct timeval now;
    gettimeofday(&now, NULL);
    if (hdfsUtime(fs, file, now.tv_sec, now.tv_sec)) {
       fprintf(stderr, "Failed to set time: %s \n", file );
       exit(-1);
    }
    fprintf(stderr, "hdfsUtime- SUCCESS!, Set time success for %s\n",file);
  14. Get file information.
    hdfsFileInfo *fileInfo = NULL;
    if((fileInfo = hdfsGetPathInfo(fs, file)) != NULL) {
       printFileInfo(fileInfo);
       hdfsFreeFileInfo(fileInfo, 1);
       fprintf(stderr, "hdfsGetPathInfo - SUCCESS!\n");
    }
  15. Variable directory.
    hdfsFileInfo *fileList = 0;
    int numEntries = 0;
    if((fileList = hdfsListDirectory(fs, dir, &numEntries)) != NULL) {
       int i = 0;
       for(i=0; i < numEntries; ++i) {
           printFileInfo(fileList+i);
       }
       hdfsFreeFileInfo(fileList, numEntries);
    }
    fprintf(stderr, "hdfsListDirectory- SUCCESS!, %s\n", dir);
  16. Stream builder interfaces.
    buffer = (char *) malloc(100);
    struct hdfsStreamBuilder *builder= hdfsStreamBuilderAlloc(fs, (char*)file, O_RDONLY);
    hdfsStreamBuilderSetBufferSize(builder,100);
    hdfsStreamBuilderSetReplication(builder,20);
    hdfsStreamBuilderSetDefaultBlockSize(builder,10485760);
    readFile = hdfsStreamBuilderBuild(builder);
    num_read = hdfsRead(fs, readFile, (void*)buffer, 100);
    fprintf(stderr, "hdfsStreamBuilderBuild- SUCCESS! File read success. Byte read : %d, File contant : %s \n", num_read ,buffer);
    free(buffer);
    
    struct hdfsReadStatistics *stats = NULL;
    hdfsFileGetReadStatistics(readFile, &stats);
    fprintf(stderr, "hdfsFileGetReadStatistics- SUCCESS! totalBytesRead : %" PRId64 ", totalLocalBytesRead : %" PRId64 ", totalShortCircuitBytesRead : %" PRId64 ", totalZeroCopyBytesRead : %" PRId64 "\n", stats->totalBytesRead , stats->totalLocalBytesRead, stats->totalShortCircuitBytesRead,  stats->totalZeroCopyBytesRead);
    hdfsFileFreeReadStatistics(stats);
  17. Disconnect the HDFS links.
    hdfsDisconnect(fs); 

Preparing Running Environment

Install a client on the node. For example, install a client in the /opt/client directory.

Compiling and Running Applications in Linux

  1. Go to the /opt/client directory and run the following command to import the environment variables of the C client:

    cd /opt/client

    source bigdata_env

  2. Go to the /opt/client/HDFS/hadoop/hdfs-c-example directory and run the following command to import the environment variables of the C client:

    cd /opt/client/HDFS/hadoop/hdfs-c-example

    source component_env_C_example

  3. Run the following command to clean the object files and executable files that are generated before:

    make clean

    The running result is displayed as follows:

    [root@10-120-85-2 hdfs-c-example]# make clean
    rm -f hdfs_test.o
    rm -f hdfs_test
  4. Run the following command to compile the new object files and executable files:

    make ( or make all )

    The running result is displayed as follows:

    [root@10-120-85-2 hdfs-c-example]# make
    cc -c -I/opt/client/HDFS/hadoop/include -Wall -o hdfs_test.o hdfs_test.c
    cc -o hdfs_test hdfs_test.o -lhdfs
  5. Run the following command to create, write and read, append, and delete files:

    make run

    The running result is displayed as follows:

    [root@10-120-85-2 hdfs-c-example]# make run
    ./hdfs_test
    hdfsConnect- SUCCESS!
    hdfsCreateDirectory- SUCCESS! : /tmp/nativeTest
    hdfsOpenFile- SUCCESS! for write : /tmp/nativeTest/testfile.txt
    hdfsWrite- SUCCESS! : /tmp/nativeTest/testfile.txt
    Flushing file data ....
    hdfsCloseFile- SUCCESS! : /tmp/nativeTest/testfile.txt
    hdfsOpenFile- SUCCESS! for read : /tmp/nativeTest/testfile.txt
    hdfsRead- SUCCESS!, Byte read : 31, File contant : Hadoop HDFS Native file write! 
    hdfsSeek- SUCCESS!, Byte read : 21, File seek contant : S Native file write! 
    hdfsPread- SUCCESS!, Byte read : 10, File pead contant : S Native f  
    hdfsCopy- SUCCESS!, File copied, src : /tmp/nativeTest/testfile.txt, des : /tmp/nativeTest/testfile1.txt 
    hdfsMove- SUCCESS!, File moved, src : /tmp/nativeTest/testfile1.txt, des : /tmp/nativeTest/testfile2.txt 
    hdfsRename- SUCCESS!, File renamed, Old name : /tmp/nativeTest/testfile2.txt, New name : /tmp/nativeTest/testfile3.txt 
    hdfsDelete- SUCCESS!, File deleted : /tmp/nativeTest/testfile3.txt
    hdfsSetReplication- SUCCESS!, Set replication 10 for /tmp/nativeTest/testfile.txt
    hdfsChown- SUCCESS!, Chown success for /tmp/nativeTest/testfile.txt
    hdfsChmod- SUCCESS!, Chmod success for /tmp/nativeTest/testfile.txt
    hdfsUtime- SUCCESS!, Set time success for /tmp/nativeTest/testfile.txt
    
    Name: hdfs://hacluster/tmp/nativeTest/testfile.txt, Type: F, Replication: 10, BlockSize: 134217728, Size: 31, LastMod: 1500345260, Owner: root, Group: root, Permissions: 511 (rwxrwxrwx)
    hdfsGetPathInfo - SUCCESS!
    
    Name: hdfs://hacluster/tmp/nativeTest/testfile.txt, Type: F, Replication: 10, BlockSize: 134217728, Size: 31, LastMod: 1500345260, Owner: root, Group: root, Permissions: 511 (rwxrwxrwx)
    hdfsListDirectory- SUCCESS!, /tmp/nativeTest
    hdfsTruncateFile- SUCCESS!, /tmp/nativeTest/testfile.txt
    Block Size : 134217728 
    hdfsGetDefaultBlockSize- SUCCESS!
    Block Size : 134217728 for file /tmp/nativeTest/testfile.txt
    hdfsGetDefaultBlockSizeAtPath- SUCCESS!
    HDFS Capacity : 102726873909
    hdfsGetCapacity- SUCCESS!
    HDFS Used : 4767076324
    hdfsGetCapacity- SUCCESS!
    hdfsExists- SUCCESS! /tmp/nativeTest/testfile.txt
    hdfsConfGetStr- SUCCESS : hdfs://hacluster 
    hdfsStreamBuilderBuild- SUCCESS! File read success. Byte read : 31, File contant : Hadoop HDFS Native file write! 
    hdfsFileGetReadStatistics- SUCCESS! totalBytesRead : 31, totalLocalBytesRead : 0, totalShortCircuitBytesRead : 0, totalZeroCopyBytesRead : 0
  6. Enter the debug mode. (Optional)

    make gdb

    The running result is displayed as follows:

    [root@10-120-85-2 hdfs-c-example]# make gdb
    gdb hdfs_test
    GNU gdb (GDB) SUSE (7.5.1-0.7.29)
    Copyright (C) 2012 Free Software Foundation, Inc.
    License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
    This is free software: you are free to change and redistribute it.
    There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
    and "show warranty" for details.
    This GDB was configured as "x86_64-suse-linux".
    For bug reporting instructions, please see:
    <http://www.gnu.org/software/gdb/bugs/>...
    Reading symbols from /opt/client/HDFS/hadoop/hdfs-c-example/hdfs_test...done.
    (gdb)