HDFS C APIs

Function Description

Users can use the C application programming interface (API) to create, read and write, append, and delete files. For details of the C API, see the following official guidelines:

http://hadoop.apache.org/docs/r3.1.1/hadoop-project-dist/hadoop-hdfs/LibHdfs.html

Code Sample

The following code snippets are used as an example. For complete code, see the HDFS C sample code hdfs-c-example/hdfs_test.c in the HDFS sample code decompression directory.

1. Configure the HDFS NameNode parameter and create the link connecting to the HDFS files.

   hdfsFS fs = hdfsConnect("default", 0);
   fprintf(stderr, "hdfsConnect- SUCCESS!\n");

2. Create the HDFS directory.

   const char* dir = "/tmp/nativeTest";
   int exitCode = hdfsCreateDirectory(fs, dir);
   if( exitCode == -1 ){
        fprintf(stderr, "Failed to create directory %s \n", dir );
        exit(-1);
   }
   fprintf(stderr, "hdfsCreateDirectory- SUCCESS! : %s\n", dir);

3. Write files.

   const char* file = "/tmp/nativeTest/testfile.txt";
   hdfsFile writeFile = openFile(fs, (char*)file, O_WRONLY |O_CREAT, 0, 0, 0);
   fprintf(stderr, "hdfsOpenFile- SUCCESS! for write : %s\n", file);
   
   if(!hdfsFileIsOpenForWrite(writeFile)){
       fprintf(stderr, "Failed to open %s for writing.\n", file);
       exit(-1);
   }
   
   char* buffer = "Hadoop HDFS Native file write!";
   
   hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer)+1);
   fprintf(stderr, "hdfsWrite- SUCCESS! : %s\n", file);
   
   printf("Flushing file data ....\n");
   if (hdfsFlush(fs, writeFile)) {
        fprintf(stderr, "Failed to 'flush' %s\n", file);
        exit(-1);
   }
   hdfsCloseFile(fs, writeFile);
   fprintf(stderr, "hdfsCloseFile- SUCCESS! : %s\n", file);

4. Read files.

   hdfsFile readFile = openFile(fs, (char*)file, O_RDONLY, 100, 0, 0);
   fprintf(stderr, "hdfsOpenFile- SUCCESS! for read : %s\n", file);
   
   if(!hdfsFileIsOpenForRead(readFile)){
       fprintf(stderr, "Failed to open %s for reading.\n", file);
       exit(-1);
   }
   
   buffer = (char *) malloc(100);
   tSize num_read = hdfsRead(fs, readFile, (void*)buffer, 100);
   fprintf(stderr, "hdfsRead- SUCCESS!, Byte read : %d, File contant : %s \n", num_read ,buffer);
   hdfsCloseFile(fs, readFile);

5. From the specified location to read the file.

   buffer = (char *) malloc(100);
   readFile = openFile(fs, file, O_RDONLY, 100, 0, 0);
   if (hdfsSeek(fs, readFile, 10)) {
        fprintf(stderr, "Failed to 'seek' %s\n", file);
        exit(-1);
   }
   num_read = hdfsRead(fs, readFile, (void*)buffer, 100);
   fprintf(stderr, "hdfsSeek- SUCCESS!, Byte read : %d, File seek contant : %s \n", num_read ,buffer);
   hdfsCloseFile(fs, readFile);

6. Copy the file.

   const char* destfile = "/tmp/nativeTest/testfile1.txt";
   if (hdfsCopy(fs, file, fs, destfile)) {
       fprintf(stderr, "File copy failed, src : %s, des : %s \n", file, destfile);
       exit(-1);
   }
   fprintf(stderr, "hdfsCopy- SUCCESS!, File copied, src : %s, des : %s \n", file, destfile);

7. Move the file.

   const char* mvfile = "/tmp/nativeTest/testfile2.txt";
   if (hdfsMove(fs, destfile, fs, mvfile )) {
       fprintf(stderr, "File move failed, src : %s, des : %s \n", destfile , mvfile);
       exit(-1);
   }
   fprintf(stderr, "hdfsMove- SUCCESS!, File moved, src : %s, des : %s \n", destfile , mvfile);

8. Rename the file.

   const char* renamefile = "/tmp/nativeTest/testfile3.txt";
   if (hdfsRename(fs, mvfile, renamefile)) {
        fprintf(stderr, "File rename failed, Old name : %s, New name : %s \n", mvfile, renamefile);
        exit(-1);
   }
   fprintf(stderr, "hdfsRename- SUCCESS!, File renamed, Old name : %s, New name : %s \n", mvfile, renamefile);

9. Delete Files.

   if (hdfsDelete(fs, renamefile, 0)) {
       fprintf(stderr, "File delete failed : %s \n", renamefile);
       exit(-1);
   }
   fprintf(stderr, "hdfsDelete- SUCCESS!, File deleted : %s\n",renamefile);

10. Set the number of replications.

    if (hdfsSetReplication(fs, file, 10)) {
        fprintf(stderr, "Failed to set replication : %s \n", file );
        exit(-1);
    }
    fprintf(stderr, "hdfsSetReplication- SUCCESS!, Set replication 10 for %s\n",file);

11. Set users, user groups.

    if (hdfsChown(fs, file, "root", "root")) {
       fprintf(stderr, "Failed to set chown : %s \n", file );
       exit(-1);
    }
    fprintf(stderr, "hdfsChown- SUCCESS!, Chown success for %s\n",file);

12. Set permissions.

    if (hdfsChmod(fs, file, S_IRWXU | S_IRWXG | S_IRWXO)) {
       fprintf(stderr, "Failed to set chmod: %s \n", file );
       exit(-1);
    }
    fprintf(stderr, "hdfsChmod- SUCCESS!, Chmod success for %s\n",file);

13. Set the file time.

    struct timeval now;
    gettimeofday(&now, NULL);
    if (hdfsUtime(fs, file, now.tv_sec, now.tv_sec)) {
       fprintf(stderr, "Failed to set time: %s \n", file );
       exit(-1);
    }
    fprintf(stderr, "hdfsUtime- SUCCESS!, Set time success for %s\n",file);

14. Get file information.

    hdfsFileInfo *fileInfo = NULL;
    if((fileInfo = hdfsGetPathInfo(fs, file)) != NULL) {
       printFileInfo(fileInfo);
       hdfsFreeFileInfo(fileInfo, 1);
       fprintf(stderr, "hdfsGetPathInfo - SUCCESS!\n");
    }

15. Variable directory.

    hdfsFileInfo *fileList = 0;
    int numEntries = 0;
    if((fileList = hdfsListDirectory(fs, dir, &numEntries)) != NULL) {
       int i = 0;
       for(i=0; i < numEntries; ++i) {
           printFileInfo(fileList+i);
       }
       hdfsFreeFileInfo(fileList, numEntries);
    }
    fprintf(stderr, "hdfsListDirectory- SUCCESS!, %s\n", dir);

16. Stream builder interfaces.

    buffer = (char *) malloc(100);
    struct hdfsStreamBuilder *builder= hdfsStreamBuilderAlloc(fs, (char*)file, O_RDONLY);
    hdfsStreamBuilderSetBufferSize(builder,100);
    hdfsStreamBuilderSetReplication(builder,20);
    hdfsStreamBuilderSetDefaultBlockSize(builder,10485760);
    readFile = hdfsStreamBuilderBuild(builder);
    num_read = hdfsRead(fs, readFile, (void*)buffer, 100);
    fprintf(stderr, "hdfsStreamBuilderBuild- SUCCESS! File read success. Byte read : %d, File contant : %s \n", num_read ,buffer);
    free(buffer);
    
    struct hdfsReadStatistics *stats = NULL;
    hdfsFileGetReadStatistics(readFile, &stats);
    fprintf(stderr, "hdfsFileGetReadStatistics- SUCCESS! totalBytesRead : %" PRId64 ", totalLocalBytesRead : %" PRId64 ", totalShortCircuitBytesRead : %" PRId64 ", totalZeroCopyBytesRead : %" PRId64 "\n", stats->totalBytesRead , stats->totalLocalBytesRead, stats->totalShortCircuitBytesRead,  stats->totalZeroCopyBytesRead);
    hdfsFileFreeReadStatistics(stats);

17. Disconnect the HDFS links.

    hdfsDisconnect(fs); 

Preparing Running Environment

Install a client on the node. For example, install a client in the /opt/client directory.

Compiling and Running Applications in Linux

1. Go to the /opt/client directory and run the following command to import the environment variables of the C client:

   cd /opt/client

   source bigdata_env

2. In the directory, run the following command as the hdfs user. For the user password, contact the cluster administrator.

   kinit hdfs

   

   The validity duration of kinit authentication is 24 hours. After 24 hours, you need to re-authenticate the sample with the kinit to restart the sample.

3. Go to the /opt/client/HDFS/hadoop/hdfs-c-example directory and run the following command to import the environment variables of the C client:

   cd /opt/client/HDFS/hadoop/hdfs-c-example

   source component_env_C_example

4. Run the following command to clean the object files and executable files that are generated before:

   make clean

   The running result is displayed as follows:

   [root@10-120-85-2 hdfs-c-example]# make clean
   rm -f hdfs_test.o
   rm -f hdfs_test

5. Run the following command to compile the new object files and executable files:

   make (or make all)

   The running result is displayed as follows:

   [root@10-120-85-2 hdfs-c-example]# make
   cc -c -I/opt/client/HDFS/hadoop/include -Wall -o hdfs_test.o hdfs_test.c 
   cc -o hdfs_test hdfs_test.o -lhdfs

6. Run the following command to create, write, read, append, and delete files:

   make run

   The running result is displayed as follows:

   [root@10-120-85-2 hdfs-c-example]# make run
   ./hdfs_test
   hdfsConnect- SUCCESS!
   hdfsCreateDirectory- SUCCESS! : /tmp/nativeTest
   hdfsOpenFile- SUCCESS! for write : /tmp/nativeTest/testfile.txt
   hdfsWrite- SUCCESS! : /tmp/nativeTest/testfile.txt
   Flushing file data ....
   hdfsCloseFile- SUCCESS! : /tmp/nativeTest/testfile.txt
   hdfsOpenFile- SUCCESS! for read : /tmp/nativeTest/testfile.txt
   hdfsRead- SUCCESS!, Byte read : 31, File contant : Hadoop HDFS Native file write! 
   hdfsSeek- SUCCESS!, Byte read : 21, File seek contant : S Native file write! 
   hdfsPread- SUCCESS!, Byte read : 10, File pead contant : S Native f   
   hdfsCopy- SUCCESS!, File copied, src : /tmp/nativeTest/testfile.txt, des : /tmp/nativeTest/testfile1.txt 
   hdfsMove- SUCCESS!, File moved, src : /tmp/nativeTest/testfile1.txt, des : /tmp/nativeTest/testfile2.txt 
   hdfsRename- SUCCESS!, File renamed, Old name : /tmp/nativeTest/testfile2.txt, New name : /tmp/nativeTest/testfile3.txt 
   hdfsDelete- SUCCESS!, File deleted : /tmp/nativeTest/testfile3.txt
   hdfsSetReplication- SUCCESS!, Set replication 10 for /tmp/nativeTest/testfile.txt
   hdfsChown- SUCCESS!, Chown success for /tmp/nativeTest/testfile.txt
   hdfsChmod- SUCCESS!, Chmod success for /tmp/nativeTest/testfile.txt
   hdfsUtime- SUCCESS!, Set time success for /tmp/nativeTest/testfile.txt
   
   Name: hdfs://hacluster/tmp/nativeTest/testfile.txt, Type: F, Replication: 10, BlockSize: 134217728, Size: 31, LastMod: 1500345260, Owner: root, Group: root, Permissions: 511 (rwxrwxrwx)
   hdfsGetPathInfo - SUCCESS!
   
   Name: hdfs://hacluster/tmp/nativeTest/testfile.txt, Type: F, Replication: 10, BlockSize: 134217728, Size: 31, LastMod: 1500345260, Owner: root, Group: root, Permissions: 511 (rwxrwxrwx)
   hdfsListDirectory- SUCCESS!, /tmp/nativeTest
   hdfsTruncateFile- SUCCESS!, /tmp/nativeTest/testfile.txt
   Block Size : 134217728 
   hdfsGetDefaultBlockSize- SUCCESS!
   Block Size : 134217728 for file /tmp/nativeTest/testfile.txt
   hdfsGetDefaultBlockSizeAtPath- SUCCESS!
   HDFS Capacity : 102726873909
   hdfsGetCapacity- SUCCESS!
   HDFS Used : 4767076324
   hdfsGetCapacity- SUCCESS!
   hdfsExists- SUCCESS! /tmp/nativeTest/testfile.txt
   hdfsConfGetStr- SUCCESS : hdfs://hacluster 
   hdfsStreamBuilderBuild- SUCCESS! File read success. Byte read : 31, File contant : Hadoop HDFS Native file write! 
   hdfsFileGetReadStatistics- SUCCESS! totalBytesRead : 31, totalLocalBytesRead : 0, totalShortCircuitBytesRead : 0, totalZeroCopyBytesRead : 0

7. Enter the debug mode (optional)

   make gdb

   The running result is displayed as follows:

   [root@10-120-85-2 hdfs-c-example]# make gdb
   gdb hdfs_test
   GNU gdb (GDB) SUSE (7.5.1-0.7.29)
   Copyright (C) 2012 Free Software Foundation, Inc.
   License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
   This is free software: you are free to change and redistribute it.
   There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
   and "show warranty" for details.
   This GDB was configured as "x86_64-suse-linux".
   For bug reporting instructions, please see:
   <http://www.gnu.org/software/gdb/bugs/>...
   Reading symbols from /opt/client/HDFS/hadoop/hdfs-c-example/hdfs_test...done.
   (gdb)