Initializing HDFS

Updated on 2024-12-10 GMT+08:00

View PDF

Function Description

Before using APIs provided by HDFS, you need to initialize HDFS. The HDFS initialization process is as follows:

Load HDFS service configuration files and perform Kerberos authentication.
Instantiate the Filesystem after the authentication succeeds.
Use HDFS APIs.

NOTE:

Obtain the keytab file for Kerberos authentication in advance.

Configuration Files

Table 1 lists the configuration files used for logging in to HDFS. These files have been imported to the conf directory of the hdfs-example project.

**Table 1** Configuration files
File Name	Function	How to Obtain
core-site.xml	Configures HDFS parameters.	MRS_Services_ClientConfig\HDFS\config\core-site.xml
hdfs-site.xml	Configures HDFS parameters.	MRS_Services_ClientConfig\HDFS\config\hdfs-site.xml
user.keytab	Provides HDFS user information for Kerberos security authentication.	If the cluster is in security mode, contact the administrator to obtain the keytab and krb5 files corresponding to the account.
krb5.conf	Contains Kerberos server configuration information.

NOTE:

The user.keytab and krb5.conf of different clusters cannot be shared.
The log4j.properties file in the conf directory can be configured based on the customer requirements.

Sample Code

The following provides code snippets. For complete codes, see the HdfsMain class in com.huawei.bigdata.hdfs.examples.

Run the initialization code of the application on the Linux client. The code example is as follows:

 /**
   * Initialization. Obtain a FileSystem instance.
   *
   * @throws IOException
   */
  private void init() throws IOException {
    confLoad();
    authentication();
    instanceBuild();
  }
  
  /**
   * 
   * If the application runs on Linux, the paths of core-site.xml and hdfs-site.xml must be 
   * modified to the absolute path of the client file on Linux.
   * 
   */
  private void confLoad() throws IOException {
    conf = new Configuration();
    // conf file
    conf.addResource(new Path(PATH_TO_HDFS_SITE_XML));
    conf.addResource(new Path(PATH_TO_CORE_SITE_XML));
   }
  
  /**
   * kerberos security authentication 
   * If the application runs on Linux, the paths of krb5.conf and keytab must be 
   * modified to the absolute path of the client file on Linux. In addition, the keytab and principal files in the sample code must be 
   * modified to the current user's keytab file name and username.
   * 
   */
  private void authentication() throws IOException {
    // Security mode
    if ("kerberos".equalsIgnoreCase(conf.get("hadoop.security.authentication"))) {
      System.setProperty("java.security.krb5.conf", PATH_TO_KRB5_CONF);
      LoginUtil.login(PRNCIPAL_NAME, PATH_TO_KEYTAB, PATH_TO_KRB5_CONF, conf);
    }
  }
  
  
  /**
   * build HDFS instance
   */
  private void instanceBuild() throws IOException {
    // get filesystem
    fSystem = FileSystem.get(conf);
  }

On Linux, the login sample code is required for the first login. For details about the code, see the LoginUtil class in com.huawei.bigdata.security.

public synchronized static void login(String userPrincipal,
      String userKeytabPath, String krb5ConfPath, Configuration conf)
      throws IOException {
    // 1. Check the input parameters.
    if ((userPrincipal == null) || (userPrincipal.length() <= 0)) {
      LOG.error("input userPrincipal is invalid.");
      throw new IOException("input userPrincipal is invalid.");
    }

    if ((userKeytabPath == null) || (userKeytabPath.length() <= 0)) {
      LOG.error("input userKeytabPath is invalid.");
      throw new IOException("input userKeytabPath is invalid.");
    }

    if ((krb5ConfPath == null) || (krb5ConfPath.length() <= 0)) {
      LOG.error("input krb5ConfPath is invalid.");
      throw new IOException("input krb5ConfPath is invalid.");
    }

    if ((conf == null)) {
      LOG.error("input conf is invalid.");
      throw new IOException("input conf is invalid.");
    }

    // 2. Check whether the file exists.
    File userKeytabFile = new File(userKeytabPath);
    if (!userKeytabFile.exists()) {
      LOG.error("userKeytabFile(" + userKeytabFile.getAbsolutePath()
          + ") does not exist.");
      throw new IOException("userKeytabFile("
          + userKeytabFile.getAbsolutePath() + ") does not exist.");
    }
    if (!userKeytabFile.isFile()) {
      LOG.error("userKeytabFile(" + userKeytabFile.getAbsolutePath()
          + ") is not a file.");
      throw new IOException("userKeytabFile("
          + userKeytabFile.getAbsolutePath() + ") is not a file.");
    }

    File krb5ConfFile = new File(krb5ConfPath);
    if (!krb5ConfFile.exists()) {
      LOG.error("krb5ConfFile(" + krb5ConfFile.getAbsolutePath()
          + ") does not exist.");
      throw new IOException("krb5ConfFile(" + krb5ConfFile.getAbsolutePath()
          + ") does not exist.");
    }
    if (!krb5ConfFile.isFile()) {
      LOG.error("krb5ConfFile(" + krb5ConfFile.getAbsolutePath()
          + ") is not a file.");
      throw new IOException("krb5ConfFile(" + krb5ConfFile.getAbsolutePath()
          + ") is not a file.");
    }

    // 3. Set and check krb5config.
    setKrb5Config(krb5ConfFile.getAbsolutePath());
    setConfiguration(conf);

    // 4. Check whether login is required.
    if (checkNeedLogin(userPrincipal)) {

    // 5. Log in to Hadoop and perform a check.
      loginHadoop(userPrincipal, userKeytabFile.getAbsolutePath());
    }

    // 6. Check and log in again.
    checkAuthenticateOverKrb();
    System.out.println("Login success!!!!!!!!!!!!!!");
  }

Parent topic: Developing an HDFS Application

Previous topic: HDFS Development Plan

Next topic: Writing Data to an HDFS File