Initializing the HDFS
Function
Hadoop distributed file system (HDFS) initialization is a prerequisite for using application programming interfaces (APIs) provided by the HDFS. The process of initializing the HDFS is
- Load the HDFS service configuration file.
- Instantiate a FileSystem.
Obtain the keytab file for Kerberos security authentication in advance.
Configuration File Description
Table 1 lists the configuration files to be used during the login to the HDFS. These files already imported to the conf directory of the hdfs-example-security project.
|
File |
Function |
|---|---|
|
core-site.xml |
Configures HDFS parameters. |
|
hdfs-site.xml |
Configures HDFS parameters. |
|
smallfs-site.xml |
Configures SmallFS parameters. |
|
user.keytab |
Provides HDFS user information for Kerberos security authentication. |
|
krb5.conf |
Contains Kerberos server configuration information. |
- Different clusters cannot share the same user.keytab and krb5.conf files.
- The log4j.properties file under the conf directory can be configured based on your needs.
Example Codes
The following is code snippets. For complete codes, see the HdfsExample class in com.huawei.bigdata.hdfs.examples.
// Complete initialization and authentication.
confLoad();
authentication();
// Creating a sample project
HdfsExample hdfs_examples = new HdfsExample("/user/hdfs-examples", "test.txt");
/**
*
* If the application is running in the Linux OS, the path of core-site.xml, hdfs-site.xml must be modified to the absolute path of the client file in the Linux OS.
*
*
*/
private static void confLoad() throws IOException {
conf = new Configuration();
// conf file
conf.addResource(new Path(PATH_TO_HDFS_SITE_XML));
conf.addResource(new Path(PATH_TO_CORE_SITE_XML));
// conf.addResource(new Path(PATH_TO_SMALL_SITE_XML));
}
/**
*Safety authentication
*
*/
private static void authentication() throws IOException {
// security mode
if ("kerberos".equalsIgnoreCase(conf.get("hadoop.security.authentication"))) {
System.setProperty("java.security.krb5.conf", PATH_TO_KRB5_CONF);
LoginUtil.login(PRNCIPAL_NAME, PATH_TO_KEYTAB, PATH_TO_KRB5_CONF, conf);
}
}
/**
*Create a sample project.
*/
public HdfsExample(String path, String fileName) throws IOException {
this.DEST_PATH = path;
this.FILE_NAME = fileName;
instanceBuild();
}
private void instanceBuild() throws IOException {
fSystem = FileSystem.get(conf);
public synchronized static void login(String userPrincipal, String userKeytabPath, String krb5ConfPath, Configuration conf)
throws IOException
{
// 1.Check the input parameters.
if ((userPrincipal == null) || (userPrincipal.length() <= 0))
{
LOG.error("input userPrincipal is invalid.");
throw new IOException("input userPrincipal is invalid.");
}
if ((userKeytabPath == null) || (userKeytabPath.length() <= 0))
{
LOG.error("input userKeytabPath is invalid.");
throw new IOException("input userKeytabPath is invalid.");
}
if ((krb5ConfPath == null) || (krb5ConfPath.length() <= 0))
{
LOG.error("input krb5ConfPath is invalid.");
throw new IOException("input krb5ConfPath is invalid.");
}
if ((conf == null))
{
LOG.error("input conf is invalid.");
throw new IOException("input conf is invalid.");
}
// 2.Check whether the file exists.
File userKeytabFile = new File(userKeytabPath);
if (!userKeytabFile.exists())
{
LOG.error("userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") does not exsit.");
throw new IOException("userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") does not exsit.");
}
if (!userKeytabFile.isFile())
{
LOG.error("userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") is not a file.");
throw new IOException("userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") is not a file.");
}
File krb5ConfFile = new File(krb5ConfPath);
if (!krb5ConfFile.exists())
{
LOG.error("krb5ConfFile(" + krb5ConfFile.getAbsolutePath() + ") does not exsit.");
throw new IOException("krb5ConfFile(" + krb5ConfFile.getAbsolutePath() + ") does not exsit.");
}
if (!krb5ConfFile.isFile())
{
LOG.error("krb5ConfFile(" + krb5ConfFile.getAbsolutePath() + ") is not a file.");
throw new IOException("krb5ConfFile(" + krb5ConfFile.getAbsolutePath() + ") is not a file.");
}
// 3.Set and check krb5config.
setKrb5Config(krb5ConfFile.getAbsolutePath());
setConfiguration(conf);
// 4.Log in to Hadoop to check items.
loginHadoop(userPrincipal, userKeytabFile.getAbsolutePath());
LOG.info("Login success!!!!!!!!!!!!!!");
}
Feedback
Was this page helpful?
Provide feedbackThank you very much for your feedback. We will continue working to improve the documentation.