Integrating Spring Boot with HDFS

Function

This example connects MRS to an HDFS cluster using Spring Boot for HDFS data read and write.

Configuration Files

Code Example

Use Spring Boot to connect to the HDFS cluster for reading and writing HDFS data.

@Repository
public class CustomerHDFSTemplate {
    protected static final Logger logger = LoggerFactory.getLogger(CustomerHDFSTemplate.class.getName());
    @Value("${spring.hdfs.config.dir}")
    private String configDir;
    @Value("${hdfs.user}")
    private String user;
    private FileSystem fSystem = null;
    @PostConstruct
    private void init() throws IOException {
        Configuration conf = new Configuration();
        // conf file
        conf.addResource(new Path(configDir + "/hdfs-site.xml"));
        conf.addResource(new Path(configDir + "/core-site.xml"));
        if ("kerberos".equalsIgnoreCase(conf.get("hadoop.security.authentication"))) {
            String krb5Conf = configDir + File.separator + "krb5.conf";
            String keytab = configDir + File.separator + "user.keytab";
            System.setProperty("java.security.krb5.conf", krb5Conf);
            LoginUtil.login(user, keytab, krb5Conf, conf);
        }
        fSystem = FileSystem.get(conf);
    }
    public void mkdir(String path) throws IOException {
        Path filePath = new Path(path);
        if (!fSystem.exists(filePath)) {
            fSystem.mkdirs(filePath);
            logger.info("mkdir path:{} success", path);
            return;
        }
        logger.info("path :{} exists", path);
    }
    public void remove(String path) throws IOException {
        Path filePath = new Path(path);
        if (fSystem.exists(filePath)) {
            fSystem.delete(filePath, true);
            logger.info("remove path:{} success", path);
            return;
        }
        logger.info("path :{} not exists", path);
    }
    public void write(String destPath, String fileName) throws IOException {
        final String content = "hi, I am bigdata. It is successful if you can see me.";
        FSDataOutputStream out = null;
        try {
            out = fSystem.create(new Path(destPath + File.separator + fileName));
            out.write(content.getBytes());
            out.hsync();
            logger.info("success to write.");
        } finally {
            // make sure the stream is closed finally.
            IOUtils.closeStream(out);
        }
    }
    public String read(String destPath, String fileName) throws IOException {
        String strPath = destPath + File.separator + fileName;
        Path path = new Path(strPath);
        FSDataInputStream in = null;
        BufferedReader reader = null;
        StringBuilder strBuffer = new StringBuilder();
        try {
            in = fSystem.open(path);
            reader = new BufferedReader(new InputStreamReader(in));
            String sTempOneLine;
            // write file
            while ((sTempOneLine = reader.readLine()) != null) {
                strBuffer.append(sTempOneLine);
            }
            logger.info("result is : " + strBuffer);
            logger.info("success to read.");
        } finally {
            // make sure the streams are closed finally.
            IOUtils.closeStream(reader);
            IOUtils.closeStream(in);
        }
        return strBuffer.toString();
    }
}

You can configure the parameters in the sample code in springboot > hdfs-examples > src > main > resources > application.properties or write them in the application.properties file in the sample running environment. Modify the parameters based on the actual environment. The meanings of the variables are as follows:

• spring.hdfs.config.dir: specifies the directory address for accessing HDFS configuration files, which include hdfs-site.xml, core-site.xml, krb5.conf, and user.keytab.
• hdfs.user: specifies the service username for accessing HDFS.