Scala Sample Code

Updated on 2022-09-14 GMT+08:00

View PDF

The following code snippets are used as an example. For complete codes, see com.huawei.spark.examples.SparkOnMultiHbase

def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("SparkOnMultiHbaseExample")
    conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
    conf.set("spark.kryo.registrator", "com.huawei.spark.examples.MyRegistrator")
    val sc = new SparkContext(conf)val tableName = "SparkOnMultiHbase"
    val clusterFlagList=List("B", "A")
    clusterFlagList.foreach{ item =>
        val hbaseConf = getConf( item )
        println(hbaseConf.get("hbase.zookeeper.quorum"))
        val hbaseUtil = new HbaseUtil(sc,hbaseConf)
        hbaseUtil.writeToHbase(tableName)
        hbaseUtil.readFromHbase(tableName)
    }
    sc.stop()
}
private def getConf(item:String):Configuration={
    val conf: Configuration = HBaseConfiguration.create()
    val url = "/opt" + File.separator + "example" + File.separator + item + File.separator + "hbase-site.xml"
    conf.addResource(new File(url).toURI.toURL) 
    conf
}

Parent topic: Concurrent Access from Spark to HBase in Two Clusters

Previous topic: Overview

Next topic: Synchronizing HBase Data from Spark to CarbonData