Help Center > > Developer Guide> Developing a DLI Datasource Connection Using a Spark Job> Interconnecting with CSS Security Clusters (By PySpark)> Complete Example Code

Complete Example Code

Updated at: Mar 17, 2020 GMT+08:00

Connecting to Datasources Through DataFrame APIs

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
# _*_ coding: utf-8 _*_
from __future__ import print_function
from pyspark.sql.types import Row, StructType, StructField, IntegerType, StringType
from pyspark.sql import SparkSession

if __name__ == "__main__":
  # Create a SparkSession session.   
  sparkSession = SparkSession.builder.appName("datasource-css").getOrCreate()
  sparkSession.conf.set("fs.s3a.access.key", ak)
  sparkSession.conf.set("fs.s3a.secret.key", sk)

  # Setting cross-source connection parameters  
  resource = "/mytest/css";
  nodes = "to-css-1174404953-hDTx3UPK.datasource.com:9200"

  # Setting schema  
  schema = StructType([StructField("id", IntegerType(), False),       
                       StructField("name", StringType(), False)])

  # Construction data 
  rdd = sparkSession.sparkContext.parallelize([Row(1, "John"), Row(2, "Bob")])

  # Create a DataFrame from RDD and schema  
  dataFrame = sparkSession.createDataFrame(rdd, schema)

  # Write data to the CSS 
  dataFrame.write.format("css")
    .option("resource", resource)
    .option("nodes", nodes)
    .option("es.net.ssl", "true")
    .option("es.net.ssl.keystore.location", "s3a://AK:SK@bucket name/path/transport-keystore.jks")
    .option("es.net.ssl.keystore.pass", "***")
    .option("es.net.ssl.truststore.location", "s3a://AK:SK@bucket name/path/truststore.jks")
    .option("es.net.ssl.truststore.pass", "***")
    .option("es.net.http.auth.user", "admin")
    .option("es.net.http.auth.pass", "***")
    .mode("Overwrite")
    .save()

  # Read data  
  jdbcDF = sparkSession.read.format("css")
    .option("resource", resource)
    .option("nodes", nodes)
    .option("es.net.ssl", "true")
    .option("es.net.ssl.keystore.location", "s3a://AK:SK@bucket name/path/transport-keystore.jks")
    .option("es.net.ssl.keystore.pass", "***")
    .option("es.net.ssl.truststore.location", "s3a://AK:SK@bucket name/path/truststore.jks")
    .option("es.net.ssl.truststore.pass", "***")
    .option("es.net.http.auth.user", "admin")
    .option("es.net.http.auth.pass", "***")
    .load()
  jdbcDF.show()

  # close session  
  sparkSession.stop()

Connecting to Datasources Through SQL APIs

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
# _*_ coding: utf-8 _*_
from __future__ import print_function
from pyspark.sql import SparkSession

if __name__ == "__main__":
  # Create a SparkSession session.  
  sparkSession = SparkSession.builder.appName("datasource-css").getOrCreate()
  sparkSession.conf.set("fs.s3a.access.key", ak)
  sparkSession.conf.set("fs.s3a.secret.key", sk)

  # Create a DLI data table for DLI-associated CSS   
  sparkSession.sql(
    "create table css_table(id long, name string) using css options( 
        'nodes'='to-css-1174404953-hDTx3UPK.datasource.com:9200',
        'nodes.wan.only'='true',
        'resource'='/mytest/css',
	'es.net.ssl'='true',
       'es.net.ssl.keystore.location'='s3a://AK:SK@bucket name/path/transport-keystore.jks',
	'es.net.ssl.keystore.pass'='***',
       'es.net.ssl.truststore.location'='s3a://AK:SK@bucket name/path/truststore.jks',
	'es.net.ssl.truststore.pass'='***',
	'es.net.http.auth.user'='admin',
	'es.net.http.auth.pass'='***')")

  # Insert data into the DLI data table  
  sparkSession.sql("insert into css_table values(3,'tom')")

  # Read data from DLI data table   
  jdbcDF = sparkSession.sql("select * from css_table")   
  jdbcDF.show()

  # close session  
  sparkSession.stop()

Did you find this page helpful?

Submit successfully!

Thank you for your feedback. Your feedback helps make our documentation better.

Failed to submit the feedback. Please try again later.

Which of the following issues have you encountered?







Please complete at least one feedback item.

Content most length 200 character

Content is empty.

OK Cancel