Common CDL Service APIs

Input Parameters

• Kafka link input parameters:

  { 
  "name": "kafka_link", 
  "description":"test kafka link", 
  "link-type":"kafka", 
  "enabled":"true", 
  "link-config-values":  { 
  "inputs": [
          { "name": "bootstrap.servers", "value": "10.244.229.232:21007,10.244.229.252:21007" },
          { "name": "sasl.kerberos.service.name", "value": "kafka" },
          { "name": "security.protocol","value": "SASL_PLAINTEXT" }// In security mode, set value to SASL_PLAINTEXT. In normal mode, set value to PLAINTEXT.
          ] 
      }
  }

• MySQL link input parameters:

  { 
  "name": "MySQL_link", // Link name, which must be globally unique
  "description":"MySQL connection", // Link description
  "link-type":"mysql", // Link type
  "enabled":"true",
  "link-config-values":  { 
  "inputs": [
          { "name": "host", "value": "10.244.224.65" }, // IP address of the node where the database is installed
          { "name": "port", "value": "3306" },// Database listening port
          { "name": "database.name", "value": "demo" }, // Name of the database to be connected
          { "name": "user", "value": "mysql" }, // Username
          { "name": "password","value": "xxx" } ,// Password
          {"name":"schema", "value":"demo"}// Same as the database name
          ] 
      }
  }

• PostgreSQL link input parameters:

  { 
   "name": "Perf_PGSQL_link", 
   "description":"Perf PGSQL connection", 
   "link-type":"pgsql", 
   "enabled":"true",
   "link-config-values":  { 
   "inputs": [
      { "name": "host", "value": "10.244.224.65" },
      { "name": "port", "value": "5432" },
  
  
      {"name" : "db.name.alias", "value" : "demo"},// Database name
      { "name": "user", "value": "postgres" }, 
      { "name": "password","value": "xxx" }
      ] 
     }
  }

• Oracle link input parameters:

  { 
   "name": "Kris_Oracle_DB", 
   "description":"Description", 
   "link-type":"oracle", 
   "enabled":"true",
   "link-config-values":  { 
   "inputs": [
      { "name": "host", "value": "10.244.224.65" },
      { "name": "port", "value": "1521" },
      { "name": "sid", "value": "orcl" }, 
      { "name": "user", "value": "demo" }, 
      { "name": "password","value": "xxx" } 
      ] 
     }
  }

• HDFS link input parameters:

  { 
   "name": "Kris_HDFS_Link", 
   "description":"test HDFS connection", 
   "link-type":"hdfs", 
   "enabled":"true",
   "link-config-values":  { 
   "inputs": [
  
      { "name": "auth.keytabFile", "value": "/opt/key/user.keytab" },// This parameter is required only for clusters in security mode.
      { "name": "user", "value": "cdc_test" }, // This parameter is required only for clusters in security mode.
      { "name": "clusterType", "value": "mrshdfs" }, 
      { "name": "auth.principal","value": "cdc_test@HADOOP.COM" },// This parameter is required only for clusters in security mode.
      { "name": "hadoop.rpc.protection","value": "privacy" },// This parameter is required only for clusters in security mode.
      { "name": "auth.type","value": "kerberos" } // In security mode, set value to kerberos. In normal mode, set value to simple.
      ] 
     }
  }

• OBS link input parameters:

  { 
   "name": "Kris_OBS_Link", 
   "description":"test OBS connection", 
   "link-type":"obs", 
   "enabled":"true",
   "link-config-values":  { 
   "inputs": [
      { "name": "endpoint", "value": "https://obs.cn-north-5.myhuaweicloud.com:443" },
      { "name": "bucket", "value": "cdc-mrs" },
      { "name": "ak", "value": "QKGD1H9TLL2VI2SVH5KN" }, 
      { "name": "sk", "value": "ULKazzduVlUlElUAU9U3TeGtW8e8DnuphGJDxhbm" } 
      ] 
     }
  }

• MySQL job configuration parameters:

  "from-config-values": { 
          "inputs": [
              {"name" : "connector.class", "value" : "com.huawei.cdc.connect.mysql.MysqlSourceConnector"},
              {"name" : "schema", "value" : "kris"},
              {"name" : "db.name.alias", "value" : "kris"},
              {"name" : "whitelist", "value" : "TESTMYSQL.*"},
              {"name" : "tables", "value" : "TESTMYSQL.*"},
              {"name" : "tasks.max", "value" : "10"},
              {"name" : "mode", "value" : "insert,update,delete"},
              {"name" : "parse.dml.data", "value" : "true"},
              {"name" : "schema.auto.creation", "value" : "false"},
              {"name" : "errors.tolerance", "value" : "all"},
    	    {"name" : "producer.override.security.protocol", "value" : "SASL_PLAINTEXT"},// In security mode, set value to SASL_PLAINTEXT. In normal mode, set value to PLAINTEXT.
  	    {"name" : "consumer.override.security.protocol", "value" : "SASL_PLAINTEXT"}// In security mode, set value to SASL_PLAINTEXT. In normal mode, set value to PLAINTEXT.
          ]
      }

• Oracle job configuration parameters:

  "from-config-values": { 
          "inputs": [
              {"name" : "connector.class", "value" : "com.huawei.cdc.connect.oracle.OracleSourceConnector"},
              {"name" : "schema", "value" : "DEMO"},
              {"name" : "start.scn", "value" : "27749085"},
              {"name" : "whitelist", "value" : "TESTDATA.*"},
              {"name" : "tables", "value" : "TESTDATA.*"},
              {"name" : "tasks.max", "value" : "1"},
              {"name" : "poll.size", "value" : "1"},
              {"name" : "db.name.alias", "value" : "orcl"},
              {"name" : "mode", "value" : "insert,update,delete"},
              {"name" : "db.fetch.size", "value" : "1"},
              {"name" : "parse.dml.data", "value" : "true"},
              {"name" : "reset.offset", "value" : "false"},
              {"name" : "multitenant", "value" : "false"},
              {"name" : "schema.auto.creation", "value" : "false"},
              {"name" : "connector.client.config.override.policy", "value" : "All"},
              {"name" : "producer.override.security.protocol", "value" : "SASL_PLAINTEXT"}// In security mode, set value to SASL_PLAINTEXT. In normal mode, set value to PLAINTEXT.
          ]
      }

• PostgreSQL job configuration parameters:

  "from-config-values": { 
          "inputs": [
              {"name" : "connector.class", "value" : "com.huawei.cdc.connect.pgsql.PgsqlSourceConnector"},
              {"name" : "schema", "value" : "public"},// Schema name of the database
              {"name" : "whitelist", "value" : "station311"},
              {"name" : "tables", "value" : "station311"},
              {"name" : "tasks.max", "value" : "30"},
              {"name" : "snapshot.mode", "value" : "never"},
              {"name" : "slot.name", "value" : "performance"},
  
              {"name" : "mode", "value" : "insert,update,delete"},
              {"name" : "schema.auto.creation", "value" : "false"},
              {"name" : "binary.handling.mode", "value":"base64"},
              {"name" : "connector.client.config.override.policy", "value" : "All"},
              {"name" : "producer.override.security.protocol", "value" : "SASL_PLAINTEXT"}// In security mode, set value to SASL_PLAINTEXT. In normal mode, set value to PLAINTEXT.
          ]
      }

• HDFS job configuration parameters:

  "to-config-values": {
          "inputs": [
              {"name" : "connector.class", "value" : "com.huawei.cdc.connect.hdfs.HdfsConnector"},
              {"name" : "tasks.max", "value" : "30"},
              {"name" : "topics", "value" : "perf_test_mysql_avro"},
              {"name" : "path", "value" : "/data/"},
  	    {"name" : "errors.tolerance", "value" : "all"},
  	    {"name" : "interval", "value" : "60"},
              {"name" : "tables", "value": "*"},
              {"name" : "mode", "value" : "insert,update,delete"},
              {"name" : "interval", "value" : "1"},	
              {"name" : "cache.size", "value" : "10000"},
              {"name" : "producer.override.security.protocol", "value" : "SASL_PLAINTEXT"},// In security mode, set value to SASL_PLAINTEXT. In normal mode, set value to PLAINTEXT.
              {"name" : "consumer.override.security.protocol", "value" : "SASL_PLAINTEXT"}// In security mode, set value to SASL_PLAINTEXT. In normal mode, set value to PLAINTEXT.
          ]
      }

• OBS job configuration parameters:

  "to-config-values": {
          "inputs": [
              {"name" : "connector.class", "value" : "com.huawei.cdc.connect.obs.ObsConnector"},
              {"name" : "tasks.max", "value" : "1"},
              {"name" : "topics", "value" : "kris_pg_obs"},
              {"name" : "obs.bucket", "value" : "cdc-mrs"},
              {"name" : "path", "value" : "/data/"},
  	    {"name" : "errors.tolerance", "value" : "all"},
  	    {"name" : "interval", "value" : "60"},
              {"name" : "tables", "value": "*"},
              {"name" : "mode", "value" : "insert,update,delete"},
              {"name" : "interval", "value" : "1"},
              {"name" : "consumer.override.security.protocol", "value" : "SASL_PLAINTEXT"}// In security mode, set value to SASL_PLAINTEXT. In normal mode, set value to PLAINTEXT.
          ]
      }