创建转储任务
调用方法
请参见如何调用API。
URI
POST /v2/{project_id}/connectors/{connector_id}/sink-tasks
请求参数
参数 |
是否必选 |
参数类型 |
描述 |
---|---|---|---|
source_type |
是 |
String |
源数据类型,目前只支持BLOB。 |
task_name |
是 |
String |
转储任务名称。 |
destination_type |
是 |
String |
转存的目标类型,当前只支持OBS。 |
obs_destination_descriptor |
是 |
ObsDestinationDescriptor object |
转存目标的描述。 |
参数 |
是否必选 |
参数类型 |
描述 |
---|---|---|---|
topics |
是 |
String |
转存的topic列表名称,支持输入多个topic,以逗号“,”分隔。同时支持正则表达式。 |
topics_regex |
否 |
String |
转存topic的正则表达式,与topics必须二选一,不能同时都设置或者“.*”。 |
consumer_strategy |
是 |
String |
转储启动偏移量:
默认是latest。 |
destination_file_type |
是 |
String |
转储文件格式。当前只支持text。 |
access_key |
是 |
String |
访问密钥AK。 |
secret_key |
是 |
String |
访问密钥SK。 |
obs_bucket_name |
是 |
String |
存储该通道数据的OBS桶名称。 |
obs_path |
否 |
String |
存储在obs的路径,默认可以不填。 取值范围:英文字母、数字、下划线、中划线和斜杠,最大长度为64个字符。 默认配置为空。 |
partition_format |
是 |
String |
将转储文件的生成时间使用“yyyy/MM/dd/HH/mm”格式生成分区字符串,用来定义写到OBS的Object文件所在的目录层次结构。
说明:
数据转储成功后,存储的目录结构为“obs_bucket_path/file_prefix/partition_format”。默认时间是GMT+8 时间 |
record_delimiter |
否 |
String |
转储文件的记录分隔符,用于分隔写入转储文件的用户数据。 取值范围:
默认值:换行符“\n”。 |
deliver_time_interval |
是 |
Integer |
根据用户配置的时间,周期性的将数据导入OBS,若某个时间段内无数据,则此时间段不会生成打包文件。 取值范围:30~900 单位:秒。
说明:
使用OBS通道转储流式数据时该参数为必选配置。 |
响应参数
状态码: 200
参数 |
参数类型 |
描述 |
---|---|---|
task_id |
String |
任务ID。 |
请求示例
创建一个转储任务,转储topic-test中的数据到OBS。
POST https://{endpoint}/v2/{project_id}/connectors/{connector_id}/sink-tasks { "source_type" : "BLOB", "task_name" : "obsTransfer-1122976956", "destination_type" : "OBS", "obs_destination_descriptor" : { "consumer_strategy" : "earliest", "destination_file_type" : "TEXT", "access_key" : "XXXXXXXXXXXXXXXXXXXX", "secret_key" : "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "obs_bucket_name" : "6666", "obs_path" : "obsTransfer-1122976956", "partition_format" : "yyyy/MM/dd/HH/mm", "record_delimiter" : "", "deliver_time_interval" : 300, "topics" : "topic-test" } }
响应示例
状态码: 200
创建转储任务成功。
{ "task_id" : "2962882a-386c-4c9d-bb59-3b4f55d82961" }
SDK代码示例
SDK代码示例如下。
创建一个转储任务,转储topic-test中的数据到OBS。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 |
package com.huaweicloud.sdk.test; import com.huaweicloud.sdk.core.auth.ICredential; import com.huaweicloud.sdk.core.auth.BasicCredentials; import com.huaweicloud.sdk.core.exception.ConnectionException; import com.huaweicloud.sdk.core.exception.RequestTimeoutException; import com.huaweicloud.sdk.core.exception.ServiceResponseException; import com.huaweicloud.sdk.kafka.v2.region.KafkaRegion; import com.huaweicloud.sdk.kafka.v2.*; import com.huaweicloud.sdk.kafka.v2.model.*; public class CreateSinkTaskSolution { public static void main(String[] args) { // The AK and SK used for authentication are hard-coded or stored in plaintext, which has great security risks. It is recommended that the AK and SK be stored in ciphertext in configuration files or environment variables and decrypted during use to ensure security. // In this example, AK and SK are stored in environment variables for authentication. Before running this example, set environment variables CLOUD_SDK_AK and CLOUD_SDK_SK in the local environment String ak = System.getenv("CLOUD_SDK_AK"); String sk = System.getenv("CLOUD_SDK_SK"); ICredential auth = new BasicCredentials() .withAk(ak) .withSk(sk); KafkaClient client = KafkaClient.newBuilder() .withCredential(auth) .withRegion(KafkaRegion.valueOf("<YOUR REGION>")) .build(); CreateSinkTaskRequest request = new CreateSinkTaskRequest(); CreateSinkTaskReq body = new CreateSinkTaskReq(); ObsDestinationDescriptor obsDestinationDescriptorbody = new ObsDestinationDescriptor(); obsDestinationDescriptorbody.withTopics("topic-test") .withConsumerStrategy(ObsDestinationDescriptor.ConsumerStrategyEnum.fromValue("earliest")) .withDestinationFileType(ObsDestinationDescriptor.DestinationFileTypeEnum.fromValue("TEXT")) .withAccessKey("XXXXXXXXXXXXXXXXXXXX") .withSecretKey("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") .withObsBucketName("6666") .withObsPath("obsTransfer-1122976956") .withPartitionFormat(ObsDestinationDescriptor.PartitionFormatEnum.fromValue("yyyy/MM/dd/HH/mm")) .withRecordDelimiter("") .withDeliverTimeInterval(300); body.withObsDestinationDescriptor(obsDestinationDescriptorbody); body.withDestinationType(CreateSinkTaskReq.DestinationTypeEnum.fromValue("OBS")); body.withTaskName("obsTransfer-1122976956"); body.withSourceType(CreateSinkTaskReq.SourceTypeEnum.fromValue("BLOB")); request.withBody(body); try { CreateSinkTaskResponse response = client.createSinkTask(request); System.out.println(response.toString()); } catch (ConnectionException e) { e.printStackTrace(); } catch (RequestTimeoutException e) { e.printStackTrace(); } catch (ServiceResponseException e) { e.printStackTrace(); System.out.println(e.getHttpStatusCode()); System.out.println(e.getRequestId()); System.out.println(e.getErrorCode()); System.out.println(e.getErrorMsg()); } } } |
创建一个转储任务,转储topic-test中的数据到OBS。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
# coding: utf-8 from huaweicloudsdkcore.auth.credentials import BasicCredentials from huaweicloudsdkkafka.v2.region.kafka_region import KafkaRegion from huaweicloudsdkcore.exceptions import exceptions from huaweicloudsdkkafka.v2 import * if __name__ == "__main__": # The AK and SK used for authentication are hard-coded or stored in plaintext, which has great security risks. It is recommended that the AK and SK be stored in ciphertext in configuration files or environment variables and decrypted during use to ensure security. # In this example, AK and SK are stored in environment variables for authentication. Before running this example, set environment variables CLOUD_SDK_AK and CLOUD_SDK_SK in the local environment ak = __import__('os').getenv("CLOUD_SDK_AK") sk = __import__('os').getenv("CLOUD_SDK_SK") credentials = BasicCredentials(ak, sk) \ client = KafkaClient.new_builder() \ .with_credentials(credentials) \ .with_region(KafkaRegion.value_of("<YOUR REGION>")) \ .build() try: request = CreateSinkTaskRequest() obsDestinationDescriptorbody = ObsDestinationDescriptor( topics="topic-test", consumer_strategy="earliest", destination_file_type="TEXT", access_key="XXXXXXXXXXXXXXXXXXXX", secret_key="XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", obs_bucket_name="6666", obs_path="obsTransfer-1122976956", partition_format="yyyy/MM/dd/HH/mm", record_delimiter="", deliver_time_interval=300 ) request.body = CreateSinkTaskReq( obs_destination_descriptor=obsDestinationDescriptorbody, destination_type="OBS", task_name="obsTransfer-1122976956", source_type="BLOB" ) response = client.create_sink_task(request) print(response) except exceptions.ClientRequestException as e: print(e.status_code) print(e.request_id) print(e.error_code) print(e.error_msg) |
创建一个转储任务,转储topic-test中的数据到OBS。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 |
package main import ( "fmt" "github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/basic" kafka "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/kafka/v2" "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/kafka/v2/model" region "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/kafka/v2/region" ) func main() { // The AK and SK used for authentication are hard-coded or stored in plaintext, which has great security risks. It is recommended that the AK and SK be stored in ciphertext in configuration files or environment variables and decrypted during use to ensure security. // In this example, AK and SK are stored in environment variables for authentication. Before running this example, set environment variables CLOUD_SDK_AK and CLOUD_SDK_SK in the local environment ak := os.Getenv("CLOUD_SDK_AK") sk := os.Getenv("CLOUD_SDK_SK") auth := basic.NewCredentialsBuilder(). WithAk(ak). WithSk(sk). Build() client := kafka.NewKafkaClient( kafka.KafkaClientBuilder(). WithRegion(region.ValueOf("<YOUR REGION>")). WithCredential(auth). Build()) request := &model.CreateSinkTaskRequest{} obsPathObsDestinationDescriptor:= "obsTransfer-1122976956" recordDelimiterObsDestinationDescriptor:= "" obsDestinationDescriptorbody := &model.ObsDestinationDescriptor{ Topics: "topic-test", ConsumerStrategy: model.GetObsDestinationDescriptorConsumerStrategyEnum().EARLIEST, DestinationFileType: model.GetObsDestinationDescriptorDestinationFileTypeEnum().TEXT, AccessKey: "XXXXXXXXXXXXXXXXXXXX", SecretKey: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", ObsBucketName: "6666", ObsPath: &obsPathObsDestinationDescriptor, PartitionFormat: model.GetObsDestinationDescriptorPartitionFormatEnum().YYYY_MM_DD_HH_MM, RecordDelimiter: &recordDelimiterObsDestinationDescriptor, DeliverTimeInterval: int32(300), } request.Body = &model.CreateSinkTaskReq{ ObsDestinationDescriptor: obsDestinationDescriptorbody, DestinationType: model.GetCreateSinkTaskReqDestinationTypeEnum().OBS, TaskName: "obsTransfer-1122976956", SourceType: model.GetCreateSinkTaskReqSourceTypeEnum().BLOB, } response, err := client.CreateSinkTask(request) if err == nil { fmt.Printf("%+v\n", response) } else { fmt.Println(err) } } |
状态码
状态码 |
描述 |
---|---|
200 |
创建转储任务成功。 |
错误码
请参见错误码。