Updated on 2024-10-21 GMT+08:00

Downloading Data

Function

This API is used to download data from DIS streams.

Calling Method

For details, see Calling APIs.

URI

GET /v2/{project_id}/records

Table 1 Path Parameters

Parameter

Mandatory

Type

Description

project_id

Yes

String

Project ID

Table 2 Query Parameters

Parameter

Mandatory

Type

Description

partition-cursor

Yes

String

Data cursor, which needs to be obtained through the API for obtaining data cursors.

Value: 1 to 512 characters.

Note:

The validity period of a cursor is 5 minutes.

max_fetch_bytes

No

Integer

Maximum number of bytes that can be obtained for each request.

Note:

If the value is less than the size of a single record in the partition, the record cannot be obtained.

Request Parameters

Table 3 Request header parameters

Parameter

Mandatory

Type

Description

X-Auth-Token

Yes

String

User token.

It can be obtained by calling the IAM API used to obtain a user token. The value of X-Subject-Token in the response header is the user token.

Response Parameters

Status code: 200

Table 4 Response body parameters

Parameter

Type

Description

records

Array of Record objects

List of downloaded records

next_partition_cursor

String

Next iterator.

Note:

The validity period of a cursor is 5 minutes.

Table 5 Record

Parameter

Type

Description

partition_key

String

Partition key set when data is being uploaded.Note:If partition_key is transferred when data is uploaded, partition_key is returned when data is downloaded. If partition_id instead of partition_key is transferred when data is uploaded, no partition_key is returned.

sequence_number

String

Sequence number of the data record

data

String

Downloaded data

The downloaded data is the serialized binary data (Base64-encoded character string).

For example, the data returned by the data download API is "ZGF0YQ==", which is "data" after Base64 decoding.

timestamp

Long

Timestamp when the record is written to DIS

timestamp_type

String

Timestamp type

  • CreateTime: creation time.

Default: CreateTime

Example Requests

Downloading Data

GET https://{Endpoint}/v2/{project_id}/records

Example Responses

Status code: 200

Normal response

{
  "records" : [ {
    "partition_key" : "0",
    "sequence_number" : "485",
    "data" : "MTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTEx",
    "timestamp" : 1527577402541,
    "timestamp_type" : "CreateTime"
  } ],
  "next_partition_cursor" : "eyJpdGVyR2VuVGltZSI6MTQ5MDk1MDE1Nzc0NywiU3RyZWFtTmFtZSI6IjY2MCIsIlNoYXJkSWQiOiIwIiwiU2hhcmRJdGVyYXRvclR5cGUiOiJBVF9TRVFVRU5DRV9OVU1CRVIiLCJTdGFydGluZ1NlcXVlbmNlTnVtYmVyIjoiMjIiLCJUaW1lU3RhbXAiOjB9"
}

SDK Sample Code

The SDK sample code is as follows.

Java

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
package com.huaweicloud.sdk.test;

import com.huaweicloud.sdk.core.auth.ICredential;
import com.huaweicloud.sdk.core.auth.BasicCredentials;
import com.huaweicloud.sdk.core.exception.ConnectionException;
import com.huaweicloud.sdk.core.exception.RequestTimeoutException;
import com.huaweicloud.sdk.core.exception.ServiceResponseException;
import com.huaweicloud.sdk.dis.v2.region.DisRegion;
import com.huaweicloud.sdk.dis.v2.*;
import com.huaweicloud.sdk.dis.v2.model.*;


public class ConsumeRecordsSolution {

    public static void main(String[] args) {
        // The AK and SK used for authentication are hard-coded or stored in plaintext, which has great security risks. It is recommended that the AK and SK be stored in ciphertext in configuration files or environment variables and decrypted during use to ensure security.
        // In this example, AK and SK are stored in environment variables for authentication. Before running this example, set environment variables CLOUD_SDK_AK and CLOUD_SDK_SK in the local environment
        String ak = System.getenv("CLOUD_SDK_AK");
        String sk = System.getenv("CLOUD_SDK_SK");

        ICredential auth = new BasicCredentials()
                .withAk(ak)
                .withSk(sk);

        DisClient client = DisClient.newBuilder()
                .withCredential(auth)
                .withRegion(DisRegion.valueOf("<YOUR REGION>"))
                .build();
        ConsumeRecordsRequest request = new ConsumeRecordsRequest();
        request.withPartitionCursor("<partition-cursor>");
        request.withMaxFetchBytes(<max_fetch_bytes>);
        try {
            ConsumeRecordsResponse response = client.consumeRecords(request);
            System.out.println(response.toString());
        } catch (ConnectionException e) {
            e.printStackTrace();
        } catch (RequestTimeoutException e) {
            e.printStackTrace();
        } catch (ServiceResponseException e) {
            e.printStackTrace();
            System.out.println(e.getHttpStatusCode());
            System.out.println(e.getRequestId());
            System.out.println(e.getErrorCode());
            System.out.println(e.getErrorMsg());
        }
    }
}

Python

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
# coding: utf-8

import os
from huaweicloudsdkcore.auth.credentials import BasicCredentials
from huaweicloudsdkdis.v2.region.dis_region import DisRegion
from huaweicloudsdkcore.exceptions import exceptions
from huaweicloudsdkdis.v2 import *

if __name__ == "__main__":
    # The AK and SK used for authentication are hard-coded or stored in plaintext, which has great security risks. It is recommended that the AK and SK be stored in ciphertext in configuration files or environment variables and decrypted during use to ensure security.
    # In this example, AK and SK are stored in environment variables for authentication. Before running this example, set environment variables CLOUD_SDK_AK and CLOUD_SDK_SK in the local environment
    ak = os.environ["CLOUD_SDK_AK"]
    sk = os.environ["CLOUD_SDK_SK"]

    credentials = BasicCredentials(ak, sk)

    client = DisClient.new_builder() \
        .with_credentials(credentials) \
        .with_region(DisRegion.value_of("<YOUR REGION>")) \
        .build()

    try:
        request = ConsumeRecordsRequest()
        request.partition_cursor = "<partition-cursor>"
        request.max_fetch_bytes = <max_fetch_bytes>
        response = client.consume_records(request)
        print(response)
    except exceptions.ClientRequestException as e:
        print(e.status_code)
        print(e.request_id)
        print(e.error_code)
        print(e.error_msg)

Go

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
package main

import (
	"fmt"
	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/basic"
    dis "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/dis/v2"
	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/dis/v2/model"
    region "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/dis/v2/region"
)

func main() {
    // The AK and SK used for authentication are hard-coded or stored in plaintext, which has great security risks. It is recommended that the AK and SK be stored in ciphertext in configuration files or environment variables and decrypted during use to ensure security.
    // In this example, AK and SK are stored in environment variables for authentication. Before running this example, set environment variables CLOUD_SDK_AK and CLOUD_SDK_SK in the local environment
    ak := os.Getenv("CLOUD_SDK_AK")
    sk := os.Getenv("CLOUD_SDK_SK")

    auth := basic.NewCredentialsBuilder().
        WithAk(ak).
        WithSk(sk).
        Build()

    client := dis.NewDisClient(
        dis.DisClientBuilder().
            WithRegion(region.ValueOf("<YOUR REGION>")).
            WithCredential(auth).
            Build())

    request := &model.ConsumeRecordsRequest{}
	request.PartitionCursor = "<partition-cursor>"
	maxFetchBytesRequest:= int32(<max_fetch_bytes>)
	request.MaxFetchBytes = &maxFetchBytesRequest
	response, err := client.ConsumeRecords(request)
	if err == nil {
        fmt.Printf("%+v\n", response)
    } else {
        fmt.Println(err)
    }
}

More

For SDK sample code of more programming languages, see the Sample Code tab in API Explorer. SDK sample code can be automatically generated.

Status Codes

Status Code

Description

200

Normal response

Error Codes

See Error Codes.