创建MySQL-GaussDB集中式版校验任务流程
场景描述
本章节指导用户通过调用多个接口,创建MySQL-GaussDB集中式版校验任务。
操作步骤
- 参考认证鉴权,通过调用IAM服务的API,获取用户的Token。
- 调用批量创建任务接口。 请求示例:https://{endpoint}/v3/5237e10fe9aa4ad5b16b6a5245248314/jobs/batch-creation
{ "jobs" : [ { "name" : "DRS-dy-test-verify-mysql2gaussdbv5ha", "node_type" : "medium", "engine_type" : "mysql-to-gaussdbv5ha", "net_type" : "vpc", "job_direction" : "up", "db_use_type" : "verify", "task_type" : "FULL_INCR_TRANS", "customize_sutnet_id" : "b57b338a-51e6-4935-9a49-84d751e96bef", "source_endpoint": { "db_type": "mysql" }, "target_endpoint": { "region": "cn-southwest-244", "db_type": "gaussdbv5ha", "inst_id": "e04a0a23727a4dcab3cc5a4daec77193in14", "project_id": "5237e10fe9aa4ad5b16b6a5245248314" } } ] }响应示例:
{ "count": 1, "results": [ { "id": "3478a6e4-e549-4589-83f5-7713398jb8d9", "name": "DRS-dy-test-verify-mysql2gaussdbv5ha", "status": "CREATING", "create_time": "1786375018920" } ] } - 调用批量查询任务详情接口,查看任务状态,等待任务状态为“CONFIGURATION”时才可进行后续步骤。
请求示例:https://{endpoint}/v3/5237e10fe9aa4ad5b16b6a5245248314/jobs/batch-detail
{ "jobs" : [ "3478a6e4-e549-4589-83f5-7713398jb8d9" ], "page_req" : { "cur_page" : 1, "per_page" : 10 } }响应示例:
{ "count": 1, "results": [ { "id": "3478a6e4-e549-4589-83f5-7713398jb8d9", "name": "DRS-dy-test-verify-mysql2gaussdbv5ha", "status": "CONFIGURATION", "create_time": "1788850045488", "task_type": "FULL_INCR_TRANS", "node_version": "26.9.0.0", "source_endpoint": { "db_type": "mysql", "ssl_link": false }, "target_endpoint": { "ip": "192.168.140.93:8000,192.168.137.23:8000,192.168.149.196:8000", "region": "cn-southwest-244", "db_type": "gaussdbv5ha", "db_port": 8000, "ssl_link": false, "inst_id": "e04a0a23727a4dcab3cc5a4daec77193in14", "project_id": "5237e10fe9aa4ad5b16b6a5245248314", "inst_name": "Auto-drs-gaussdbv5HA-tar-2", "vpc_id": "7cf67bea-2465-4117-b1b9-a7683624a1ea", "subnet_id": "b57b338a-51e6-4935-9a49-84d751e96bef", "cluster_mode": "Ha" }, "net_type": "vpc", "inst_info": { "ip": "192.168.144.26", "status": "ACTIVE", "inst_type": "medium", "engine_type": "mysql-to-gaussdbv5ha", "volume_size": 500 }, "update_time": "1788850361460", "job_direction": "up", "db_use_type": "verify", "need_restart": false, "schema_type": "Tungsten", "object_switch": false, "sync_schema": false, "az_code": "cn-southwest-244a", "vpc_id": "7cf67bea-2465-4117-b1b9-a7683624a1ea", "vpc_name": "vpc-target-192", "subnet_id": "b57b338a-51e6-4935-9a49-84d751e96bef", "issue_coupon": false, "multi_write": false, "support_ip_v6": false, "is_restart_required": false, "enterprise_project_id": "0", "expired_days": "14", "fail_release_flag": "Y", "az_name": "az1", "is_multi_az": false, "support_many_to_one_compare": [ "lines", "object" ], "job_level_function_list": { "support_config_line_filter": "true", "support_re_edit_line_filter": "true", "support_replay_error_classification": "true", "support_compare_new_feature": "true", "is_support_new_periodic_compare": "true", "support_replay_export_in_kernel": "true", "support_gaussdbv5ha_pdbName": "true", "drs_support_sync_policy_mysql_to_gaussdbv5": "true", "job_start_support_version": "2307", "support_query_db_name_use_refresh": "true", "support_set_customized_dns": "true", "support_full_transfer_ignore_error": "true", "support_case_trans": "true", "support_compare_advance_config": "true", "support_ddl_history": "true", "support_compare_config_line_filter": "true", "drs_feature_guassdb_kafka_object_select_support_database_sync": "true", "support_append_many_columns": "new", "support_pause_or_restart_compare": "true", "support_ddl_alarm_policy": "true", "support_pg_more_version": "true", "drs_feature_support_ddl": "true", "support_replay_show_progress_details": "true", "support_dirty_data": "true", "drs_feature_support_modify_parameters": "true", "drs_feature_guassdb_kafka_object_select_support_schema_sync": "true" }, "pay_mode": "0", "original_job_direction": "up", "object_infos": [], "delay_start_time": 3, "is_required_password": true } ] } - 调用批量操作指定ID任务接口,进行源库、目标库测试连接。
请求示例:https://{endpoint}/v5/5237e10fe9aa4ad5b16b6a5245248314/jobs/action
{ "jobs" : [ { "job_id": "3478a6e4-e549-4589-83f5-7713398jb8d9", "action_name": "network", "action_params": { "endpoints": [ { "db_type": "mysql", "endpoint_type": "cloud", "endpoint_role": "so", "endpoint": { "instance_id": "d95ac396f0b24d40a7f46573d896a49ain01", "ip": "192.168.61.50", "db_port": 3306, "db_user": "xxx", "db_password": "xxx" }, "cloud": { "region": "cn-southwest-244", "project_id": "5237e10fe9aa4ad5b16b6a5245248314" }, "vpc": {}, "ssl": { "ssl_link": false } } ] } }, { "job_id": "3478a6e4-e549-4589-83f5-7713398jb8d9", "action_name": "network", "action_params": { "endpoints": [ { "db_type": "gaussdbv5ha", "endpoint_type": "cloud", "endpoint_role": "ta", "endpoint": { "instance_id": "e04a0a23727a4dcab3cc5a4daec77193in14", "ip": "192.168.140.93:8000,192.168.137.23:8000,192.168.149.196:8000", "db_port": 0, "db_user": "xxx", "db_password": "xxx", "pdb_name": "", "kerberos": null }, "cloud": { "region": "cn-southwest-244", "project_id": "5237e10fe9aa4ad5b16b6a5245248314" }, "vpc": {}, "ssl": { "ssl_link": false } } ] } } ] }响应示例:
{ "jobs": [ { "id": "3478a6e4-e549-4589-83f5-7713398jb8d9", "name": "DRS-dy-test-verify-mysql2gaussdbv5ha", "query_id": "64334c3b-faed-41c0-a0a4-2f163ffcefd0", "status": "success" }, { "id": "3478a6e4-e549-4589-83f5-7713398jb8d9", "name": "DRS-dy-test-verify-mysql2gaussdbv5ha", "query_id": "dd017ef9-fbc0-4ef2-8361-5c1e00e6151c", "status": "success" } ] } - 根据步骤4中响应的query_id调用查询任务详情接口查看测试连接结果,以源库测试连接为例:
请求示例:https://{endpoint}v5/5237e10fe9aa4ad5b16b6a5245248314/jobs/3478a6e4-e549-4589-83f5-7713398jb8d9?type=network&query_id=64334c3b-faed-41c0-a0a4-2f163ffcefd0
响应示例:
{ "job": { "network_results": [ { "ip": "192.168.61.50", "success": true, "status": "success", "db_port": "3306" } ] } } - 调用设置批量数据库对象选择接口,进行对象选择。
请求示例:https://{endpoint}/v3/5237e10fe9aa4ad5b16b6a5245248314/jobs/batch-select-objects
{ "jobs": [ { "job": [ { "id": "dws_drop-*-*-www", "object_alias_name": "www_tar", "object_name": "www", "object_type": "table", "parent_id": "dws_drop", "select": "true" }, { "id": "dws_drop", "object_name": "dws_drop", "object_type": "database", "select": "partial" } ], "job_id": "3478a6e4-e549-4589-83f5-7713398jb8d9", "selected": true, "target_root_db": { "db_encoding": "utf8", "db_name": "jrho" } } ] }响应示例:
{ "all_counts": 1, "results": [ { "job_id": "d2977945-e6a9-4def-a029-a7e40eajb8d9", "status": true } ] } - 调用查询数据库对象信息接口,查看对象选择信息。
请求示例:https://{endpoint}/v5.1/5237e10fe9aa4ad5b16b6a5245248314/jobs/3478a6e4-e549-4589-83f5-7713398jb8d9/db-object?type=modified
响应示例:
{ "object_scope": "table", "target_root_db": { "syncAll": null, "db_name": "jrho", "db_encoding": "utf8" }, "object_info": { "dws_drop": { "name": "dws_drop", "all": false, "tables": { "www": { "name": "www_tar", "type": "table", "all": true, "is_synchronized": false } }, "is_synchronized": false, "total_table_num": 1 } }, "max_table_num": 50000, "id": "3478a6e4-e549-4589-83f5-7713398jb8d9" } - 根据业务需要调用更新指定任务数据加工规则接口,指定附加列信息。无业务需要可跳转至步骤11。
说明:
- 此接口可设置数据过滤、列加工和附加列信息,若需要通过此接口设置附加列,需要先于数据过滤、列加工操作,否则会丢失数据过滤/列加工信息。
- 仅部分链路支持设置附加列,需要确认当前链路是否支持。其中,MySQL-GaussDB 集中式支持,故使用此接口指定附加列信息。
请求示例:https://{endpoint}/v5/5237e10fe9aa4ad5b16b6a5245248314/jobs/3478a6e4-e549-4589-83f5-7713398jb8d9/data-processing-rules
{ "data_process_info": [ { "add_columns": [ { "column_name": "append", "column_type": "ADDITIONALCOLUMN,default_value", "data_type": "int", "column_value": "1" } ], "db_object": { "object_scope": "table", "object_info": { "dws_drop": { "name": "dws_drop", "all": false, "tables": { "www": { "name": "www_tar", "all": true, "columns": { "append": { "name": "append", "type": "int", "additional": true, "operation_type": "ADDITIONALCOLUMN,default_value", "value": "1" } } } } } } } } ], "source": "job" }响应示例:
{ "id": "260316b0-1216-432f-9476-30d33538ce41", "job_id": "3478a6e4-e549-4589-83f5-7713398jb8d9", "status": "pending" } - 根据步骤8返回的id调用获取指定任务数据加工规则更新结果接口,查看指定附加列操作是否成功。
请求示例:https://{endpoint}/v5/5237e10fe9aa4ad5b16b6a5245248314/job/3478a6e4-e549-4589-83f5-7713398jb8d9/data-processing-rules/result?query_id=260316b0-1216-432f-9476-30d33538ce41
响应示例:
{ "id": "260316b0-1216-432f-9476-30d33538ce41", "job_id": "3478a6e4-e549-4589-83f5-7713398jb8d9", "status": "success" } - 调用查询数据加工对象的附加列信息接口,获取附加列信息
请求示例:https://{endpoint}/v5/5237e10fe9aa4ad5b16b6a5245248314/jobs/3478a6e4-e549-4589-83f5-7713398jb8d9/extra-column-info
响应示例:
{ "column_process_objects": [ { "object_alias_name": "dws_drop-*-*-www_tar", "object_source_names": [ "dws_drop-*-*-www" ], "is_sent": false, "extra_column_infos": [ { "column_name": "append", "column_type": "ADDITIONALCOLUMN,default_value", "data_type": "int", "column_value": "1" } ] } ], "total_count": 1 } - 根据业务需要调用校验数据过滤规则接口,进行数据过滤规则校验。无数据过滤业务需要需要可跳转至步骤17。
说明:仅部分链路支持设置数据过滤功能,需要确认当前链路是否支持。其中,MySQL-GaussDB 集中式支持,故使用此接口指定数据过滤规则。
请求示例:https://{endpoint}/v5/5237e10fe9aa4ad5b16b6a5245248314/jobs/3478a6e4-e549-4589-83f5-7713398jb8d9/check-filter-rules
{ "general_filtering_list": [ { "filter_object_list": [ { "id": "dws_drop-*-*-www", "parent_id": "dws_drop", "object_name": "www", "object_type": "table" } ], "source_filter_condition": "id > 1" } ], "source": "job", "data_filter_type": "common" }响应示例:
{ "id": "1d8eb393-5e34-4309-ad64-ce4f8591adf4" } - 根据步骤11返回的id调用获取数据过滤校验结果接口,查看数据过滤校验结果。
请求示例:https://{endpoint}/v5/5237e10fe9aa4ad5b16b6a5245248314/job/3478a6e4-e549-4589-83f5-7713398jb8d9/data-filtering/result?query_id=1d8eb393-5e34-4309-ad64-ce4f8591adf4
响应示例:
{ "success_count": 1, "failed_count": 0, "db_object_filtering_result": [ { "db_name": "dws_drop", "table_name": "www", "is_success": true, "source": "job" } ] } - 调用保存数据过滤规则接口,指定数据过滤规则。
请求示例:https://{endpoint}/v5/5237e10fe9aa4ad5b16b6a5245248314/jobs/3478a6e4-e549-4589-83f5-7713398jb8d9/filter-rules
{ "general_filtering_list": [ { "filter_object_list": [ { "id": "dws_drop-*-*-www", "parent_id": "dws_drop", "object_name": "www", "object_type": "table" } ], "source_filter_condition": "id > 1" } ], "source": "job", "data_filter_type": "common" } - 调用采集数据过滤规则信息接口,查询已设置的数据过滤规则。
请求示例:https://{endpoint}/v5/5237e10fe9aa4ad5b16b6a5245248314/jobs/3478a6e4-e549-4589-83f5-7713398jb8d9/filter-rules
{ "filter_type" : "common", "is_all_rules" : true }响应示例:
{ "id": "4f79a05a-121b-49ba-a1f7-727be83170ea" } - 根据步骤14返回的id调用获取数据过滤规则信息结果接口,获取查看数据过滤规则结果。
请求示例:https://{endpoint}/v5/5237e10fe9aa4ad5b16b6a5245248314/jobs/3478a6e4-e549-4589-83f5-7713398jb8d9/filter-rules/result?query_id=4f79a05a-121b-49ba-a1f7-727be83170ea
响应示例:
{ "status": "SUCCESS", "total_count": 1, "rules": [ { "batch_id": 1, "filter_object_list": [ { "batch_id": 1, "id": "dws_drop-*-*-www-*-*---conditionFilter--", "parent_id": "dws_drop-*-*-www", "object_name": "--conditionFilter--", "object_alias_name": null, "object_type": "column", "db_name": "dws_drop", "schema_name": null, "table_name": "www", "is_data_filter": "true", "data_filter_type": "contentConditionalFilter", "filter_condition": "id > 1", "is_synchronized": false } ], "filter_rules_resp": { "source_filter_condition": "id > 1", "target_filter_condition": null, "condition_separated": "false" }, "advanced_setting_resp": null } ] } - 根据需要调用删除数据过滤规则信息接口,删除数据过滤规则。
请求示例:https://{endpoint}/v5/5237e10fe9aa4ad5b16b6a5245248314/jobs/3478a6e4-e549-4589-83f5-7713398jb8d9/filter-rules
{ "rule_ids" : [ 1 ], "type" : "common" } - 根据业务需要调用查询对象的列信息(列映射、列过滤)接口,查询列信息。无列加工业务需要需要可跳转至步骤20。
说明:仅部分链路支持设置列加工功能,需要确认当前链路是否支持。其中,MySQL-GaussDB 集中式支持,故使用此接口指定列加工规则。
请求示例:https://{endpoint}/v5/5237e10fe9aa4ad5b16b6a5245248314/job/3478a6e4-e549-4589-83f5-7713398jb8d9/column-info
{ "object_ids": [ "dws_drop-*-*-www" ], "refresh": false }响应示例:
{ "column_map_infos": [ { "object_id": "dws_drop-*-*-www", "column_info_lists": [ { "status": true, "partition": false, "column_name": "id", "column_type": "tinyint(3) unsigned", "key_type": "PRI", "column_mapped_name": null, "desensitization_algorithm": null } ] } ], "object_with_column_infos": [] } - 调用修改对象的列信息(列映射、列过滤)接口,指定列加工规则。
请求示例:https://{endpoint}/v5/5237e10fe9aa4ad5b16b6a5245248314/job/3478a6e4-e549-4589-83f5-7713398jb8d9/column-info-modification
{ "column_map_infos": [ { "object_id": "dws_drop-*-*-www", "column_info_lists": [ { "status": true, "partition": false, "column_name": "id", "column_type": "tinyint(3) unsigned", "key_type": "PRI", "column_mapped_name": "id_tar", "desensitization_algorithm": null } ] } ], "object_with_column_infos": [] }响应示例:
{ "modify_result": true } - 根据需要调用删除对象的列信息(列映射、列过滤)接口,删除数据过滤规则。
请求示例:https://{endpoint}/v5/5237e10fe9aa4ad5b16b6a5245248314/job/3478a6e4-e549-4589-83f5-7713398jb8d9/column-info
{ "table_ids": [ "dws_drop-*-*-www" ], "db_ids" : [ ], "schema_ids" : [ ] }响应示例:
{ "delete_result": true } - 调用批量操作指定ID任务接口,下发预检查。
请求示例:https://{endpoint}/v5/5237e10fe9aa4ad5b16b6a5245248314/jobs/action
{ "jobs" : [ { "job_id": "3478a6e4-e549-4589-83f5-7713398jb8d9", "action_name": "precheck", "action_params": { "precheck_mode": "forStartJob" } } ] }响应示例:
{ "jobs": [ { "id": "3478a6e4-e549-4589-83f5-7713398jb8d9", "name": "DRS-dy-test-verify-mysql2gaussdbv5ha", "query_id": "3478a6e4-e549-4589-83f5-7713398jb8d9", "status": "success" } ] } - 根据步骤20中响应的query_id调用查询任务详情接口查看预检查结果。预检查通过率100%后才可启动任务。
请求示例:https://{endpoint}/v5/5237e10fe9aa4ad5b16b6a5245248314/jobs/3478a6e4-e549-4589-83f5-7713398jb8d9?type=precheck&query_id=3478a6e4-e549-4589-83f5-7713398jb8d9
响应示例:
{ "job": { "precheck_result": { "result": true, "process": "100%", "total_passed_rate": "100%", "rds_instance_id": "e04a0a23727a4dcab3cc5a4daec77193in14", "job_direction": "up", "precheck_results": [ { "item": "dstDbConnection", "result": "PASSED", "group": "network", "is_support_skip": false }, { "item": "srcDbConnection", "result": "PASSED", "group": "network", "is_support_skip": false } ] } } } - 调用批量操作指定ID任务接口启动任务。
请求示例:https://{endpoint}/v5/5237e10fe9aa4ad5b16b6a5245248314/jobs/action
{ "jobs" : [ { "job_id": "3478a6e4-e549-4589-83f5-7713398jb8d9", "action_name": "start" } ] }响应示例:
{ "jobs": [ { "id": "3478a6e4-e549-4589-83f5-7713398jb8d9", "name": "DRS-dy-test-verify-mysql2gaussdbv5ha", "status": "success" } ] }