Heartbeat Data Formats

For a link from MySQL, PostgreSQL, GaussDB, or Oracle to Kafka, you can configure export of heartbeat table data to a topic. Data in a heartbeat table can be exported in the format configured for the link, which can be Canal-JSON or Debezium-JSON. Compared with common data formats, the two formats include the isHeartbeat field, for which the value is true. Based on this field, downstream data processing programs can filter heartbeat table data.

• Canal-Json

  {
  	"id": 39,
  	"serverId": "1753240249",
  	"data": [
  		{
  			"dlf_job_id": "1129476",
  			"external_job_id": "158663",
  			"heartbeat": 1768910859110
  		}
  	],
  	"old": [
  		{
  			"dlf_job_id": "1129476",
  			"external_job_id": "158663",
  			"heartbeat": 1768910829110
  		}
  	],
  	"es": 1768910859000,
  	"table": "your_heartbeat_table_name",
  	"type": "UPDATE",
  	"database": "your_database_name",
  	"ts": 1768910859116,
  	"mysqlType": {
  		"dlf_job_id": "VARCHAR(128)",
  		"external_job_id": "VARCHAR(128)",
  		"heartbeat": "BIGINT(20)"
  	},
  	"sqlType": {
  		"dlf_job_id": 12,
  		"external_job_id": 12,
  		"heartbeat": -5
  	},
  	"dbType": "mysql",
  	"isDdl": false,	"pkNames": [
  		"dlf_job_id"
  	],
  	"dataSourceName": "mysql_binlog_source",
  	"isHeartbeat": true
  }

• Debezium-Json

  {
  	"before": {
  		"dlf_job_id": "1129476",
  		"external_job_id": "158703",
  		"heartbeat": 1768987457610
  	},
  	"after": {
  		"dlf_job_id": "1129476",
  		"external_job_id": "158703",
  		"heartbeat": 1768987487610
  	},
  	"source": {
  		"name": "mysql_binlog_source",
  		"db": "your_database_name",
  		"table": "your_heartbeat_table_name",
  		"pkNames": [
  			"dlf_job_id"
  		],
  		"sqlType": {
  			"dlf_job_id": 12,
  			"external_job_id": 12,
  			"heartbeat": -5
  		},
  		"mysqlType": {
  			"dlf_job_id": "VARCHAR(128)",
  			"external_job_id": "VARCHAR(128)",
  			"heartbeat": "BIGINT(20)"
  		}
  	},
  	"tableChange": {
  		"type": "CREATE",
  		"previousId": null,
  		"id": {},
  		"table": {}
  	},
  	"tsMs": 1768987487000,
  	"op": "u",
  	"dataSourceName": "mysql_binlog_source",
  	"dbType": "mysql",
  	"isHeartbeat": true
  }