DataX supports reader and writer plug-ins for most common data sources, and provides detailed user guides.
Reader and writer plug-ins for CSV files
CSV files are text files that are read and written by using TXTReader and TXTWriter. For detailed syntax of the configuration file, see DataX product page.
Example of TXTReader configuration:
"reader":{
"name":"txtfilereader",
"parameter":{
"path":["Full path of the file"],
"encoding":"UTF-8",
"column":[
{ "index":0, "type":"long" }
,{ "index":1, "type":"long" }
,{ "index":2, "type":"string" }
,{ "index":3, "type":"double" }
,{ "index":4, "type":"string" }
],
"fieldDelimiter":"||",
"fileFormat":"text"
}
}
Example of TXTWriter Configuration example:
"writer":{
"name":"txtfilewriter",
"parameter":{
"path":"Full path of the file",
"fileName":"File name",
"writeMode":"truncate",
"dateFormat":"yyyy-MM-dd",
"charset":"UTF-8",
"nullFormat":"",
"fileDelimiter":"||"
}
}
Reader and writer plug-ins for MySQL databases
MysqlReader and MysqlWriter are reader and writer plug-ins for MySQL databases.
Configuration example of MysqlReader :
"reader": {
"name": "mysqlreader",
"parameter": {
"username": "**u***",
"password": "******",
"column": [
"id",
"name"
],
"splitPk": "db_id",
"connection": [
{
"table": [
"table"
],
"jdbcUrl": [
"jdbc:mysql://10.0.0.0:3306/database"
]
}
]
}
}
Configuration example of MysqlWriter :
"writer": {
"name": "mysqlwriter",
"parameter": {
"writeMode": "insert",
"username": "**u***",
"password": "******",
"column": [
"id",
"name"
],
"session": [
"set session sql_mode='ANSI'"
],
"preSql": [
"delete from test"
],
"connection": [
{
"jdbcUrl": "jdbc:mysql://10.0.0.0:3306/datax?useUnicode=true&characterEncoding=gbk",
"table": [
"test"
]
}
]
}
}
Reader and writer plug-ins for Oracle databases
OracleReader and OracleWriter are reader and writer plug-ins for Oracle databases.
Configuration example of OracleReader:
"reader": {
"name": "oraclereader",
"parameter": {
// The username that is used to access the database.
"username": "**u***",
// The password of the username that is used to access the database.
"password": "******",
"column": [
"id","name"
],
// Split the primary key
"splitPk": "db_id",
"connection": [
{
"table": [
"table"
],
"jdbcUrl": [
"jdbc:oracle:thin:@[HOST_NAME]:PORT:[DATABASE_NAME]"
]
}
]
}
}
Configuration example of OracleWriter :
"writer": {
"name": "oraclewriter",
"parameter": {
"username": "**u***",
"password": "******",
"column": [
"id",
"name"
],
"preSql": [
"delete from test"
],
"connection": [
{
"jdbcUrl": "jdbc:oracle:thin:@[HOST_NAME]:PORT:[DATABASE_NAME]",
"table": [
"test"
]
}
]
}
}
Reader and writer plug-ins for DB2 databases
Configuration example of DB2Reader :
"reader":{
"name":"db2reader",
"parameter":{
"username":"**u***",
"password":"******",
"column":[
"SRC_COLUMN_LIST"
],
"connection":[
{
"table":[
"SRC_TABLE_NAME"
],
"jdbcUrl":[
"jdbc:db2://SRC_DB_IP:SRC_DB_PORT/SRC_DB_NAME"
]
}
]
}
}
Configuration example of DB2Writer :
Reader and writer plug-ins for OceanBase databases
oceanbasev10reader and oceanbasev10writer are reader and writer plug-ins for OceanBase Database. These plug-ins are provided by the OceanBase product team.
- Configuration example of oceanbasev10reader
"reader":{
"name":"oceanbasev10reader",
"parameter":{
"where":"",
"timeout":10000,
"readBatchSize":100000,
"readByPartition":"true",
"column": [
"Column name 1","Column name 2"
],
"connection":[
{
"jdbcUrl":["||_dsc_ob10_dsc_||Cluster name:Tenant name||_dsc_ob10_dsc_||jdbc:oceanbase://Connection IP:Connection port/Mode or database name"],
"table":["The name of the table."]
}
],
"username":"The username of the tenant.",
"password":"******"
}
}
Example: Exporting the ware table from OceanBase to a CSV file
[admin@*** /home/admin/datax3]
$cat job/ob_tpcc_ware_2_csv.json
{
"job":{
"setting":{
"speed":{
"channel":10
},
"errorLimit":{
"record":0, "percentage": 0.02
}
},
"content":[
{
"reader":{
"name":"oceanbasev10reader",
"parameter":{
"where":"",
"timeout":10000,
"readBatchSize":100000,
"readByPartition":"true",
"column": [
"W_ID","W_YTD","W_TAX","W_NAME","W_STREET_1","W_STREET_2","W_CITY","W_STATE","W_ZIP"
],
"connection":[
{
"jdbcUrl":["||_dsc_ob10_dsc_||obdemo:obbmsql||_dsc_ob10_dsc_||jdbc:oceanbase://127.1:2883/tpcc"],
"table":["ware"]
}
],
"username":"tpcc",
"password":"******"
}
},
"writer":{
"name":"txtfilewriter",
"parameter":{
"path":"/home/admin/csvdata/",
"fileName":"ware",
"writeMode":"truncate",
"dateFormat":"yyyy-MM-dd",
"charset":"UTF-8",
"nullFormat":"",
"fileDelimiter":"||"
}
}
}
]
}
}
[admin@*** /home/admin/datax3]
$bin/datax.py job/ob_tpcc_ware_2_csv.json
- Configuration example of oceanbasev10writer :
When you use DataX to write data to OceanBase Database, control the writing speed to avoid using up all available incremental memory of OceanBase. We recommend that you set a writing speed limit in the DataX configuration file. You can use the memstoreThreshold keyword:
"writer": {
"name": "oceanbasev10writer",
"parameter": {
"username": "The username of the tenant.",
"password": "******",
"writeMode": "insert",
"column": [
"Column name 1","Column name 2"
],
"preSql": [
""
],
"connection": [
{
"jdbcUrl": "||_dsc_ob10_dsc_||Cluster name:Tenant name||_dsc_ob10_dsc_||jdbc:oceanbase://Connection IP:Connection port(Default value: 2883)/Mode or database name",
"table": [
"Table name"
]
}
],
"batchSize": 1024,
"memstoreThreshold": "90"
}
}
Example: Importing data from a CSV file to an OceanBase table
[admin@*** /home/admin/datax3]
$cat job/csv_2_ob_tpcc_ware2.json
{
"job":{
"setting":{
"speed":{
"channel":32
},
"errorLimit":{
"record":0, "percentage": 0.02
}
},
"content":[
{
"reader":{
"name":"txtfilereader",
"parameter":{
"path":["/home/admin/csvdata/ware*"],
"encoding":"UTF-8",
"column":[
{ "index":0, "type":"long" }
,{ "index":1, "type":"long" }
,{ "index":2, "type":"long" }
,{ "index":3, "type":"string" }
,{ "index":4, "type":"string" }
,{ "index":5, "type":"string" }
,{ "index":6, "type":"string" }
,{ "index":7, "type":"string" }
,{ "index":8, "type":"string" }
],
"fieldDelimiter":",",
"fileFormat":"text"
}
},
"writer":{
"name":"oceanbasev10writer",
"parameter":{
"writeMode":"insert",
"column":[
"W_ID","W_YTD","W_TAX","W_NAME","W_STREET_1","W_STREET_2","W_CITY","W_STATE","W_ZIP"
],
"connection":[
{
"jdbcUrl":"||_dsc_ob10_dsc_||obdemo:obbmsql||_dsc_ob10_dsc_||jdbc:oceanbase://127.1:2883/tpcc",
"table":["WARE2"]
}
],
"username":"tpcc",
"password":"******",
"batchSize":256,
" memstoreThreshold":"90"
}
}
}
]
}
}
[admin@*** /home/admin/datax3]
$bin/datax.py job/csv_2_ob_tpcc_ware2.json