DataX supports reader and writer plug-ins for most common data sources, and provides detailed user guides.
Reader and writer plug-ins for CSV files
CSV files are text files that are read and written by using TXTReader and TXTWriter. For more information about the configuration file syntax, see the instructions on the official website of DataX.
Example of configuring the txtreader plug-in:
"reader":{
"name":"txtfilereader",
"parameter":{
"path":["Full path of the file"],
"encoding":"UTF-8",
"column":[
{ "index":0, "type":"long" }
,{ "index":1, "type":"long" }
,{ "index":2, "type":"string" }
,{ "index":3, "type":"double" }
,{ "index":4, "type":"string" }
],
"fieldDelimiter":"||",
"fileFormat":"text"
}
}
Example of configuring the txtwriter plug-in:
"writer":{
"name":"txtfilewriter",
"parameter":{
"path":"Full path of the file",
"fileName":"File name",
"writeMode":"truncate",
"dateFormat":"yyyy-MM-dd",
"charset":"UTF-8",
"nullFormat":"",
"fileDelimiter":"||"
}
}
Reader and writer plug-ins for MySQL databases
MysqlReader and MysqlWriter are reader and writer plug-ins for MySQL databases.
Example of configuring the mysqlreader plug-in:
"reader": {
"name": "mysqlreader",
"parameter": {
"username": "**u***",
"password": "**1***",
"column": [
"id",
"name"
],
"splitPk": "db_id",
"connection": [
{
"table": [
"table"
],
"jdbcUrl": [
"jdbc:mysql://10.0.0.0:3306/database"
]
}
]
}
}
Example of configuring the mysqlwriter plug-in:
"writer": {
"name": "mysqlwriter",
"parameter": {
"writeMode": "insert",
"username": "**u***",
"password": "**1***",
"column": [
"id",
"name"
],
"session": [
"set session sql_mode='ANSI'"
],
"preSql": [
"delete from test"
],
"connection": [
{
"jdbcUrl": "jdbc:mysql://10.0.0.0:3306/datax?useUnicode=true&characterEncoding=gbk",
"table": [
"test"
]
}
]
}
}
Reader and writer plug-ins for Oracle databases
OracleReader and OracleWriter are reader and writer plug-ins for Oracle databases.
Configuration example of OracleReader:
"reader": {
"name": "oraclereader",
"parameter": {
// The username that is used to access the database.
"username": "**u***",
// The password of the username that is used to access the database.
"password": "**1***",
"column": [
"id","name"
],
// Split the primary key.
"splitPk": "db_id",
"connection": [
{
"table": [
"table"
],
"jdbcUrl": [
"jdbc:oracle:thin:@[HOST_NAME]:PORT:[DATABASE_NAME]"
]
}
]
}
}
Example of configuring the oraclewriter plug-in:
"writer": {
"name": "oraclewriter",
"parameter": {
"username": "**u***",
"password": "**1***",
"column": [
"id",
"name"
],
"preSql": [
"delete from test"
],
"connection": [
{
"jdbcUrl": "jdbc:oracle:thin:@[HOST_NAME]:PORT:[DATABASE_NAME]",
"table": [
"test"
]
}
]
}
}
Reader and writer plug-ins for Db2 databases
Example of configuring the db2reader plug-in:
"reader":{
"name":"db2reader",
"parameter":{
"username":"**u***",
"password":"**1***",
"column":[
"SRC_COLUMN_LIST"
],
"connection":[
{
"table":[
"SRC_TABLE_NAME"
],
"jdbcUrl":[
"jdbc:db2://SRC_DB_IP:SRC_DB_PORT/SRC_DB_NAME"
]
}
]
}
}
Reader and writer plug-ins for OceanBase Database
oceanbasev10reader and oceanbasev10writer are reader and writer plug-ins for OceanBase Database. These plug-ins are provided by the OceanBase product team.
- Example of configuring the oceanbasev10reader plug-in:
"reader":{
"name":"oceanbasev10reader",
"parameter":{
"where":"",
"timeout":10000,
"readBatchSize":100000,
"readByPartition":"true",
"column": [
"Column name 1","Column name 2"
],
"connection":[
{
"jdbcUrl":["||_dsc_ob 10_dsc_||Cluster name:Tenant name||_dsc_ob10_dsc_||jdbc:oceanbase://Connection IP address:Connection port/Mode or database name"],
"table":["The name of the table."]
}
],
"username":"The username of the tenant.",
"password":"**1***"
}
}
Example: Exporting the ware table from OceanBase to a CSV file
[admin@*** /home/admin/datax3]
$cat job/ob_tpcc_ware_2_csv.json
{
"job":{
"setting":{
"speed":{
"channel":10
},
"errorLimit":{
"record":0, "percentage": 0.02
}
},
"content":[
{
"reader":{
"name":"oceanbasev10reader",
"parameter":{
"where":"",
"timeout":10000,
"readBatchSize":100000,
"readByPartition":"true",
"column": [
"W_ID","W_YTD","W_TAX","W_NAME","W_STREET_1","W_STREET_2","W_CITY","W_STATE","W_ZIP"
],
"connection":[
{
"jdbcUrl":["||_dsc_ob10_dsc_||obdemo:obbmsql||_dsc_ob10_dsc_||jdbc:oceanbase://127.1:2883/tpcc"],
"table":["ware"]
}
],
"username":"tpcc",
"password":"**1***"
}
},
"writer":{
"name":"txtfilewriter",
"parameter":{
"path":"/home/admin/csvdata/",
"fileName":"ware",
"writeMode":"truncate",
"dateFormat":"yyyy-MM-dd",
"charset":"UTF-8",
"nullFormat":"",
"fileDelimiter":"||"
}
}
}
]
}
}
[admin@*** /home/admin/datax3]
$bin/datax.py job/ob_tpcc_ware_2_csv.json
- Example of configuring the oceanbasev10writer plug-in:
"writer": {
"name": "oceanbasev10writer",
"parameter": {
"username": "The username of the tenant.",
"password": "**1***",
"writeMode": "insert",
"column": [
"Column name 1","Column name 2"
],
"preSql": [
""
],
"connection": [
{
"jdbcUrl": "||_dsc_ob10_dsc_||Cluster name:Tenant name||_dsc_ob10_dsc_||jdbc:oceanbase://Connection IP address:Connection port (2883 by default)/Mode or database name",
"table": [
"Table name"
]
}
],
"batchSize": 1024,
"activeMemPercent": "90"
}
}
Example: Importing data from a CSV file to an OceanBase table
[admin@*** /home/admin/datax3]
$cat job/csv_2_ob_tpcc_ware2.json
{
"job":{
"setting":{
"speed":{
"channel":32
},
"errorLimit":{
"record":0, "percentage": 0.02
}
},
"content":[
{
"reader":{
"name":"txtfilereader",
"parameter":{
"path":["/home/admin/csvdata/ware*"],
"encoding":"UTF-8",
"column":[
{ "index":0, "type":"long" }
,{ "index":1, "type":"long" }
,{ "index":2, "type":"long" }
,{ "index":3, "type":"string" }
,{ "index":4, "type":"string" }
,{ "index":5, "type":"string" }
,{ "index":6, "type":"string" }
,{ "index":7, "type":"string" }
,{ "index":8, "type":"string" }
],
"fieldDelimiter":",",
"fileFormat":"text"
}
},
"writer":{
"name":"oceanbasev10writer",
"parameter":{
"writeMode":"insert",
"column":[
"W_ID","W_YTD","W_TAX","W_NAME","W_STREET_1","W_STREET_2","W_CITY","W_STATE","W_ZIP"
],
"connection":[
{
"jdbcUrl":"||_dsc_ob10_dsc_||obdemo:obbmsql||_dsc_ob10_dsc_||jdbc:oceanbase://127.1:2883/tpcc",
"table":["WARE2"]
}
],
"username":"tpcc",
"password":"**1***",
"batchSize":256,
" memstoreThreshold":"90"
}
}
}
]
}
}