Datax导入(from mysql)
下载安装
#下载解压datax
wget http://datax-opensource.oss-cn-hangzhou.aliyuncs.com/datax.tar.gz
tar -zvxf datax.tar.gz
# writer推荐使用最新插件，下载地址：https://github.com/StarRocks/DataX/releases
#下载解压doriswrite
wget 'http://dorisdb-release.oss-cn-zhangjiakou.aliyuncs.com/doriswriter.tar.gz?Expires=1988133021&OSSAccessKeyId=LTAI4GFYjbX9e7QmFnAAvkt8&Signature=9RWd5APw26q%2B7aL8sJpu1h2namU%3D' -O doriswriter.tar.gz
tar -zvxf doriswriter.tar.gz
#将dorisdbwriter放至datax插件目录
mv doriswriter datax/plugin/writer/
DDL
StarRocks建表
CREATE TABLE dataxtest(
name1
tinyint(4) NULL COMMENT “”,
name2 tinyint(4) NULL COMMENT “”
) ENGINE=OLAP
duplicate KEY( name1 )
COMMENT “OLAP”
DISTRIBUTED BY HASH( name1 ) BUCKETS 3
PROPERTIES (
“replication_num” = “1”,
“in_memory” = “false”,
“storage_format” = “DEFAULT”
);
mysql建表
CREATE TABLE dataxtest(
name1
tinyint(4) NULL COMMENT “”,
name2 tinyint(4) NULL COMMENT “”
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT=‘订单表’
mysql插入数据
Insert into dataxtest values(1,1),(2,2),(3,3),(4,4)
SR创建用户:
CREATE USER datax@'%' IDENTIFIED BY '123456';
grant all on . to 'datax'@'%';
编辑任务：
vim job/mysql.json
{
"job": {
"setting": {
"speed": {
"channel": 1
},
"errorLimit": {
"record": 0,
"percentage": 0
}
},
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"username": "root",
"password": "123456",
"column": ["name1", "name2"],
"connection": [
{
"table": [ "dataxtest"],
"jdbcUrl": [
"jdbc:mysql://127.0.0.1:3306/canal"
]
}
]
}
},
"writer": {
"name": "doriswriter",
"parameter": {
"username": "datax",
"password": "123456",
"database": "qcy",
"table": "dataxtest",
"column": ["name1","name2"],
"preSql": [],
"postSql": [],
"jdbcUrl": "jdbc:mysql://127.0.0.1:8012/",
"loadUrl": ["127.0.0.1:8011"],
"loadProps": {}
}
}
}
]
}
}
执行任务
python bin/datax.py --jvm="-Xms6G -Xmx6G" --loglevel=debug job/mysql.json
写入成功，查看数据
