TuGraph-DataX
此文档主要介绍 TuGraph DataX 的安装编译和使用示例
1.简介
TuGraph 在阿里开源的 DataX 基础上添加了 TuGraph 的写插件以及 TuGraph jsonline 数据格式的支持，其他数据源可以通过 DataX 往 TuGraph 里面写数据。 TuGraph DataX 介绍 https://github.com/TuGraph-family/DataX，支持的功能包括：
从 MySQL、SQL Server、Oracle、PostgreSQL、HDFS、Hive、HBase、OTS、ODPS、Kafka 等各种异构数据源导入 TuGraph
将 TuGraph 导入相应的目标源 （待开发）
DataX 原始项目介绍参考 https://github.com/alibaba/DataX
2.编译安装
git clone https://github.com/TuGraph-family/DataX.git
mvn -U clean package assembly:assembly -Dmaven.test.skip=true
编译出来的 DataX 文件在 target 目录下
3.文本数据通过DataX导入TuGraph
我们以 TuGraph 手册中导入工具 lgraph_import 章节举的数据为例子，有三个 csv 数据文件，如下：
actors.csv
nm015950,Stephen Chow
nm0628806,Man-Tat Ng
nm0156444,Cecilia Cheung
nm2514879,Yuqi Zhang
movies.csv
tt0188766,King of Comedy,1999,7.3
tt0286112,Shaolin Soccer,2001,7.3
tt4701660,The Mermaid,2016,6.3
roles.csv
nm015950,Tianchou Yin,tt0188766
nm015950,Steel Leg,tt0286112
nm0628806,,tt0188766
nm0628806,coach,tt0286112
nm0156444,PiaoPiao Liu,tt0188766
nm2514879,Ruolan Li,tt4701660
然后建三个 DataX 的 job 配置文件：
job_actors.json
{
"job": {
"setting": {
"speed": {
"channel": 1
}
},
"content": [
{
"reader": {
"name": "txtfilereader",
"parameter": {
"path": ["actors.csv"],
"encoding": "UTF-8",
"column": [
{
"index": 0,
"type": "string"
},
{
"index": 1,
"type": "string"
}
],
"fieldDelimiter": ","
}
},
"writer": {
"name": "tugraphwriter",
"parameter": {
"host": "127.0.0.1",
"port": 7071,
"username": "admin",
"password": "73@TuGraph",
"graphName": "default",
"schema": [
{
"label": "actor",
"type": "VERTEX",
"properties": [
{ "name": "aid", "type": "STRING" },
{ "name": "name", "type": "STRING" }
],
"primary": "aid"
}
],
"files": [
{
"label": "actor",
"format": "JSON",
"columns": ["aid", "name"]
}
]
}
}
}
]
}
}
job_movies.json
{
"job": {
"setting": {
"speed": {
"channel": 1
}
},
"content": [
{
"reader": {
"name": "txtfilereader",
"parameter": {
"path": ["movies.csv"],
"encoding": "UTF-8",
"column": [
{
"index": 0,
"type": "string"
},
{
"index": 1,
"type": "string"
},
{
"index": 2,
"type": "string"
},
{
"index": 3,
"type": "string"
}
],
"fieldDelimiter": ","
}
},
"writer": {
"name": "tugraphwriter",
"parameter": {
"host": "127.0.0.1",
"port": 7071,
"username": "admin",
"password": "73@TuGraph",
"graphName": "default",
"schema": [
{
"label": "movie",
"type": "VERTEX",
"properties": [
{ "name": "mid", "type": "STRING" },
{ "name": "name", "type": "STRING" },
{ "name": "year", "type": "STRING" },
{ "name": "rate", "type": "FLOAT", "optional": true }
],
"primary": "mid"
}
],
"files": [
{
"label": "movie",
"format": "JSON",
"columns": ["mid", "name", "year", "rate"]
}
]
}
}
}
]
}
}
job_roles.json
{
"job": {
"setting": {
"speed": {
"channel": 1
}
},
"content": [
{
"reader": {
"name": "txtfilereader",
"parameter": {
"path": ["roles.csv"],
"encoding": "UTF-8",
"column": [
{
"index": 0,
"type": "string"
},
{
"index": 1,
"type": "string"
},
{
"index": 2,
"type": "string"
}
],
"fieldDelimiter": ","
}
},
"writer": {
"name": "tugraphwriter",
"parameter": {
"host": "127.0.0.1",
"port": 7071,
"username": "admin",
"password": "73@TuGraph",
"graphName": "default",
"schema": [
{
"label": "play_in",
"type": "EDGE",
"properties": [{ "name": "role", "type": "STRING" }]
}
],
"files": [
{
"label": "play_in",
"format": "JSON",
"SRC_ID": "actor",
"DST_ID": "movie",
"columns": ["SRC_ID", "role", "DST_ID"]
}
]
}
}
}
]
}
}
./lgraph_server -c lgraph_standalone.json -d 'run' 启动 TuGraph 后依次执行如下三个命令：
python3 datax/bin/datax.py job_actors.json
python3 datax/bin/datax.py job_movies.json
python3 datax/bin/datax.py job_roles.json
4.MySQL数据通过DataX导入TuGraph
我们在 test database 下建立如下电影 movies 表
CREATE TABLE `movies` (
`mid` varchar(200) NOT NULL,
`name` varchar(100) NOT NULL,
`year` int(11) NOT NULL,
`rate` float(5,2) unsigned NOT NULL,
PRIMARY KEY (`mid`)
);
往表中插入几条数据
insert into
test.movies (mid, name, year, rate)
values
('tt0188766', 'King of Comedy', 1999, 7.3),
('tt0286112', 'Shaolin Soccer', 2001, 7.3),
('tt4701660', 'The Mermaid', 2016, 6.3);
建立一个 DataX 的 job 配置文件
job_mysql_to_tugraph.json
配置字段方式
{
"job": {
"setting": {
"speed": {
"channel": 1
}
},
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"username": "root",
"password": "root",
"column": ["mid", "name", "year", "rate"],
"splitPk": "mid",
"connection": [
{
"table": ["movies"],
"jdbcUrl": ["jdbc:mysql://127.0.0.1:3306/test?useSSL=false"]
}
]
}
},
"writer": {
"name": "tugraphwriter",
"parameter": {
"host": "127.0.0.1",
"port": 7071,
"username": "admin",
"password": "73@TuGraph",
"graphName": "default",
"schema": [
{
"label": "movie",
"type": "VERTEX",
"properties": [
{ "name": "mid", "type": "STRING" },
{ "name": "name", "type": "STRING" },
{ "name": "year", "type": "STRING" },
{ "name": "rate", "type": "FLOAT", "optional": true }
],
"primary": "mid"
}
],
"files": [
{
"label": "movie",
"format": "JSON",
"columns": ["mid", "name", "year", "rate"]
}
]
}
}
}
]
}
}
写简单 sql 方式
{
"job": {
"setting": {
"speed": {
"channel": 1
}
},
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"username": "root",
"password": "root",
"connection": [
{
"querySql": [
"select mid, name, year, rate from test.movies where year > 2000;"
],
"jdbcUrl": ["jdbc:mysql://127.0.0.1:3306/test?useSSL=false"]
}
]
}
},
"writer": {
"name": "tugraphwriter",
"parameter": {
"host": "127.0.0.1",
"port": 7071,
"username": "admin",
"password": "73@TuGraph",
"graphName": "default",
"schema": [
{
"label": "movie",
"type": "VERTEX",
"properties": [
{ "name": "mid", "type": "STRING" },
{ "name": "name", "type": "STRING" },
{ "name": "year", "type": "STRING" },
{ "name": "rate", "type": "FLOAT", "optional": true }
],
"primary": "mid"
}
],
"files": [
{
"label": "movie",
"format": "JSON",
"columns": ["mid", "name", "year", "rate"]
}
]
}
}
}
]
}
}
./lgraph_server -c lgraph_standalone.json -d 'run' 启动 TuGraph 后执行如下命令：
python3 datax/bin/datax.py job_mysql_to_tugraph.json