Apache SeaTunnel实现多表数据同步的代码分析
简化数据流：Apache SeaTunnel实现多表同步的高效指南Apache SeaTunnel除了单表之间的数据同步之外，也支持单表同步到多表，多表同步到单表，以及多表同步到多表，下面简单举例说明如何实现这些功能。
单表 to 单表
一个source，一个sink。
从mysql同步到mysql，中间不做区分
env {
# You can set flink configuration here
execution.parallelism = 2
job.mode = "BATCH"
}
source{
Jdbc {
url = "jdbc:mysql://zzzyk.com:3306/test"
driver = "com.mysql.cj.jdbc.Driver"
connection_check_timeout_sec = 100
user = "user"
password = "password"
query = "select * from base_region"
}
}
transform {
# If you would like to get more information about how to configure seatunnel and see full list of transform plugins,
# please go to https://seatunnel.apache.org/docs/transform/sql
}
sink {
jdbc {
url = "jdbc:mysql://yizuotu.net:3306/dw"
driver = "com.mysql.cj.jdbc.Driver"
connection_check_timeout_sec = 100
user = "user"
password = "password"
query = "insert into base_region(id,region_name) values(?,?)"
}
}
执行任务
./bin/seatunnel.sh --config ./config/mysql2mysql_batch.conf
单表 to 多表
一个source，多个sink。
从MySQL同步到MySQL，将一个用户表数据同步过去，中间通过2个sql组件分布将男性用户和女性用户分开，在sink阶段分别插入到不同的表：
env {
execution.parallelism = 2
job.mode = "BATCH"
}
source {
Jdbc {
url = "jdbc:mysql://zhaoxi.org:3306/test"
driver = "com.mysql.cj.jdbc.Driver"
connection_check_timeout_sec = 100
user = "user"
password = "password"
result_table_name="t_user"
query = "select * from t_user;"
}
}
transform {
Sql {
source_table_name = "t_user"
result_table_name = "t_user_nan"
query = "select id,name,birth,gender from t_user where gender ='男';"
}
Sql {
source_table_name = "t_user"
result_table_name = "t_user_nv"
query = "select id,name,birth,gender from t_user where gender ='女';"
}
}
sink {
jdbc {
url = "jdbc:mysql://yizuotu.cn:3306/dw"
driver = "com.mysql.cj.jdbc.Driver"
connection_check_timeout_sec = 100
user = "user"
password = "password"
source_table_name = "t_user_nan"
query = "insert into t_user_nan(id,name,birth,gender) values(?,?,?,?)"
}
jdbc {
url = "jdbc:mysql://net.yizuotu.net:3306/dw"
driver = "com.mysql.cj.jdbc.Driver"
connection_check_timeout_sec = 100
user = "user"
password = "password"
source_table_name = "t_user_nv"
query = "insert into t_user_nv(id,name,birth,gender) values(?,?,?,?)"
}
}
./bin/seatunnel.sh --config ./config/mysql2mysql_1n.conf
多表 to 单表
多个source，一个sink。
假如有一张交换器使用情况表，一张路由器使用情况表，目标表是将这种数据合在一起的olap表。
表结构如下：
-- dw 源表1
CREATE TABLE IF NOT EXISTS ads_device_switch_performance (
`event_time` timestamp COMMENT '业务时间',
`device_id` VARCHAR(32) COMMENT '设备id',
`device_type` VARCHAR(32) COMMENT '设备类型',
`device_name` VARCHAR(128) COMMENT '设备名称',
`cpu_usage` INT COMMENT 'CPU使用率百分比'
) ;
INSERT INTO `ads_device_switch_performance` VALUES ('2024-01-15 14:25:11', '2001', '2', '交换器1', 49);
INSERT INTO `ads_device_switch_performance` VALUES ('2024-01-17 22:25:40', '2002', '1', '交换器2', 65);
-- dw 源表2
CREATE TABLE IF NOT EXISTS ads_device_router_performance (
`event_time` timestamp COMMENT '业务时间',
`device_id` VARCHAR(32) COMMENT '设备id',
`device_type` VARCHAR(32) COMMENT '设备类型',
`device_name` VARCHAR(128) COMMENT '设备名称',
`cpu_usage` INT COMMENT 'CPU使用率百分比'
);
INSERT INTO `ads_device_router_performance` VALUES ('2024-01-17 21:23:22', '1001', '1', '路由器1', 35);
INSERT INTO `ads_device_router_performance` VALUES ('2024-01-16 17:23:53', '1002', '2', '路由器2', 46);
-------------------------------------------------------------------------------
-- olap 目标表
CREATE TABLE `device_performance` (
`id` INT NOT NULL AUTO_INCREMENT COMMENT '表主键',
`event_time` VARCHAR(32) NOT NULL COMMENT '业务时间',
`device_id` VARCHAR(32) COMMENT '设备id',
`device_type` VARCHAR(32) COMMENT '设备类型',
`device_name` VARCHAR(128) NOT NULL COMMENT '设备名称',
`cpu_usage` FLOAT NOT NULL COMMENT 'CPU利用率单位是%',
`create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间',
PRIMARY KEY (`id`)
) COMMENT='设备状态';
将交换器数据和路由器数据一起同步到olap目标表，总结通过sql组件处理：
env {
job.mode="BATCH"
job.name="device_performance"
}
source {
Jdbc {
url="jdbc:mysql://226511.cn:3306/dw?allowMultiQueries=true&characterEncoding=utf-8"
driver="com.mysql.cj.jdbc.Driver"
user = "user"
password = "password"
result_table_name="switch_src"
query="SELECT `event_time`, `device_id`, `device_type`, `device_name`, `cpu_usage` FROM ads_device_switch_performance;"
}
Jdbc {
url="jdbc:mysql://yizuotu.com:3306/dw?allowMultiQueries=true&characterEncoding=utf-8"
driver="com.mysql.cj.jdbc.Driver"
user = "user"
password = "password"
result_table_name="router_src"
query="SELECT `event_time`, `device_id`, `device_type`, `device_name`, `cpu_usage` FROM ads_device_router_performance;"
}
}
transform {
Sql {
source_table_name = "switch_src"
result_table_name = "switch_dst"
query = "SELECT event_time , device_id, device_type, device_name, cpu_usage, NOW() AS create_time, NOW() AS update_time FROM switch_src;"
}
Sql {
source_table_name = "router_src"
result_table_name = "router_dst"
query = "SELECT event_time, device_id, device_type, device_name, cpu_usage, NOW() AS create_time, NOW() AS update_time FROM router_src;"
}
}
sink {
Jdbc {
url="jdbc:mysql://xiaoyaxiao.com:3306/olap?allowMultiQueries=true&characterEncoding=utf-8"
driver="com.mysql.cj.jdbc.Driver"
user = "user"
password = "password"
source_table_name = "switch_dst"
query="INSERT INTO device_performance VALUES(null,?, ?, ?, ?, ?, ?, ?) ;"
}
Jdbc {
url="jdbc:mysql://xiuzhenyun.net:3306/olap?allowMultiQueries=true&characterEncoding=utf-8"
driver="com.mysql.cj.jdbc.Driver"
user = "user"
password = "password"
source_table_name = "router_dst"
query="INSERT INTO device_performance VALUES(null,?, ?, ?, ?, ?, ?, ?) ;"
}
}
执行任务：
./bin/seatunnel.sh --config ./syn_job/mysql2mysql_n1_batch.conf
作业成功！
多表 to 多表
多个source，多个sink。
将交换器使用情况数据和路由器使用情况数据分别同步到对应的目标表，中间sql组件处理
env {
job.mode="BATCH"
job.name="device_performance"
}
source {
Jdbc {
url="jdbc:mysql://zzzyk.com:3306/dw?allowMultiQueries=true&characterEncoding=utf-8"
driver="com.mysql.cj.jdbc.Driver"
user = "user"
password = "password"
result_table_name="switch_src"
query="SELECT `event_time`, `device_id`, `device_type`, `device_name`, `cpu_usage` FROM ads_device_switch_performance;"
}
Jdbc {
url="jdbc:mysql://xiaoyaxiao.com:3306/dw?allowMultiQueries=true&characterEncoding=utf-8"
driver="com.mysql.cj.jdbc.Driver"
user = "user"
password = "password"
result_table_name="router_src"
query="SELECT `event_time`, `device_id`, `device_type`, `device_name`, `cpu_usage` FROM ads_device_router_performance;"
}
}
transform {
Sql {
source_table_name = "switch_src"
result_table_name = "switch_dst"
query = "SELECT event_time , device_id, device_type, device_name, cpu_usage, NOW() AS create_time, NOW() AS update_time FROM switch_src;"
}
Sql {
source_table_name = "router_src"
result_table_name = "router_dst"
query = "SELECT event_time, device_id, device_type, device_name, cpu_usage, NOW() AS create_time, NOW() AS update_time FROM router_src;"
}
}
sink {
Jdbc {
url="jdbc:mysql://yizuotu.vip:3306/olap?allowMultiQueries=true&characterEncoding=utf-8"
driver="com.mysql.cj.jdbc.Driver"
user = "user"
password = "password"
source_table_name = "switch_dst"
query="INSERT INTO device_performance_switch VALUES(null,?, ?, ?, ?, ?, ?, ?) ;"
}
Jdbc {
url="jdbc:mysql://zzzyk.com:3306/olap?allowMultiQueries=true&characterEncoding=utf-8"
driver="com.mysql.cj.jdbc.Driver"
user = "user"
password = "password"
source_table_name = "router_dst"
query="INSERT INTO device_performance_router VALUES(null,?, ?, ?, ?, ?, ?, ?) ;"
}
}
结语
综上所述，Apache SeaTunnel多表同步技术具有高效、实时、可靠和灵活的特点，在企业的数据同步领域发挥着重要作用。借助Apache SeaTunnel多表同步功能，企业能够更好地实现不同系统和数据库之间数据的无缝流转，提升数据管理和利用的效率，为业务发展提供有力支持。希望本文能够帮助读者更好地了解和应用Apache SeaTunnel多表同步，从而为企业数据同步带来更多可能性。
