From a71d0ce1124aa45ad8bf82e4473bd758d369c7a4 Mon Sep 17 00:00:00 2001 From: Breeze0806 Date: Tue, 8 Oct 2024 03:07:23 +0800 Subject: [PATCH] docs: fix some errors in config file and document --- datax/plugin/reader/csv/resources/plugin.json | 2 +- .../reader/oracle/resources/plugin.json | 2 +- .../oracle/resources/plugin_job_template.json | 15 ++++++++++- .../resources/plugin_job_template.json | 22 ++++++++------- .../db2/resources/plugin_job_template.json | 25 ++++++++--------- datax/plugin/writer/mysql/README.md | 7 ++++- .../oracle/resources/plugin_job_template.json | 16 ++++++++++- .../resources/plugin_job_template.json | 1 - datax/plugin/writer/sqlite3/README.md | 7 +++-- datax/plugin/writer/sqlite3/README_zh-CN.md | 14 +++++----- .../resources/plugin_job_template.json | 27 ++++++++++--------- datax/plugin/writer/xlsx/README_zh-CN.md | 2 +- .../xlsx/resources/plugin_job_template.json | 10 +++---- 13 files changed, 92 insertions(+), 58 deletions(-) diff --git a/datax/plugin/reader/csv/resources/plugin.json b/datax/plugin/reader/csv/resources/plugin.json index 9485cae..9054917 100644 --- a/datax/plugin/reader/csv/resources/plugin.json +++ b/datax/plugin/reader/csv/resources/plugin.json @@ -2,5 +2,5 @@ "name" : "csvreader", "developer":"Breeze0806", "opener":"csv", - "description":"" + "description":"CsvReader leverages the os and encoding/csv standard libraries to read files" } \ No newline at end of file diff --git a/datax/plugin/reader/oracle/resources/plugin.json b/datax/plugin/reader/oracle/resources/plugin.json index 6c15ceb..d667b6a 100644 --- a/datax/plugin/reader/oracle/resources/plugin.json +++ b/datax/plugin/reader/oracle/resources/plugin.json @@ -2,5 +2,5 @@ "name" : "oraclereader", "developer":"Breeze0806", "dialect":"oracle", - "description":"" + "description":"OracleReader connects to remote Oracle databases using Oracle Instant Client via github.com/godror/godror" } \ No newline at end of file diff --git a/datax/plugin/reader/oracle/resources/plugin_job_template.json b/datax/plugin/reader/oracle/resources/plugin_job_template.json index 2cb5c34..6176c08 100644 --- a/datax/plugin/reader/oracle/resources/plugin_job_template.json +++ b/datax/plugin/reader/oracle/resources/plugin_job_template.json @@ -1,6 +1,19 @@ { "name": "oraclereader", "parameter": { - + "connection": { + "url": "connectString=\"192.168.15.130:1521/xe\" heterogeneousPool=false standaloneConnection=true", + "table": { + "schema":"TEST", + "name":"SRC" + } + }, + "username": "system", + "password": "oracle", + "column": ["*"], + "split" : { + "key":"id" + }, + "where": "" } } \ No newline at end of file diff --git a/datax/plugin/reader/sqlite3/resources/plugin_job_template.json b/datax/plugin/reader/sqlite3/resources/plugin_job_template.json index 7d6ce5f..f87e876 100644 --- a/datax/plugin/reader/sqlite3/resources/plugin_job_template.json +++ b/datax/plugin/reader/sqlite3/resources/plugin_job_template.json @@ -1,14 +1,16 @@ { "name": "sqlite3reader", "parameter": { - "column": ["*"], - "connection": { - "url": "", - "table": { - "db":"", - "name":"" - } - }, - "where": "" + "column": [ + "*" + ], + "connection": { + "url": "E:\\Sqlite3\\test.db", + "table": { + "db": "main", + "name": "type_table" + } + }, + "where": "" } -} \ No newline at end of file + } \ No newline at end of file diff --git a/datax/plugin/writer/db2/resources/plugin_job_template.json b/datax/plugin/writer/db2/resources/plugin_job_template.json index 92da0fe..95f6622 100644 --- a/datax/plugin/writer/db2/resources/plugin_job_template.json +++ b/datax/plugin/writer/db2/resources/plugin_job_template.json @@ -1,19 +1,20 @@ { - "name": "db2writer", - "parameter": { - "username": "", - "password": "", - "writeMode": "", - "column": [], - "preSql": [], - "connection": { - "url": "", + "name": "db2writer", + "parameter": { + "connection": { + "url": "HOSTNAME=127.0.0.1;PORT=50000;DATABASE=db", "table": { - "schema":"", - "name":"" + "schema":"SOURCE", + "name":"TEST" } }, + "username": "root", + "password": "12345678", + "writeMode": "insert", + "column": ["*"], + "preSql": ["create table a like b"], + "postSql": ["drop table a"], "batchTimeout": "1s", "batchSize":1000 - } + } } \ No newline at end of file diff --git a/datax/plugin/writer/mysql/README.md b/datax/plugin/writer/mysql/README.md index 0a98ed7..41428af 100644 --- a/datax/plugin/writer/mysql/README.md +++ b/datax/plugin/writer/mysql/README.md @@ -10,7 +10,12 @@ MysqlWriter connects to remote Mysql databases using github.com/go-sql-driver/my MysqlWriter implements specific queries by invoking go-etl's custom DBWrapper from storage/database, using the query process defined in dbmswriter. DBWrapper encapsulates many interfaces from database/sql and abstracts the database dialect, Dialect. For Mysql, the Dialect implemented by storage/database/mysql is used. -Based on the configured `writeMode`, MysqlWriter generates either an `insert into...` statement (which will not insert conflicting rows in case of primary key/unique index conflicts) or a `replace into...` statement (which behaves like `insert into` when no conflicts occur, but replaces the entire row with new values when conflicts arise). Data is buffered in memory and written in batches to optimize performance. +Based on the configured `writeMode`, MysqlWriter generates: +an `insert into...` statement (which will not insert conflicting rows in case of primary key/unique index conflicts) + +**or** + +a `replace into...` statement (which behaves like `insert into` when no conflicts occur, but replaces the entire row with new values when conflicts arise). Data is buffered in memory and written in batches to optimize performance. ## Functionality Description diff --git a/datax/plugin/writer/oracle/resources/plugin_job_template.json b/datax/plugin/writer/oracle/resources/plugin_job_template.json index fa0815f..dc26c85 100644 --- a/datax/plugin/writer/oracle/resources/plugin_job_template.json +++ b/datax/plugin/writer/oracle/resources/plugin_job_template.json @@ -1,6 +1,20 @@ { "name": "oraclewriter", "parameter": { - + "connection": { + "url": "connectString=\"192.168.15.130:1521/xe\" heterogeneousPool=false standaloneConnection=true", + "table": { + "schema":"TEST", + "name":"DEST" + } + }, + "username": "system", + "password": "oracle", + "writeMode": "insert", + "column": ["*"], + "preSql": ["create table a like b"], + "postSql": ["drop table a"], + "batchTimeout": "1s", + "batchSize":1000 } } \ No newline at end of file diff --git a/datax/plugin/writer/postgres/resources/plugin_job_template.json b/datax/plugin/writer/postgres/resources/plugin_job_template.json index 8333f12..b50f237 100644 --- a/datax/plugin/writer/postgres/resources/plugin_job_template.json +++ b/datax/plugin/writer/postgres/resources/plugin_job_template.json @@ -9,7 +9,6 @@ "connection": { "url": "postgres://192.168.15.130:5432/postgres?sslmode=disable", "table": { - "db":"postgres", "schema":"destination", "name":"type_table" } diff --git a/datax/plugin/writer/sqlite3/README.md b/datax/plugin/writer/sqlite3/README.md index 0b88a61..43c0b45 100644 --- a/datax/plugin/writer/sqlite3/README.md +++ b/datax/plugin/writer/sqlite3/README.md @@ -36,7 +36,6 @@ Configuring a job to synchronously write data to a Sqlite3 database: "connection": { "url": "E:\\Sqlite3\\test.db", "table": { - "db": "main", "name": "type_table_copy" } }, @@ -59,11 +58,11 @@ Configuring a job to synchronously write data to a Sqlite3 database: - Required: Yes - Default: None -#### name +#### table Describes the Sqlite3 table information. -##### table +##### name - Description: Primarily used to configure the table name of the Sqlite3 table. - Required: Yes @@ -120,7 +119,7 @@ Below is a conversion table for Sqlite3Writer with regards to Sqlite3 types: | go-etl的类型 | sqlite3数据类型 | | ------------ | -------------------------------------------------------- | -| string | INTEGER、TEXT、REAL、BLOB | +| string | INTEGER,TEXT,REAL,BLOB,NUMERIC| ## Performance Report diff --git a/datax/plugin/writer/sqlite3/README_zh-CN.md b/datax/plugin/writer/sqlite3/README_zh-CN.md index 595606a..b412217 100644 --- a/datax/plugin/writer/sqlite3/README_zh-CN.md +++ b/datax/plugin/writer/sqlite3/README_zh-CN.md @@ -10,10 +10,9 @@ Sqlite3Writer通过github.com/mattn/go-sqlite3连接远程sqlite3数据库，并 sqlite3通过使用dbmswriter中定义的查询流程调用go-etl自定义的storage/database的DBWrapper来实现具体的查询。DBWrapper封装了database/sql的众多接口，并且抽象出了数据库方言Dialect。其中sqlite3采取了storage/database/sqlite3实现的Dialect。 +根据你配置的 `writeMode` 生成 -**或者** - -- `copy in ...` 与 insert into 行为一致，速度比insert into方式迅速。出于性能考虑，将数据缓冲到内存 中，当 内存累计到预定阈值时，才发起写入请求。 +- `insert into...`(当主键/唯一性索引冲突时会写不进去冲突的行) ## 功能说明 @@ -35,7 +34,6 @@ sqlite3通过使用dbmswriter中定义的查询流程调用go-etl自定义的sto "connection": { "url": "E:\\Sqlite3\\test.db", "table": { - "db": "main", "name": "type_table_copy" } }, @@ -56,11 +54,11 @@ sqlite3通过使用dbmswriter中定义的查询流程调用go-etl自定义的sto - 必选：是 - 默认值: 无 -#### name +#### table 描述sqlite3表信息 -##### table +##### name - 描述 主要用于配置sqlite3表的表名 - 必选：是 @@ -82,7 +80,7 @@ sqlite3通过使用dbmswriter中定义的查询流程调用go-etl自定义的sto #### writeMode -- 描述：写入模式，insert代表insert into方式写入数据，copyIn代表copy in方式写入数据。 +- 描述：写入模式，insert代表insert into方式写入数据。 - 必选：否 - 默认值: insert @@ -118,7 +116,7 @@ sqlite3通过使用dbmswriter中定义的查询流程调用go-etl自定义的sto | go-etl的类型 | sqlite3数据类型 | | ------------ | -------------------------------------------------------- | -| string | INTEGER、TEXT、REAL、BLOB | +| string | INTEGER,TEXT,REAL,BLOB,NUMERIC| ## 性能报告 diff --git a/datax/plugin/writer/sqlite3/resources/plugin_job_template.json b/datax/plugin/writer/sqlite3/resources/plugin_job_template.json index 5d55505..464025e 100644 --- a/datax/plugin/writer/sqlite3/resources/plugin_job_template.json +++ b/datax/plugin/writer/sqlite3/resources/plugin_job_template.json @@ -1,16 +1,19 @@ { "name": "sqlite3writer", "parameter": { - "writeMode": "insert", - "column": ["*"], - "connection": { - "url": "", - "table": { - "db":"", - "name":"" - } - }, - "batchTimeout": "1s", - "batchSize":1000 + "writeMode": "insert", + "column": [ + "*" + ], + "connection": { + "url": "E:\\Sqlite3\\test.db", + "table": { + "name": "type_table_copy" + } + }, + "preSql": ["create table a like b"], + "postSql": ["drop table a"], + "batchTimeout": "1s", + "batchSize": 1000 } -} \ No newline at end of file + } \ No newline at end of file diff --git a/datax/plugin/writer/xlsx/README_zh-CN.md b/datax/plugin/writer/xlsx/README_zh-CN.md index 41e4651..4a7108c 100644 --- a/datax/plugin/writer/xlsx/README_zh-CN.md +++ b/datax/plugin/writer/xlsx/README_zh-CN.md @@ -110,7 +110,7 @@ XlsxWriter通过使用file.Task中定义的写入流程调用go-etl自定义的s - 默认值：无 #### sheetRow -- 描述：sheet最大的列数,最大为1048576。 +- 描述：sheet最大的行数,最大为1048576。 - 必选：否 - 默认值：1048576 diff --git a/datax/plugin/writer/xlsx/resources/plugin_job_template.json b/datax/plugin/writer/xlsx/resources/plugin_job_template.json index 068f523..9d7aae1 100644 --- a/datax/plugin/writer/xlsx/resources/plugin_job_template.json +++ b/datax/plugin/writer/xlsx/resources/plugin_job_template.json @@ -1,5 +1,5 @@ { - "name": "csvwriter", + "name": "xlsxwriter", "parameter": { "column" :[ { @@ -10,11 +10,11 @@ ], "xlsxs":[ { - "path":"", - "sheets":["",""] + "path":"Book1.xlsx", + "sheets":["Sheet1"] } ], - "batchSize":1000, - "batchTimeout":"1s" + "batchTimeout": "1s", + "batchSize":1000 } } \ No newline at end of file