{"id":18088,"date":"2026-03-11T15:14:11","date_gmt":"2026-03-11T07:14:11","guid":{"rendered":"https:\/\/kbwp.transwarp.cn\/?p=18088"},"modified":"2026-03-25T12:03:07","modified_gmt":"2026-03-25T04:03:07","slug":"datax%e8%af%bb%e5%86%99argodb","status":"publish","type":"post","link":"https:\/\/kbwp.transwarp.cn\/?p=18088","title":{"rendered":"datax\u8bfb\u5199ArgoDB\uff08\u81ea\u5b9a\u4e49argodbwriter\/argodbreader\u63d2\u4ef6\uff09"},"content":{"rendered":"<h3>\u6982\u8981\u63cf\u8ff0<\/h3>\n<hr \/>\n<p>\u672c\u6587\u6863\u4e3b\u8981\u4ecb\u7ecdArgodb\u5982\u4f55\u901a\u8fc7datax\u7684\u81ea\u5b9a\u4e49argodbwriter\/argodbreader\u63d2\u4ef6\uff0c\u8bfb\u5199argodb\u8868\u3002<\/p>\n<blockquote>\n<p>\u7248\u672c\u8981\u6c42\uff1a<br \/>\nArgoDB\u7248\u672c&gt;=6\uff0c Patch\u7248\u672c&gt;=20250930<\/p>\n<\/blockquote>\n<h3>\u8be6\u7ec6\u4ecb\u7ecd<\/h3>\n<hr \/>\n<h4>\u4e00. \u914d\u7f6e\u4e0e\u51c6\u5907<\/h4>\n<h5>1. DataX \u7a0b\u5e8f\u5305<\/h5>\n<p>\u5b98\u65b9 github \u4e0b\u8f7d\u5e76\u89e3\u538b\uff0c\u6309\u8981\u6c42\u51c6\u5907 DataX \u7684\u8fd0\u884c\u73af\u5883\u4f9d\u8d56<\/p>\n<h5>2. ArgoDB DataX \u63d2\u4ef6\u5305<\/h5>\n<p>\u6ce8\uff1a${version} \u6839\u636e\u5b9e\u9645\u4f7f\u7528\u7684\u5305\u8fdb\u884c\u8c03\u6574<br \/>\n\u5728 ArgoDB patch \u7684\u914d\u5957\u5de5\u5177\u91cc\u83b7\u53d6\u4ee5\u4e0b\u5305\uff1a<\/p>\n<pre><code class=\"language-shell\">argodb_client\/argodb-datax\/argodbwriter-${version}.jar\nargodb_client\/argodb-datax\/argodbreader-${version}.jar \nargodb_client\/argodb-sink-client\/quark-driver-${version}.jar\nargodb_client\/argodb-sink-client\/argodb-sink-client-${version}-all-in-one.jar<\/code><\/pre>\n<p>\u5728 datax \u8def\u5f84\u4e0b\u521b\u5efa\u76ee\u5f55\u5e76\u653e\u7f6e\u9700\u8981\u7684\u5305<\/p>\n<pre><code class=\"language-shell\">cd datax\n\n# \u5982\u9700\u3010\u5199\u3011ArgoDB \u7684\u529f\u80fd\uff0c\u6267\u884c\u4ee5\u4e0b\u64cd\u4f5c\nmkdir -p plugin\/writer\/argodbwriter\/libs\ncp argodbwriter-${version}.jar plugin\/writer\/argodbwriter\/\ncp argodb-sink-client-${version}-all-in-one.jar plugin\/writer\/argodbwriter\/libs\/\ncp quark-driver-${version}.jar plugin\/writer\/argodbwriter\/libs\/\n\n# \u5982\u9700\u3010\u8bfb\u3011ArgoDB \u7684\u529f\u80fd\uff0c\u6267\u884c\u4ee5\u4e0b\u64cd\u4f5c\nmkdir -p plugin\/reader\/argodbreader\/libs\ncp argodbreader-${version}.jar plugin\/reader\/argodbreader\/\n# \u5c06\u5bf9\u5e94\u7248\u672c quark \u955c\u50cf\u4e2d \/usr\/lib\/inceptor\/lib \u7684\u6240\u6709 jar \u5305\u51c6\u5907\u597d\uff08\u540e\u7eed\u4f1a\u4f18\u5316\u9700\u8981\u5305\u7684\u5217\u8868\uff09\ncp \/usr\/lib\/inceptor\/lib\u4e0b\u7684\u6240\u6709jar\u5305 plugin\/reader\/argodbreader\/libs\/<\/code><\/pre>\n<h5>3. ArgoDB DataX \u63d2\u4ef6\u914d\u7f6e<\/h5>\n<p>\u5c06 writer_plugin.json \u6587\u4ef6\u653e\u7f6e\u5728 datax\/plugin\/writer\/argodbwriter\/ \u76ee\u5f55\u4e0b\u5e76\u91cd\u547d\u540d\u4e3a plugin.json<br \/>\n\u5c06 reader_plugin.json \u6587\u4ef6\u653e\u7f6e\u5728 datax\/plugin\/reader\/argodbreader\/ \u76ee\u5f55\u4e0b\u5e76\u91cd\u547d\u540d\u4e3a plugin.json<\/p>\n<h5>4. \u7f51\u7edc\u914d\u7f6e<\/h5>\n<p>DataX \u8fdb\u7a0b\u9700\u8981\u80fd\u8bbf\u95ee\u4ee5\u4e0b\u7aef\u53e3\uff0c\u9700\u8981\u5f00\u653e\u7684\u7aef\u53e3\u6e05\u5355\uff1a<\/p>\n<p>\u8bbf\u95ee Quark<\/p>\n<ul>\n<li>\u914d\u7f6e\u9879 hive.server2.thrift.port \u6307\u5b9a\u7684\u7aef\u53e3<\/li>\n<\/ul>\n<p>\u8bbf\u95ee TDDMS<\/p>\n<ul>\n<li>\u4ee5\u914d\u7f6e\u9879 master.rpc_service.master_service_port  \u5f00\u59cb\u7684\u8fde\u7eed4\u4e2a\u7aef\u53e3<\/li>\n<li>\u4ee5\u914d\u7f6e\u9879 tabletserver.rpc_service.manage_service_port  \u5f00\u59cb\u7684\u8fde\u7eed4\u4e2a\u7aef\u53e3<\/li>\n<\/ul>\n<h4>\u4e8c. DataX Job \u914d\u7f6e<\/h4>\n<h5>1. \u5199\u4efb\u52a1<\/h5>\n<p>\u90e8\u5206\u53c2\u6570\u8bf4\u660e\uff1a<\/p>\n<pre><code class=\"language-shell\"># preSqlList - \u6267\u884c\u5199\u5165\u524d\u9700\u8981\u6267\u884c\u7684 SQL \u5217\u8868\uff08\u5982\u521b\u5efa\u5206\u533a\u3001\u6e05\u7406\u7b49\u64cd\u4f5c\uff09\n# postSqlList - \u5199\u5165\u5b8c\u6210\u540e\u9700\u8981\u6267\u884c\u7684 SQL \u5217\u8868\n# compressionType - \u5199\u5165\u6570\u636e\u7684\u538b\u7f29\u65b9\u5f0f\uff1b\u53ef\u9009\u503c\u6709\uff1a SNAPPY, NOTCOMPRESSION, LZF, ZLIB\uff0c\u9ed8\u8ba4\u4e3a SNAPPY\n# largeStringEnabled - \u662f\u5426\u5f00\u542f\u5927 string \u5bf9\u8c61\u63d2\u5165\u529f\u80fd\uff0c\u9ed8\u8ba4 false\uff1b\u7b49\u4ef7\u4e8e\u6267\u884c SQL \u65f6\u7684\u53c2\u6570  argodb.insert.large.string.enabled\n# maxLargeStringSize - \u652f\u6301\u7684\u6700\u5927\u7684 string \u5927\u5c0f\uff0c\u5355\u4f4d\u4e3a bytes\uff0c\u9ed8\u8ba4 3145728\uff0c\u53ea\u6709\u5f53 largeStringEnabled \u4e3a true \u65f6\u624d\u6709\u6548\uff1b\u7b49\u4ef7\u4e8e\u6267\u884c SQL \u65f6\u7684\u53c2\u6570 argodb.max.large.string.size\n# columnList - \u5199\u5165 Holodesk \u8868\u65f6\uff0c\u6307\u5b9a\u63d2\u5165\u54ea\u4e9b\u5217<\/code><\/pre>\n<pre><code class=\"language-json\">{\n    \"name\": \"argodbwriter\",\n    \"parameter\": {\n        \"quarkTableName\": \"\",\n        \"connection\": {\n          \"quarkUrl\": \"\",\n          \"quarkUserName\": \"\",\n          \"quarkPasswd\": \"\"\n        },\n        \"sink\": {\n          \"largeStringEnabled\": false,\n          \"maxLargeStringSize\": 3145728,\n          \"compressionType\": \"SNAPPY\",\n          \"columnList\": []\n        },\n        \"sql\": {\n          \"preSqlList\": [],\n          \"postSqlList\": []\n        }\n    }\n}<\/code><\/pre>\n<h5>2. \u8bfb\u4efb\u52a1<\/h5>\n<p>\u90e8\u5206\u53c2\u6570\u8bf4\u660e\uff1a<\/p>\n<pre><code class=\"language-shell\"># neededColumns - \u9700\u8981\u8bfb\u54ea\u4e9b\u5217\uff0c\u4f7f\u7528\u534a\u89d2\u9017\u53f7\u5c06\u591a\u4e2a\u5217\u540d\u4e32\u8fde\uff1b\u5982\u679c\u9700\u8981\u8bfb\u6240\u6709\u5217\uff0c\u8bbe\u7f6e\u4e3a *\uff1b\n# \u6570\u636e\u8f93\u51fa\u65f6\uff0c\u4f1a\u6309\u7167\u8868\u539f\u59cb\u7684\u5217\u987a\u5e8f\u8f93\u51fa\uff0c\u4e0d\u8bba neededColumns \u53c2\u6570\u5982\u4f55\u5b9a\u4e49<\/code><\/pre>\n<pre><code class=\"language-json\">{\n    \"name\": \"argodbreader\",\n    \"parameter\": {\n        \"quarkTableName\": \"\",\n        \"neededColumns\": \"\",\n        \"connection\": {\n          \"quarkUrl\": \"\",\n          \"quarkUserName\": \"\",\n          \"quarkPasswd\": \"\"\n        }\n    }\n}<\/code><\/pre>\n<h4>\u4e09. \u573a\u666f\u4e3e\u4f8b<\/h4>\n<h5>\u573a\u666f1. \u8bfbargodb\u5199argodb<\/h5>\n<p>\u4ee5\u4e0b\u4efb\u52a1\u4ece default.out_tbl \u8868\u4e2d\u8bfb\u53d6\u6570\u636e\uff0c\u5199\u5165\u5230 default.in_tbl \u8868\u4e2d\u3002\uff08\u4e24\u5f20\u8868\u5747\u4e3a Holodesk \u8868\uff09<\/p>\n<p>\u521b\u5efa\u6d4b\u8bd5\u8868<\/p>\n<pre><code class=\"language-sql\">DROP TABLE IF EXISTS default.out_tbl;\nDROP TABLE IF EXISTS default.in_tbl;\nCREATE TABLE  default.out_tbl (a INT,b STRING ) STORED AS HOLODESK ;\nCREATE TABLE  default.in_tbl (a INT,b STRING ) STORED AS HOLODESK ;\nINSERT INTO default.out_tbl SELECT 1,'a' FROM system.dual;\nINSERT INTO default.out_tbl SELECT 2,'b' FROM system.dual;<\/code><\/pre>\n<p>job.json \u7684\u5185\u5bb9\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-json\">{\n    \"job\": {\n        \"setting\": {\n            \"speed\": {\n                \"channel\": 10\n            },\n            \"errorLimit\": {\n                \"record\": 0,\n                \"percentage\": 0.02\n            }\n        },\n        \"content\": [{\n                \"reader\": {\n                    \"name\": \"argodbreader\",\n                    \"parameter\": {\n                        \"quarkTableName\": \"default.out_tbl\",\n                        \"neededColumns\": \"*\",\n                        \"connection\": {\n                            \"quarkUrl\": \"jdbc:hive2:\/\/kv1:10000\/default;guardianToken=so1GPbLhVmYJ459orice-KTDH?use.linac=false\",\n                            \"quarkUserName\": \"\",\n                            \"quarkPasswd\": \"\"\n                        }\n                    }\n                },\n                \"writer\": {\n                    \"name\": \"argodbwriter\",\n                    \"parameter\": {\n                        \"quarkTableName\": \"default.in_tbl\",\n                        \"connection\": {\n                            \"quarkUrl\": \"jdbc:hive2:\/\/kv1:10000\/default;guardianToken=so1GPbLhVmYJ459orice-KTDH?use.linac=false\",\n                            \"quarkUserName\": \"\",\n                            \"quarkPasswd\": \"\"\n                        },\n                        \"sink\": {\n                            \"largeStringEnabled\": false,\n                            \"maxLargeStringSize\": 3145728,\n                            \"compressionType\": \"SNAPPY\",\n                            \"columnList\": []\n                        },\n                        \"sql\": {\n                            \"preSqlList\": [],\n                            \"postSqlList\": []\n                        }\n                    }\n                }\n            }\n        ]\n    }\n}\n<\/code><\/pre>\n<p>Job \u542f\u52a8\u547d\u4ee4\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-shell\">python datax\/bin\/datax.py job.json<\/code><\/pre>\n<p>\u6267\u884c\u5b8c\u6210\u4e4b\u540e\uff0c\u68c0\u67e5default.in_tbl\u8868\u5bfc\u5165\u6210\u529f<\/p>\n<div style=\"box-shadow: 1px 1px 10px rgba(0,0,0,0.1); padding: 1px; display: inline-block; width: auto; margin-bottom: 10px;\">\n  <img decoding=\"async\" src=\"\/wp-content\/uploads\/2026\/03\/image-1773211222566.png\" style=\"display: block;\">\n<\/div>\n<h5>\u573a\u666f2. \u8bfboracle\u5199argodb<\/h5>\n<p>\u4ee5\u4e0b\u4efb\u52a1\u4eceoracle 11g \u7684emp\u8868\u4e2d\u8bfb\u53d6\u6570\u636e\uff0c\u5199\u5165\u5230 default.emp_holo \u8868\u4e2d\u3002<\/p>\n<p>oracle\u7aef \u521b\u5efa\u6d4b\u8bd5\u8868<\/p>\n<pre><code class=\"language-sql\">DROP TABLE EMP;\nCREATE TABLE EMP(\n       EMPNO int,\n       ENAME varchar2(255),\n       JOB varchar2(255),\n       MGR INT,\n       HIREDATE DATE,\n       SAL INT,\n       COMM INT,\n       DEPTNO INT\n);\n\nINSERT INTO EMP VALUES (7369,'SMITH','CLERK',7902,to_date('17-12-1980','dd-mm-yyyy'),800,NULL,20);\nINSERT INTO EMP VALUES (7499,'ALLEN','SALESMAN',7698,to_date('20-2-1981','dd-mm-yyyy'),1600,300,30);\nINSERT INTO EMP VALUES (7521,'WARD','SALESMAN',7698,to_date('22-2-1981','dd-mm-yyyy'),1250,500,30);\nINSERT INTO EMP VALUES (7566,'JONES','MANAGER',7839,to_date('2-4-1981','dd-mm-yyyy'),2975,NULL,20);\nINSERT INTO EMP VALUES (7654,'MARTIN','SALESMAN',7698,to_date('28-9-1981','dd-mm-yyyy'),1250,1400,30);<\/code><\/pre>\n<p>oracleToArgodb.json \u7684\u5185\u5bb9\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-json\">{\n    \"job\": {\n        \"setting\": {\n            \"speed\": {\n                \"channel\": 10\n            },\n            \"errorLimit\": {\n                \"record\": 0,\n                \"percentage\": 0.02\n            }\n        },\n        \"content\": [{\n                \"reader\": {\n                    \"name\": \"oraclereader\",\n                    \"parameter\": {\n                        \"username\": \"lkw\",\n                        \"password\": \"123456\",\n                        \"connection\": [{\n                                \"querySql\": [\"select * from emp\"],\n                                \"jdbcUrl\": [\"jdbc:oracle:thin:@172.18.131.175:1521\/helowin\"]\n                            }\n                        ],\n                    }\n                },\n                \"writer\": {\n                    \"name\": \"argodbwriter\",\n                    \"parameter\": {\n                        \"quarkTableName\": \"default.emp_holo\",\n                        \"connection\": {\n                            \"quarkUrl\": \"jdbc:hive2:\/\/kv1:10000\/default;guardianToken=so1GPbLhVmYJ459orice-KTDH?use.linac=false\",\n                            \"quarkUserName\": \"\",\n                            \"quarkPasswd\": \"\"\n                        },\n                        \"sink\": {\n                            \"largeStringEnabled\": false,\n                            \"maxLargeStringSize\": 3145728,\n                            \"compressionType\": \"SNAPPY\",\n                            \"columnList\": []\n                        },\n                        \"sql\": {\n                            \"preSqlList\": [],\n                            \"postSqlList\": []\n                        }\n                    }\n                }\n            }\n        ]\n    }\n}<\/code><\/pre>\n<p>Job \u542f\u52a8\u547d\u4ee4\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-shell\">python .\/datax.py oracleToArgodb.json<\/code><\/pre>\n<p>\u6267\u884c\u5b8c\u6210\u4e4b\u540e\uff0c\u68c0\u67e5emp_holo\u8868\u5bfc\u5165\u6210\u529f<\/p>\n<div style=\"box-shadow: 1px 1px 10px rgba(0,0,0,0.1); padding: 1px; display: inline-block; width: auto; margin-bottom: 10px;\">\n  <img decoding=\"async\" src=\"\/wp-content\/uploads\/2026\/03\/image-1773213300954.png\" style=\"display: block;\">\n<\/div>\n<h4>\u6ce8\u610f\u4e8b\u9879\uff08\u5fc5\u8bfb\uff09<\/h4>\n<p>1.\u5199\u5165 ArgoDB \u65f6\uff0c\u4e0d\u4fdd\u8bc1\u539f\u5b50\u6027\uff0c\u9700\u8981\u7528\u6237\u81ea\u884c\u5904\u7406\u9519\u8bef\u60c5\u51b5\uff08\u5982\uff1a\u90e8\u5206 Task \u5931\u8d25\uff0c\u90a3\u4e48\u6210\u529f\u7684 Task \u4f1a\u628a\u6570\u636e\u5199\u5165 ArgoDB\uff0c\u9700\u8981\u7528\u6237\u80fd\u81ea\u5df1\u8fdb\u884c\u9519\u8bef\u6062\u590d\uff09\uff1b<br \/>\n2.\u5bf9\u4e8e blob\/clob \u7c7b\u578b\uff0c\u4ec5\u652f\u6301 performance \u8868\uff0c\u5efa\u8868\u65f6\u9700\u8981\u8bbe\u7f6e set argodb.performance.complex.types.enabled=true\uff1b\u5bf9\u4e8e blob \u7c7b\u578b\u7684\u5199\u5165\uff0c\u76ee\u524d\u4ec5\u652f\u6301\u80fd\u8f6c\u6210 string \u7684\u4e8c\u8fdb\u5236\u503c\uff08\u540e\u7eed\u4f1a\u5b8c\u5584\uff09\uff1b<br \/>\n3.\u76ee\u524d\u4e0d\u652f\u6301\u8bfb\u5206\u533a\u8868\uff1b<br \/>\n4.\u5728\u4f7f\u7528 writer \u7684\u65f6\uff0c\u9700\u8981\u7528\u6237\u4fdd\u8bc1\u4ece reader \u8bfb\u5230\u7684\u8f93\u51fa\uff0c\u5217\u6570\u91cf\u548c\u987a\u5e8f\u4e0e\u5199\u5165\u7684\u76ee\u6807\u8868\u4e00\u81f4\u3002\u5728\u672a\u6307\u5b9a columnList \u53c2\u6570\u65f6\uff0creader \u8bfb\u51fa\u7684\u5217\u7684\u6570\u91cf\u548c\u987a\u5e8f\u5fc5\u987b\u548c Holodesk \u8868\u7684\u8868\u5b9a\u4e49\u4e00\u81f4\uff1b\u6307\u5b9a columnList \u53c2\u6570\u65f6\uff0creader \u8bfb\u51fa\u7684\u5217\u7684\u6570\u91cf\u548c\u987a\u5e8f\u5fc5\u987b\u548c  columnList \u6307\u5b9a\u7684\u4e00\u81f4\u3002<br \/>\n5.udf\u4e0d\u652f\u6301linac\uff0c\u6240\u4ee5\u9047\u5230 <code>Check Dependency Version failed: EXECUTION FAILED: Task MAPRED-SPARK error PlanException: [Error 1] Found no matched plan<\/code> \u7684\u62a5\u9519\uff0c\u6709\u53ef\u80fd\u662f\u5168\u5c40\u5f00\u542f\u4e86linac\uff0c\u6267\u884c SELECT showArgodbVersion() \u62a5\u9519\u5bfc\u81f4\u7684\uff0c\u53ef\u4ee5\u5728jdbc\u8fde\u63a5\u4e32\u540e\u9762\u52a0\u4e0a<code>?use.linac=false<\/code> session\u7ea7\u5173\u95edlinac\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u6982\u8981\u63cf\u8ff0 \u672c\u6587\u6863\u4e3b\u8981\u4ecb\u7ecdArgodb\u5982\u4f55\u901a\u8fc7datax\u7684\u81ea\u5b9a\u4e49argodbwriter\/argodbreade ..<\/p>\n<div class=\"clear-fix\"><\/div>\n<p><a href=\"https:\/\/kbwp.transwarp.cn\/?p=18088\" title=\"read more...\">Read more<\/a><\/p>\n","protected":false},"author":12,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-18088","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"acf":[],"_links":{"self":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/18088","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/users\/12"}],"replies":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=18088"}],"version-history":[{"count":4,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/18088\/revisions"}],"predecessor-version":[{"id":18194,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/18088\/revisions\/18194"}],"wp:attachment":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=18088"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=18088"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=18088"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}