https:\/\/drive.weixin.qq.com\/s?k=ABoAuwfGAAk2aC1njT<\/a> (\u4f01\u4e1a\u7f51\u76d8) \u83b7\u53d6<\/li>\n<\/ul>\n\u8be6\u7ec6\u8bf4\u660e<\/h3>\n
\n1. \u5c06\u6570\u636e\u6587\u4ef6\u653e\u7f6e\u5230\u53ef\u4ee5\u90e8\u7f72TDH-Client\u5ba2\u6237\u7aef\u7684\u8282\u70b9\u76ee\u5f55\u4e0b<\/h4>\n
\u8fd9\u91cc\u6211\u4eec\u5c06people\u8868\u76841000w\u6761\u6570\u636e\u6587\u4ef6\uff0c\u653e\u7f6e\u5230\/root\/people<\/code>\u76ee\u5f55\u4e0b<\/p>\n![\"file\"](\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2022\/08\/image-1659693587878.png\")
<\/p>\n
\u53ef\u4ee5\u770b\u5230\u6570\u636e\u6587\u4ef6\u7684\u5206\u9694\u7b26\u662f\uff0c<\/code>\u9017\u53f7\uff0c\u6240\u4ee5\u540e\u9762\u4fee\u6539\u914d\u7f6e\u6587\u4ef6\u7684\u65f6\u5019argodb.import.ext.table.delimiter=,<\/code> \u4e0e\u4e4b\u4fdd\u6301\u4e00\u81f4\u3002<\/p>\n2. Argodb\u521b\u5efa\u76ee\u6807\u8868<\/h4>\nDROP TABLE IF EXISTS people_import;\nCREATE TABLE people_import (\n name string DEFAULT NULL,\n sex string DEFAULT NULL,\n nation string DEFAULT NULL,\n id string DEFAULT NULL,\n borndate STRING,\n phonenumber string DEFAULT NULL,\n email string DEFAULT NULL,\n homeaddress string DEFAULT NULL )\nSTORED AS HOLODESK;<\/code><\/pre>\n3.\u4fee\u6539\u914d\u7f6e\u6587\u4ef6<\/h4>\n
\u914d\u7f6e\u6587\u4ef6\u5728TDH\u5ba2\u6237\u7aef\u7684TDH-Client\/impexp\/argodb-impexp.properties<\/code><\/strong>\u76ee\u5f55\u4e0b\uff0c\u91cd\u70b9\u5173\u6ce8\u5982\u4e0b\u5907\u6ce8\u8fdb\u884c\u4fee\u6539<\/p>\n# ====== COMMON Used for Export and Import========\n# \u4ed3\u5e93\u8bbf\u95ee\uff08\u4e00\u822c\u662fHDFS\uff09 \u914d\u7f6e\u6587\u4ef6\u8def\u5f84\nargodb.repository.conf.dir=\/etc\/hdfs1\/conf\n# \u4ed3\u5e93\u8bbf\u95ee\uff08\u4e00\u822c\u662fHDFS\uff09 \u662f\u5426\u5f00\u542f kerberos\nargodb.repository.kerberos.enabled=true\nargodb.repository.kerberos.user=hdfs\/argodb1@ARGODBTDH\n# \u6587\u4ef6\u626b\u63cf\u5e76\u63d0\u4ea4\u7ed3\u675f\u540e\uff0c\u68c0\u67e5\u4efb\u52a1\u6267\u884c\u8fdb\u5ea6\u7684\u95f4\u9694\u65f6\u95f4\nargodb.impexp.check.complete.interval.ms=30000\n\nargodb.server.hostname=172.22.23.1\nargodb.server.port=10000\n# \u5982\u679c\u4f7f\u7528LDAP\u8ba4\u8bc1\uff0c\u6dfb\u52a0\u5982\u4e0b2\u9879\u914d\u7f6e\uff0c\u5426\u5219\u4e0d\u52a0\nargodb.server.ldap.username=hive\nargodb.server.ldap.password=123456\n\n# ====== EXPORT ======\n# \u5f85\u5378\u8f7d\u7684\u6e90\u8def\u5f84\uff08\u4e00\u822c\u662fHDFS\uff09\nargodb.export.repository.sourcedir=\/tmp\/hive\/impexp\/export\n# \u5f85\u5b58\u50a8\u7684\u76ee\u6807\u8def\u5f84\nargodb.export.local.targetdir=\/tmp\/impexp\n# \u662f\u5426\u5bfc\u51fa\u4e00\u4e2a\u6587\u4ef6\uff0c\u8bbe\u7f6e\u4e3a true\u5bfc\u51fa\u4e00\u4e2a\u6587\u4ef6\nargodb.export.mergeOneFile=false\n# If argodb.export.mergeOneFile=true, this property is useful.\nargodb.export.local.mergefile.name=single_one.file\n# \u7f51\u7edc\u4f20\u8f93\u7684buffersize\uff0c\u9ed8\u8ba4 1MB\nargodb.export.buffersize=1048576\n# \u4e0b\u8f7d\u6587\u4ef6\u65f6\u7684\u5e76\u53d1\u7ebf\u7a0b\u6c60\u5927\u5c0f\nargodb.export.thread.pool.size=10\nargodb.export.thread.pool.queue.size=1000\n# \u5f85\u4e0b\u8f7d\u7684\u6587\u4ef6\u5148\u653e\u5165queue\u4e2d\nargodb.export.file.queue.size=1000\n# \u538b\u7f29\u6587\u4ef6 \u89e3\u538b\u4e0b\u8f7d\u65f6 \u53bb\u6389 \u538b\u7f29\u540e\u7f00\nargodb.export.compressfile.remove.suffix=true\n# \u76d1\u542c\u76ee\u5f55\u65f6\u7684\u7ec8\u7ed3\u6587\u4ef6\uff0c\u53d1\u73b0\u8be5\u6587\u4ef6\uff0c\u76d1\u542c\u7ed3\u675f\uff0c\u4e0b\u8f7d\u7ed3\u675f\u540e\u4f1a\u81ea\u52a8\u5220\u9664\u8be5\u6587\u4ef6\nargodb.export.listen.endfile.flag=_END_.FLAG\n# \u76d1\u542c\u7ebf\u7a0b\u7684\u95f4\u9694\u65f6\u95f4\nargodb.export.listen.interval.ms=5000\n# \u76d1\u542c\u7ebf\u7a0b\u542f\u52a8\u65f6\uff0c\u662f\u5426\u5ffd\u89c6\u6389\u76ee\u5f55\u5f53\u524d\u5df2\u6709\u7684\u6587\u4ef6\uff0cfalse\u8868\u793a\u4e0d\u5ffd\u89c6\nargodb.export.listen.init.skipfile=false\n# \u5bfc\u51fa\u6e05\u5355\u6587\u4ef6\uff0c\u5982\u679c\u811a\u672c\u6307\u5b9a\u8868\u4fe1\u606f \uff0c\u5219\u5ffd\u7565\u8be5\u53c2\u6570\nargodb.export.list.file=\n\n# ====== IMPORT ======\nargodb.import.repository.targetdir=\/tmp\/hive\/impexp\/import\nargodb.import.local.sourcedir=\/tmp\/impexp\nargodb.import.thread.pool.size=2\nargodb.import.thread.pool.queue.size=2\nargodb.import.file.queue.size=1000\nargodb.import.buffersize=1048576\n# \u4e0d\u6307\u5b9a\u5916\u8868\u65f6\uff0c\u81ea\u52a8\u751f\u6210\u5916\u8868\u7684\u5217\u5206\u5272\u7b26\nargodb.import.ext.table.delimiter=,\n# \u5bfc\u5165\u6e05\u5355\u6587\u4ef6\uff0c\u5982\u679c\u811a\u672c\u6307\u5b9a\u8868\u4fe1\u606f \uff0c\u5219\u5ffd\u7565\u8be5\u53c2\u6570\nargodb.import.list.file=<\/code><\/pre>\n4.\u6267\u884c\u5bfc\u5165\u547d\u4ee4<\/h4>\n#\u9996\u5148\u9700\u8981\u521d\u59cb\u5316\u5ba2\u6237\u7aef\nsource TDH-Client\/init.sh\n#\u5982\u679c\u662fkerberos\u5b89\u5168\u8ba4\u8bc1\u7684\u8bdd\uff0c\u9700\u8981\u6267\u884ckinit\u547d\u4ee4\n#\u901a\u8fc7\u6267\u884cargodb_import_export.py\u7ed3\u5408\u53c2\u6570\u8fdb\u884c\u8c03\u7528\npython argodb_import_export.py import default.people_import \/root\/people<\/code><\/pre>\n\u811a\u672c\u4f7f\u7528\u8bf4\u660e\uff1a<\/p>\n
# \u5355\u8868\npython argodb_import_export.py import ${destTableName} ${srcPath}\npython argodb_import_export.py import ${extTableName} ${destTableName} ${srcPath}\n# \u8868\u6e05\u5355\npython argodb_import_export.py import -Dargodb.import.list.file=import.list\n\n# \u8868\u6e05\u5355\u6587\u4ef6(import.list)\u683c\u5f0f\u5982\u4e0b\n \n \n \n# \u6216\u6307\u5b9a\u5916\u8868\n \n \n \n\n\u53c2\u6570\u8bf4\u660e\uff1a\n\n${destTableName} - \u76ee\u6807ArgoDB\u8868\u540d\uff08\u5e93\u540d.\u8868\u540d \u5f62\u5f0f\uff09\n${srcPath} - \u5bfc\u5165\u6570\u636e\u6587\u4ef6\u8def\u5f84\uff08\u5355\u6587\u4ef6\u6216\u76ee\u5f55\uff09\n${extTableName} - \u4e0e\u76ee\u6807\u8868\u7ed3\u6784\u76f8\u540c\u7684\u5916\u8868\uff08\u5e93\u540d.\u8868\u540d \u5f62\u5f0f\uff09\n\n\u4e3e\u4f8b\uff1a\n# \u5c06 \/demo\/data\/ \u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6570\u636e\u5bfc\u5165\u81f3 demo_db.demo_tbl \u8868\npython argodb_import_export.py import demo_db.demo_tbl \/demo\/data\/\n# \u5c06 \/demo\/data\/ \u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6570\u636e\u5bfc\u5165\u81f3 demo_db.demo_tbl \u8868\uff0c\u4f7f\u7528 demo_db.ext_tbl \u4f5c\u4e3a\u4e2d\u95f4\u5916\u8868\npython argodb_import_export.py import demo_db.ext_tbl demo_db.demo_tbl \/demo\/data\/<\/code><\/pre>\n\u811a\u672c\u6267\u884c\u65e5\u5fd7\u5982\u4e0b\uff1a<\/p>\n
ARGODB_IMEXPORT_OPTS=\nAnalyze argument finished.\nShow create table command: [beeline -u \"jdbc:hive2:\/\/172.22.23.1:10000\/default\" -n hive -p 123456 -e 'show create table default.people_import' | grep -E '^\\|']\nWARNING: Use \"yarn jar\" to launch YARN applications.\nSLF4J: Class path contains multiple SLF4J bindings.\nSLF4J: Found binding in [jar:file:\/root\/TDH-Client\/hadoop\/hadoop\/lib\/slf4j-log4j12-1.7.10.jar!\/org\/slf4j\/impl\/StaticLoggerBinder.class]\nSLF4J: Found binding in [jar:file:\/root\/TDH-Client\/inceptor\/lib\/shiva-client-shade-1.11.5.jar!\/org\/slf4j\/impl\/StaticLoggerBinder.class]\nSLF4J: See http:\/\/www.slf4j.org\/codes.html#multiple_bindings for an explanation.\nSLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]\nscan complete in 17ms\nConnecting to jdbc:hive2:\/\/172.22.23.1:10000\/default\nlog4j:WARN No appenders could be found for logger (org.apache.hive.jdbc.HiveConnection).\nlog4j:WARN Please initialize the log4j system properly.\nlog4j:WARN See http:\/\/logging.apache.org\/log4j\/1.2\/faq.html#noconfig for more info.\nConnected to: Apache Hive (version 8.16.1)\nDriver: Inceptor JDBC (version 8.16.1)\nTransaction isolation: TRANSACTION_REPEATABLE_READ\n22 rows selected (0.339 seconds)\nBeeline version 8.16.1 by Apache Hive\nClosing: 0: jdbc:hive2:\/\/172.22.23.1:10000\/default\nCreate external table sql:\n[CREATE EXTERNAL TABLE default.people_import_ext_491be5d8 (name<\/code> string, sex<\/code> string, nation<\/code> string, id<\/code> string, borndate<\/code> string, phonenumber<\/code> string, email<\/code> string, homeaddress<\/code> string) ROW FORMAT DELIMITED FIELDS TERMINATED BY \",\"]\n========================================\n Start import at: [05\/08\/2022 17:48:39]\n========================================\n\u5bfc\u5165\u8868 default.people_import \u5f00\u59cb...\n\u4e3a default.people_import \u521b\u5efahdfs\u5bfc\u5165\u76ee\u5f55\nhadoop fs -mkdir -p hdfs:\/\/nameservice1\/tmp\/hive\/importTable\/default.people_import\/default.people_import_ext_491be5d8\n\u6e05\u7a7a\u76ee\u5f55\u4e0b\u6570\u636e\nhadoop fs -rmr hdfs:\/\/nameservice1\/tmp\/hive\/importTable\/default.people_import\/default.people_import_ext_491be5d8\/*\nUploading source path: [\/root\/people].\nUploading target path: [hdfs:\/\/nameservice1\/tmp\/hive\/importTable\/default.people_import\/default.people_import_ext_491be5d8].\n\u6267\u884c\u547d\u4ee4\uff1a.\/impexpTool.sh superImport \/root\/people hdfs:\/\/nameservice1\/tmp\/hive\/importTable\/default.people_import\/default.people_import_ext_491be5d8\n\u4e0a\u4f20\u6587\u4ef6 \/root\/people to hdfs:\/\/nameservice1\/tmp\/hive\/importTable\/default.people_import\/default.people_import_ext_491be5d8 spend time 85.1135439873 s\n\u5bfc\u5165\u6a21\u677f\uff1ainsert into default.people_import select * from default.people_import_ext_491be5d8;\n\n\u7b49\u5f85\u6587\u4ef6\u4e0a\u4f20\u5b8c\u6210\n\u6587\u4ef6\u4e0a\u4f20\u5b8c\u6210\uff0c\u5f00\u59cb\u6267\u884cSQL\u63d2\u5165\nSQL\u8bed\u53e5\uff1a\ndrop table if exists default.people_import_ext_491be5d8_import_idx0;\nCREATE EXTERNAL TABLE default.people_import_ext_491be5d8_import_idx0 LIKE default.people_import_ext_491be5d8 LOCATION 'hdfs:\/\/nameservice1\/tmp\/hive\/importTable\/default.people_import\/default.people_import_ext_491be5d8\/people';\ninsert into default.people_import select * from default.people_import_ext_491be5d8_import_idx0;\ndrop table if exists default.people_import_ext_491be5d8_import_idx0;\n\n\u6267\u884cSQL\u547d\u4ee4\uff1a\nbeeline -u \"jdbc:hive2:\/\/172.22.23.1:10000\/default\" -n hive -p 123456 -f default.people_import_1659693014.72_execSQL.sql\n\u5220\u9664\u4e34\u65f6sql\u6587\u4ef6:default.people_import_1659693014.72_execSQL.sql\n\u6267\u884cSQL\u6210\u529f\nsql\u6267\u884c\u7528\u65f6\uff1a114.712487936\nImport data completed, clean start_flag_file.\nhadoop fs -rm hdfs:\/\/nameservice1\/tmp\/hive\/importTable\/default.people_import\/default.people_import_ext_491be5d8\/_START_.FLAG\n\u5bfc\u5165\u8868 default.people_import \u7ed3\u675f\uff0c\u603b\u7528\u65f6 216.777380943 s\n<\/code><\/pre>\n\u5927\u81f4\u7684\u903b\u8f91\u5176\u5b9e\u5c31\u662f \u6570\u636e\u6d41\u5411\uff1a\u672c\u5730\u6587\u4ef6 -> HDFS(\u5916\u8868) -> ArgoDB\u8868<\/strong> \u8fd9\u4e48\u4e2a\u8fc7\u7a0b\u3002<\/p>\n\u6700\u7ec8\u68c0\u67e5people_import<\/code>\u8868\u6570\u636e\u5b8c\u6574\u6027\uff0c\u9a8c\u8bc1\u901a\u8fc7\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"\u6982\u8981\u63cf\u8ff0 \u672c\u6587\u6863\u65e8\u5728\u63d0\u4f9b\u4e00\u4efd impexp \u5de5\u5177\u7684\u5feb\u901f\u4e0a\u624b\u6307\u5357\uff0c\u6709\u5173\u5de5\u5177\u7684\u66f4\u8be6\u7ec6\u7684\u914d\u7f6e\u3001\u4f7f\u7528\u3001\u539f\u7406\u3001\u6ce8\u610f\u4e8b\u9879\u3002 ..<\/p>\n
<\/div>\n
Read more<\/a><\/p>\n","protected":false},"author":12,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[47],"tags":[],"class_list":["post-8676","post","type-post","status-publish","format-standard","hentry","category-third_part"],"acf":[],"_links":{"self":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/8676","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/users\/12"}],"replies":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=8676"}],"version-history":[{"count":1,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/8676\/revisions"}],"predecessor-version":[{"id":15689,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/8676\/revisions\/15689"}],"wp:attachment":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=8676"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=8676"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=8676"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}