{"id":3023,"date":"2020-05-21T09:25:13","date_gmt":"2020-05-21T01:25:13","guid":{"rendered":"https:\/\/nj.transwarp.cn:8180\/?p=3023"},"modified":"2020-05-21T09:25:13","modified_gmt":"2020-05-21T01:25:13","slug":"studio1-2%e4%bb%a5%e4%b8%8b%e7%89%88%e6%9c%ac%ef%bc%88%e4%b8%8d%e5%90%ab1-2%ef%bc%89canaltdt%e6%95%b0%e6%8d%ae%e5%90%8c%e6%ad%a5","status":"publish","type":"post","link":"https:\/\/kbwp.transwarp.cn\/?p=3023","title":{"rendered":"studio1.2\u4ee5\u4e0b\u7248\u672c\uff08\u4e0d\u542b1.2\uff09Canal+TDT\u6570\u636e\u540c\u6b65"},"content":{"rendered":"<h3>\u6982\u8981\u63cf\u8ff0<\/h3>\n<hr \/>\n<p>\u8be5\u5de5\u5177\u7528\u4e8e\u4ecemysql\u4e2d\u5b9e\u65f6\u83b7\u53d6\u589e\u91cf\u7684\u6570\u636e\uff08Insert\uff0cUpdate\uff0cDelete\uff09\uff0c\u7136\u540e\u5c06\u589e\u91cf\u6570\u636e\u4ee5OGG JSON\u7684\u683c\u5f0f\u5199\u5230hdfs\u4e2d\uff0c\u4f9btdt\u8bfb\u53d6\u5e76\u540c\u6b65\u5230orc\u4e8b\u52a1\u8868\u3002<\/p>\n<p>\u57fa\u4e8ecanal\u5b9e\u73b0canal\u662falibaba\u5f00\u6e90\u7684binlog\u62bd\u53d6\u89e3\u6790\u5de5\u5177\uff0ccanal\u63d0\u4f9bserver\u7aef\u5b9e\u73b0\uff0c\u8fd9\u91cc\u57fa\u4e8ecanal\u5b9e\u73b0\u4e00\u79cdc\/s\u6a21\u5f0f\u7684binlog\u8ba2\u9605\uff0c\u7136\u540e\u653e\u5230hdfs\u4e2d\uff0c\u4f9btdt\u8bfb\u53d6\u3002<\/p>\n<ol>\n<li>canal server \uff1a \u7528\u6237\u7684DML\uff0cDDL\u8bed\u53e5\u5728mysql\u4e2d\u90fd\u4f1a\u4ea7\u751fbinlog\u65e5\u5fd7\uff0ccanal server\u80fd\u591f\u8fde\u63a5mysql\u5e76\u53d1\u9001dump\u8bf7\u6c42\u53bb\u83b7\u5f97binlog\u65e5\u5fd7\u3002<\/li>\n<li>instance \uff1a \u8fd9\u91cc\u79f0\u4e3a\u4e00\u4e2a\u540c\u6b65\u5b9e\u4f8b\uff0c\u4e00\u4e2acanal server\u53ef\u4ee5\u52a0\u8f7dn\u4e2a\u540c\u6b65\u5b9e\u4f8b\uff0c\u540c\u6b65\u5b9e\u4f8b\u4e2d\u914d\u7f6e\u4e86\u8981\u540c\u6b65\u7684mysql\u6e90\u76f8\u5173\u4fe1\u606f\u3002client\u8fde\u63a5server\u540e\uff0c\u5e94\u5f53\u8ba2\u9605\u5176\u4e2d\u67d0\u4e00\u4e2ainstance\uff0c\u4ece\u8fd9\u4e2ainstance\u83b7\u53d6\u6570\u636e\u3002<\/li>\n<li>client client\u8fde\u63a5server\uff0c\u8ba2\u9605instance\uff0c\u83b7\u53d6\u83b7\u5f97canal server\u89e3\u6790\u8fc7\u7684binlog\u6570\u636e\u3002\u6709\u591a\u79cdclient\uff0c\u6bd4\u5982\u540c\u6b65\u81f3\u5173\u7cfb\u578b\u6570\u636e\u5e93\u7684client\uff0c\u540c\u6b65\u81f3hbase\u7684client\uff0c\u540c\u6b65\u81f3kafka\u7684client\uff0c\u8fd9\u91cc\u8981\u4f7f\u7528\u7684\u662f\u540c\u6b65\u81f3inceptor\u7684client<\/li>\n<li>\u6574\u4f53\u7406\u89e3\uff1a canal server\u8d1f\u8d23\u52a0\u8f7dinstance\uff0c\u6839\u636einstance\u7684\u914d\u7f6e\u4ecemysql\u83b7\u53d6binlog\u6570\u636e\uff0c\u4e00\u4e2acanal server\u53ef\u4ee5\u52a0\u8f7dn\u4e2a\u540c\u6b65\u5b9e\u4f8b\u3002\u800cclient\u8d1f\u8d23\u8ba2\u9605instance\uff0c\u6839\u636e\u4e1a\u52a1\u9700\u6c42\u5c06\u6570\u636e\u540c\u6b65\u81f3\u4e0d\u540c\u5730\u65b9\u3002canal server\u548cclient\u89e3\u8026\uff0c\u53ef\u4ee5\u5206\u5f00\u90e8\u7f72\u3002<\/li>\n<\/ol>\n<p><strong>\u6280\u672f\u6982\u89c8\uff1a<\/strong><\/p>\n<ul>\n<li>\u5f00\u542fMySQL\u7684binary log\u65e5\u5fd7\u8bb0\u5f55<\/li>\n<li>\u4fee\u6539MySQL\u7684binary log\u6a21\u5f0f\u4e3aROW<\/li>\n<li>canal-server\u5145\u5f53MySQL\u96c6\u7fa4\u7684\u4e00\u4e2aslave\uff0c\u83b7\u53d6master\u7684binary log\u4fe1\u606f<\/li>\n<li>canal-server\u5c06\u62ff\u5230\u7684binary log\u4fe1\u606f\u63a8\u9001\u7ed9sync-client<\/li>\n<li>canal-server\u548csync-client\u91c7\u7528\u591a\u8282\u70b9\u90e8\u7f72\u7684\u65b9\u5f0f\u63d0\u9ad8\u53ef\u7528\u6027<\/li>\n<li>sync-client\u5c06json\u6587\u4ef6\u540c\u6b65\u5230hdfs\u96c6\u7fa4<\/li>\n<li>transporter\u7ec4\u4ef6\u8bfb\u53d6json\u6587\u4ef6\u540c\u6b65\u5230orc\u8868<\/li>\n<li>workflow\u7ec4\u4ef6\u5bf9transporter\u5de5\u4f5c\u6d41\u5b9e\u73b0\u51c6\u5b9e\u65f6\u8c03\u5ea6<\/li>\n<\/ul>\n<p><strong>\u67b6\u6784\u56fe<\/strong><br \/>\n<img decoding=\"async\" src=\"\/wp-content\/uploads\/2020\/05\/image-1589177900036.png\" alt=\"file\" \/><\/p>\n<h3>\u8be6\u7ec6\u8bf4\u660e<\/h3>\n<hr \/>\n<p>\u73af\u5883\u51c6\u5907\uff1a<\/p>\n<ul>\n<li>TDH 5.2.2<\/li>\n<li>mysql 5.7<\/li>\n<\/ul>\n<h4>1. \u914d\u7f6ecanal server<\/h4>\n<h5>1.1 \u51c6\u5907\u5de5\u4f5c<\/h5>\n<h6>1.1.1 canal\u57fa\u4e8emysql\u7684binlog\u529f\u80fd\uff0c\u6240\u4ee5mysql\u9700\u8981\u5f00\u542fbinlog\u7684\u5199\u5165\u529f\u80fd\uff0c\u5e76\u4e14\u914d\u7f6ebinlog\u6a21\u5f0f\u4e3arow\u3002\u9700\u8981\u4fee\u6539mysql\u7684\u914d\u7f6e\u6587\u4ef6[mysqld\u6587\u4ef6]\uff0c\u914d\u7f6e\u5b8c\u6210\u540e\u91cd\u542fmysql<\/h6>\n<pre><code class=\"language-shell\">[mysqld]\nlog-bin=mysql-bin #\u6dfb\u52a0\u8fd9\u4e00\u884c\u5c31ok\nbinlog-format=ROW #\u9009\u62e9row\u6a21\u5f0f\nserver_id=1 #\u914d\u7f6emysql replaction\u9700\u8981\u5b9a\u4e49\uff0c\u4e0d\u80fd\u548ccanal\u7684slaveId\u91cd\u590d<\/code><\/pre>\n<h6>1.1.2 \u5728mysql\u5ba2\u6237\u7aef\u901a\u8fc7\u4ee5\u4e0b\u8bed\u53e5\u8fdb\u884c\u67e5\u8be2\u786e\u8ba4<\/h6>\n<pre><code class=\"language-sql\">show variables like 'binlog_format'; \/\/ \u6b63\u786e\u7ed3\u679c\u4e3a row\nshow variables like 'log_bin'; \/\/ \u6b63\u786e\u7ed3\u679c\u4e3a on<\/code><\/pre>\n<h6>1.1.3 \u5728mysql\u4e0a\u6267\u884c, canal\u662f\u5c06\u81ea\u5df1\u4f2a\u88c5\u6210slave\u5411master\u8bf7\u6c42binlog\u6570\u636e\uff0c\u56e0\u6b64\u9700\u8981\u7ed9canal\u590d\u5236\u7684\u6743\u9650<\/h6>\n<pre><code class=\"language-sql\">CREATE USER canal IDENTIFIED BY '1qaz!QAZ'; \/\/ \u521b\u5efacanal\u7528\u6237\nGRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'canal'@'%'; \/\/ \u8d4b\u4e88canal \u7528\u6237\u6743\u9650\n-- GRANT ALL PRIVILEGES ON *.* TO 'canal'@'%' ;\nFLUSH PRIVILEGES; \/\/ \u8ba9\u6743\u9650\u8bbe\u7f6e\u751f\u6548<\/code><\/pre>\n<h6>1.1.4 \u5728mysql\u5ba2\u6237\u7aef\u901a\u8fc7\u4ee5\u4e0b\u8bed\u53e5\u8fdb\u884c\u67e5\u8be2\u786e\u8ba4\uff1a<\/h6>\n<pre><code class=\"language-sql\">show grants for 'canal';<\/code><\/pre>\n<h5>1.2 canal server\u914d\u7f6e<\/h5>\n<p>canal server\u7684\u76ee\u5f55\u7ed3\u6784\u5982\u4e0b<\/p>\n<pre><code class=\"language-shell\">canal server -\n|- bin \/\/ canal server\u7684\u542f\u52a8\u3001\u505c\u6b62\u811a\u672c\u76ee\u5f55\n|- conf \/\/ canal server\u548cinstance\u7684\u914d\u7f6e\u76ee\u5f55\n|- lib \/\/ \u76f8\u5173\u4f9d\u8d56\u5305\u5b58\u653e\u76ee\u5f55\n|- logs \/\/ canal server\u548cinstance\u7684\u65e5\u5fd7\u76ee\u5f55<\/code><\/pre>\n<p>\u53ef\u4ee5\u770b\u5230canal server\u548cinstance\u90fd\u5728conf\u76ee\u5f55\u4e0b\u8fdb\u884c\u914d\u7f6e\uff0c\u73b0\u5728\u5148\u5173\u6ce8canal server\u7684\u914d\u7f6e\uff0cconf\u76ee\u5f55\u4e0b\u7684canal.properties\u5373\u4e3acanal server\u7684\u914d\u7f6e\u6587\u4ef6\uff0c\u4e3b\u8981\u5173\u6ce8\u4ee5\u4e0b\u914d\u7f6e\u9879\uff1a<\/p>\n<ol>\n<li>canal.id \u9ed8\u8ba4\u662f1 \uff0c\u542f\u52a8\u591a\u4e2aserver\u65f6\uff0c\u53ef\u4ee5\u4f9d\u6b21\u4f7f\u75281\uff0c2\uff0c&#8230;<\/li>\n<li>canal.ip canal server\u5de5\u4f5c\u7684ip\uff0c\u9ed8\u8ba4127.0.0.1\uff0c\u8bbe\u7f6e\u6210server\u6240\u5728\u4e3b\u673a\u7684ip<\/li>\n<li>canal.port canal server\u5de5\u4f5c\u7684\u7aef\u53e3\uff0c\u9ed8\u8ba411111\uff0c\u53ef\u4ee5\u8bbe\u7f6e\u6210\u4efb\u4f55\u53ef\u7528\u7aef\u53e3<\/li>\n<li>canal.instance.global.spring.xml instance\u7684bean\u9009\u62e9\uff0c\u9ed8\u8ba4\u4e3aclasspath:spring\/file-instance.xml\uff0c\u4f1a\u5c06\u540c\u6b65\u4f4d\u70b9\u5199\u5165\u672c\u5730\u6587\u4ef6\u7cfb\u7edf<\/li>\n<\/ol>\n<p>\u5176\u4f59\u914d\u7f6e\u9879\u4fdd\u6301\u9ed8\u8ba4\u5373\u53ef\uff0c\u53ef\u4ee5\u5230\u5b98\u7f51\u67e5\u770b\u5176\u5b83\u914d\u7f6e\u9879\u7684\u542b\u4e49\uff1a<a href=\"https:\/\/github.com\/alibaba\/canal\/wiki\/AdminGuide\">https:\/\/github.com\/alibaba\/canal\/wiki\/AdminGuide<\/a><\/p>\n<h4>2. \u914d\u7f6einstance<\/h4>\n<p>server\u7684\u52a0\u8f7d\u57fa\u672c\u5bf9\u8c61\u662finstance\uff0cinstance\u914d\u7f6e\u6587\u4ef6\u4e2d\u914d\u7f6e\u4e86\u540c\u6b65\u7684\u5177\u4f53\u6570\u636e\u5e93\u3001\u8868\u3002 conf\u76ee\u5f55\u5185\u5bb9\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-shell\">conf -\n|- canal.properties \/\/ canal server\u7684\u914d\u7f6e\u6587\u4ef6\n|- logback.xml \/\/ \u65e5\u5fd7\u914d\u7f6e\u6587\u4ef6\n|- spring \/\/ java bean\u7684\u914d\u7f6e\u6587\u4ef6\u5b58\u653e\u76ee\u5f55\uff0c\u4e0d\u8981\u79fb\u52a8\u6216\u4fee\u6539\n|- metrics \/\/ \u6682\u65f6\u4e0d\u7528\u5173\u6ce8\n|- example \/\/ \u5b98\u65b9\u7ed9\u51fa\u7684\u4e00\u4e2a\u9ed8\u8ba4instance\u7684\u6837\u4f8b\u914d\u7f6e\n|- instance.properties \/\/ example\u8fd9\u4e2ainstance\u7684\u914d\u7f6e\u6587\u4ef6<\/code><\/pre>\n<p>\u53ef\u4ee5\u770b\u5230\u5b98\u65b9\u7ed9\u51fa\u4e86\u4e00\u4e2a\u6837\u4f8b\u7684instance\uff0c\u5373\u4e00\u4e2a\u540d\u4e3aexample\u7684\u6587\u4ef6\u5939\uff0c\u6587\u4ef6\u5939\u4e2d\u4fdd\u5b58\u7740\u4e00\u4e2a\u540d\u4e3ainstance.properties\u7684\u6587\u4ef6\u3002<strong><font color=red>\u6309\u7167canal server\u7684\u9ed8\u8ba4\u914d\u7f6e\uff0ccanal server\u4f1a\u626b\u63cfconf\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\u5939\uff0c\u5f53\u53d1\u73b0\u67d0\u4e2a\u6587\u4ef6\u5939\u4e0b\u9762\u5b58\u5728instance.properties\u8fd9\u4e2a\u6587\u4ef6\u65f6\uff0c\u5c31\u5c06\u8be5\u6587\u4ef6\u5939\u8bc6\u522b\u4e3a\u4e00\u4e2ainstance\uff0c\u8be5instance\u7684\u540d\u5b57\u5373\u4e3a\u6587\u4ef6\u5939\u7684\u540d\u5b57\u3002\u6240\u4ee5\u5b98\u65b9\u7ed9\u51fa\u7684\u6837\u4f8binstance\u7684\u540d\u5b57\u5373\u4e3aexample\uff0c\u5176\u4e2d\u7684instance.properties\u5373\u4e3a\u8fd9\u4e2ainstance\u7684\u914d\u7f6e\u6587\u4ef6\u3002<\/font><\/strong>\u8be5\u914d\u7f6e\u6587\u4ef6\u4e2d\u5173\u6ce8\u4ee5\u4e0b\u914d\u7f6e\u9879\uff1a<\/p>\n<pre><code>1. canal.instance.master.address\uff0c \u5fc5\u586b\u9879\uff0c\u88ab\u540c\u6b65\u7684\u6570\u636e\u5e93ip\u548c\u7aef\u53e3\n2. canal.instance.master.journal.name\uff0c\u9009\u586b\u9879\uff0c\u6307\u5b9a\u8981\u540c\u6b65\u7684binlog\u539f\u59cb\u6587\u4ef6\u540d\uff0ccanal\u5c06\u4ece\u8be5\u6587\u4ef6\u5f00\u59cb\u5f80\u540e\u89e3\u6790\uff0c\u53ef\u4ee5\u5728mysql\u5ba2\u6237\u7aef\u4e2d\u6267\u884cshow binary logs\u6765\u67e5\u770b\u6240\u6709\u7684binlog\u6587\u4ef6\n3. canal.instance.master.position\uff0c\u9009\u586b\u9879\uff0c\u6307\u5b9abinlog\u539f\u59cb\u6587\u4ef6\u4e2d\u7684\u540c\u6b65\u4f4d\u70b9\uff0ccanal \u5c06\u4ece\u6307\u5b9a\u7684binlog\u539f\u59cb\u6587\u4ef6\u4e2d\u7684\u6307\u5b9a\u4f4d\u70b9\u5f00\u59cb\u5f80\u540e\u89e3\u6790\uff0c\u53ea\u6709\u5728\u586b\u5199\u4e86\u4e0a\u4e00\u4e2a\u914d\u7f6e\u9879\u4e4b\u540e\u624d\u6709\u610f\u4e49\u3002\u53ef\u4ee5\u5728mysql\u5ba2\u6237\u7aef\u6267\u884cshow binlog events in <your choosed binlog file>\u67e5\u770b\u539f\u59cbbinlog\u6587\u4ef6\u4e2d\u7684\u5177\u4f53\u5185\u5bb9,\u53ef\u4ee5\u67e5\u5230\u4f4d\u70b9\u4fe1\u606f\uff0c\u4f46\u662f\u4e0d\u80fd\u67e5\u5230\u65f6\u95f4\n4. canal.instance.master.timestamp\uff0c \u9009\u586b\u9879\uff0c\u6307\u5b9a\u4e00\u4e2a\u65f6\u95f4\u6233\uff0c\u8be5\u65f6\u95f4\u6233\u8981\u5728\u6240\u6709binlog\u539f\u59cb\u6587\u4ef6\u4e2d\u8bb0\u5f55\u7684\u65f6\u95f4\u8303\u56f4\u5185\uff0ccanal\u5c06\u4ece\u8fd9\u4e2a\u65f6\u95f4\u6233\u6700\u8fd1\u7684\u4f4d\u7f6e\u5f80\u540e\u89e3\u6790\u3002\u5728mysql\u6240\u5728\u673a\u5668\u7684\u7ec8\u7aef\u6267\u884cmysqlbinlog <path>\/<binlog file>\u547d\u4ee4\u6765\u67e5\u770bbinglog\u6587\u4ef6\u4e2d\u7684\u4fe1\u606f\uff0c\u53ef\u67e5\u770b\u5230\u4f4d\u70b9\u548c\u65f6\u95f4\u4fe1\u606f\n5. canal.instance.standby.address\uff0c \u9009\u586b\u9879\uff0cmysql slave\u7684\u5730\u5740\uff0c master\u5d29\u6e83\u65f6\u4f1a\u8f6c\u5411slave\uff08\u76ee\u524dcanal server\u53ea\u652f\u6301\u4e00\u4e2aslave\uff09\n6. canal.instance.dbUsername\uff0c\u5fc5\u586b\u9879\uff0c\u8bbf\u95ee\u6570\u636e\u5e93\u7684\u7528\u6237\u540d\n7. canal.instance.dbPassword, \u5fc5\u586b\u9879\uff0c\u8bbf\u95ee\u6570\u636e\u5e93\u7684\u5bc6\u7801\n8. canal.instance.filter.regex\uff0c\u4e8c\u9009\u4e00\u5fc5\u586b\u9879\uff0c\u767d\u540d\u5355\u6b63\u5219\u8868\u8fbe\u5f0f\uff0c\u80fd\u5339\u914d\u8be5\u6b63\u5219\u7684\u5e93\u8868\u90fd\u5c06\u8fdb\u884c\u540c\u6b65\n9. canal.instance.filter.black.regex\uff0c\u4e8c\u9009\u4e00\u5fc5\u586b\u9879\uff0c\u9ed1\u540d\u5355\u6b63\u5219\u8868\u8fbe\u5f0f\uff0c\u5339\u914d\u8be5\u6b63\u5219\u7684\u5e93\u8868\u4e4b\u5916\u7684\u6240\u6709\u5e93\u8868\u90fd\u5c06\u8fdb\u884c\u540c\u6b65<\/code><\/pre>\n<p><strong>\u91cd\u8981\u8bf4\u660e<\/strong>\uff1a<\/p>\n<ol>\n<li>\n<p>\u4ee5\u4e0a2\u30013\u30014\u4e09\u4e2a\u914d\u7f6e\u9879\u51b3\u5b9acanal server\u4ece\u4ec0\u4e48\u4f4d\u7f6e\u5f00\u59cb\u540c\u6b65\u30022\u30013\u53ef\u4ee5\u7cbe\u786e\u7684\u786e\u5b9a\u4e00\u4e2a\u4f4d\u7f6e\uff1b4\u53ef\u4ee5\u6a21\u7cca\u786e\u5b9a\u4e00\u4e2a\u4f4d\u7f6e\u3002\u4f46\u5982\u679c\u8fd9\u4e09\u4e2a\u9009\u9879\u90fd\u4e0d\u586b\uff0c\u5219\u9ed8\u8ba4\u4ececanal server\u542f\u52a8\u65f6\u7684\u6700\u65b0\u4f4d\u70b9\u5f00\u59cb\u5f80\u540e\u540c\u6b65\u3002\u9644\uff1a\u5173\u4e8emydql binlog\u7684\u5e38\u7528\u547d\u4ee4\u53c2\u8003binlog<\/p>\n<\/li>\n<li>\n<p>\u914d\u7f6e\u98798\u548c9\u4e2d\u586b\u5199\u7684\u662f\u5b9a\u5236\u7684java\u6b63\u5219\u8868\u8fbe\u5f0f\uff0c\u5206\u4e3a\u4e09\u90e8\u5206\uff1a[\u5e93\u6b63\u5219][\u5206\u9694\u7b26][\u8868\u6b63\u5219]\uff0c\u5206\u9694\u7b26\u56fa\u5b9a\u4e3a[.]\uff0c\u591a\u4e2a\u6b63\u5219\u4e4b\u95f4\u4ee5\u9017\u53f7[,]\u5206\u9694\uff0c \u5e38\u89c1\u4f8b\u5b50\uff1a\u6240\u6709\u5e93\u6240\u6709\u8868\uff1a.<em>\\..<\/em>;test_db\u5e93\u4e0b\u7684\u6240\u6709\u8868\uff1atest\\..<em>\uff1btest_db\u5e93\u4e0b\u7684\u6240\u6709\u4ee5canal\u6253\u5934\u7684\u8868\uff1atest_db\\.canal.<\/em>\uff1btest_db\u5e93\u4e0b\u7684\u4e00\u5f20\u5177\u4f53\u8868test_table1\uff1atest_db\\.test_table1\uff1b\u591a\u4e2a\u89c4\u5219\u7ec4\u5408\u4f7f\u7528\uff1atest_db1\\..*,test_db2\\.test_table1,test_db2\\.test_table2 (\u9017\u53f7\u5206\u9694)<\/p>\n<\/li>\n<\/ol>\n<p>\u53ef\u4ee5\u5230\u5b98\u7f51\u67e5\u770binstance\u6240\u6709\u914d\u7f6e\u9879\u542b\u4e49\uff1a<a href=\"https:\/\/github.com\/alibaba\/canal\/wiki\/AdminGuide\">https:\/\/github.com\/alibaba\/canal\/wiki\/AdminGuide<\/a><\/p>\n<h4>3. \u914d\u7f6esync client<\/h4>\n<p>sync client\u7528\u4e8e\u8ba2\u9605instance\uff0c\u5e76\u5c06binlog\u6570\u636e\u89e3\u6790\u6210\u7279\u5b9a\u683c\u5f0f\u5199\u5165\u6587\u4ef6\u5e76\u4e0a\u4f20\u5230hdfs\u4e0a\uff0c\u4f9bTDT\u4f7f\u7528\u3002sync client\u4e0ecanal server\u53ef\u4ee5\u90e8\u7f72\u5230\u4e0d\u540c\u673a\u5668\u4e0a\uff0c\u53ea\u8981\u4fdd\u8bc1\u673a\u5668\u80fd\u4e92\u76f8ping\u901a\u5373\u53ef\u3002 sync client\u7684\u76ee\u5f55\u7ed3\u6784\u5982\u4e0b\u6240\u793a\uff1a<\/p>\n<pre><code class=\"language-shell\">sync client -\n|- bin \/\/ sync client\u7684\u542f\u52a8\u3001\u505c\u6b62\u811a\u672c\u76ee\u5f55\n|- conf \/\/ sync client\u7684\u914d\u7f6e\u6587\u4ef6\u76ee\u5f55\n|- client-conf \/\/ client conf\u7684\u914d\u7f6e\u6587\u4ef6\u5b58\u653e\u7684\u76ee\u5f55\n|- sync-client.conf \/\/ \u9ed8\u8ba4\u7ed9\u51fa\u7684client\u914d\u7f6e\u6587\u4ef6\u6837\u4f8b\n|- logback.xml \/\/ \u65e5\u5fd7\u914d\u7f6e\u6587\u4ef6\n|- hdfs-site.xml \/\/ \u8fde\u63a5hdfs\u6240\u9700\u8981\u7684\u914d\u7f6e\u6587\u4ef6\n|- core-site.xml \/\/ \u8fde\u63a5hdfs\u6240\u9700\u8981\u7684\u914d\u7f6e\u6587\u4ef6\n|- lib \/\/ \u4e3b\u8fd0\u884c\u7a0b\u5e8f\u5b58\u653e\u76ee\u5f55\n|- libext \/\/ \u4f9d\u8d56\u5305\u5b58\u653e\u76ee\u5f55\n|- logs \/\/ \u65e5\u5fd7\u5b58\u653e\u76ee\u5f55\uff0c\u7a0b\u5e8f\u8fd0\u884c\u540e\u81ea\u52a8\u751f\u6210\uff0c\u8bb0\u5f55\u8fd0\u884c\u65e5\u5fd7<\/code><\/pre>\n<p>sync client\u9700\u8981\u8fde\u63a5hdfs\uff0c\u8fde\u63a5hdfs\u6709\u4e24\u79cd\u65b9\u5f0f\uff0c<br \/>\n\u7b2c\u4e00\u79cd\u662fRPC\u8fde\u63a5\u65b9\u5f0f\uff0c\u9002\u7528\u4e8eTDH\u96c6\u7fa4\u7684hdfs\uff0c\u8fd9\u79cd\u8fde\u63a5\u65b9\u5f0f\u9700\u8981\u5c06hdfs-site.xml\u548ccore-site.xml\u6587\u4ef6\u653e\u5165\u5230conf\u76ee\u5f55\u4e0b\uff0c\u5982\u679chdfs\u5f00\u542f\u4e86kerberos\u8ba4\u8bc1\uff0c\u8fd8\u9700\u8981\u5c06krb5.conf\u6587\u4ef6\u653e\u5165\u5230\/etc\u76ee\u5f55\u4e0b\uff0c\u9700\u8981\u5c06keytab\u6587\u4ef6\u653e\u5165\u5230\u67d0\u4e2a\u76ee\u5f55\u5907\u7528\uff0c\u4f7f\u7528klist\u547d\u4ee4\u67e5\u8be2keytab\u6587\u4ef6\u4e2d\u7684principal\uff08\u8be5principal\u5728hdfs\u4e0a\u5fc5\u987b\u5177\u6709\u8bfb\u5199\u7684\u6743\u9650\uff09\uff0c\u4e4b\u540e\u9700\u8981\u5728sync client\u914d\u7f6e\u6587\u4ef6\u4e2d\u586b\u5199\u8fd9\u4e9b\u4fe1\u606f\u3002klist\u547d\u4ee4\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-shell\">klist -kt <path to keytab><\/code><\/pre>\n<p>\u7b2c\u4e8c\u79cd\u8fde\u63a5\u65b9\u5f0f\u662fTCP\u8fde\u63a5\u65b9\u5f0f\uff0c\u5373\u4f7f\u7528httpfs\uff0c\u9002\u7528\u4e8eTDC\u96c6\u7fa4\u7684hdfs\uff0c\u8fd9\u79cd\u8fde\u63a5\u65b9\u5f0f\u9700\u8981hdfs\u5f00\u542fhttpfs\u670d\u52a1\uff0c\u9700\u8981\u5728\u914d\u7f6e\u6587\u4ef6\u4e2d\u586b\u5199\u5916\u90e8\u53ef\u8bbf\u95ee\u7684httpfs\u7684\u5730\u5740\uff08\u53ef\u5728TDC\u7ba1\u7406\u754c\u9762\u7684hdfs\u670d\u52a1\u4fe1\u606f\u83b7\u5f97\uff09\uff0c\u5982\u679chdfs\u5f00\u542f\u4e86kerberos\u8ba4\u8bc1\uff0c\u5219\u8fd8\u9700\u8981\u586b\u5199guardian_token\uff08\u53ef\u5728guardian\u7ba1\u7406\u754c\u9762\u83b7\u5f97\uff09\u3002<\/p>\n<p>\u6839\u636e\u4e1a\u52a1\u573a\u666f\u9009\u62e9\u8fde\u63a5\u65b9\u5f0f\uff0c\u7136\u540e\u5f00\u59cb\u586b\u5199sync client\u7684\u914d\u7f6e\u6587\u4ef6\u3002\u652f\u6301\u542f\u52a8\u591a\u4e2aclient\uff0c\u6bcf\u4e2aclient\u4f7f\u7528\u4e0d\u540c\u7684\u914d\u7f6e\u6587\u4ef6\u3002 conf\/client-conf\u76ee\u5f55\u4e0b\u5b58\u653e\u4e86\u8fd9\u4e9b\u914d\u7f6e\u6587\u4ef6\uff0c\u9ed8\u8ba4\u7ed9\u51fa\u4e86\u4e00\u4e2a\u914d\u7f6e\u6587\u4ef6\u6837\u4f8bsync-client.conf\uff0c\u5404\u914d\u7f6e\u9879\u542b\u4e49\u5982\u4e0b<\/p>\n<pre><code>1. canal.server.ip canal server\u7684ip\uff0c\u5982\u679ccanal server\u914d\u7f6e\u4e86HA\uff0c\u53ef\u4ee5\u586b\u591a\u4e2a\uff0c\u4ee5\u5206\u53f7[;]\u8fdb\u884c\u5206\u9694\n2. canal.server.port canal server\u7684\u7aef\u53e3\u53f7\uff0c\u5982\u679ccanal server\u914d\u7f6e\u4e86HA\uff0c\u53ef\u4ee5\u586b\u591a\u4e2a\uff0c\u4ee5\u5206\u53f7[;]\u5206\u9694\uff0c\u4f46\u5fc5\u987b\u4e0eip\u4fdd\u6301\u4e00\u4e00\u5bf9\u5e94\n3. canal.instance.list client\u8ba2\u9605\u7684instance\u5217\u8868\uff0c\u7528\u5206\u53f7[;]\u8fdb\u884c\u5206\u9694\n4. canal.instance.max.number client\u80fd\u8ba2\u9605\u7684instance\u6700\u5927\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a100\n5. canal.db.table.merge.rule \u5206\u5e93\u5206\u8868\u5408\u5e76\u89c4\u5219\n6. canal.client.retry.time client\u9047\u5230\u9519\u8bef\u540e\u91cd\u8bd5\u6b21\u6570\uff0c\u9ed8\u8ba4\u4e3a3\u6b21\n7. canal.file.max.records \u751f\u6210\u7684\u89e3\u6790\u6587\u4ef6\u8bb0\u5f55\u6761\u6570\u4e0a\u9650\uff0c\u9ed8\u8ba4\u4e3a10\u4e07\u6761\n8. canal.file.flush.time \u9ed8\u8ba4\u4ea7\u751f\u65b0\u7684\u89e3\u6790\u6587\u4ef6\u7684\u9891\u7387\uff0c\u9ed8\u8ba4\u4e3a60\u79d2\n9. canal.file.check.interval \u9ed8\u8ba4\u7684\u5237\u65b0\u9891\u7387\uff0c\u9ed8\u8ba4\u4e3a1\u79d2\uff0c\u4e0d\u9700\u8981\u5173\u6ce8\n10. canal.file.local.reserve.time \u89e3\u6790\u6587\u4ef6\u672c\u5730\u4fdd\u7559\u65f6\u95f4\uff0c\u9ed8\u8ba4\u662f0\uff0c\u5373\u4e0d\u4fdd\u7559\n11. canal.file.local.reserve.dir \u89e3\u6790\u6587\u4ef6\u7684\u672c\u5730\u5907\u4efd\u76ee\u5f55\uff0c\u9700\u8981\u8bbe\u7f6e\u4e3a777\u7684\u6743\u9650\n12. canal.file.local.store.dir \u89e3\u6790\u6587\u4ef6\u7684\u672c\u5730\u4e34\u65f6\u5b58\u50a8\u76ee\u5f55\uff0c\u9ed8\u8ba4\u4e3a\/tmp\/binlogsync\uff0c\u9700\u8981\u8bbe\u7f6e\u4e3a777\u7684\u6743\u9650\n13. canal.file.hdfs.target.dir \u4e0a\u4f20\u5230hdfs\u7684\u76ee\u5f55\uff0c\u5f00\u542f\u5b89\u5168\u60c5\u51b5\u4e0b\u9700\u8981\u5bf9principal\u5177\u6709\u5199\u6743\u9650\uff0c\u6ca1\u5f00\u5b89\u5168\u60c5\u51b5\u4e0b\u9700\u8981\u5177\u6709\u5199\u6743\u9650\n14. canal.hdfs.auth.mode hdfs\u7684\u8ba4\u8bc1\u6a21\u5f0f\uff0c\u6709\u4e09\u79cd\u8ba4\u8bc1\u6a21\u5f0fsimple\u3001kerberos\u4ee5\u53catoken\uff0c\u9ed8\u8ba4\u4e3asimple\n15. canal.hdfs.principal \u5982\u679c\u8ba4\u8bc1\u6a21\u5f0f\u586b\u5199kerberos\uff0c\u5219\u5fc5\u987b\u586b\u5199principal\n16. canal.hdfs.keytab \u5982\u679c\u8ba4\u8bc1\u6a21\u5f0f\u586b\u5199kerberos\uff0c\u5219\u5fc5\u987b\u586b\u5199keytab\u6587\u4ef6\u8def\u5f84\n17. canal.httpfs.address \u5982\u679c\u8ba4\u8bc1\u6a21\u5f0f\u586b\u5199token\uff0c\u5c06\u4f1a\u4ee5httpfs\u7684\u65b9\u5f0f\u8bbf\u95eehdfs\uff0c\u5fc5\u987b\u586b\u5199\u5916\u90e8\u80fd\u8bbf\u95ee\u7684httpfs\u5730\u5740\n18. canal.httpfs.token \u5982\u679c\u8ba4\u8bc1\u6a21\u5f0f\u586b\u5199token\uff0c\u5e76\u4e14\u96c6\u7fa4\u5f00\u542f\u4e86guardian\uff0c\u5219\u5fc5\u987b\u586b\u5199guardian_token\uff0c\u5982\u679c\u96c6\u7fa4\u6ca1\u6709\u5f00\u542fguardian\uff0c\u5219\u4e0d\u7528\u586b\u5199<\/code><\/pre>\n<p><strong>\u91cd\u8981\u8bf4\u660e<\/strong><br \/>\n1\u3001\u7b2c5\u6761\u914d\u7f6e\u9879\uff0c\u5206\u5e93\u5206\u8868\u5408\u5e76\u89c4\u5219\u9700\u8981\u586b\u5199\u5b9a\u5236\u7684\u6b63\u5219\u8868\u8fbe\u5f0f\uff0c\u5206\u4e3a\u4e09\u90e8\u5206:[\u5408\u5e76\u540e\u7684\u5e93\u8868\u540d][\u5206\u9694\u7b26][\u5e93\u8868\u6b63\u5219\u8868\u8fbe\u5f0f]\uff0c\u5206\u9694\u7b26\u9ed8\u8ba4\u4e3a[:]\uff0c\u591a\u4e2a\u89c4\u5219\u4e4b\u95f4\u4ee5[;]\u5206\u9694\uff0c\u6ce8\u610f\u5408\u5e76\u540e\u7684\u5e93\u8868\u540d\u5c3d\u91cf\u4e0d\u542b\u6709\u7279\u6b8a\u5b57\u7b26\uff0c\u4e14\u662fschema.table\u7684\u683c\u5f0f\u3002\u4e0b\u9762\u8fdb\u884c\u4e3e\u4f8b\u8bf4\u660e\uff0c\u5047\u5982\u73b0\u5728\u6709\u4e24\u4e2a\u5206\u5e93sync_db_01\u548csync_db_02\uff0c\u5206\u8868\u60c5\u51b5\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-shell\">sync_db_01\n|- table_a_01\n|- table_b_01\nsync_db_02\n|- table_a_02\n|- table_b_02<\/code><\/pre>\n<p>\u8981\u5c06sync_db_01.table_a_01\u548csync_db_02.table_a_02\u5408\u5e76\u6210sync_db.table_a\uff0c\u5c06sync_db_01.table_b_02\u548csync_db_02.table_b_02\u5408\u5e76\u4e3async_db.table_b\uff0c\u90a3\u4e48\u5408\u5e76\u89c4\u5219\u4e3a\uff1a<\/p>\n<pre><code class=\"language-shell\">sync_db.table_a:sync_db_[0-9]*.table_a_[0-9]*;sync_db.table_b:sync_db_[0-9]*.table_b_[0-9]*\n<\/code><\/pre>\n<p>2\u3001\u7b2c7\u548c\u7b2c8\u6761\u914d\u7f6e\u9879\u5171\u540c\u51b3\u5b9a\u4e86\u89e3\u6790\u6587\u4ef6\u7684\u751f\u6210\u9891\u7387\uff0c\u4e24\u4e2a\u6761\u4ef6\u6ee1\u8db3\u5176\u4e2d\u4e00\u4e2a\u5c31\u4f1a\u751f\u6210\u4e00\u4e2a\u65b0\u7684\u89e3\u6790\u6587\u4ef6\uff0c\u73b0\u573a\u5e94\u8be5\u6309\u7167\u540c\u6b65\u65e5\u5fd7\u7684\u751f\u6210\u901f\u5ea6\u81ea\u884c\u628a\u63e1\uff0c\u4fdd\u6301\u9ed8\u8ba4\u4e5f\u53ef\u4ee5<\/p>\n<h4>4. \u542f\u52a8\u6b65\u9aa4<\/h4>\n<h5>4.1 \u542f\u52a8canal server<\/h5>\n<p>\u5728\u914d\u7f6e\u5b8ccanal server\u548cinstance\u4e4b\u540e\u5c31\u53ef\u4ee5\u542f\u52a8canal server\u4e86\uff0c\u5728canal server\u7684bin\u76ee\u5f55\u4e0b\u9762\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4<\/p>\n<pre><code class=\"language-shell\">bash startup.sh<\/code><\/pre>\n<p>\u53ef\u4ee5\u5728logs\u76ee\u5f55\u4e2d\u67e5\u770b\u65e5\u5fd7\u6765\u786e\u8ba4canal server\u662f\u5426\u542f\u52a8\u6210\u529f\u3002 \u8981\u505c\u6b62canal server\u5219\u8fd0\u884c\u547d\u4ee4bash stop.sh<\/p>\n<h5>4.2 \u542f\u52a8sync client<\/h5>\n<p>\u5728\u542f\u52a8\u5b8ccanal server\u4e4b\u540e\uff0c\u5c31\u53ef\u4ee5\u542f\u52a8sync client\u4e86\uff0c\u5728sync client\u7684bin\u76ee\u5f55\u4e0b\u9762\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4<\/p>\n<pre><code class=\"language-shell\">nohup bash start-client.sh -f ..\/conf\/client-conf\/<client conf> > client.log 2>&1 &<\/code><\/pre>\n<p>\u53ef\u4ee5\u5728logs\u76ee\u5f55\u67e5\u770b\u65e5\u5fd7\u6765\u786e\u8ba4sync client\u662f\u5426\u542f\u52a8\u6210\u529f client\u4ee3\u7801\u4e2d\u6ce8\u518c\u65f6jvm hook\u5b9e\u73b0\u4f18\u96c5\u505c\u673a\uff0c \u76f4\u63a5\u4f7f\u7528kill jvmid\u5373\u53ef\uff0c \u4e0d\u5efa\u8bae\u4f7f\u7528kill -9 jvmid<\/p>\n<h4>5. \u914d\u7f6etdt\u6570\u636e\u6d41\u8bfb\u53d6hdfs\u6587\u4ef6<\/h4>\n<p>\u53c2\u8003\u5982\u4e0b\u6570\u636e\u6d41\u8fdb\u884c\u914d\u7f6e\uff0cJSON_SYNC_READER\u9009\u62e9\u5bf9\u5e94\u7684hdfs\u76ee\u5f55\uff0cORCTRANSACTIONSYNC_GENERIC_WRITER\u9009\u62e9\u5bf9\u5e94\u7684\u8868<br \/>\n<img decoding=\"async\" src=\"\/wp-content\/uploads\/2020\/05\/image-1589164456199.png\" alt=\"file\" \/><\/p>\n<h4>6. workflow\u5b9a\u65f6\u8c03\u5ea6\u5b9e\u73b0\u51c6\u5b9e\u65f6\u5bfc\u5165<\/h4>\n<p>\u5bf9\u53d1\u5e03\u7684TDT\u5de5\u4f5c\u6d41\u8fdb\u884c\u5b9a\u65f6\u8c03\u5ea6\uff0c\u53ef\u81ea\u5b9a\u4e49\u540c\u6b65\u9891\u7387\u5b9e\u73b0\u51c6\u5b9e\u65f6<\/p>\n<h4>7. \u9644\u5f55\uff1a<\/h4>\n<p><strong>sync hdfs wirter\u4e2d\u7684\u76ee\u6807\u8def\u5f84\u586b\u5199\u89c4\u5219\u793a\u4f8b\uff1a<\/strong><br \/>\n<code>.*..*:\/tmp\/test<\/code><br \/>\n\u89e3\u91ca\uff1a\u4ee5\u5192\u53f7\u5206\u9694\uff0c\u5206\u4e3a\u4e24\u90e8\u5206\uff0c\u524d\u9762 .<em>..<\/em>\u662f\u5e93\u8868\u5339\u914d\u7684\u6b63\u5219\u8868\u8fbe\u5f0f\uff0c\u8868\u793a\u5339\u914d\u6240\u6709\u5e93\u6240\u6709\u8868\uff0c \u540e\u9762\/tmp\/test\u662fhdfs\u76ee\u5f55\uff0c\u6240\u4ee5\u8fd9\u4e2a\u7684\u542b\u4e49\u662f\u5c06\u6240\u6709\u5e93\u6240\u6709\u8868\u7684\u540c\u6b65\u65e5\u5fd7\u90fd\u5199\u5230\/tmp\/hdfs\u76ee\u5f55\u4e0b<\/p>\n<p><strong>\u591a\u4e2a\u8bbe\u7f6e\u53ef\u4ee5\u7528\u9017\u53f7\u5206\u9694\uff0c\u793a\u4f8b\uff1a<\/strong><br \/>\n<code>canal_db..*:\/tmp\/canal,tdt_db..*:\/tmp\/tdt<\/code><br \/>\n\u89e3\u91ca\uff1a\u5c06canal_db\u8fd9\u4e2a\u5e93\u4e0b\u7684\u6240\u6709\u8868\u7684\u540c\u6b65\u65e5\u5fd7\u5199\u5230\/tmp\/canal\u76ee\u5f55\u4e0b\uff0c\u5c06tdt_db\u8fd9\u4e2a\u5e93\u4e0b\u7684\u6240\u6709\u8868\u5199\u5165\u5230\/tmp\/tdt\u8fd9\u4e2a\u76ee\u5f55\u4e0b<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u6982\u8981\u63cf\u8ff0 \u8be5\u5de5\u5177\u7528\u4e8e\u4ecemysql\u4e2d\u5b9e\u65f6\u83b7\u53d6\u589e\u91cf\u7684\u6570\u636e\uff08Insert\uff0cUpdate\uff0cDelete\uff09\uff0c\u7136\u540e\u5c06\u589e\u91cf\u6570 ..<\/p>\n<div class=\"clear-fix\"><\/div>\n<p><a href=\"https:\/\/kbwp.transwarp.cn\/?p=3023\" title=\"read more...\">Read more<\/a><\/p>\n","protected":false},"author":12,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-3023","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"acf":[],"_links":{"self":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/3023","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/users\/12"}],"replies":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=3023"}],"version-history":[{"count":2,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/3023\/revisions"}],"predecessor-version":[{"id":3709,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/3023\/revisions\/3709"}],"wp:attachment":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=3023"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=3023"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=3023"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}