{"id":2017,"date":"2020-10-02T17:07:32","date_gmt":"2020-10-02T09:07:32","guid":{"rendered":"https:\/\/nj.transwarp.cn:8180\/?p=2017"},"modified":"2020-10-02T17:07:32","modified_gmt":"2020-10-02T09:07:32","slug":"morphling%e5%92%8cmapred%e7%9a%84%e5%8c%ba%e5%88%ab","status":"publish","type":"post","link":"https:\/\/kbwp.transwarp.cn\/?p=2017","title":{"rendered":"morphling\u548cmapred\u7684\u533a\u522b"},"content":{"rendered":"

\u6982\u8981\u63cf\u8ff0<\/h3>\n
\n

\u672c\u6587\u65e8\u5728\u8bf4\u660eslipstream\u4e8b\u4ef6\u9a71\u52a8\u548c\u5fae\u6279\u6a21\u5f0f\u7684\u5f02\u540c
\nslipstream\u5b98\u7f51\u4ecb\u7ecd\u94fe\u63a5\uff1aslipstream\u5b98\u7f51\u4ecb\u7ecd<\/a><\/p>\n

\u8be6\u7ec6\u8bf4\u660e<\/h3>\n

Slipstream\u63a8\u8350\u4e8b\u4ef6\u9a71\u52a8\u6a21\u578b, \u4e8b\u4ef6\u9a71\u52a8\u6a21\u578b\u662fTDH4.8\u4e4b\u540e\u65b0\u589e\u7684\u6d41\u5904\u7406\u6a21\u578b\uff0c5.1\u53ca\u4e4b\u540e\u9ed8\u8ba4\u5f00\u542f\u4e8b\u4ef6\u9a71\u52a8\u6a21\u578b\u3002ngmr.engine.mode\u53c2\u6570\u51b3\u5b9a\u4e86Slipstream\u9009\u62e9\u4f55\u79cd\u5f15\u64ce\uff0cngmr.engine.mode=mapred\u4e3a\u5fae\u6279\u6a21\u578b, ngmr.engine.mode=morphling\u4e3a\u4e8b\u4ef6\u9a71\u52a8\u6a21\u578b\u3002<\/p>\n

\u4e5f\u53ef\u4ee5\u901a\u8fc7\u521b\u5efajob\u7684\u65f6\u5019\u6307\u5b9a\u9a71\u52a8\u6a21\u578b\uff1a<\/p>\n

CREATE STREAMJOB  as (\"\")\nJOBPROPERTIES(\"streamsql.use.eventmode\"=\"true\"[,\"key\"=\"value\"...]);<\/code><\/pre>\n
\u4e8b\u4ef6\u9a71\u52a8\u6a21\u578b\u66f4\u63a5\u8fd1\u539f\u751f\u7684\u6d41\u5904\u7406,\u4e00\u6709\u6570\u636e\u8fdb\u6765\u5c31\u5904\u7406. \u5c31\u597d\u6bd4\u6d41\u6c34\u7ebf\u4e0a\u7684\u96f6\u4ef6, \u6765\u4e00\u4e2a\u5904\u7406\u4e00\u4e2a
\n\"file\"<\/p>\n
\u5fae\u6279\u6a21\u578b\u987e\u540d\u601d\u4e49\u8fd8\u662f\u57fa\u4e8e"\u6279"\u7684\u601d\u60f3. \u5c06Input Stream\u6309\u65f6\u95f4\u6bb5\u5206\u5272\u6210\u4e00\u4e2a\u4e2a\u5c0f\u5757\u5904\u7406. \u5c31\u50cf\u662f\u7ed9\u6d41\u6c34\u7ebf\u589e\u52a0\u4e00\u4e2a\u7f13\u51b2, \u6bcf\u9694\u4e00\u6bb5\u65f6\u95f4\u53bb\u53d6\u4e00\u6279\u96f6\u4ef6\u52a0\u5de5
\n\"file\"<\/p>\n
\u76f8\u6bd4\u8f83\u5fae\u6279\u6a21\u578b, \u4e8b\u4ef6\u9a71\u52a8\u6a21\u578b\u5177\u6709\u4ee5\u4e0b\u7279\u70b9:<\/p>\n
    \n
  1. \u5ef6\u8fdf\u66f4\u4f4e: \u5fae\u6279\u662f\u79d2\u7ea7\u7684\u5ef6\u8fdf,\u800c\u4e8b\u4ef6\u9a71\u52a8\u6a21\u578b\u7684\u5ef6\u8fdf\u662f\u6beb\u79d2\u7ea7\u7684.<\/li>\n
  2. \u529f\u80fd\u66f4\u5168:
    \na. \u652f\u6301\u4f1a\u8bdd\u7a97\u53e3(Session Window),
    \nb. \u590d\u6742\u4e8b\u4ef6\u7684\u5904\u7406(CEP)
    \nc. \u66f4\u52a0\u5b8c\u5584\u7684\u9ad8\u53ef\u7528\u6027(HA)\uff089.3\u8282\uff09
    \nd. StreamJob\u7684Auto-failover<\/li>\n<\/ol>\n
    Stream<\/strong>
    \n\u4e8b\u4ef6\u9a71\u52a8\u6a21\u578b\u4e2d,\u5f53\u6570\u636e\u6e90\u5904\u6709\u6570\u636e\u8fdb\u6765\u5c31\u7acb\u5373\u5904\u7406,\u6765\u4e00\u6761\u5904\u7406\u4e00\u6761.
    \n\"file\"<\/p>\n
    \u5fae\u6279\u6a21\u578b\u4e0bInput Stream\u88ab\u5206\u5272\u6210\u4e00\u7cfb\u5217 \u5c0f\u6570\u636e\u5757\uff08Batch\uff09 \u6765\u5904\u7406,Batch\u5728\u83b7\u53d6\u6570\u636e\u65f6\u751f\u6210: \u5728\u4e00\u6bb5\u5355\u4f4d\u65f6\u95f4\u5185\u83b7\u53d6\u7684\u6570\u636e\u90fd\u653e\u8fdb\u4e00\u4e2abatch\u4e2d\uff0c\u8fd9\u6bb5\u5355\u4f4d\u65f6\u95f4\u79f0\u4e3a Batch Interval\uff0c\u5b83\u7684\u957f\u5ea6\uff08Batch Duration\uff09\u53ef\u4ee5\u8bbe\u7f6e,\u5982\u4e0b\u56fe\u6240\u793a\u3002<\/p>\n
    \"file\"<\/p>\n
    Derived Stream<\/strong>
    \n\u4e8b\u4ef6\u9a71\u52a8\u6a21\u578b\u4e0b, \u6bcf\u5f97\u5230\u4e00\u6761\u6570\u636e,\u5c31\u5bf9\u5176\u8fdb\u884c\u53d8\u5f62,\u5f97\u5230Derived Stream:
    \n\"file\"<\/p>\n
    \u800c\u5fae\u6279\u6a21\u578b\u4e0b,\u56e0\u4e3aInput Stream\u662f\u4e00\u7cfb\u5217batch,\u90a3\u4e48Stream\u7684\u53d8\u5f62\u5b9e\u9645\u4e0a\u662f\u5bf9batch\u8ba1\u7b97\u5f97\u5230\u65b0batch\u7684
    \n\u8fc7\u7a0b\uff1a
    \n\"file\"<\/p>\n
    Window Stream<\/strong>
    \n\u4e8b\u4ef6\u9a71\u52a8\u6a21\u578b\u4e0b\u652f\u6301\u79d2\u7ea7\u7684window stream
    \n\"file\"<\/p>\n
    \u4e0a\u56fe\u4e2d\u7a97\u53e3\u957f\u5ea6(Length)\u4e3a4s,\u6ed1\u52a8\u95f4\u9694(Slide)\u4e3a3s. \u5982\u6b64\u56fe\u4e2d\u7684\u7b2c\u4e00\u4e2a\u7a97\u53e3\u4e3a[t,t+4s),\u7b2c\u4e8c\u4e2a\u7a97\u53e3\u4e3a[t+3s,t+7s),\u4ee5\u6b64\u7c7b\u63a8. \u7a97\u53e3\u7684\u957f\u5ea6\u548c\u6ed1\u52a8\u7684\u95f4\u9694\u53ea\u8981\u662f1s\u7684\u6574\u6570\u500d\u5373\u53ef. \u9700\u8981\u6ce8\u610f\u7684\u662f, \u5f53t=0s,\u5373Stream\u521b\u5efa\u7684\u521d\u59cb\u65f6\u523b, \u7b2c\u4e00\u4e2a\u7a97\u53e3\u4e3a[0,3s),\u6bd4\u7a97\u53e3\u957f\u5ea6\u4f1a\u5c111\u79d2,\u8fd9\u662f\u56e0\u4e3aSlide\u662f\u6267\u884c\u7a97\u53e3\u64cd\u4f5c\u7684\u95f4\u9694, Slide\u65f6\u95f4\u5230\u4e86\u5c31\u4f1a\u6267\u884c\u7a97\u53e3\u8ba1\u7b97.<\/p>\n
    \u5fae\u6279\u6a21\u578b\u4e0b,\u7a97\u53e3\u957f\u5ea6(Length)\u548c\u6ed1\u52a8\u95f4\u9694(Slide)\u7684\u957f\u5ea6\u90fd\u5fc5\u987b\u662fbatch duration\u7684\u6574\u6570\u500d\u3002
    \n\"file\"
    \n\u4e0a\u56fe\u7684\u7a97\u53e3\u53d8\u5f62\u4e2d\uff0clength\u662fbatch duration\u76842\u500d\uff08SET streamsql.hdfs.batchflush.interval.ms=2;),slide\u662fbatch duration\u76841\u500d\u3002\u5982\u6b64\u56fe\u4e2d\u7684\u7b2c\u4e00\u4e2a\u7a97\u53e3\u4e3a[t,t+4s),\u7b2c\u4e8c\u4e2a\u7a97\u53e3\u4e3a[t+2s,t+6s),\u4ee5\u6b64\u7c7b\u63a8. \u4e0e\u4e8b\u4ef6\u9a71\u52a8\u6a21\u578b\u76f8\u540c\u7684\u662f,\u5f53t=0\u65f6,\u7b2c\u4e00\u4e2a\u7a97\u53e3\u7684\u957f\u5ea6\u7b49\u4e8eSlide\uff082s).<\/p>\n
    \u4eceTDH 5.1\u5f00\u59cb, \u5728\u4e8b\u4ef6\u9a71\u52a8\u6a21\u5f0f\u4e0b\u652f\u6301\u7528\u6237\u81ea\u5b9a\u4e49\u8f93\u5165\u6e90. \u6dfb\u52a0\u81ea\u5b9a\u4e49\u8f93\u5165\u6e90\u6b65\u9aa4\u5982\u4e0b:<\/p>\n
      \n
    1. \u7ee7\u627f io.transwarp.slipstream.api.source.CustomDataSource \u62bd\u8c61\u7c7b\n
      def init(id: Int, params: Properties): Unit\n\/**\n* non-blocking\n* return null if get nothing\n*\/\ndef poll(): Array[Byte]\ndef close(): Unit<\/code><\/pre>\n<\/li>\n<\/ol>\n
      \u793a\u4f8b\u4ee3\u7801\u53c2\u8003slipstream\u624b\u518c6.2.6.1.4\u7ae0\u8282<\/p>\n
      Stream DML \u6ce8\u610f\u4e8b\u9879<\/strong><\/p>\n
      \u4e8b\u4ef6\u9a71\u52a8\u6a21\u5f0f\u4e0b, \u53ea\u5728 window stream \u4e0b\u652f\u6301\u805a\u5408\u64cd\u4f5c<\/p>\n
      \u5fae\u6279\u6a21\u5f0f: \u652f\u6301\u6d41\u4e0e\u6d41\u7684join,\u652f\u6301\u6d41\u4e0e\u8868\u7684join.
      \n\u4e8b\u4ef6\u9a71\u52a8\u6a21\u5f0f\u4e0b:
      \n\u652f\u6301mapjoin\u548cglobal lookup join.
      \n\u6d41\u4e0e\u6d41\u7684Join\u8981\u6c42join\u7684\u4e24\u4e2a\u6d41\u5fc5\u987b\u662fwindow stream,\u4e14window\u7684slide(\u6ed1\u52a8\u95f4\u9694)\u5fc5\u987b\u4e00\u81f4<\/p>\n
      \u4eceTDH5.1\u5f00\u59cb,\u5728\u4e8b\u4ef6\u9a71\u52a8\u6a21\u5f0f\u4e0b\u652f\u6301\u81ea\u5b9a\u4e49\u8f93\u51fa\uff0cdemo\u53c2\u8003\u624b\u518c6.3.8.3\u7ae0\u8282<\/p>\n
      \u6279\u91cfFlush\u5230HDFS<\/strong><\/p>\n
      \u652f\u6301\u6279\u91cf\u5c06\u6570\u636eflush\u5230HDFS\u4e0a\u7684TEXT\u8868\u548cORC\u8868\uff0c\u6279\u91cfflush\u53ef\u4ee5\u907f\u514d\u4ea7\u751f\u8fc7\u591a\u5c0f\u6587\u4ef6\u3002\u6ce8\u610f\uff0c\u76ee\u6807\u8868\u4e0d\u80fd\u4e3a\u4e8b\u52a1\u8868\u3001\u5206\u533a\u8868\uff085.1.2\u4ee5\u524d\u7248\u672c\uff09\u6216\u8005\u5206\u6876\u8868\u3002\u8fdb\u884c\u6279\u91cfflush\u9700\u8981\u8bbe\u7f6e\u4e0b\u9762\u51e0\u4e2a job\u7ea7\u522b \u7684\u53c2\u6570\uff1a
      \n\u5fae\u6279\u6a21\u5f0f
      \nSET streamsql.enable.hdfs.batchflush = true
      \n–\u6253\u5f00\u6279\u91cfflush\u5f00\u5173
      \nSET streamsql.hdfs.batchflush.size = 
      \n–\u8bbe\u7f6e\u4e00\u6b21flush\u7684\u6d88\u606f\u4e2a\u6570\uff0c\u6d88\u606f\u91cf\u8fbe\u5230\u8be5\u53c2\u6570\u65f6flush\u4e00\u6b21
      \nSET streamsql.hdfs.batchflush.interval.ms = 
      \n–\u8bbe\u7f6e\u6bcf\u8fc7\u591a\u957f\u65f6\u95f4\uff08\u5355\u4f4d\u4e3a\u6beb\u79d2\uff09flush\u4e00\u6b21
      \n\u53ea\u9700\u6ee1\u8db3 batchflush.size \u548c batchflush.interval.ms \u5176\u4e2d\u7684\u4e00\u4e2a\u6761\u4ef6\u5373\u4f1a\u89e6\u53d1\u4e00\u6b21flush\u3002
      \n\u4e8b\u4ef6\u9a71\u52a8\u6a21\u5f0f
      \nSET morphling.result.auto.flush=true
      \n–\u6765\u4e00\u6761\u6570\u636e\u5237\u4e00\u6761\u5230\u8f93\u51fa\u8868, \u53ef\u5728\u6d4b\u8bd5\u65f6\u4f7f\u7528,\u4e0d\u63a8\u8350\u5728\u751f\u4ea7\u4e2d\u4f7f\u7528.<\/p>\n
      \u6279\u91cfFlush\u9ed8\u8ba4\u5f00\u542f
      \n\u2022 TDH5.1\u524d
      \nSET morphling.hdfs.flush.size = 
      \n–\u8bbe\u7f6e\u4e00\u6b21flush\u7684\u6d88\u606f\u4e2a\u6570\uff0c\u6d88\u606f\u91cf\u8fbe\u5230\u8be5\u53c2\u6570\u65f6flush\u4e00\u6b21, \u9ed8\u8ba4100000
      \nSET morphling.hdfs.flush.interval.ms = 
      \n–\u8bbe\u7f6e\u6bcf\u8fc7\u591a\u957f\u65f6\u95f4\uff08\u5355\u4f4d\u4e3a\u6beb\u79d2\uff09flush\u4e00\u6b21,\u9ed8\u8ba460000
      \n\u2022 TDH5.1+
      \nSET morphling.record.flush.size = 
      \nSET morphling.record.flush.interval.ms = 
      \n\u540c\u5fae\u6279\u6a21\u5f0f, flush size \u548c flush interval \u6ee1\u8db3\u5176\u4e2d\u4e4b\u4e00\u5c31\u4f1a\u89e6\u53d1\u4e00\u6b21Flush.<\/p>\n
      \u4e8b\u4ef6\u9a71\u52a8\u6a21\u5f0f\u4e0b\u652f\u6301\u5206\u533a\u8868\u51fd\u6570\uff08Partition Table Function\uff0cPTF)\u3002 PTF\u5c06\u8f93\u5165\u6d41\u89c6\u4e3a\u4e00\u5f20\u5206\u533a\u8868\uff0c\u6839\u636e\u5b9a\u4e49\u7684\u5206\u533a\u5b57\u6bb5\u5206\u6210\u4e0d\u540c\u7684partition\uff0c\u901a\u8fc7\u65f6\u95f4\u5212\u5206\u7a97\u53e3\uff0c\u7136\u540e\u5728\u8fd9\u4e2a\u7a97\u53e3\uff08\u6709\u9650\u6570\u636e\u96c6\uff09\u4e0a\u505a\u5206\u6790\u3002 \u4e0e\u5728Stream Window\u4e0a\u505a\u805a\u5408\u8ba1\u7b97\u4e0d\u540c\u7684\u662f\uff0cPTF\u529f\u80fd\u66f4\u591a\uff0c\u8ba1\u7b97\u66f4\u7075\u6d3b\uff0c\u8be6\u60c5\u53c2\u8003\u624b\u518c6.9.1\u7ae0\u8282<\/p>\n
      \u5fae\u6279\u6a21\u5f0f\u4e0b\u53c2\u6570<\/strong>
      \n\"file\"
      \n\"file\"
      \n\"file\"<\/p>\n
      \u4e8b\u4ef6\u9a71\u52a8\u6a21\u5f0f\u4e0b\u53c2\u6570<\/strong><\/p>\n
      \"file\"
      \n\"file\"
      \n\"file\"<\/p>\n","protected":false},"excerpt":{"rendered":"
      \u6982\u8981\u63cf\u8ff0 \u672c\u6587\u65e8\u5728\u8bf4\u660eslipstream\u4e8b\u4ef6\u9a71\u52a8\u548c\u5fae\u6279\u6a21\u5f0f\u7684\u5f02\u540c slipstream\u5b98\u7f51\u4ecb\u7ecd\u94fe\u63a5\uff1aslip ..<\/p>\n
      <\/div>\n
      Read more<\/a><\/p>\n","protected":false},"author":9,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-2017","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"acf":[],"_links":{"self":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/2017","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/users\/9"}],"replies":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=2017"}],"version-history":[{"count":1,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/2017\/revisions"}],"predecessor-version":[{"id":2508,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/2017\/revisions\/2508"}],"wp:attachment":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=2017"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=2017"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=2017"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}