{"id":3818,"date":"2021-03-03T19:32:02","date_gmt":"2021-03-03T11:32:02","guid":{"rendered":"https:\/\/nj.transwarp.cn:8180\/?p=3818"},"modified":"2025-05-09T14:37:03","modified_gmt":"2025-05-09T06:37:03","slug":"%e5%a6%82%e4%bd%95%e4%bb%8e%e6%b5%81%ef%bc%88slipstream%ef%bc%89%e4%b8%ad%e5%8a%a8%e6%80%81%e6%8f%92%e5%85%a5orc%e5%8d%95%e5%80%bc%e5%88%86%e5%8c%ba%e4%ba%8b%e5%8a%a1%e8%a1%a8","status":"publish","type":"post","link":"https:\/\/kbwp.transwarp.cn\/?p=3818","title":{"rendered":"\u5982\u4f55\u4ece\u6d41\uff08slipstream\uff09\u4e2d\u52a8\u6001\u63d2\u5165orc\u5355\u503c\u5206\u533a\u8868"},"content":{"rendered":"<h3>\u6982\u8981\u8bf4\u660e<\/h3>\n<hr \/>\n<p>\u672c\u6848\u4f8b\u63cf\u8ff0\u5982\u4f55\u5c06\u6d41\u4e0a\u6570\u636e\u843d\u5165\u5230\u5206\u533a\u5206\u6876\u8868\u4e2d\u3002<br \/>\n\u6ce8\u610f\u4e8b\u9879\uff1a<\/p>\n<ol>\n<li>\u5206\u533a\u8868\u76ee\u524d\u53ea\u652f\u6301\u5bf9TEXT\u8868\u3001ORC\u8868\u3001CSV\u8868\u548cHolodesk\u8868\u5206\u533a\uff1b<\/li>\n<li>holodesk\u8868\u7684\u5206\u533a\u9700\u8981\u63d0\u524d\u521b\u5efa\uff0c\u6682\u4e0d\u652f\u6301holodesk\u8868\u5199\u5165\u65f6\u81ea\u52a8\u521b\u5efa\u5206\u533a\uff1b<br \/>\n3.\u5b9e\u9a8c\u73af\u5883\u5df2\u5f00\u542f\u5b89\u5168\u3002<\/li>\n<\/ol>\n<h3>\u8be6\u7ec6\u8bf4\u660e<\/h3>\n<hr \/>\n<h4>1\u3001\u901a\u8fc7kafka\u83b7\u53d6\u5b9e\u65f6\u7684\u6d41\u6570\u636e<\/h4>\n<p>\u5f00\u542f\u5b89\u5168\u7684\u73af\u5883\uff0cslipstream\u5982\u4f55\u63a5\u6536kafka\u6570\u636e\uff0c\u53c2\u8003\u6587\u6863\uff1a<a href=\"\/?p=720\" title=\"kafka\u5f00\u542fGuardian\u5b89\u5168\u540eSlipstream\u63a5\u6536Kafka\u6d88\u606f\u64cd\u4f5c\u6b65\u9aa4\">kafka\u5f00\u542fGuardian\u5b89\u5168\u540eSlipstream\u63a5\u6536Kafka\u6d88\u606f\u64cd\u4f5c\u6b65\u9aa4<\/a><\/p>\n<h4>2\u3001\u8fde\u63a5slipstream\uff0c\u901a\u8fc7SQL\u6d88\u8d39\u6570\u636e<\/h4>\n<p>\u672c\u5b9e\u9a8c\u7528waterdrop\u5de5\u5177\u8fde\u63a5slipstream\uff0c\u6ce8\u610f\u7aef\u53e3\u53f7\u662f <strong>10010<\/strong>\u3002<br \/>\n<img decoding=\"async\" src=\"\/wp-content\/uploads\/2020\/08\/image-1597653275693.png\" alt=\"file\" \/><\/p>\n<h4>3\u3001\u521b\u5efa\u8f93\u5165\u6d41<\/h4>\n<pre><code class=\"language-sql\">CREATE STREAM test_bucket(id INT,sex STRING, letter STRING)\n  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','\n  TBLPROPERTIES(\"topic\"=\"luoyang\",\n  \"kafka.zookeeper\"=\"tdh524-03:2181,tdh524-02:2181,tdh524-01:2181\",\n  \"kafka.broker.list\"=\"tdh524-03:9092,tdh524-02:9092\uff0ctdh524-01:9092\",\n  \"transwarp.consumer.security.protocol\"=\"SASL_PLAINTEXT\",\n  \"transwarp.consumer.sasl.kerberos.service.name\"=\"kafka\",\n  \"transwarp.consumer.sasl.jaas.config\"=\"com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true  keyTab=\\\"\/etc\/slipstream1\/conf\/kafka.keytab\\\" principal=\\\"kafka@TDH\\\"\"\n);<\/code><\/pre>\n<h4>4\u3001\u521b\u5efaorc\u5355\u503c\u5206\u533a\u8868<\/h4>\n<pre><code class=\"language-sql\">CREATE TABLE orc_bucket_table(id INT,sex STRING,letter string) \nPARTITIONED BY (p_sex string) \nCLUSTERED BY (id) INTO 3 BUCKETS STORED AS ORC;<\/code><\/pre>\n<h4>5\u3001\u521b\u5efaStreamJob<\/h4>\n<pre><code class=\"language-sql\">CREATE STREAMJOB o1 AS (\n    \"insert into orc_bucket_table PARTITION(p_sex) \n    select id,sex,letter,sex from test_bucket\")\n    jobPROPERTIES(\"hive.exec.dynamic.partition\"=\"TRUE\",\"hive.enforce.bucketing\"=\"true\",\"morphling.result.auto.flush\"=\"true\");<\/code><\/pre>\n<p>\u8bf4\u660e\uff1a<\/p>\n<ul>\n<li>hive.exec.dynamic.partition \uff1a \u5f00\u542f\u52a8\u6001\u5206\u533a\uff1b<\/li>\n<li>hive.enforce.bucketing \uff1a \u6253\u5f00enforce bucketing\u5f00\u5173\uff0c\u5f3a\u5236Slipstream\u5206\u6876\u3002\u8fd9\u4e2a\u5f00\u5173\u9ed8\u8ba4\u60c5\u51b5\u4e0b\u662f\u6253\u5f00\u7684\uff0c\u4f46\u662f\u53ef\u80fd\u5728Slipstream\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u88ab\u5173\u6389\uff0c\u6240\u4ee5\u5411\u5206\u6876\u8868\u5bfc\u6570\u636e\u524d\uff0c\u8bf7\u786e\u4fdd\u5f00\u5173\u662f\u6253\u5f00\u7684\uff1b<\/li>\n<li>morphling.result.auto.flush \uff1a \u662f\u5426\u81ea\u52a8Sink\u3002\u5f00\u542f\u8be5\u5f00\u5173\u540e\uff0c\u53ea\u8981\u6709\u6570\u636e\u4ea7\u751f\u5c31\u7acb\u523bflush\u5230\u76ee\u6807\u8868\uff1b<\/li>\n<\/ul>\n<h4>6\u3001\u9a8c\u8bc1<\/h4>\n<ul>\n<li>\n<p>\u767b\u5f554044\u67e5\u770bstreamjob\u6b63\u5e38\u8fd0\u884c<br \/>\n<img decoding=\"async\" src=\"\/wp-content\/uploads\/2020\/08\/image-1597653787228.png\" alt=\"file\" \/><\/p>\n<\/li>\n<li>\n<p>kafka\u751f\u4ea7\u6570\u636e<br \/>\n<img decoding=\"async\" src=\"\/wp-content\/uploads\/2020\/08\/image-1597653925241.png\" alt=\"file\" \/><\/p>\n<\/li>\n<li>\n<p>\u8868\u4e2d\u67e5\u770b\u6570\u636e\u662f\u5426\u5df2\u843d\u8868<br \/>\n<img decoding=\"async\" src=\"\/wp-content\/uploads\/2020\/08\/image-1597654032153.png\" alt=\"file\" \/><\/p>\n<\/li>\n<li>\n<p>hdfs\u4e0a\u9a8c\u8bc1\u5206\u533a\u5206\u6876\u6587\u4ef6\u5b58\u50a8\u6b63\u5e38<br \/>\n<img decoding=\"async\" src=\"\/wp-content\/uploads\/2020\/08\/image-1597654397105.png\" alt=\"file\" \/><\/p>\n<\/li>\n<\/ul>\n","protected":false},"excerpt":{"rendered":"<p>\u6982\u8981\u8bf4\u660e \u672c\u6848\u4f8b\u63cf\u8ff0\u5982\u4f55\u5c06\u6d41\u4e0a\u6570\u636e\u843d\u5165\u5230\u5206\u533a\u5206\u6876\u8868\u4e2d\u3002 \u6ce8\u610f\u4e8b\u9879\uff1a \u5206\u533a\u8868\u76ee\u524d\u53ea\u652f\u6301\u5bf9TEXT\u8868\u3001ORC\u8868\u3001C ..<\/p>\n<div class=\"clear-fix\"><\/div>\n<p><a href=\"https:\/\/kbwp.transwarp.cn\/?p=3818\" title=\"read more...\">Read more<\/a><\/p>\n","protected":false},"author":14,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-3818","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"acf":[],"_links":{"self":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/3818","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/users\/14"}],"replies":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=3818"}],"version-history":[{"count":5,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/3818\/revisions"}],"predecessor-version":[{"id":16521,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/3818\/revisions\/16521"}],"wp:attachment":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=3818"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=3818"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=3818"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}