{"id":13001,"date":"2024-08-12T11:35:42","date_gmt":"2024-08-12T03:35:42","guid":{"rendered":"https:\/\/nj.transwarp.cn:8180\/?p=13001"},"modified":"2024-08-12T11:35:41","modified_gmt":"2024-08-12T03:35:41","slug":"%e5%bc%80%e6%ba%90-spark-%e5%af%b9%e6%8e%a5-inceptor-%e6%95%b0%e6%8d%ae%e5%ba%93","status":"publish","type":"post","link":"https:\/\/kbwp.transwarp.cn\/?p=13001","title":{"rendered":"\u5f00\u6e90 Spark \u5bf9\u63a5 Inceptor \u6570\u636e\u5e93"},"content":{"rendered":"<h3>\u6982\u8981\u63cf\u8ff0<\/h3>\n<p>\u901a\u8fc7\u5f00\u6e90 Spark sql \u76f4\u63a5\u8fde\u63a5 inceptor \u53ea\u80fd\u8bfb\u53d6\u6709\u9650\u7684\u51e0\u79cd\u8868\u7c7b\u578b\uff1a\u666e\u901atext\u8868\u3001\u975e\u5206\u533a\u3001\u975e\u5206\u6876\u666e\u901aorc\u8868\u3001\u666e\u901aorc\u5206\u6876\u8868\u3001orc\u5355\u503c\u5206\u533a\u8868\uff1b<\/p>\n<p>\u5bf9\u4e8eorc\u8303\u56f4\u5206\u533a\u548corc\u4e8b\u52a1\u8868\u5747\u4e0d\u652f\u6301\uff1b<\/p>\n<p>\u672c\u6587\u63d0\u4f9b\u4e86\u4e00\u4e9b\u9002\u914d jar \u5305\uff0c\u5b9e\u73b0\u5bf9 orc\u8303\u56f4\u5206\u533a\u548c orc\u4e8b\u52a1\u8868\u7684\u8bfb\u5199\u64cd\u4f5c\u3002<\/p>\n<h3>\u8be6\u7ec6\u8bf4\u660e<\/h3>\n<ol>\n<li>\u5b89\u88c5 spark \u548c scala<\/li>\n<li>\u6dfb\u52a0 Inceptor \u4f9d\u8d56\u5305<\/li>\n<li>yarn \u53c2\u6570\u914d\u7f6e<\/li>\n<li>spark \u53c2\u6570\u914d\u7f6e<\/li>\n<li>\u542f\u52a8 spark-shell \u6216\u8005 \u63d0\u4ea4 spark \u7a0b\u5e8f<\/li>\n<\/ol>\n<h4>1. \u5b89\u88c5 spark \u548c scala<\/h4>\n<p>spark \u5b98\u7f51\u4e0b\u8f7d\uff1a<a href=\"https:\/\/spark.apache.org\/downloads.html\">https:\/\/spark.apache.org\/downloads.html<\/a> \uff0c \u5386\u53f2\u6240\u6709\u7248\u672c\uff1a<a href=\"https:\/\/archive.apache.org\/dist\/spark\/\">https:\/\/archive.apache.org\/dist\/spark\/<\/a><\/p>\n<p>scala \u5b98\u7f51\u6240\u6709\u7248\u672c\u4e0b\u8f7d\uff1a<a href=\"https:\/\/www.scala-lang.org\/download\/all.html\">https:\/\/www.scala-lang.org\/download\/all.html<\/a><\/p>\n<p><img decoding=\"async\" src=\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2024\/05\/image-1716185822762.png\" alt=\"file\" \/><\/p>\n<h4>2. \u6dfb\u52a0 Inceptor \u4f9d\u8d56\u5305<\/h4>\n<p>\u76f4\u63a5\u4e0b\u8f7d\u94fe\u63a5\uff1a<a href=\"https:\/\/pan.baidu.com\/s\/1MmkpEaCBg5T-o3OYyXYmLQ?pwd=1234\">https:\/\/pan.baidu.com\/s\/1MmkpEaCBg5T-o3OYyXYmLQ?pwd=1234<\/a> \u63d0\u53d6\u7801\uff1a1234<\/p>\n<p>\u5c06 torc \u7684jar \u5305\u590d\u5236\u5230 jars\u4e2d<\/p>\n<p>spark2.1.3 \u9009\u62e9 inceptor-torc-2.1.3.jar<br \/>\nspark3.1.1 \u9009\u62e9 inceptor-torc-3.1.1.jar<\/p>\n<p>cp xxx.jar spark-2.1.3-bin-hadoop2.7\/jars<\/p>\n<h4>3. \u914d\u7f6e yarn<\/h4>\n<p>3.1 \u62f7\u8d1dspark\u8fd0\u884c\u65f6\u4f9d\u8d56\u7684jars\u5230hdfs<br \/>\nhadoop fs -put \/spark-2.4.5-bin-hadoop2.7\/jars \/tmp\/spark-yarn\/<\/p>\n<p>3.2 spark-env.sh \u914d\u7f6e<\/p>\n<pre><code class=\"language-shell\"># \u4ee5\u4e0b\u914d\u7f6e\u8bf7\u6839\u636e\u672c\u5730\u5b9e\u9645\u73af\u5883\u914d\u7f6e \uff08\u5fc5\u8981\uff09\nHADOOP_USER_NAME=hive\nHADOOP_CONF_DIR=\/etc\/hdfs1\/conf<\/code><\/pre>\n<p>3.3 spark-defaults.conf\u57fa\u7840\u914d\u7f6e<\/p>\n<pre><code class=\"language-shell\">\n# \u4ee5\u4e0b\u914d\u7f6e\u8bf7\u6839\u636e\u672c\u5730\u5b9e\u9645\u73af\u5883\u914d\u7f6e\uff08\u5fc5\u8981\uff09\n\n# spark on yarn \u4f9d\u8d56\u7684jars\uff0c4.1 \u8282\u4e0a\u4f20\u7684jar\u5305\u8def\u5f84\nspark.yarn.jars           hdfs:\/\/172.22.22.10:8020\/tmp\/spark-yarn\/jars\/*\n\n# \u6307\u5b9aArgodb inceptor server IP\u5730\u5740\u6216hostname\nspark.argodb.server.addr 172.22.22.10\n\n# \u6307\u5b9aspark\u8fd0\u884c\u5728yarn\u73af\u5883\u7684java\u7248\u672c\uff08\u9ed8\u8ba4yarn\u7684jdk\u7248\u672c\u4e3ajdk 1.7\uff09\n# \u5177\u4f53jdk\u8def\u5f84\u53ef\u4ee5\u67e5\u770byarn image\u4e2d\u7684jdk\u8def\u5f84\nspark.executorEnv.JAVA_HOME  \/usr\/java\/jdk1.8.0_25\nspark.yarn.appMasterEnv.JAVA_HOME \/usr\/java\/jdk1.8.0_25<\/code><\/pre>\n<p>3.4 spark-defaults.conf\u5b89\u5168\u76f8\u5173\u914d\u7f6e<br \/>\n\u6839\u636eArgodb\u7684\u5b89\u5168\u8bbe\u7f6e\uff0c\u6dfb\u52a0\u4e0b\u9762\u7684\u53c2\u6570<\/p>\n<pre><code class=\"language-shell\"># Argodb\u7684\u5b89\u5168\u914d\u7f6e\n# spark.argodb.auth.type : none(default) [kerberos or ldap or none]\nspark.argodb.auth.type : ldap\n\n# KERBEROS \u6a21\u5f0f\n# \u4f7f\u7528SparkSQL\u65b9\u5f0f\u6267\u884c\u65f6\u9700\u8981\u914d\u7f6e\n# spark.kerberos.principal <kerberos principal name>\n# spark.kerberos.keytab <keytab>\n\n# principal\u548ckeytab\u672a\u6307\u5b9a\u65f6\uff0c\u5c06\u4f7f\u7528spark on yarn kerberos\u53c2\u6570\u914d\u7f6e \u5373\uff1aspark.kerberos.principal\u548cspark.kerberos.keytab\n# spark.argodb.krb.principal <kerberos principal name>\n# spark.argodb.krb.keytab <keytab>\n\n# \u800cuser\u4e3aspark.kerberos.principal\n# spark.argodb.krb.user <kerberos user name>\n# spark.argodb.krb.conf \/etc\/krb5.conf(default)\n\n# LDAP \u6a21\u5f0f\nspark.argodb.ldap.user hive\nspark.argodb.ldap.password 123456\n<\/code><\/pre>\n<p>3.5 \u62f7\u8d1d\u914d\u7f6e\u6587\u4ef6\u5230spark conf<\/p>\n<pre><code class=\"language-shell\">\n# \u62f7\u8d1dinceptor hive-site.xml\u914d\u7f6e,\u7528\u4e8e\u8bfb\u53d6argodb\u7684\u914d\u7f6e\u4fe1\u606f\ncp \/etc\/inceptor1\/conf\/hive-site.xml \/opt\/gts-project\/src\/test-app\/spark\/spark-2.1.3-bin-hadoop2.7\/conf\n\n# \u62f7\u8d1dyarn yarn-site.xml & core-site.xml\u914d\u7f6e,\u7528\u4e8e\u8bfb\u53d6yarn\u7684\u914d\u7f6e\u4fe1\u606f\ncp \/etc\/yarn1\/conf\/yarn-site.xml \/opt\/gts-project\/src\/test-app\/spark\/spark-2.1.3-bin-hadoop2.7\/conf\ncp \/etc\/yarn1\/conf\/core-site.xml \/opt\/gts-project\/src\/test-app\/spark\/spark-2.1.3-bin-hadoop2.7\/conf\n\n# \u62f7\u8d1dHDFS hdfs-site.xml\u914d\u7f6e,\u7528\u4e8e\u8bfb\u53d6HDFS\u7684\u914d\u7f6e\u4fe1\u606f\ncp \/etc\/hdfs1\/conf\/hdfs-site.xml \/opt\/gts-project\/src\/test-app\/spark\/spark-2.1.3-bin-hadoop2.7\/conf\n<\/code><\/pre>\n<h4>\u542f\u52a8 spark-shell<\/h4>\n<p>\u4e94\u3001spark-shell on yarn\u8bfb\u53d6torc<br \/>\n\u57fa\u4e8espark 2.1.3\u8fdb\u884c\u6d4b\u8bd5\u3002 spark3.1.1 \u540c\u7406\u3002<\/p>\n<pre><code class=\"language-shell\">cd .\/spark-2.1.3-bin-hadoop2.7\n\n.\/bin\/spark-shell --master yarn --conf spark.ui.port=4444 --driver-memory 1g --executor-memory 2g --num-executors 2 --executor-cores 1 --principal hive@FYFTDH622  --keytab hive2210.keytab\n\n# \u53c2\u6570\u8bf4\u660e\uff1a\n# --conf spark.ui.port=4444 spark-web ui\u7aef\u53e3\n# --driver-memory 8g Spark driver\u5185\u5b58\n# --executor-memory 32g Spark executor\u7684\u5185\u5b58\n# --num-executors 4 Spark executor\u7684\u6570\u91cf\n# --executor-cores 24 \u6bcf\u4e2aexecutor\u5206\u914d\u7684cpu\u6838\u6570\n# --principal hive\/tx-node1@TDH \u96c6\u7fa4\u5f00\u542fkerberos\u540eprincipal\uff08\u53ef\u4ee5\u4f7f\u7528\/etc\/argodbcomputing*\/conf\/hive-site.xml\u6587\u4ef6\u5185hive.server2.authentication.kerberos.principal\u53c2\u6570\u503c\uff09\n# --keytab \/etc\/inceptor1\/conf\/hive.keytab  \u96c6\u7fa4\u5f00\u542fkerberos\u540e\uff0cinceptor \u7684 keytab\u6587\u4ef6\n<\/code><\/pre>\n<p><img decoding=\"async\" src=\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2024\/05\/image-1716186336456.png\" alt=\"file\" \/><\/p>\n<pre><code class=\"language-scala\"># \u8fdb\u5165Spark\uff0c\u8f93\u5165scala\u6d4b\u8bd5\u8bed\u53e5\uff1a\nval sqlContext=new org.apache.spark.sql.SQLContext(sc)\nsqlContext.sql(\"show databases\").show\n# \u51fa\u73b0ArgoDB\u7684databaseName\uff0c\u5219\u8868\u660e\u8bbf\u95ee\u6210\u529f\n<\/code><\/pre>\n<p><img decoding=\"async\" src=\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2024\/05\/image-1716186433604.png\" alt=\"file\" \/><\/p>\n<h4>\u63d0\u4ea4 spark-submit<\/h4>\n<pre><code class=\"language-shell\">.\/bin\/spark-submit --master yarn --deploy-mode client --class org.apache.spark.examples.JavaSparkPi  --executor-memory 2g --driver-memory 2g --principal hive@FYFTDH622 --keytab hive2210.keytab \/opt\/gts-project\/src\/test-app\/spark\/spark-2.1.3-bin-hadoop2.7\/examples\/jars\/spark-examples_2.11-2.1.3.jar\n<\/code><\/pre>\n<p>\u51c6\u786e\u8ba1\u7b97\u51fa\u4e86 \u03c0 \u7684\u503c<\/p>\n<p><img decoding=\"async\" src=\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2024\/05\/image-1716199010066.png\" alt=\"file\" \/><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u6982\u8981\u63cf\u8ff0 \u901a\u8fc7\u5f00\u6e90 Spark sql \u76f4\u63a5\u8fde\u63a5 inceptor \u53ea\u80fd\u8bfb\u53d6\u6709\u9650\u7684\u51e0\u79cd\u8868\u7c7b\u578b\uff1a\u666e\u901atext\u8868\u3001 ..<\/p>\n<div class=\"clear-fix\"><\/div>\n<p><a href=\"https:\/\/kbwp.transwarp.cn\/?p=13001\" title=\"read more...\">Read more<\/a><\/p>\n","protected":false},"author":11,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-13001","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"acf":[],"_links":{"self":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/13001","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/users\/11"}],"replies":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=13001"}],"version-history":[{"count":3,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/13001\/revisions"}],"predecessor-version":[{"id":13912,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/13001\/revisions\/13912"}],"wp:attachment":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=13001"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=13001"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=13001"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}