{"id":1829,"date":"2020-10-16T10:29:45","date_gmt":"2020-10-16T02:29:45","guid":{"rendered":"https:\/\/nj.transwarp.cn:8180\/?p=1829"},"modified":"2021-03-17T15:15:50","modified_gmt":"2021-03-17T07:15:50","slug":"%e9%9b%86%e7%be%a4%e5%a4%96%e9%83%a8%e7%bd%b2spark%e8%bf%9e%e6%8e%a5%e5%bc%80%e4%ba%86%e5%ae%89%e5%85%a8%e7%9a%84tdh","status":"publish","type":"post","link":"https:\/\/kbwp.transwarp.cn\/?p=1829","title":{"rendered":"\u96c6\u7fa4\u5916\u90e8\u7f72spark\u8fde\u63a5\u5f00\u4e86\u5b89\u5168\u7684TDH"},"content":{"rendered":"<h3>\u6982\u8981\u63cf\u8ff0<\/h3>\n<hr \/>\n<p>\u672c\u6587\u63cf\u8ff0\u5982\u4f55\u5728TDH\u96c6\u7fa4\u5916\u90e8\u7f72spark\u5ba2\u6237\u7aef\uff0c\u8fde\u63a5\u5f00\u4e86\u5b89\u5168\u8ba4\u8bc1\u7684TDH\u96c6\u7fa4<\/p>\n<h3>\u73af\u5883\u51c6\u5907<\/h3>\n<hr \/>\n<table>\n<thead>\n<tr>\n<th>HOSTNAME<\/th>\n<th>IP<\/th>\n<th>\u5907\u6ce8<\/th>\n<\/tr>\n<\/thead>\n<tbody>\n<tr>\n<td>tdh1-600<\/td>\n<td>172.22.16.88<\/td>\n<td>TDH6.0.2\u8282\u70b91\uff0cdiscover notebook\u8282\u70b9<\/td>\n<\/tr>\n<tr>\n<td>tdh2-600<\/td>\n<td>172.22.16.88<\/td>\n<td>TDH6.0.2\u8282\u70b92<\/td>\n<\/tr>\n<tr>\n<td>tdh3-600<\/td>\n<td>172.22.16.88<\/td>\n<td>TDH6.0.2\u8282\u70b93<\/td>\n<\/tr>\n<tr>\n<td>xiannv<\/td>\n<td>172.22.16.188<\/td>\n<td>\u96c6\u7fa4\u5916\u5355\u673a<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u9700\u8981\u4fdd\u8bc1\u96c6\u7fa4\u5916\u5355\u673a\u548c\u96c6\u7fa4\u5185\u8282\u70b9\u7684\u7f51\u7edc\u653e\u901a<\/p>\n<h3>\u8be6\u7ec6\u8bf4\u660e<\/h3>\n<hr \/>\n<h4>1 \u96c6\u7fa4\u5916\u90e8\u7f72TDH Client\u5ba2\u6237\u7aef<\/h4>\n<p>\u4e0b\u8f7d\u6216\u8005\u62f7\u8d1dTDH-Client\u5ba2\u6237\u7aef\u81f3\u96c6\u7fa4172.22.16.188<\/p>\n<pre><code class=\"language-shell\">source init.sh<\/code><\/pre>\n<p>\u53bb\u5b89\u88c5krb5\u7b49\u76f8\u5173\u547d\u4ee4\uff0c\u4ee5\u53ca\u5199\u5165TDH\u96c6\u7fa4\u7684hostname\u7b49\u5bf9\u5e94\u4fe1\u606f\u8fdb\/etc\/hosts\u91cc<br \/>\n\u8be6\u89c1TDH\u5b89\u88c5\u624b\u518c \u201c8.6. \u5b89\u88c5TDH\u5ba2\u6237\u7aef\u201d\u7ae0\u8282<br \/>\n<a href=\"https:\/\/www.warpcloud.cn\/#\/documents-support\/docs-detail\/document\/TDH-OPS\/6.2\/010InstallManual?docType=docs%3Fcategory%3DTDH%26index%3D0&docName=TDH%E5%AE%89%E8%A3%85%E6%89%8B%E5%86%8C\">https:\/\/www.warpcloud.cn\/#\/documents-support\/docs-detail\/document\/TDH-OPS\/6.2\/010InstallManual?docType=docs%3Fcategory%3DTDH%26index%3D0&#038;docName=TDH%E5%AE%89%E8%A3%85%E6%89%8B%E5%86%8C<\/a><\/p>\n<h4>2 \u8fdb\u5165discover notebook\u7684pod\u5185\u62f7\u8d1dspark\u76f8\u5173\u6587\u4ef6\u5939<\/h4>\n<p>\u5728TDH\u96c6\u7fa4\u4e0a\u8fdb\u5165discover notebook\u7684pod<\/p>\n<h5>1\uff09\u62f7\u8d1d\u6574\u4e2a\/usr\/lib\/discover\u5230\u96c6\u7fa4\u5916\u8282\u70b9172.22.16.188<\/h5>\n<p><img decoding=\"async\" src=\"\/wp-content\/uploads\/2019\/12\/image-1576580739915.png\" alt=\"file\" \/><\/p>\n<h5>2\uff09\u7531\u4e8eSpark\uff0cscala\u7b49\u5bf9\u4e8ejava\u7684\u7248\u672c\u6bd4\u8f83\u654f\u611f\uff0c\u5efa\u8bae\u76f4\u63a5\u4ecepod\u5185\u62f7\u8d1djava\u73af\u5883\u5230\u96c6\u7fa4\u5916\u8282\u70b9<\/h5>\n<p><img decoding=\"async\" src=\"\/wp-content\/uploads\/2019\/12\/image-1576580949186.png\" alt=\"file\" \/><\/p>\n<h4>3 \u62f7\u8d1d\u96c6\u7fa4\u914d\u7f6e\u6587\u4ef6\u5230\u96c6\u7fa4\u5916\u670d\u52a1\u5668<\/h4>\n<p>\u5c06\u96c6\u7fa4\u7684<code>hive-site.xml<\/code>,<code>yarn-site.xml<\/code>,<code>hdfs-site.xml<\/code>,<code>core-site.xml<\/code>\u4ee5\u53cayarn\u7684keytab\u6587\u4ef6\uff0c\u653e\u5230\u96c6\u7fa4\u5916\u8282\u70b9172.22.16.188\u7684<code>discover\/conf<\/code>\u8def\u5f84\u4e0b<br \/>\n<img decoding=\"async\" src=\"\/wp-content\/uploads\/2019\/12\/image-1576581271815.png\" alt=\"file\" \/><\/p>\n<h4>4 \u96c6\u7fa4\u5916\u8282\u70b9\u914d\u7f6ejava\u73af\u5883<\/h4>\n<pre><code class=\"language-shell\">vim \/etc\/profile\nexport JAVA_HOME=\/usr\/java\/jdk1.8.0_25<\/code><\/pre>\n<h4>5 \u786e\u8ba4\u96c6\u7fa4\u5916spark-defaults.conf\u7684\u914d\u7f6e<\/h4>\n<p>\u4e3b\u8981\u662f\u786e\u8ba4java\u73af\u5883\u7684\u914d\u7f6e\u662f\u5426\u6b63\u786e\uff0c\u8fd9\u91cc\u662fyarn\u7684pod\u5185\u7684jdk\u8def\u5f84\u3002<br \/>\n\u5982\u679c\u5728\u8fd9\u91cc\u4e0d\u6307\u5b9a\u7684\u8bdd\uff0c\u53ef\u4ee5\u901a\u8fc7spark-submit\u547d\u4ee4\u6765\u6307\u5b9a<br \/>\n<code>--conf spark.yarn.appMasterEnv.JAVA_HOME=\/usr\/java\/jdk1.8.0_25 \\<\/code><br \/>\n<code>--conf spark.executorEnv.JAVA_HOME=\/usr\/java\/jdk1.8.0_25 \\<\/code><\/p>\n<p><img decoding=\"async\" src=\"\/wp-content\/uploads\/2019\/12\/image-1576581395479.png\" alt=\"file\" \/><\/p>\n<h4>6 \u4e0a\u4f20\u6216\u8005\u4e0b\u8f7d\u76f8\u5173\u6d4b\u8bd5jar\u5305<\/h4>\n<pre><code class=\"language-shell\">cd \/discover\/jars\nwget http:\/\/central.maven.org\/maven2\/io\/snappydata\/snappy-spark-examples_2.11\/2.1.1.1\/snappy-spark-examples_2.11-2.1.1.1.jar<\/code><\/pre>\n<h4>7 \u8fd0\u884cspark\u547d\u4ee4\uff0c\u63d0\u4ea4\u4efb\u52a1\u53bbyarn\u8fdb\u884c\u9a8c\u8bc1<\/h4>\n<pre><code class=\"language-shell\">cd \/discover\/bin\n.\/spark-submit --class org.apache.spark.examples.JavaSparkPi --principal yarn\/tdh1-600@TDH  --keytab \/discover\/conf\/yarn.keytab  --master yarn --deploy-mode client \/discover\/jars\/<\/code><\/pre>\n<p><img decoding=\"async\" src=\"\/wp-content\/uploads\/2019\/12\/image-1576582226128.png\" alt=\"file\" \/><br \/>\n\u53bbYARN\u4e0a\u67e5\u770b\u4efb\u52a1\uff1a<br \/>\n<img decoding=\"async\" src=\"\/wp-content\/uploads\/2019\/12\/image-1576582464399.png\" alt=\"file\" \/><\/p>\n<h3>\u5e38\u89c1\u9519\u8bef<\/h3>\n<hr \/>\n<h4><strong>\u95ee\u98981\uff1ayarn\u65e5\u5fd7\u62a5\u9519<code> \/var\/log\/discover\/spark.log (No such file or directory)<\/code><\/strong><\/h4>\n<pre><code class=\"language-shell\">log4j:ERROR setFile(null,true) call failed.\njava.io.FileNotFoundException: \/var\/log\/discover\/spark.log (No such file or directory)\nat java.io.FileOutputStream.open(Native Method)\nat java.io.FileOutputStream.<init>(FileOutputStream.java:213)\nat java.io.FileOutputStream.<init>(FileOutputStream.java:133)\nat org.apache.log4j.FileAppender.setFile(FileAppender.java:294)\nat org.apache.log4j.RollingFileAppender.setFile(RollingFileAppender.java:207)\nat org.apache.log4j.FileAppender.activateOptions(FileAppender.java:165)\nat org.apache.log4j.config.PropertySetter.activate(PropertySetter.java:307)\nat org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:172)\nat org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:104)\nat org.apache.log4j.PropertyConfigurator.parseAppender(PropertyConfigurator.java:842)\nat org.apache.log4j.PropertyConfigurator.parseCategory(PropertyConfigurator.java:768)\nat org.apache.log4j.PropertyConfigurator.configureRootCategory(PropertyConfigurator.java:648)\nat org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:514)\nat org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:580)\nat org.apache.log4j.helpers.OptionConverter.selectAndConfigure(OptionConverter.java:526)\nat org.apache.log4j.LogManager.<clinit>(LogManager.java:127)\nat org.apache.spark.internal.Logging$class.initializeLogging(Logging.scala:117)\nat org.apache.spark.internal.Logging$class.initializeLogIfNecessary(Logging.scala:102)\nat org.apache.spark.deploy.yarn.ApplicationMaster$.initializeLogIfNecessary(ApplicationMaster.scala:737)\nat org.apache.spark.internal.Logging$class.log(Logging.scala:46)\nat org.apache.spark.deploy.yarn.ApplicationMaster$.log(ApplicationMaster.scala:737)\nat org.apache.spark.deploy.yarn.ApplicationMaster$.main(ApplicationMaster.scala:752)\nat org.apache.spark.deploy.yarn.ExecutorLauncher$.main(ApplicationMaster.scala:786)\nat org.apache.spark.deploy.yarn.ExecutorLauncher.main(ApplicationMaster.scala)\nSLF4J: Class path contains multiple SLF4J bindings.\nSLF4J: Found binding in [jar:file:\/vdir\/mnt\/disk1\/hadoop\/yarn\/local\/usercache\/yarn\/filecache\/13\/__spark_libs__3613082673827864887.zip\/slf4j-log4j12-1.7.16.jar!\/org\/slf4j\/impl\/StaticLoggerBinder.class]\nSLF4J: Found binding in [jar:file:\/usr\/lib\/hadoop\/lib\/slf4j-log4j12-1.7.10.jar!\/org\/slf4j\/impl\/StaticLoggerBinder.class]\nSLF4J: See http:\/\/www.slf4j.org\/codes.html#multiple_bindings for an explanation.\nSLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]<\/code><\/pre>\n<p>\u89e3\u51b3\u65b9\u6848\uff1a\u5b9e\u9645\u4e0a\u5e76\u6ca1\u6709\u4f7f\u7528spark\u7684log4j\u914d\u7f6e\u6587\u4ef6\uff0c\u5c06\/discover\/conf\/log4j.properties\u6587\u4ef6\u632a\u51fa\u3002<\/p>\n<h4><strong>\u95ee\u98982\uff1aspark-submit\u6267\u884c\u62a5\u9519<code>java.lang.ClassNotFoundException:org.apache.hadoop.hive.ql.metadata.Hive<\/code><\/strong><\/h4>\n<p><img decoding=\"async\" src=\"\/wp-content\/uploads\/2019\/12\/image-1576582875648.png\" alt=\"file\" \/><br \/>\n\u6392\u67e5\/discover\/jars\u6587\u4ef6\u5939\u91cc\u662f\u5426\u6709hive\u76f8\u5173\u7684jar\u5305<\/p>\n<h5>1\uff09\u5982\u679c\u6ca1\u6709\uff0c\u53ef\u4ee5\u4ece<code>TDH-Client\/inceptor\/lib<\/code>\u91cc\u62f7\u8d1d\u76f8\u5e94jar\u5305\u8fc7\u6765<\/h5>\n<p><img decoding=\"async\" src=\"\/wp-content\/uploads\/2019\/12\/image-1576583002982.png\" alt=\"file\" \/><\/p>\n<h5>2\uff09\u5982\u679c\u5df2\u7ecf\u5b58\u5728jar\u5305\uff0c\u786e\u8ba4jar\u5305\u540e\u9762\u7684\u7248\u672c\u53f7\u662f\u5426\u8ddf\u96c6\u7fa4\u7248\u672c\u53f7\u5339\u914d<\/h5>\n<p><img decoding=\"async\" src=\"\/wp-content\/uploads\/2019\/12\/image-1576583160308.png\" alt=\"file\" \/><br \/>\n\u4e00\u822c\u51fa\u73b0<code>ClassNotFoundException:org.apache.hadoop.hive.ql.metadata.Hive<\/code>\u4f1a\u662f\u96c6\u7fa4\u7248\u672c\u4e0d\u5339\u914d\u5bfc\u81f4\u7684\uff0c\u6b64\u65f6\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a<\/p>\n<pre><code class=\"language-shell\">mkdir \/tmp\/hive_jars\ncd \/discover\/jars\nmv hive* \/tmp\/hive_jars\n<\/code><\/pre>\n<p>\u7136\u540e\u518d\u5c06TDH-Client\/inceptor\/lib\u91cc\u62f7\u8d1d\u76f8\u5e94hive\u7684jar\u5305\u8fc7\u6765<\/p>\n<h5>3\uff09\u5982\u679c\u76f4\u63a5\u62f7\u8d1d<code>TDH-Client\/inceptor\/lib<\/code>\u91cc\u7684jar\u5305\u8fc7\u6765\uff0c\u4f1a\u5bfc\u81f4jar\u5305\u51b2\u7a81\u62a5\u9519<\/h5>\n<p><code>Exception in thread &quot;main&quot; java.lang.NoSuchMethodError: scala.Predef $.$conf<\/code><br \/>\n<img decoding=\"async\" src=\"\/wp-content\/uploads\/2019\/12\/image-1576583562655.png\" alt=\"file\" \/><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u6982\u8981\u63cf\u8ff0 \u672c\u6587\u63cf\u8ff0\u5982\u4f55\u5728TDH\u96c6\u7fa4\u5916\u90e8\u7f72spark\u5ba2\u6237\u7aef\uff0c\u8fde\u63a5\u5f00\u4e86\u5b89\u5168\u8ba4\u8bc1\u7684TDH\u96c6\u7fa4 \u73af\u5883\u51c6\u5907 HOSTNA ..<\/p>\n<div class=\"clear-fix\"><\/div>\n<p><a href=\"https:\/\/kbwp.transwarp.cn\/?p=1829\" title=\"read more...\">Read more<\/a><\/p>\n","protected":false},"author":15,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-1829","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"acf":[],"_links":{"self":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/1829","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/users\/15"}],"replies":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1829"}],"version-history":[{"count":2,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/1829\/revisions"}],"predecessor-version":[{"id":2539,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/1829\/revisions\/2539"}],"wp:attachment":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1829"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1829"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1829"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}