{"id":4809,"date":"2020-12-09T11:28:56","date_gmt":"2020-12-09T03:28:56","guid":{"rendered":"https:\/\/nj.transwarp.cn:8180\/?p=4809"},"modified":"2021-10-21T13:51:32","modified_gmt":"2021-10-21T05:51:32","slug":"discover-notebook%e7%9a%84pyspark%e8%a7%a3%e9%87%8a%e5%99%a8%e6%89%a7%e8%a1%8c%e6%8a%a5%e9%94%99","status":"publish","type":"post","link":"https:\/\/kbwp.transwarp.cn\/?p=4809","title":{"rendered":"discover notebook\u7684pyspark\u89e3\u91ca\u5668\u6267\u884c\u62a5\u9519"},"content":{"rendered":"<h3>\u95ee\u9898\u63cf\u8ff0<\/h3>\n<hr \/>\n<p>\u5ba2\u6237\u901a\u8fc7discover-notebook\u4e2d\u7684%spark.pyspark\u89e3\u91ca\u5668\uff0c\u6267\u884c\u62a5\u9519\uff0c\u62a5\u9519\u5185\u5bb9\u89c1\u622a\u56fe<\/p>\n<p><img decoding=\"async\" src=\"\/wp-content\/uploads\/2020\/12\/image-1607483588075.png\" alt=\"file\" \/><\/p>\n<h3>\u89e3\u51b3\u65b9\u6cd5<\/h3>\n<hr \/>\n<h4>1. \/var\/log\/discoverX\/interpreter.sh-XXX.log\u65e5\u5fd7\u62a5\u9519Only one of &#8211;proxy-user or &#8211;principal can be provided<\/h4>\n<pre><code class=\"language-ruby\">++ \/usr\/lib\/discover\/bin\/spark-submit --class org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer --driver-class-path ':\/usr\/lib\/notebook\/\/interpreter\/spark\/*:\/usr\/lib\/notebook\/\/lib\/interpreter\/*::\/usr\/lib\/notebook\/\/interpreter\/spark\/zeppelin-spark_2.11-0.8.0-SNAPSHOT.jar:\/etc\/yarn1\/conf' --driver-java-options ' -Dfile.encoding=UTF-8 -Dlog4j.configuration=file:\/\/\/etc\/discover1\/conf\/notebook\/log4j.properties -Dzeppelin.log.file=\/var\/log\/discover1\/zeppelin-interpreter-spark-gaopan--nari-08.log' --keytab \/home\/gaopan\/.keytab --principal gaopan@TDH --master 'local[*]' --conf spark.yarn.dist.archives=\/usr\/lib\/discover\/R\/lib\/sparkr.zip --conf spark.app.name=discover1-notebook --proxy-user gaopan \/usr\/lib\/notebook\/\/interpreter\/spark\/zeppelin-spark_2.11-0.8.0-SNAPSHOT.jar 10.238.0.87 10000 :\nError: Only one of --proxy-user or --principal can be provided.\nRun with --help for usage help or --verbose for debug output<\/code><\/pre>\n<p><code>Error: Only one of --proxy-user or --principal can be provided.<\/code> \u8be5\u62a5\u9519\u4e3a <code>-keytab<\/code>\uff0c<code>--principal<\/code>\u548c<code>proxy-user<\/code>\u4e0d\u80fd\u5171\u5b58\u7684\u95ee\u9898\uff0c<br \/>\n\u9700\u8981\u5230manager\u8282\u70b9\u4fee\u6539<code>\/var\/lib\/transwarp-manager\/master\/content\/meta\/services\/DISCOVER\/transwarp-X.X.X-final\/templates\/zeppelin-env.sh<\/code>(\u6ce8\u610f\u8fd9\u91cc\u9700\u8981\u5230\u5bf9\u5e94discover\u955c\u50cf\u7248\u672c\u7684\u76ee\u5f55)\uff0c\u6dfb\u52a0\u5982\u4e0b\u4e00\u884c<\/p>\n<pre><code class=\"language-ruby\">export ZEPPELIN_IMPERSONATE_SPARK_PROXY_USER=false<\/code><\/pre>\n<p>\u7136\u540ediscover\u7ec4\u4ef6\u91cd\u65b0\u5173\u95ed\u3001\u5f00\u542fkerberos\u5b89\u5168\uff0c<code>\/etc\/discover1\/conf\/notebook\/zeppelin-env.sh<\/code> \u4fbf\u4f1a\u5237\u65b0\u3002<\/p>\n<h4>2. \u518d\u6b21\u6267\u884c\uff0c\u62a5\u9519\u63d0\u793akeytab does not exists<\/h4>\n<pre><code class=\"language-ruby\">Keytab file: \/home\/crm\/.keytab does not exists<\/code><\/pre>\n<p>\u9700\u8981\u5230notebook\u7684\u5bb9\u5668\u5185\u4fee\u6539<code>\/usr\/lib\/notebook\/bin\/interpreter.sh<\/code>,\u4f4d\u7f6e\u5927\u6982\u5728234\u884c\u5de6\u53f3\uff0c<\/p>\n<p><code>\/home\/${ZEPPELIN_IMPERSONATE_USER}\/.keytab --principal ${ZEPPELIN_IMPERSONATE_USER}@${TDH_REALM} <\/code><\/p>\n<p>\u5efa\u8bae\u4fee\u6539\u4e3a<br \/>\n<code>\/etc\/discover1\/conf\/${ZEPPELIN_IMPERSONATE_USER}.keytab<\/code><\/p>\n<p>\u9009\u62e9\/etc\/discover1\/conf\/\u8fd9\u4e2a\u6301\u4e45\u5316\u76ee\u5f55\u4f5c\u4e3akeytab\u6587\u4ef6\u7684\u5b58\u50a8\u76ee\u5f55\u3002<\/p>\n<p>\u7136\u540e\uff0c\u5c06guardian\u9875\u9762\u4e0b\u8f7d\u7684\u5bf9\u5e94\u79df\u6237\uff08notebook\u767b\u9646\u79df\u6237\uff09\u7684keytab\u6587\u4ef6\uff0c\u653e\u5230\/etc\/discover1\/conf\/\u76ee\u5f55\u4e0b<\/p>\n<p><font color=red>\u6ce8\u610f\uff01<br \/>\n\u8fd9\u91cc\u4f7f\u7528\u7684keytab\u6587\u4ef6\u7edf\u4e00\u662f\u7531guardian\u7684\u79df\u6237\u9875\u9762\u8fdb\u884c\u4e0b\u8f7d\u7684\uff0c\u9875\u9762\u4e0b\u8f7d\u7684\u65b9\u5f0f\u751f\u6210\u7684keytab\u6587\u4ef6\u662f\u4e0d\u5305\u542bhostname\u7684\uff0c\u6240\u4ee5\u5728interpreter.sh\u811a\u672c\u4e2d\u6211\u4eec\u56fa\u8bdd\u6210${ZEPPELIN_IMPERSONATE_USER}@${TDH_REALM}\u7684\u683c\u5f0f\u3002<\/font><\/p>\n<p><font color=red>discover\u79df\u6237\u5728\u8fd9\u91cc\u4f1a\u6bd4\u8f83\u7279\u6b8a\uff0cdiscover\u7ec4\u4ef6\u5173\u95ed\u3001\u5f00\u542fkerberos\u65f6\u4f1a\u91cd\u65b0\u751f\u6210\/etc\/discover1\/conf\/discover.keytab\u3002\u800c\u8fd9\u4e2aprincipal\u662f\u5e26hostname\u7684\u683c\u5f0f\uff0c\u6240\u4ee5\u9700\u8981\u4eceguardian\u9875\u9762\u4e0b\u8f7d\u4f7f\u7528\uff0c\u8981\u5c24\u5176\u6ce8\u610f\uff01<\/font><\/p>\n<h4>3. \u518d\u6b21\u9a8c\u8bc1\uff0c\u95ee\u9898\u89e3\u51b3<\/h4>\n<p><strong><font color=red>\u6b65\u9aa42\u4fee\u6539\u7684\u662f\u5bb9\u5668\u5185\u7684\u975e\u6301\u4e45\u5316\u76ee\u5f55\uff0c\u6240\u4ee5\u9700\u505a\u955c\u50cf\u6301\u4e45\u5316\u52a8\u4f5c\u3002<\/font><\/strong><\/p>\n<p><img decoding=\"async\" src=\"\/wp-content\/uploads\/2020\/12\/image-1607484124263.png\" alt=\"file\" \/><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u95ee\u9898\u63cf\u8ff0 \u5ba2\u6237\u901a\u8fc7discover-notebook\u4e2d\u7684%spark.pyspark\u89e3\u91ca\u5668\uff0c\u6267\u884c\u62a5\u9519\uff0c\u62a5\u9519\u5185\u5bb9 ..<\/p>\n<div class=\"clear-fix\"><\/div>\n<p><a href=\"https:\/\/kbwp.transwarp.cn\/?p=4809\" title=\"read more...\">Read more<\/a><\/p>\n","protected":false},"author":12,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[41,39],"tags":[],"class_list":["post-4809","post","type-post","status-publish","format-standard","hentry","category-other_issue","category-usage_issues"],"acf":[],"_links":{"self":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/4809","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/users\/12"}],"replies":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=4809"}],"version-history":[{"count":1,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/4809\/revisions"}],"predecessor-version":[{"id":5108,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/4809\/revisions\/5108"}],"wp:attachment":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=4809"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=4809"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=4809"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}