{"id":7137,"date":"2023-10-18T14:33:59","date_gmt":"2023-10-18T06:33:59","guid":{"rendered":"https:\/\/nj.transwarp.cn:8180\/?p=7137"},"modified":"2023-10-18T14:33:59","modified_gmt":"2023-10-18T06:33:59","slug":"spark-on-yarn%e4%bd%9c%e4%b8%9a%e8%bf%90%e8%a1%8c%e7%9a%84jar%e5%8c%85%e7%bc%93%e5%ad%98%e4%bc%98%e5%8c%96","status":"publish","type":"post","link":"https:\/\/kbwp.transwarp.cn\/?p=7137","title":{"rendered":"Spark on yarn\u4f5c\u4e1a\u8fd0\u884c\u7684jar\u5305\u7f13\u5b58\u4f18\u5316"},"content":{"rendered":"<h3>\u6982\u8981\u63cf\u8ff0<\/h3>\n<hr \/>\n<p>\u7528spark-submit\u63d0\u4ea4\u5230yarn\u540e\uff0c\u4efb\u52a1\u6267\u884c\u5b8c\u6210\u540e\uff0c\u5728yanr\u7684yarn.nodemanager.local-dirs\u53c2\u6570\u914d\u7f6e\u7684\u7f13\u5b58\u76ee\u5f55\u4e0b\u751f\u6210\u7684\u6587\u4ef6\u5374\u6ca1\u6709\u6e05\u7406<br \/>\n<img decoding=\"async\" src=\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2021\/11\/image-1637649209528.png\" alt=\"file\" \/><\/p>\n<p>\u867d\u7136\u5728nodemanager\u7684\u65e5\u5fd7\u4e2d\u770b\u5230\u4e86\u8fd9\u4e2aremoved\uff0c\u4f46\u662f\u5b9e\u9645ls\u8fd8\u662f\u80fd\u770b\u5230jar\u5305\u8fd8\u5728<\/p>\n<p><img decoding=\"async\" src=\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2021\/11\/image-1637649255805.png\" alt=\"file\" \/><\/p>\n<h3>\u8be6\u7ec6\u8bf4\u660e<\/h3>\n<hr \/>\n<h4><strong>\u6dfb\u52a0\u6e05\u7406\u65e5\u5fd7\u7684\u53c2\u6570<\/strong><\/h4>\n<p><a href=\"https:\/\/blog.csdn.net\/mtj66\/article\/details\/51841930\">https:\/\/blog.csdn.net\/mtj66\/article\/details\/51841930<\/a><br \/>\n\u6839\u636e\u8fd9\u7bc7\u6587\u7ae0\u914d\u7f6e\u4e86\u53c2\u6570yarn.nodemanager.localizer.cache.target-size-mb\u8ddfyarn.nodemanager.localizer.cache.cleanup.interval-ms\u4f46\u662f\u6ca1\u6709\u8fbe\u5230\u9884\u671f\u6548\u679c<\/p>\n<p><img decoding=\"async\" src=\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2021\/11\/image-1637649336051.png\" alt=\"file\" \/><br \/>\n<img decoding=\"async\" src=\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2021\/11\/image-1637649499117.png\" alt=\"file\" \/><\/p>\n<p>\u7f13\u5b58\u76ee\u5f55\u4e0b\u8fd8\u662f\u6709jar\u5305\uff0c\u800c\u4e14\u6bcf\u4e2a\u7f13\u5b58\u76ee\u5f55\u7684\u90fd\u4e00\u6837\uff1a<\/p>\n<p><img decoding=\"async\" src=\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2021\/11\/image-1637649534272.png\" alt=\"file\" \/><\/p>\n<p>usercache\u7f13\u5b58\u5927\u5c0f\u534a\u5929\u65f6\u95f4\u5df2\u7ecf\u4e0a\u767e\u4e2aG\u4e86\uff0c\u4fee\u6539\u53c2\u6570yarn.nodemanager.localizer.cache.cleanup.interval-ms\u5e0c\u671b\u6e05\u7406\u7684\u9891\u7387\u66f4\u5927\u4e9b\u4f46\u662f\u6ca1\u6709\u6548\u679c<\/p>\n<p><img decoding=\"async\" src=\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2021\/11\/image-1637649585316.png\" alt=\"file\" \/><\/p>\n<h4><strong>\u539f\u56e0\u4e0e\u89e3\u51b3<\/strong><\/h4>\n<p>yarn.nodemanager.delete.debug-delay-sec \u8fd9\u4e2a\u914d\u7f6e\u9879\u5728yarn-site.xml\u91cc\u9762<\/p>\n<p>\u8fd9\u4e2a\u914d\u7f6e\u9879\u662f\u8c03\u5ea6task\u5220\u9664\u76ee\u5f55\u7684\u65f6\u95f4\uff0c\u9ed8\u8ba4\u662f24h\uff0c\u5b83\u662f\u771f\u6b63\u5220\u9664\u76ee\u5f55\u7684\u65f6\u95f4\uff0c\u68c0\u67e5\u4e86\u7f13\u5b58\u6587\u4ef6\uff0c\u786e\u5b9e\u6ca1\u6709\u8d85\u8fc724h\u7684\uff0c\u4fee\u6539\u8fd9\u4e2a\u53c2\u6570\u662f1h\uff0c\u9a8c\u8bc1\u4e86\u7f13\u5b58\u786e\u5b9e1h\u540e\u6e05\u7406\u4e86\u3002<\/p>\n<p>\u4e4b\u524d\u6539\u8fc7\u7684\u90a3yarn.nodemanager.localizer.cache.cleanup.interval-ms\u53c2\u6570\uff0c\u5b83\u4f1a\u5e72\u4e24\u4ef6\u4e8b\u60c5\uff1a<\/p>\n<ul>\n<li>\n<p>a) \u8c03\u5ea6\u5220\u9664task\uff0c\u5220\u9664XXX_DEL_\u6587\u4ef6\u5939\uff0c\u8c03\u5ea6\u6267\u884c\u7684\u65f6\u95f4\u662fyarn.nodemanager.delete.debug-delay-sec\uff0c\u4e5f\u5c31\u662f24\u5c0f\u65f6\u540e\u6267\u884c<\/p>\n<\/li>\n<li>\n<p>a) \u68c0\u67e5cache\u6587\u4ef6\uff0c\u8d85\u8fc7yarn.nodemanager.localizer.cache.target-size-mb\u7684\u6587\u4ef6\uff0c\u4f1a\u79fb\u52a8\u5230XXX_DEL_\u6587\u4ef6\u5939\u91cc\u9762\u3002<\/p>\n<\/li>\n<\/ul>\n","protected":false},"excerpt":{"rendered":"<p>\u6982\u8981\u63cf\u8ff0 \u7528spark-submit\u63d0\u4ea4\u5230yarn\u540e\uff0c\u4efb\u52a1\u6267\u884c\u5b8c\u6210\u540e\uff0c\u5728yanr\u7684yarn.nodemanag ..<\/p>\n<div class=\"clear-fix\"><\/div>\n<p><a href=\"https:\/\/kbwp.transwarp.cn\/?p=7137\" title=\"read more...\">Read more<\/a><\/p>\n","protected":false},"author":70,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[121],"tags":[],"class_list":["post-7137","post","type-post","status-publish","format-standard","hentry","category-121"],"acf":[],"_links":{"self":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/7137","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/users\/70"}],"replies":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=7137"}],"version-history":[{"count":2,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/7137\/revisions"}],"predecessor-version":[{"id":7144,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/7137\/revisions\/7144"}],"wp:attachment":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=7137"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=7137"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=7137"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}