{"id":5314,"date":"2024-05-16T16:35:42","date_gmt":"2024-05-16T08:35:42","guid":{"rendered":"https:\/\/nj.transwarp.cn:8180\/?p=5314"},"modified":"2024-05-16T16:35:42","modified_gmt":"2024-05-16T08:35:42","slug":"mapreduce-on-yarn-%e4%bd%9c%e4%b8%9a%e6%97%a5%e5%bf%97%e7%ba%a7%e5%88%ab%e8%ae%be%e7%bd%ae","status":"publish","type":"post","link":"https:\/\/kbwp.transwarp.cn\/?p=5314","title":{"rendered":"MapReduce on Yarn \u4f5c\u4e1a\u65e5\u5fd7\u7ea7\u522b\u8bbe\u7f6e"},"content":{"rendered":"<h3>\u6982\u8981\u63cf\u8ff0<\/h3>\n<hr \/>\n<p>\u5728 TDH \u4f7f\u7528\u4e2d\uff0c\u4f1a\u6709\u63d0\u4ea4 MapReduce \u4efb\u52a1\u5230 Yarn \u4e0a\u7684\u573a\u666f\uff0c\u6709\u65f6\u5728\u8c03\u8bd5\u4efb\u52a1\u65f6\u4e3a\u4e86\u5206\u6790\u5b9a\u4f4d\u539f\u56e0\u9700\u8981\u8c03\u6574\u4f5c\u4e1a\u7684\u65e5\u5fd7\u7ea7\u522b\u3002Yarn \u4f5c\u4e1a\u9ed8\u8ba4\u65e5\u5fd7\u7ea7\u522b\u662f INFO\uff0c\u6211\u4eec\u53ef\u4ee5\u6839\u636e\u672c\u6587\u65b9\u6cd5\u6765\u8c03\u6574\u4f5c\u4e1a\u7684 Map\u3001Reduce\u3001ApplicationMaster \u7684\u65e5\u5fd7\u7ea7\u522b\u4e3a\u5176\u4ed6\u3002<\/p>\n<h3>\u8be6\u7ec6\u8bf4\u660e<\/h3>\n<hr \/>\n<ul>\n<li>\n<p>\u672c\u6587\u5728 Centos 7.2\uff0cTDH 6.2.2 \u73af\u5883\u4e0a\u6d4b\u8bd5\u901a\u8fc7<\/p>\n<\/li>\n<li>\n<p>\u6d89\u53ca\u914d\u7f6e\u9879<\/p>\n<\/li>\n<\/ul>\n<table>\n<thead>\n<tr>\n<th>\u914d\u7f6e\u9879<\/th>\n<th>\u8bf4\u660e<\/th>\n<th>\u914d\u7f6e\u6587\u4ef6<\/th>\n<th>\u9ed8\u8ba4\u503c<\/th>\n<th>\u53ef\u9009\u503c<\/th>\n<\/tr>\n<\/thead>\n<tbody>\n<tr>\n<td>mapreduce.map.log.level<\/td>\n<td>map task \u65e5\u5fd7\u7ea7\u522b<\/td>\n<td>mapred-site.xml<\/td>\n<td>INFO<\/td>\n<td>OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL<\/td>\n<\/tr>\n<tr>\n<td>mapreduce.reduce.log.level<\/td>\n<td>reduce task \u65e5\u5fd7\u7ea7\u522b<\/td>\n<td>mapred-site.xml<\/td>\n<td>INFO<\/td>\n<td>OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL<\/td>\n<\/tr>\n<tr>\n<td>yarn.app.mapreduce.am.log.level<\/td>\n<td>MR ApplicationMaster \u65e5\u5fd7\u7ea7\u522b<\/td>\n<td>mapred-site.xml<\/td>\n<td>INFO<\/td>\n<td>OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<ul>\n<li>\u793a\u4f8b<br \/>\n\u4fee\u6539 MR \u4efb\u52a1\u4f5c\u4e1a\u65e5\u5fd7\u7ea7\u522b\u4e3a WARN<\/li>\n<\/ul>\n<ol>\n<li>\n<p>\u6253\u5f00 TDH Manager \u5e76\u8fdb\u5165 Yarn \u7684\u914d\u7f6e\u9875\u9762\uff0c\u901a\u8fc7 \u201c\u6dfb\u52a0\u81ea\u5b9a\u4e49\u53c2\u6570\u201d \u6dfb\u52a0\u4ee5\u4e0a 3 \u4e2a\u53c2\u6570<br \/>\n<img decoding=\"async\" src=\"\/wp-content\/uploads\/2021\/02\/image-1614251912130.png\" alt=\"file\" \/><br \/>\n<img decoding=\"async\" src=\"\/wp-content\/uploads\/2021\/02\/image-1614251937543.png\" alt=\"file\" \/><\/p>\n<\/li>\n<li>\n<p>\u914d\u7f6e\u670d\u52a1\u5e76\u91cd\u542f Yarn<br \/>\n<img decoding=\"async\" src=\"\/wp-content\/uploads\/2021\/02\/image-1614252039444.png\" alt=\"file\" \/><\/p>\n<\/li>\n<li>\n<p>\u91cd\u65b0\u4e0b\u8f7d TDH-Client\uff0c\u5e76\u63d0\u4ea4\u6d4b\u8bd5\u4efb\u52a1\u9a8c\u8bc1<br \/>\n\u5982\u679c\u9700\u8981\u4f7f\u7528 TDH-Client \u901a\u8fc7\u547d\u4ee4\u63d0\u4ea4 MapReduce \u4efb\u52a1\u6765\u9a8c\u8bc1\uff0c\u4e00\u5b9a\u8981\u91cd\u65b0\u4e0b\u8f7d TDH-Client \u6765\u66f4\u65b0\u5ba2\u6237\u7aef\u914d\u7f6e\uff1b<br \/>\n\u5982\u679c\u901a\u8fc7\u5176\u4ed6\u65b9\u5f0f\u6765\u9a8c\u8bc1\uff0c\u4e5f\u8bf7\u540c\u6b65\u66f4\u65b0\u5bf9\u5e94 Client \u7684 mapred-site.xml \u914d\u7f6e\u6587\u4ef6\uff1b<br \/>\n\u5982\u56fe\u53ef\u4ee5\u770b\u5230\u65e5\u5fd7\u7ea7\u522b\u6210\u529f\u8c03\u6574\u4e3a WARN\u3002<br \/>\n<img decoding=\"async\" src=\"\/wp-content\/uploads\/2021\/02\/image-1614252244621.png\" alt=\"file\" \/><\/p>\n<\/li>\n<\/ol>\n<p><\/br><\/p>\n<p>\u9644\uff1a<a href=\"http:\/\/hadoop.apache.org\/docs\/stable\/hadoop-mapreduce-client\/hadoop-mapreduce-client-core\/mapred-default.xml\">mapred-default \u53c2\u6570\u8bf4\u660e<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u6982\u8981\u63cf\u8ff0 \u5728 TDH \u4f7f\u7528\u4e2d\uff0c\u4f1a\u6709\u63d0\u4ea4 MapReduce \u4efb\u52a1\u5230 Yarn \u4e0a\u7684\u573a\u666f\uff0c\u6709\u65f6\u5728\u8c03\u8bd5\u4efb\u52a1\u65f6\u4e3a\u4e86\u5206 ..<\/p>\n<div class=\"clear-fix\"><\/div>\n<p><a href=\"https:\/\/kbwp.transwarp.cn\/?p=5314\" title=\"read more...\">Read more<\/a><\/p>\n","protected":false},"author":10,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[38],"tags":[187,188,189],"class_list":["post-5314","post","type-post","status-publish","format-standard","hentry","category-configuration","tag-yarn-","tag-container-"],"acf":[],"_links":{"self":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/5314","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/users\/10"}],"replies":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=5314"}],"version-history":[{"count":3,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/5314\/revisions"}],"predecessor-version":[{"id":12971,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/5314\/revisions\/12971"}],"wp:attachment":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=5314"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=5314"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=5314"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}