{"id":450,"date":"2019-08-07T17:17:19","date_gmt":"2019-08-07T09:17:19","guid":{"rendered":"https:\/\/nj.transwarp.cn:8180\/?p=450"},"modified":"2021-02-03T14:46:47","modified_gmt":"2021-02-03T06:46:47","slug":"yarn%e4%b8%8acompact%e4%bd%9c%e4%b8%9a%e4%bb%bb%e5%8a%a1%e5%a4%b1%e8%b4%a5%e9%97%ae%e9%a2%98%e5%88%86%e6%9e%90","status":"publish","type":"post","link":"https:\/\/kbwp.transwarp.cn\/?p=450","title":{"rendered":"YARN\u4e0acompact\u4f5c\u4e1a\u4efb\u52a1\u5931\u8d25"},"content":{"rendered":"

\u6982\u8981\u63cf\u8ff0<\/h3>\n
\n

\u8dd1MapReduce\u4efb\u52a1\uff0c\u4efb\u52a1\u63d0\u4ea4\u5230yarn\u540e\u4e00\u76f4\u5361\u5728\u67d0\u4e2areduce\u9636\u6bb5\uff0c
\n\u67e5\u770byarn\u7684nodemanager\u65e5\u5fd7\uff08\/var\/log\/yarn1\/hadoop-yarn-nodemanager-mll01.log\uff09\uff0c\u6709\u7c7b\u4f3cOPERATION=Container Finished – Killed<\/font> \u62a5\u9519\uff0c\u8be6\u7ec6\u65e5\u5fd7\u4fe1\u606f\u5982\u4e0b\uff1a<\/p>\n

INFO\uff1aorg.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch: Cleaning up container container_e32_1558929087194_0228_01_000002\nWARN\uff1aorg.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor: Exit code from container container_e32_1558929087194_0228_01_000002 is : 143\nINFO\uff1aorg.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl: Container container_e32_1558929087194_0228_01_000002 transitioned from KILLING to CONTAINER_CLEANEDUP_AFTER_KILL\nINFO:org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger: USER=hive OPERATION=Container Finished - Killed   TARGET=ContainerImpl    RESULT=SUCCESS  APPID=application_1558929087194_0228    CONTAINERID=container_e32_1558929087194_0228_01_000002\nINFO:org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl: Container container_e32_1558929087194_0228_01_000002 transitioned from CONTAINER_CLEANEDUP_AFTER_KILL to DONE\nINFO:org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl: Removing container_e32_1558929087194_0228_01_000002 from application application_1558929087194_0228\nINFO :org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl: Neither virutal-memory nor physical-memory monitoring is needed. Not running the monitor-thread\nINFO:org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AppLogAggregatorImpl: Considering container container_e32_1558929087194_0228_01_000002 for log-aggregation<\/code><\/pre>\n
\u8be6\u7ec6\u8bf4\u660e<\/h3>\n
\n
\u4eceyarn\u7684nodemanager\u65e5\u5fd7\u4fe1\u606f\uff0c\u53ef\u4ee5\u770b\u51fa\u5728\u6267\u884creduce\u9636\u6bb5container\u88abkill\u6389\u4e86\uff0c\u5bfc\u81f4\u4efb\u52a1\u5931\u8d25\u3002
\n\u5927\u90e8\u5206\u6b64\u7c7b\u60c5\u51b5\u90fd\u53ef\u80fd\u662f\u7531\u4e8e\u7269\u7406\u5185\u5b58\u4e0d\u8db3\u5bfc\u81f4\u7684\uff0c\u53ef\u4ee5\u5c1d\u8bd5\u589e\u5927yarn\u7684\u5185\u5b58\u914d\u7f6e\uff1a<\/p>\n
    \n
  1. \u5728beeline\u4e0a\uff08\u70b9\u51fb\u67e5\u770bbeeline\u4f7f\u7528\u65b9\u6cd5<\/a><\/em>\uff09\u8fd0\u884c\u5982\u4e0b\u4e24\u4e2a\u547d\u4ee4\u67e5\u770b\u5173\u4e8emapreduce container\u5185\u5b58\u7684\u5f53\u524d\u914d\u7f6e\uff1a
    \nSET mapreduce.map.memory.mb;<\/strong>
    \nSET mapreduce.reduce.memory.mb;<\/strong>
    \n\u4f8b\u5982\uff1a<\/p>\n
    0: jdbc:hive2:\/\/bryan1:10000> SET mapreduce.map.memory.mb;\n+-------------------------------+\n|              set              |\n+-------------------------------+\n| mapreduce.map.memory.mb=1024  |\n+-------------------------------+\n1 row selected (0.266 seconds)\n0: jdbc:hive2:\/\/bryan1:10000> SET mapreduce.reduce.memory.mb;\n+----------------------------------+\n|               set                |\n+----------------------------------+\n| mapreduce.reduce.memory.mb=1024  |\n+----------------------------------+\n1 row selected (0.007 seconds)<\/code><\/pre>\n
    TDH 5.2.x\uff0c\u9ed8\u8ba4\u5185\u5b58\u8bbe\u7f6e\u4e3a1G\uff0cTDH 6.0.x\uff0c\u9ed8\u8ba4\u5185\u5b58\u8bbe\u7f6e\u4e3amap 1G\uff0creduce 2G<\/em><\/p>\n<\/li>\n
  2. \u6839\u636e\u5f53\u524d\u96c6\u7fa4\u7684\u7269\u7406\u5185\u5b58\u4f7f\u7528\u60c5\u51b5\uff0c\u51b3\u5b9a\u8c03\u6574\u540e\u7684\u5185\u5b58\u914d\u7f6e\uff08\u4f8b\u5982\u589e\u5927\u52304G<\/em>\uff09<\/li>\n
  3. \u767b\u5f55 Manager<\/strong> \u9875\u9762 (http:\/\/manager_ip:8180<\/a>)<\/li>\n
  4. \u70b9\u51fb YARN<\/strong> \u7ec4\u4ef6\uff0c\u5e76\u70b9\u51fb \u914d\u7f6e<\/strong> \u6309\u94ae\u8fdb\u5165YARN\u7ec4\u4ef6\u7684\u914d\u7f6e\u53c2\u6570\u9875\u9762
    \n\"file\"<\/li>\n
  5. \u70b9\u51fb \u6dfb\u52a0\u81ea\u5b9a\u4e49\u53c2\u6570<\/strong> \u6309\u94ae\uff0c\u6dfb\u52a0\u5982\u4e0b\u81ea\u5b9a\u4e49\u5c5e\u6027\u53c2\u6570\uff1a
    \nmapreduce.map.memory.mb<\/strong>
    \nmapreduce.reduce.memory.mb<\/strong>
    \n\u503c\u4e3a\u7b2c2\u6b65\u51b3\u5b9a\u7684\u5185\u5b58\u5927\u5c0f\uff0c4G\u5bf9\u5e94\u503c\u4e3a4096\uff0c\u914d\u7f6e\u6587\u4ef6\uff1amapred_site<\/strong>\uff0c\u53c2\u8003\u4e0b\u56fe\uff1a
    \n\"\"<\/li>\n
  6. \u53c2\u6570\u6dfb\u52a0\u5b8c\u6210\u540e\uff0c\u70b9\u51fb \u66f4\u591a\u64cd\u4f5c<\/strong> > \u914d\u7f6e\u670d\u52a1<\/strong>
    \n\"file\"<\/li>\n
  7. \u914d\u7f6e\u5b8c\u6210\u540e\uff0c\u91cd\u542f YARN<\/strong> \u7ec4\u4ef6\u670d\u52a1<\/li>\n<\/ol>\n
    \u82e5\u95ee\u9898\u4f9d\u7136\u5b58\u5728\uff0c\u8bf7\u8054\u7cfb\u661f\u73af\u79d1\u6280\u6280\u672f\u652f\u6301\u4e2d\u5fc3<\/em><\/p>\n","protected":false},"excerpt":{"rendered":"
    \u6982\u8981\u63cf\u8ff0 \u8dd1MapReduce\u4efb\u52a1\uff0c\u4efb\u52a1\u63d0\u4ea4\u5230yarn\u540e\u4e00\u76f4\u5361\u5728\u67d0\u4e2areduce\u9636\u6bb5\uff0c \u67e5\u770byarn\u7684node ..<\/p>\n
    <\/div>\n
    Read more<\/a><\/p>\n","protected":false},"author":14,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[41,38,39],"tags":[],"class_list":["post-450","post","type-post","status-publish","format-standard","hentry","category-other_issue","category-configuration","category-usage_issues"],"acf":[],"_links":{"self":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/450","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/users\/14"}],"replies":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=450"}],"version-history":[{"count":4,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/450\/revisions"}],"predecessor-version":[{"id":454,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/450\/revisions\/454"}],"wp:attachment":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=450"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=450"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=450"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}