{"id":13660,"date":"2024-08-01T15:10:01","date_gmt":"2024-08-01T07:10:01","guid":{"rendered":"https:\/\/nj.transwarp.cn:8180\/?p=13660"},"modified":"2024-08-01T15:09:58","modified_gmt":"2024-08-01T07:09:58","slug":"spark3-on-hadoop3-%e5%9c%a8-tdh-%e9%9b%86%e7%be%a4%e7%9a%84%e4%bd%bf%e7%94%a8%e6%96%b9%e6%b3%95","status":"publish","type":"post","link":"https:\/\/kbwp.transwarp.cn\/?p=13660","title":{"rendered":"Spark3 on hadoop3 \u5728 TDH \u96c6\u7fa4\u7684\u4f7f\u7528\u65b9\u6cd5"},"content":{"rendered":"

\u6982\u8981\u63cf\u8ff0<\/h3>\n

TDH 9.3 \u4e4b\u540e\u7684\u7248\u672c\uff0chadoop \u7248\u672c\u5347\u7ea7\u4e3a hadoop3 \u4e86\uff0c\u672c\u6587\u4ecb\u7ecd\u4e00\u4e0b Spark \u5728 hadoop3 \u73af\u5883\u4e0a\u7684\u4f7f\u7528\u65b9\u6cd5\uff1b<\/p>\n

\u8be6\u7ec6\u8bf4\u660e<\/h3>\n
    \n
  1. \u4e0b\u8f7dspark<\/li>\n
  2. \u914d\u7f6e YARN \u4ee5\u53ca Spark \u73af\u5883<\/li>\n
  3. \u5220\u9664 Guardian \u8ba4\u8bc1<\/li>\n
  4. Spark-shell \u4ee5\u53ca spark-submit \u9a8c\u8bc1<\/li>\n<\/ol>\n

    1\u3001\u4e0b\u8f7d spark<\/h4>\n

    spark \u5b98\u7f51\u4e0b\u8f7d\uff1ahttps:\/\/spark.apache.org\/downloads.html<\/a> \uff0c \u5386\u53f2\u6240\u6709\u7248\u672c\uff1ahttps:\/\/archive.apache.org\/dist\/spark\/<\/a><\/p>\n

    2\u3001\u914d\u7f6eyarn<\/h4>\n

    2.1 \u62f7\u8d1dspark\u8fd0\u884c\u65f6\u4f9d\u8d56\u7684jars\u5230hdfs<\/p>\n

    hadoop fs -put \/jars \/tmp\/spark3\/<\/code><\/pre>\n
    2.2 spark-env.sh \u914d\u7f6e<\/p>\n
    # \u4ee5\u4e0b\u914d\u7f6e\u8bf7\u6839\u636e\u672c\u5730\u5b9e\u9645\u73af\u5883\u914d\u7f6e \uff08\u5fc5\u8981\uff09\nHADOOP_USER_NAME=hive\nHADOOP_CONF_DIR=\/etc\/hdfs1\/conf<\/code><\/pre>\n
    2.3 spark-defaults.conf\u57fa\u7840\u914d\u7f6e<\/p>\n
    # \u4ee5\u4e0b\u914d\u7f6e\u8bf7\u6839\u636e\u672c\u5730\u5b9e\u9645\u73af\u5883\u914d\u7f6e\uff08\u5fc5\u8981\uff09\n\n# spark on yarn \u4f9d\u8d56\u7684jars\uff0c2.1 \u8282\u4e0a\u4f20\u7684jar\u5305\u8def\u5f84\nspark.yarn.jars           hdfs:\/\/172.22.37.171:8020\/tmp\/spark3\/jars\/*\n\n# \u6307\u5b9aspark\u8fd0\u884c\u5728yarn\u73af\u5883\u7684java\u7248\u672c\uff08\u9ed8\u8ba4yarn\u7684jdk\u7248\u672c\u4e3ajdk 1.7\uff09\n# \u5177\u4f53jdk\u8def\u5f84\u53ef\u4ee5\u67e5\u770byarn image\u4e2d\u7684jdk\u8def\u5f84\nspark.executorEnv.JAVA_HOME  \/usr\/java\/jdk1.8.0_25\nspark.yarn.appMasterEnv.JAVA_HOME \/usr\/java\/jdk1.8.0_25\n<\/code><\/pre>\n
    2.4 \u62f7\u8d1d\u914d\u7f6e\u6587\u4ef6\u5230spark conf\u76ee\u5f55<\/p>\n
    # \u62f7\u8d1dinceptor hive-site.xml\u914d\u7f6e,\u7528\u4e8e\u8bfb\u53d6argodb\u7684\u914d\u7f6e\u4fe1\u606f\ncp \/etc\/inceptor1\/conf\/hive-site.xml \/conf\n\n# \u62f7\u8d1dyarn yarn-site.xml & core-site.xml\u914d\u7f6e,\u7528\u4e8e\u8bfb\u53d6yarn\u7684\u914d\u7f6e\u4fe1\u606f\ncp \/etc\/yarn1\/conf\/yarn-site.xml \/conf\ncp \/etc\/yarn1\/conf\/core-site.xml \/conf\n\n# \u62f7\u8d1dHDFS hdfs-site.xml\u914d\u7f6e,\u7528\u4e8e\u8bfb\u53d6HDFS\u7684\u914d\u7f6e\u4fe1\u606f\ncp \/etc\/hdfs1\/conf\/hdfs-site.xml \/conf<\/code><\/pre>\n
    3\u3001\u5220\u9664 guardian \u8ba4\u8bc1<\/h4>\n
    \u5220\u9664\u914d\u7f6e\u6587\u4ef6\u4e2d guardian \u7684\u8ba4\u8bc1\uff0c\u4e3b\u8981\u662f yarn-site.xml \u4ee5\u53ca hdfs-site.xml\uff0c\u50cf\u8fd9\u6837\uff1a<\/p>\n
    yarn-site.xml\uff1a<\/strong><\/p>\n
    \"file\"<\/p>\n
    hdfs-site.xml\uff1a<\/strong><\/p>\n
    \"file\"<\/p>\n
    4\u3001\u6d4b\u8bd5\u9a8c\u8bc1<\/h4>\n
     .\/bin\/spark-shell --master yarn --conf spark.ui.port=4444 --driver-memory 1g --executor-memory 2g --num-executors 2 --executor-cores 1 --principal admin@GTSARGODB5  --keytab admin_171.keytab<\/code><\/pre>\n
    \"file\"<\/p>\n
    .\/bin\/spark-submit --master yarn --deploy-mode client --class org.apache.spark.examples.JavaSparkPi  --executor-memory 2g --driver-memory 2g --principal admin@GTSARGODB5 --keytab admin_171.keytab examples\/jars\/spark-examples_2.12-3.5.1.jar<\/code><\/pre>\n
    \"file\"<\/p>\n","protected":false},"excerpt":{"rendered":"
    \u6982\u8981\u63cf\u8ff0 TDH 9.3 \u4e4b\u540e\u7684\u7248\u672c\uff0chadoop \u7248\u672c\u5347\u7ea7\u4e3a hadoop3 \u4e86\uff0c\u672c\u6587\u4ecb\u7ecd\u4e00\u4e0b Spark  ..<\/p>\n
    <\/div>\n
    Read more<\/a><\/p>\n","protected":false},"author":11,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-13660","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"acf":[],"_links":{"self":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/13660","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/users\/11"}],"replies":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=13660"}],"version-history":[{"count":4,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/13660\/revisions"}],"predecessor-version":[{"id":13767,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/13660\/revisions\/13767"}],"wp:attachment":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=13660"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=13660"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=13660"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}