{"id":18293,"date":"2026-04-29T16:17:16","date_gmt":"2026-04-29T08:17:16","guid":{"rendered":"https:\/\/kbwp.transwarp.cn\/?p=18293"},"modified":"2026-04-29T16:17:16","modified_gmt":"2026-04-29T08:17:16","slug":"java%e5%86%99%e5%85%a5orc%e6%95%b0%e6%8d%ae%e6%96%87%e4%bb%b6","status":"publish","type":"post","link":"https:\/\/kbwp.transwarp.cn\/?p=18293","title":{"rendered":"java\u5199\u5165orc\u6570\u636e\u6587\u4ef6"},"content":{"rendered":"

\u6982\u8981\u63cf\u8ff0<\/h2>\n

\u5728 Windows \u73af\u5883\u4e0b\uff0c\u4f7f\u7528 Java \u7a0b\u5e8f\u901a\u8fc7 Kerberos \u8ba4\u8bc1\u8fde\u63a5 TDH \u96c6\u7fa4\uff0c\u5c06\u6570\u636e\u4ee5 ORC \u683c\u5f0f\u5199\u5165 HDFS\uff0c\u4f9b Quark\/Inceptor \u7684 ORC \u8868\u76f4\u63a5\u8bfb\u53d6\u3002<\/p>\n

\u524d\u7f6e\u6761\u4ef6<\/h2>\n\n\n\n\n\n\n\n\n
\u9879\u76ee<\/th>\n\u8981\u6c42<\/th>\n<\/tr>\n<\/thead>\n
JDK<\/td>\n1.8\uff081.8.0_391 \u9a8c\u8bc1\u901a\u8fc7\uff09<\/td>\n<\/tr>\n
\u7f51\u7edc<\/td>\nWindows \u673a\u5668\u9700\u80fd\u8bbf\u95ee HDFS NameNode \u7aef\u53e3\uff08\u9ed8\u8ba4 8020\uff09<\/td>\n<\/tr>\n
hosts<\/td>\n\u672c\u5730 C:\\Windows\\System32\\drivers\\etc\\hosts<\/code> \u9700\u914d\u7f6e\u96c6\u7fa4\u8282\u70b9\u6620\u5c04<\/td>\n<\/tr>\n
TDH \u670d\u52a1\u5668<\/td>\n\u9700\u80fd SSH \u767b\u5f55\u5230\u4efb\u4e00\u96c6\u7fa4\u8282\u70b9\uff0c\u62f7\u8d1d\u4f9d\u8d56\u6587\u4ef6<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n

\u64cd\u4f5c\u6b65\u9aa4<\/h2>\n

\u6b65\u9aa4\u4e00\uff1a\u914d\u7f6e\u672c\u5730 hosts<\/h3>\n

\u5728 Windows \u7684 C:\\Windows\\System32\\drivers\\etc\\hosts<\/code> \u4e2d\u6dfb\u52a0\u96c6\u7fa4\u8282\u70b9\u6620\u5c04\uff1a<\/p>\n

172.18.131.171 kv1\n172.18.131.172 kv2\n172.18.131.173 kv3\n172.18.131.174 kv4<\/code><\/pre>\n
\n
\u6ce8\u610f<\/strong>\uff1a\u8282\u70b9\u4e3b\u673a\u540d\u9700\u4e0e\u96c6\u7fa4\u5b9e\u9645\u914d\u7f6e\u4e00\u81f4\uff0c\u53ef\u5728\u670d\u52a1\u5668\u7684 \/etc\/hosts<\/code> \u4e2d\u786e\u8ba4\u3002<\/p>\n<\/blockquote>\n
\u6b65\u9aa4\u4e8c\uff1a\u4ece\u670d\u52a1\u5668\u62f7\u8d1d\u4f9d\u8d56\u6587\u4ef6<\/h3>\n
\u5728 Windows \u672c\u5730\u521b\u5efa\u76ee\u5f55\u7ed3\u6784\uff1a<\/p>\n
C:\\Users\\<\u7528\u6237\u540d>\\tdh-libs\\\n\u251c\u2500\u2500 conf\\          \u2190 HDFS \u914d\u7f6e\u6587\u4ef6\n\u2514\u2500\u2500 jars\\          \u2190 \u4f9d\u8d56 JAR \u5305<\/code><\/pre>\n
2.1 \u62f7\u8d1d\u914d\u7f6e\u6587\u4ef6<\/h4>\n
\u4ece TDH \u670d\u52a1\u5668 \/etc\/hdfs1\/conf\/<\/code> \u76ee\u5f55\u4e0b\u8f7d\u4ee5\u4e0b\u6587\u4ef6\u5230\u672c\u5730 conf\\<\/code> \u76ee\u5f55\uff1a<\/p>\n\n\n\n\n\n\n\n\n
\u6587\u4ef6<\/th>\n\u6765\u6e90\u8def\u5f84<\/th>\n<\/tr>\n<\/thead>\n
core-site.xml<\/code><\/td>\n\/etc\/hdfs1\/conf\/core-site.xml<\/code><\/td>\n<\/tr>\n
hdfs-site.xml<\/code><\/td>\n\/etc\/hdfs1\/conf\/hdfs-site.xml<\/code><\/td>\n<\/tr>\n
krb5.conf<\/code><\/td>\n\/etc\/hdfs1\/conf\/krb5.conf<\/code><\/td>\n<\/tr>\n
hdfs.keytab<\/code><\/td>\n\/etc\/hdfs1\/conf\/hdfs.keytab<\/code><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
2.2 \u62f7\u8d1d\u4f9d\u8d56 JAR<\/h4>\n
\u4ece TDH \u670d\u52a1\u5668\u7684 TDH Client \u76ee\u5f55\u4e0b\u8f7d\u4ee5\u4e0b JAR \u5230\u672c\u5730 jars\\<\/code> \u76ee\u5f55\uff1a<\/p>\n
\u6838\u5fc3 JAR\uff08\u5fc5\u987b\uff09\uff1a<\/strong><\/p>\n\n\n\n\n\n\n\n\n\n\n\n
JAR \u6587\u4ef6<\/th>\n\u670d\u52a1\u5668\u8def\u5f84<\/th>\n<\/tr>\n<\/thead>\n
hadoop-common-transwarp-9.3.3.jar<\/code><\/td>\n\/root\/TDH-Client\/hadoop\/hadoop\/share\/common\/<\/code><\/td>\n<\/tr>\n
inceptor-exec-8.43.2.jar<\/code><\/td>\n\/root\/TDH-Client\/inceptor\/lib\/<\/code><\/td>\n<\/tr>\n
hadoop-hdfs-transwarp-9.3.3.jar<\/code><\/td>\n\/root\/TDH-Client\/hadoop\/hadoop\/share\/hdfs\/<\/code><\/td>\n<\/tr>\n
hadoop-hdfs-client-transwarp-9.3.3.jar<\/code><\/td>\n\/root\/TDH-Client\/hadoop\/hadoop\/share\/hdfs\/<\/code><\/td>\n<\/tr>\n
hadoop-mapreduce-client-core-transwarp-9.3.3.jar<\/code><\/td>\n\/root\/TDH-Client\/hadoop\/hadoop\/share\/mapreduce\/<\/code><\/td>\n<\/tr>\n
hadoop-mapreduce-client-common-transwarp-9.3.3.jar<\/code><\/td>\n\/root\/TDH-Client\/hadoop\/hadoop\/share\/mapreduce\/<\/code><\/td>\n<\/tr>\n
hadoop-auth-transwarp-9.3.3.jar<\/code><\/td>\n\/root\/TDH-Client\/hadoop\/hadoop\/share\/common\/lib\/<\/code><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
\n
\u5feb\u6377\u65b9\u5f0f<\/strong>\uff1a\u5982\u679c\u4e0d\u60f3\u9010\u4e2a\u4e0b\u8f7d\uff0c\u53ef\u4ee5\u76f4\u63a5\u5c06\u670d\u52a1\u5668\u4e0a\u4ee5\u4e0b\u4e09\u4e2a\u76ee\u5f55\u7684\u6240\u6709 JAR \u62f7\u8d1d\u5230\u672c\u5730 jars\\<\/code> \u76ee\u5f55\uff1a<\/p>\n
    \n
  • \/root\/TDH-Client\/hadoop\/hadoop\/share\/common\/lib\/*.jar<\/code><\/li>\n
  • \/root\/TDH-Client\/hadoop\/hadoop\/share\/hdfs\/lib\/*.jar<\/code><\/li>\n
  • \/root\/TDH-Client\/inceptor\/lib\/inceptor-exec-8.43.2.jar<\/code><\/li>\n<\/ul>\n<\/blockquote>\n
    \u6b65\u9aa4\u4e09\uff1a\u7f16\u5199 Java \u4ee3\u7801<\/h3>\n
    \u521b\u5efa HdfsOrcWriter.java<\/code>\uff1a<\/p>\n
    import org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.FileSystem;\nimport org.apache.hadoop.fs.Path;\nimport org.apache.hadoop.hive.ql.io.orc.OrcFile;\nimport org.apache.hadoop.hive.ql.io.orc.Writer;\nimport org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;\nimport org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;\nimport org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;\nimport org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;\nimport org.apache.hadoop.io.IntWritable;\nimport org.apache.hadoop.io.Text;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\npublic class HdfsOrcWriter {\n    public static void main(String[] args) throws Exception {\n        \/\/ \u914d\u7f6e\u6587\u4ef6\u76ee\u5f55\uff08\u901a\u8fc7\u547d\u4ee4\u884c\u53c2\u6570\u6307\u5b9a\uff0c\u9ed8\u8ba4\u4f7f\u7528\u672c\u5730\u8def\u5f84\uff09\n        String confDir = args.length > 0 ? args[0] : \"C:\\\\Users\\\\17171\\\\tdh-libs\\\\conf\";\n\n        Configuration conf = new Configuration();\n        conf.addResource(new Path(new java.io.File(confDir, \"core-site.xml\").getAbsolutePath()));\n        conf.addResource(new Path(new java.io.File(confDir, \"hdfs-site.xml\").getAbsolutePath()));\n        \/\/ \u7981\u7528 OAuth2\uff08TDH Guardian \u63d2\u4ef6\u5728\u672c\u5730\u5ba2\u6237\u7aef\u73af\u5883\u65e0\u6cd5\u5de5\u4f5c\uff09\n        conf.set(\"hadoop.security.authentication.oauth2.enabled\", \"false\");\n        conf.set(\"hadoop.security.authentication.web.oauth2.enabled\", \"false\");\n\n        \/\/ Kerberos \u8ba4\u8bc1\n        org.apache.hadoop.security.UserGroupInformation.setConfiguration(conf);\n        String keytabPath = new java.io.File(confDir, \"hdfs.keytab\").getAbsolutePath();\n        org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytab(\"hdfs\/kv1@KTDH\", keytabPath);\n\n        FileSystem fs = FileSystem.get(conf);\n\n        \/\/ HDFS \u8f93\u51fa\u8def\u5f84\n        Path hdfsDir = new Path(\"\/user\/hdfs\/orc_test\");\n        fs.mkdirs(hdfsDir);\n        Path hdfsOrcPath = new Path(hdfsDir, \"output.orc\");\n        if (fs.exists(hdfsOrcPath)) {\n            fs.delete(hdfsOrcPath, false);\n        }\n\n        \/\/ \u5b9a\u4e49 Schema\uff1aid INT, name STRING, age INT\n        List colNames = new ArrayList();\n        colNames.add(\"id\");\n        colNames.add(\"name\");\n        colNames.add(\"age\");\n\n        List colOIs = new ArrayList();\n        colOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);\n        colOIs.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);\n        colOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);\n\n        StructObjectInspector soi = ObjectInspectorFactory.getStandardStructObjectInspector(colNames, colOIs);\n\n        \/\/ \u521b\u5efa ORC Writer\n        Writer orcWriter = OrcFile.createWriter(\n            hdfsOrcPath,\n            OrcFile.writerOptions(conf)\n                .inspector(soi)\n                .stripeSize(67108864)\n                .bufferSize(262144)\n                .rowIndexStride(10000)\n                .fileSystem(fs)\n        );\n\n        \/\/ \u5199\u5165\u6570\u636e\n        Object[][] data = {\n            {new IntWritable(1), new Text(\"zhangsan\"), new IntWritable(20)},\n            {new IntWritable(2), new Text(\"lisi\"), new IntWritable(25)},\n            {new IntWritable(3), new Text(\"wangwu\"), new IntWritable(30)},\n            {new IntWritable(4), new Text(\"zhaoliu\"), new IntWritable(28)},\n            {new IntWritable(5), new Text(\"tianqi\"), new IntWritable(22)}\n        };\n        for (Object[] row : data) {\n            orcWriter.addRow(row);\n        }\n\n        orcWriter.close();\n        fs.close();\n        System.out.println(\"ORC file written to HDFS successfully!\");\n        System.out.println(\"Path: \" + hdfsOrcPath.toString());\n    }\n}<\/code><\/pre>\n
    \n
    \u6ce8\u610f<\/strong>\uff1a\u4ee3\u7801\u4e2d loginUserFromKeytab<\/code> \u7684 principal\uff08hdfs\/kv1@KTDH<\/code>\uff09\u9700\u6839\u636e\u5b9e\u9645\u96c6\u7fa4\u914d\u7f6e\u4fee\u6539\uff0c\u53ef\u901a\u8fc7\u5728\u670d\u52a1\u5668\u4e0a\u6267\u884c klist -kt \/etc\/hdfs1\/conf\/hdfs.keytab<\/code> \u67e5\u770b keytab \u4e2d\u5305\u542b\u7684 principal\u3002<\/p>\n<\/blockquote>\n
    \u6b65\u9aa4\u56db\uff1a\u7f16\u8bd1\u4e0e\u8fd0\u884c<\/h3>\n
    \u6784\u5efaclasspath<\/h4>\n
    \u5c06 jars\\<\/code> \u76ee\u5f55\u4e0b\u6240\u6709 JAR \u548c conf\\<\/code> \u76ee\u5f55\u52a0\u5165 classpath\uff1a<\/p>\n
    $jarDir = \"C:\\Users\\<\u7528\u6237\u540d>\\tdh-libs\\jars\"\n$confDir = \"C:\\Users\\<\u7528\u6237\u540d>\\tdh-libs\\conf\"\n$jars = Get-ChildItem -Path $jarDir -Filter \"*.jar\" | Select-Object -ExpandProperty FullName\n$cp = ($jars -join \";\") + \";\" + $confDir<\/code><\/pre>\n
    \u7f16\u8bd1<\/h4>\n
    javac -cp $cp HdfsOrcWriter.java<\/code><\/pre>\n
    \u8fd0\u884c<\/h4>\n
    java -cp $cp `\n  -Djava.security.krb5.conf=\"$confDir\\krb5.conf\" `\n  -Dhadoop.home.dir=C:\\Users\\<\u7528\u6237\u540d>\\tdh-libs `\n  HdfsOrcWriter $confDir<\/code><\/pre>\n\n\n\n\n\n\n
    JVM \u53c2\u6570<\/th>\n\u8bf4\u660e<\/th>\n<\/tr>\n<\/thead>\n
    -Djava.security.krb5.conf<\/code><\/td>\n\u6307\u5b9a Kerberos \u914d\u7f6e\u6587\u4ef6\u8def\u5f84<\/td>\n<\/tr>\n
    -Dhadoop.home.dir<\/code><\/td>\nHadoop \u4e3b\u76ee\u5f55\uff08\u907f\u514d Windows \u4e0b\u627e\u4e0d\u5230 WINUTILS \u7684\u8b66\u544a\uff09<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
    \u8fd0\u884c\u6210\u529f\u540e\u8f93\u51fa\uff1a<\/p>\n
    ORC file written to HDFS successfully!\nPath: \/user\/hdfs\/orc_test\/output.orc<\/code><\/pre>\n
    \u6b65\u9aa4\u4e94\uff1a\u9a8c\u8bc1\u7ed3\u679c<\/h3>\n
    5.1 \u901a\u8fc7 Beeline \u521b\u5efa ORC \u8868\u5e76\u67e5\u8be2<\/h4>\n
    \u5728 TDH \u670d\u52a1\u5668\u4e0a\u6267\u884c\uff1a<\/p>\n
    source \/root\/TDH-Client\/init.sh n n\nbeeline -u \"jdbc:hive2:\/\/:10000\/default\" -n hive -p <\u5bc6\u7801> \\\n  -e \"CREATE EXTERNAL TABLE IF NOT EXISTS test_orc_table(\n        id INT,\n        name STRING,\n        age INT\n      ) STORED AS ORC\n      LOCATION '\/user\/hdfs\/orc_test';\n\n      SELECT * FROM test_orc_table;\"<\/code><\/pre>\n
    \u67e5\u8be2\u7ed3\u679c\uff1a<\/p>\n
    +-----+-----------+------+\n| id  |   name    | age  |\n+-----+-----------+------+\n| 1   | zhangsan  | 20   |\n| 2   | lisi      | 25   |\n| 3   | wangwu    | 30   |\n| 4   | zhaoliu   | 28   |\n| 5   | tianqi    | 22   |\n+-----+-----------+------+<\/code><\/pre>\n
    5.2 \u76f4\u63a5\u67e5\u770b HDFS \u6587\u4ef6<\/h4>\n
    kinit -kt \/etc\/hdfs1\/conf\/hdfs.keytab hdfs\/kv1@KTDH\nhdfs dfs -ls -h \/user\/hdfs\/orc_test\/<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"
    \u6982\u8981\u63cf\u8ff0 \u5728 Windows \u73af\u5883\u4e0b\uff0c\u4f7f\u7528 Java \u7a0b\u5e8f\u901a\u8fc7 Kerberos \u8ba4\u8bc1\u8fde\u63a5 TDH \u96c6\u7fa4\uff0c\u5c06\u6570 ..<\/p>\n
    <\/div>\n
    Read more<\/a><\/p>\n","protected":false},"author":12,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-18293","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"acf":[],"_links":{"self":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/18293","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/users\/12"}],"replies":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=18293"}],"version-history":[{"count":3,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/18293\/revisions"}],"predecessor-version":[{"id":18335,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/18293\/revisions\/18335"}],"wp:attachment":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=18293"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=18293"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=18293"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}