{"id":2565,"date":"2020-07-31T16:23:27","date_gmt":"2020-07-31T08:23:27","guid":{"rendered":"https:\/\/nj.transwarp.cn:8180\/?p=2565"},"modified":"2020-07-31T16:24:08","modified_gmt":"2020-07-31T08:24:08","slug":"object-store%e5%ad%98%e5%82%a8%e9%9d%9e%e7%bb%93%e6%9e%84%e5%8c%96%e6%95%b0%e6%8d%ae%ef%bc%88%e5%9b%be%e7%89%87%ef%bc%89%e7%a4%ba%e4%be%8b","status":"publish","type":"post","link":"https:\/\/kbwp.transwarp.cn\/?p=2565","title":{"rendered":"Object Store\u5b58\u50a8\u975e\u7ed3\u6784\u5316\u6570\u636e\uff08\u56fe\u7247\uff09\u793a\u4f8b"},"content":{"rendered":"<h3>\u6982\u8981\u63cf\u8ff0<\/h3>\n<hr \/>\n<p>\u4f7f\u7528\u987b\u77e5\uff1a<\/p>\n<ol>\n<li>\u672c\u5b9e\u9a8c\u662f\u5728<strong>\u6ca1\u6709\u5f00\u542fguardian\u5b89\u5168\u8ba4\u8bc1<\/strong>\u7684\u73af\u5883\u4e2d\u6d4b\u8bd5\u7684\uff1b<\/li>\n<li>\u901a\u8fc7 <strong>API<\/strong> \u65b9\u5f0f\u6765\u4f7f\u7528Object Store<\/li>\n<\/ol>\n<p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0cHBase\u7528\u76f8\u540c\u7684\u65b9\u6cd5\u5904\u7406\u7ed3\u6784\u5316\u548c\u975e\u7ed3\u6784\u5316\u7684\u6570\u636e\u3002\u4f46\u662f\uff0c\u7531\u4e8e\u56fe\u7247\u7b49\u975e\u7ed3\u6784\u5316\u6570\u636e\u975e\u5e38\u5927\uff0c\u5728\u5411HBase\u5bfc\u5165\u65f6\u4f1a\u8ba9Region\u5927\u5c0f\u589e\u957f\u8fc5\u901f\uff0c\u9891\u7e41\u89e6\u53d1Region\u7684Split\u548cCompaction\uff0c\u5728\u4e00\u5b9a\u7a0b\u5ea6\u4e0a\u5361\u4f4f\u5ba2\u6237\u7aef\u7684\u5199\u5165\uff0c\u5f71\u54cdHBase\u7684\u63d2\u5165\u6027\u80fd\u3002\u56e0\u6b64\uff0c\u5bf9\u975e\u7ed3\u6784\u5316\u7684\u5927\u5bf9\u8c61\u91c7\u7528\u4e0d\u540c\u65b9\u5f0f\u5904\u7406\uff0c\u5728\u63d2\u5165\u65f6\u964d\u4f4eRegion\u7684Split\u548cCompaction\u9891\u7387\uff0c\u63d0\u9ad8\u63d2\u5165\u6027\u80fd\uff0cObject Store\u5c31\u662f\u4e3a\u4e86\u89e3\u51b3\u8fd9\u4e2a\u95ee\u9898\u800c\u8bbe\u8ba1\u7684\u3002<\/p>\n<h3>\u8be6\u7ec6\u8bf4\u660e<\/h3>\n<hr \/>\n<h4>\u83b7\u53d6Hbase api\u8fd0\u884c\u4f9d\u8d56\u7684jar\u5305<\/h4>\n<ol>\n<li>\n<p>HBase API\u7684\u4f9d\u8d56jar\u5305\u90fd\u5728\u5982\u4e0b\u5217\u8868\u4e2d<br \/>\njar\u540d\u79f0\u4e2d * \u4f4d\u7f6e\u662f\u5305\u7684\u7248\u672c\u53f7\uff0c\u8fd9\u4e9bjar\u7684\u7248\u672c\u53f7\u6839\u636e\u60a8\u4f7f\u7528\u7684TDH\u7248\u672c\u7684\u4e0d\u540c\u4f1a\u6709\u6240\u4e0d\u540c\uff0c\u4ee5\u60a8\u96c6\u7fa4\u4e0a\u6b63\u5728\u4f7f\u7528\u7684\u5305\u4e3a\u51c6\uff1a<\/p>\n<pre><code class=\"language-shell\">hbase-client-*-transwarp.jar\nhbase-bson-*-transwarp.jar\nhbase-common-*-transwarp.jar\nhbase-protocol-*-transwarp.jar\nhadoop-common-*-transwarp.jar\nhadoop-auth-*-transwarp.jar\nhadoop-annotations-*-transwarp.jar\nhadoop-mapreduce-client-core-*-transwarp.jar\ncommons-codec-*.jar\ncommons-io-*.jar\ncommons-lang-*.jar\ncommons-logging-*.jar\ncommons-collections-*.jar\nguava-*.jar\nprotobuf-java-*.jar\nnetty-*.Final.jar\nhtrace-core-*.jar\nelasticsearch-*-transwarp.jar\njackson-mapper-asl-*.jar\nfindbugs-annotations-*.jar\nzookeeper-*-transwarp.jar\nlog4j-*.jar\ncommons-configuration-*.jar\nslf4j-api-*.jar\nslf4j-log4j12-*.jar\njsch-*.jar\njzlib-*.jar\nhbase-hyperbase-*-transwarp.jar<\/code><\/pre>\n<\/li>\n<li>\n<p>\u8fdb\u5165\u5bb9\u5668\u5185\u90e8\u5c06jar\u5305\u4f7f\u7528\u547d\u4ee4scp\u83b7\u53d6<\/p>\n<ul>\n<li>\n<p>\u83b7\u53d6hyperbase\u670d\u52a1\u6240\u5728pod<\/p>\n<pre><code class=\"language-shell\">kubectl get pod|grep hyperbase<\/code><\/pre>\n<\/li>\n<li>\n<p>\u8fdb\u5165\u4efb\u610f\u4e00\u4e2a\u8be5\u670d\u52a1pod\u4e2d\uff0c\u8fd9\u91cc\u4ee5regionserver\u4e3a\u4f8b<\/p>\n<pre><code class=\"language-shell\">kubectl exec -it hyperbase-regionserver-hyperbase1-2285948569-0b7cl bash<\/code><\/pre>\n<\/li>\n<li>\n<p>\u53d6\u51fajar\u5305<\/p>\n<pre><code class=\"language-shell\">scp \/usr\/lib\/hbase\/lib\/*.jar 172.22.33.1:\/mnt\/disk3<\/code><\/pre>\n<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n<blockquote>\n<p>\u82e5\u9879\u76ee\u914d\u7f6e\u4e86maven\u4ed3\u5e93\u4e3aTDH\u7684SDK\uff0c\u53ef\u4f7f\u7528\u5982\u4e0bpom\u7247\u6bb5 <strong>\u7248\u672c\u53f7\uff0c\u8bf7\u53c2\u8003\u5b9e\u9645\u73af\u5883\u4e2d\u7684\u7248\u672c<\/strong><\/p>\n<\/blockquote>\n<pre><code><dependency>\n    <groupId>org.apache.hadoop<\/groupId>\n    <artifactId>hadoop-common<\/artifactId>\n    <version>2.7.2-transwarp-5.2.2<\/version>\n<\/dependency>\n\n<dependency>\n    <groupId>org.apache.hbase<\/groupId>\n    <artifactId>hbase-client<\/artifactId>\n    <version>0.98.6-transwarp-5.2.2<\/version>\n<\/dependency><\/code><\/pre>\n<h4>Object Store API \u793a\u4f8b<\/h4>\n<p>demo\u4f7f\u7528\u6ce8\u610f\u4e8b\u9879\uff1a<\/p>\n<ol>\n<li><strong><code>conf.set<\/code><\/strong> \u4e2d\u7684 <strong><code>hbase.zookeeper.quorum \u3001 zookeeper.znode.parent \u3001hbase.zookeeper.property.clientPort<\/code><\/strong> \u6539\u6210\u5bf9\u5e94\u7684\u96c6\u7fa4\u4e0a\u9762\u7684\u914d\u7f6e\uff0c<strong><code>hbase.zookeeper.quorum<\/code><\/strong>\u662f\u96c6\u7fa4zookeeper\u76f8\u5173\u914d\u7f6e\uff0c\u540e\u9762\u4e24\u4e2a\u53c2\u6570\u6ca1\u6709\u4fee\u6539\u7684\u8bdd\u4f7f\u7528\u9ed8\u8ba4\u7684\u5373\u53ef<\/li>\n<li><strong><code>testLOBGet()<\/code><\/strong> \u4e2d\u7684 <strong><code>path<\/code><\/strong> \u662f\u56fe\u7247\u5728windows\u4e0a\u4fdd\u5b58\u7684\u4f4d\u7f6e<\/li>\n<\/ol>\n<p>\u53c2\u8003demo:<\/p>\n<pre><code class=\"language-java\">package transwarp;\n\nimport static org.junit.Assert.assertTrue;\n\nimport java.io.File;\nimport java.io.FileInputStream;\nimport java.io.IOException;\nimport java.util.concurrent.TimeUnit;\n\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.hbase.*;\nimport org.apache.hadoop.hbase.client.Get;\nimport org.apache.hadoop.hbase.client.HTable;\nimport org.apache.hadoop.hbase.client.Put;\nimport org.apache.hadoop.hbase.client.Result;\nimport org.apache.hadoop.hbase.protobuf.generated.HyperbaseProtos;\nimport org.apache.hadoop.hbase.util.Bytes;\nimport org.apache.hadoop.hyperbase.client.HyperbaseAdmin;\nimport org.apache.hadoop.hyperbase.metadata.HyperbaseMetadata;\nimport org.apache.hadoop.hyperbase.secondaryindex.IndexedColumn;\nimport org.apache.hadoop.hyperbase.secondaryindex.LOBIndex;\n\npublic class Test {\n\n  protected static HyperbaseAdmin admin = null;\n  protected static Configuration conf = null;\n  static {\n    conf = HBaseConfiguration.create();\n    \/\/\u6539\u6210\u5bf9\u5e94\u7684\u96c6\u7fa4\u4e0a\u9762\u7684\u914d\u7f6e\uff01\n    conf.set(\"hbase.zookeeper.quorum\",\"mll01,mll02,mll03\");\n    conf.set(\"zookeeper.znode.parent\", \"\/hyperbase1\");\n    conf.set(\"hbase.zookeeper.property.clientPort\", \"2181\");\n    try {\n      admin = new HyperbaseAdmin(conf);\n    } catch (IOException e) {\n      \/\/ TODO Auto-generated catch block\n      e.printStackTrace();\n    }\n  }\n\n  public static void main(String[] args) throws Exception {\n    testLOBGet();\n  }\n\n  public static void testLOBGet() throws Exception {\n    byte[] row = Bytes.toBytes(\"rowkey01\");\n    byte[] tableName = Bytes\n        .toBytes(\"SIMPLE_TEST_PUT_SCAN\" + System.nanoTime());\n    byte[] indexName = Bytes.toBytes(\"IDX\");\n    byte[] family1 = Bytes.toBytes(\"f1\");\n    byte[] family2 = Bytes.toBytes(\"f2\");\n    String path = \"D:\\\\work\\\\java\\\\objectstore\\\\test\\\\980f51924e9d409d2949cef6ea47d324.jpg\";\n    createTable(TableName.valueOf(tableName), family1, family2);\n    addLOB(tableName, family1, indexName);\n    byte[] value01 = getFileBytes(path);\n\n    HTable htable = new HTable(conf, tableName);\n\n    Put put = new Put(row);\n    put.add(family1, Bytes.toBytes(\"q1\"), value01);\n    htable.put(put);\n    htable.flushCommits();\n    TimeUnit.SECONDS.sleep(1);\n\n    Get get = new Get(row);\n    Result rs = htable.get(get);\n    CellScanner cs = rs.cellScanner();\n    while(cs.advance()){\n      assertTrue(Bytes.equals(CellUtil.cloneValue(cs.current()), value01));\n      System.out.println(Bytes.toString(CellUtil.cloneValue(cs.current())));\n    }\n    htable.close();\n\/\/    admin.deleteTable(TableName.valueOf(tableName));\n  }\n\n  public static byte[] getFileBytes(String path) throws IOException {\n    FileInputStream fis = new FileInputStream(new File(path));\/\/ \u65b0\u5efa\u4e00\u4e2aFileInputStream\u5bf9\u8c61\n    byte[] b = new byte[fis.available()];\/\/ \u65b0\u5efa\u4e00\u4e2a\u5b57\u8282\u6570\u7ec4\n    fis.read(b);\/\/ \u5c06\u6587\u4ef6\u4e2d\u7684\u5185\u5bb9\u8bfb\u53d6\u5230\u5b57\u8282\u6570\u7ec4\u4e2d\n    fis.close();\n    return b;\n  }\n\n  public static void createTable(TableName tableName, byte[]... families) throws Exception {\n    HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);\n    for (byte[] family : families) {\n      tableDescriptor.addFamily(new HColumnDescriptor(family));\n    }\n    \/\/ \u6ce8\u610f\uff0cobject store\u4e00\u5b9a\u8981\u9884\u5206\u914dregion\uff0c\u6bcf\u4e2aregion\u6700\u597d\u4e0d\u8981\u8d85\u8fc7500G\u7684\u6570\u636e\u3002\n    byte[][] splitKeys = new byte[100][];\n    for (int i = 0; i < 100; i++) {\n      splitKeys[i] =  Bytes.toBytes(\"rowkey\" + i);\n    }\n    \/\/ create table succ\n    admin.createTable(tableDescriptor, null, splitKeys);\n    HyperbaseMetadata metadata = admin.getTableMetadata(tableName);\n    assertTrue(metadata != null);\n    \/\/ check metadata\n    assertTrue(metadata.getFulltextMetadata() == null);\n    assertTrue(metadata.getGlobalIndexes().isEmpty());\n    assertTrue(metadata.getLocalIndexes().isEmpty());\n    assertTrue(metadata.getLobs().isEmpty());\n    assertTrue(metadata.isTransactionTable() == false);\n  }\n\n  public static void addLOB(byte[] tableName, byte[] family, byte[] LOBFamily) throws\n      IOException {\n    HyperbaseProtos.SecondaryIndex.Builder LOBBuilder = HyperbaseProtos.SecondaryIndex\n        .newBuilder();\n    LOBBuilder.setClassName(LOBIndex.class.getName());\n    LOBBuilder.setUpdate(true);\n    LOBBuilder.setDcop(true);\n    IndexedColumn column = new IndexedColumn(family, Bytes.toBytes(\"q1\"));\n    LOBBuilder.addColumns(column.toPb());\n    admin.addLob(TableName.valueOf(tableName), new LOBIndex(LOBBuilder.build()), LOBFamily, false, 2);\/\/ \u8fd9\u91cc\u7684'2'\u8868\u793a\u5728hdfs\u4e0a\u521b\u5efa\uff12\u4e2a\u526f\u672c\uff0c\u5728\u751f\u4ea7\u73af\u5883\u4e2d\u8bf7\u52a1\u5fc5\u786e\u4fdd\u8be5\u53c2\u6570\u5927\u4e8e1\uff01\n  }\n\n}<\/code><\/pre>\n<ol>\n<li>\n<p>\u901a\u8fc7\u4e0a\u8ff0\u4ee3\u7801\u5c06\u56fe\u7247\u5df2\u7ecf\u4fdd\u5b58\u5728\u5bf9\u5e94\u7684hbase\u8868\u4e2d<br \/>\n\u8fdb\u5165hbase shell \u67e5\u770b\u751f\u6210\u7684hbase\u8868\uff1a<\/p>\n<pre><code class=\"language-shell\">[root@mll01 ~]# hbase shell\n2020-03-04 15:17:36,080 INFO Configuration.deprecation: hadoop.native.lib is deprecated. Instead, use io.native.lib.available\n....\nhbase(main):001:0> list\nTABLE\nSLF4J: Class path contains multiple SLF4J bindings.\n...\n=> [\"SIMPLE_TEST_PUT_SCAN1134099364716900\", \"SIMPLE_TEST_PUT_SCAN1136461722430500\"]<\/code><\/pre>\n<p><img decoding=\"async\" src=\"\/wp-content\/uploads\/2020\/03\/image-1583306205641.png\" alt=\"file\" \/><\/p>\n<\/li>\n<li>\n<p>\u5728inceptor\u4e2d\u521b\u5efahbase\u5916\u8868<br \/>\n<strong>\u6ce8\u610f<\/strong>\uff1a<\/p>\n<ul>\n<li><strong><code>hbase.columns.mapping<\/code><\/strong> \u662f\u5b9a\u4e49\u6620\u5c04\u8868\u548cHBase\u8868\u5217\u4e4b\u95f4\u7684\u6620\u5c04\u5173\u7cfb\uff1b<\/li>\n<li><strong><code>hbase.table.name<\/code><\/strong> \u662f\u5b9a\u4e49\u6620\u5c04\u8868\u5bf9\u5e94\u7684HBase\u8868\u7684\u8868\u540d\n<pre><code class=\"language-sql\">CREATE EXTERNAL TABLE object_test(\nkey1 string,\nex1 BLOB\n)\nSTORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'\nWITH SERDEPROPERTIES (\"hbase.columns.mapping\"=\":key,f1:q1\") \nTBLPROPERTIES (\"hbase.table.name\"=\"SIMPLE_TEST_PUT_SCAN1136461722430500\");<\/code><\/pre>\n<\/li>\n<\/ul>\n<\/li>\n<li>\n<p>\u5728inceptor\u4e2d\u67e5\u8be2\u4fdd\u5b58\u7684\u56fe\u7247<\/p>\n<pre><code class=\"language-sql\">SELECT * FROM object_test;<\/code><\/pre>\n<p><img decoding=\"async\" src=\"\/wp-content\/uploads\/2020\/03\/image-1583307706257.png\" alt=\"file\" \/><\/p>\n<\/li>\n<\/ol>\n","protected":false},"excerpt":{"rendered":"<p>\u6982\u8981\u63cf\u8ff0 \u4f7f\u7528\u987b\u77e5\uff1a \u672c\u5b9e\u9a8c\u662f\u5728\u6ca1\u6709\u5f00\u542fguardian\u5b89\u5168\u8ba4\u8bc1\u7684\u73af\u5883\u4e2d\u6d4b\u8bd5\u7684\uff1b \u901a\u8fc7 API \u65b9\u5f0f\u6765\u4f7f\u7528Ob ..<\/p>\n<div class=\"clear-fix\"><\/div>\n<p><a href=\"https:\/\/kbwp.transwarp.cn\/?p=2565\" title=\"read more...\">Read more<\/a><\/p>\n","protected":false},"author":14,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[38],"tags":[],"class_list":["post-2565","post","type-post","status-publish","format-standard","hentry","category-configuration"],"acf":[],"_links":{"self":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/2565","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/users\/14"}],"replies":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=2565"}],"version-history":[{"count":1,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/2565\/revisions"}],"predecessor-version":[{"id":2595,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/2565\/revisions\/2595"}],"wp:attachment":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=2565"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=2565"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=2565"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}