{"id":11962,"date":"2024-02-22T09:30:18","date_gmt":"2024-02-22T01:30:18","guid":{"rendered":"https:\/\/nj.transwarp.cn:8180\/?p=11962"},"modified":"2024-02-22T09:30:18","modified_gmt":"2024-02-22T01:30:18","slug":"inceptor-server-jmap-%e4%b8%ad%e5%b8%b8%e8%a7%81-fgc-%e9%97%ae%e9%a2%98","status":"publish","type":"post","link":"https:\/\/kbwp.transwarp.cn\/?p=11962","title":{"rendered":"inceptor server jmap \u4e2d\u5e38\u89c1 FGC \u95ee\u9898"},"content":{"rendered":"<h3>\u6982\u8981\u63cf\u8ff0<\/h3>\n<p>\u65e9\u671f\u7248\u672c\u7684 inceptor server \u4f1a\u6709FGC \u5f02\u5e38\u7684\u60c5\u51b5\uff0c\u672c\u6587 \u8bb0\u5f55\u4e00\u4e0b jmap \u4e2d\u7279\u5b9a\u5173\u952e\u5b57\u5bf9\u5e94\u7684\u7279\u5b9a\u95ee\u9898\uff1b<\/p>\n<h3>\u8be6\u7ec6\u8bf4\u660e<\/h3>\n<p>\u53ea\u9488\u5bf9 TDH 6.2.2 \u53ca\u4e4b\u524d\u7248\u672c\u7684 inceptor server<\/p>\n<h4>Inceptor Server \u5e38\u89c1\u7684 GC \u76f8\u5173\u95ee\u9898<\/h4>\n<ol>\n<li>PL\/SQL\u6b7b\u5faa\u73afprofile\u4fe1\u606f\u79ef\u7d2f\n<pre><code class=\"language-java\">org.apache.hadoop.hive.ql.pl.runtime.PLProfile$PLFuncProfile\norg.apache.hadoop.hive.ql.hooks.PerfStatistics<\/code><\/pre>\n<p>\u89e3\u51b3\u65b9\u6848\uff1a<\/p>\n<ul>\n<li>\u4f4e\u7248\u672c\u624b\u52a8 set plsql.runtime.profile\u8bbe\u7f6e\u4e3afalse\uff0c\u53c2\u8003\u94fe\u63a5\uff1a<a href=\"http:\/\/172.16.0.244:8080\/browse\/WARP-29255\">http:\/\/172.16.0.244:8080\/browse\/WARP-29255<\/a><\/li>\n<\/ul>\n<\/li>\n<\/ol>\n<hr \/>\n<ol start=\"2\">\n<li>orc\u8868\u7684\u5217\u7edf\u8ba1\u4fe1\u606f\u8fc7\u591a\n<pre><code class=\"language-java\">org.apache.hadoop.hive.ql.io.orc.OrcProto$ColumnStatistics\norg.apache.hadoop.hive.ql.io.orc.OrcProto$StringStatistics\norg.apache.hadoop.hive.ql.io.orc.OrcProto$DecimalStatistics\norg.apache.hadoop.fs.FileSystem$Statistics$StatisticsData<\/code><\/pre>\n<p>\u89e3\u51b3\u65b9\u6848\uff1a<\/p>\n<ul>\n<li>hive-server2.log \u91cc\u9762\u641c FooterCacheHitRatio\uff0c \u5982\u679c\u7edd\u5927\u90e8\u5206\u662f0\uff0c\u53ef\u4ee5\u8003\u8651\u628ahive.orc.cache.stripe.details.size\u8bbe\u62100\u3002\u4f5c\u4e3a\u4e00\u4e2acache\uff0c\u5982\u679c\u547d\u4e2d\u7387\u6ca1\u6709\u8fbe\u523050%\u4ee5\u4e0a\uff0c\u8fd9\u4e2acache\u7684\u6574\u4f53\u6548\u679c\u4e0d\u4f1a\u5f88\u597d\u3002<\/li>\n<li>\u53e6\u5916\u4e00\u79cd\u60c5\u51b5\u53ef\u80fd\u662f orc \u8868\u7684\u5c0f\u6587\u4ef6\u6570\u91cf\u5f88\u591a\uff0c\u9700\u8981\u5408\u5e76\u5c0f\u6587\u4ef6\uff1b<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n<hr \/>\n<ol start=\"3\">\n<li>4040 Spark UI \u76d1\u63a7\u4fe1\u606fbug\n<pre><code class=\"language-java\">org.apache.spark.executor.TaskMetrics\norg.apache.spark.scheduler.TaskInfo\norg.apache.spark.storge.RDDInfo<\/code><\/pre>\n<p>\u89e3\u51b3\u65b9\u6848\uff1a<\/p>\n<ul>\n<li>\u91cd\u542f Inceptor \u7f13\u89e3\uff0c\u6216\u8005\u662f\u5347\u7ea7\u5230 dbaservice<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n<hr \/>\n<ol start=\"4\">\n<li>\u8fde\u63a5\u6570\u91cf\u592a\u591a\uff0cHiveConf \u5b9e\u4f8b\u6570\u91cf\u8d85\u8fc7 1w\n<pre><code class=\"language-java\">org.apache.hadoop.hive.conf.HiveConf<\/code><\/pre>\n<p>\u89e3\u51b3\u65b9\u6848\uff1a<\/p>\n<ul>\n<li>\u67e5\u770bbeeline \u8fde\u63a5\u7684 session \u4e2a\u6570\uff0c\u662f\u5426\u5b58\u5728\u8fde\u63a5\u6ca1\u6709\u91ca\u653e\uff1bctc\u8868\u592a\u5927\uff08\u5783\u573e\u592a\u591a\uff09HiveConf\u4e2d\u4f1a\u5305\u542bCTC\u8868\u7684\u4fe1\u606f\uff0c\u5982\u679c\u8bb0\u5f55\u6570\u76ee\u592a\u591a\uff0c\u5360\u7528\u5185\u5b58\u4f1a\u6bd4\u8f83\u591a\uff1b<\/li>\n<li>SELECT count(1) FROM hive_metastore.COMPLETED_TXN_COMPONENTS\uff0c\u8c03\u5927inceptor server\u5185\u5b58\uff0c\u9002\u5f53\u8c03\u5927ttl\u65f6\u95f4\uff0c\u6bd4\u5982server&amp;executor\u914d\u7f6e-Dspark.cleaner.ttl.BROADSACT_FAST\u53c2\u6570\u4e3a900 (\u6539\u53c2\u6570\u6ca1\u6709\u526f\u4f5c\u7528)\uff0c\u8be5\u53c2\u6570\u5f71\u54cd\u8303\u56f4\u5185\u5bb9\uff1a\u5f53Inceptor\u4ee5Cluster\u6a21\u5f0f\u201c\u9ad8\u5e76\u53d1\u201d\u8fde\u7eed\u6267\u884c\u6279\u5904\u7406\u4f5c\u4e1a\u65f6\uff08\u4e00\u822c\u662f\u5e73\u57471\u5206\u949f\u6267\u884c20\u4e2aSQL\u4ee5\u4e0a\uff09\u8fde\u7eed\u6267\u884c\u6570\u5c0f\u65f6\uff0c\u4f1a\u5bfc\u81f4Executor\u4e25\u91cdFullGC\uff0c\u5e76\u4e14\u65e0\u6cd5\u81ea\u7136\u7f13\u89e3<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n<hr \/>\n<ol start=\"5\">\n<li>\u8868\u7684 partition \u592a\u591a\uff08\u591a\u7ea7\u5206\u533a\uff09\uff0c\u5bfc\u81f4\u7684properity string\u7d2f\u79ef\u9020\u6210server gc\n<pre><code class=\"language-java\">org.apache.hadoop.hive.ql.io.orc.OrcSplit\norc.apache.spark.rdd.HadoopValueOnlyPartition\nio.transwarp.inceptor.inceptor.execution.HadoopTableReader$$annofun$10<\/code><\/pre>\n<p>\u89e3\u51b3\u65b9\u6848\uff1a<\/p>\n<ul>\n<li>\u4f18\u5316\u5206\u533a\u8868\uff0c\u4e0d\u5efa\u8bae\u4f7f\u7528\u591a\u7ea7\u5206\u533a\uff0c\u4f1a\u5bfc\u81f4\u5c0f\u6587\u4ef6\u95ee\u9898\u4ee5\u53ca\u52a0\u8f7d\u7684 fspath \u592a\u957f\uff0c\u6bcf\u4e00\u4e2aHadoopTableReader hold \u4e24\u4e2a property(partProp, tableProp)\u3002Properties \u6570\u91cf\u4e3a 54w<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n<hr \/>\n<ol start=\"6\">\n<li>\n<p>\u5c0f\u6587\u4ef6\u6570\u91cf\u8fc7\u591a\uff0c\u67e5\u8be2\u5bfc\u81f4 server gc<\/p>\n<pre><code class=\"language-java\">org.apache.hadoop.fs.Path\norg.apache.hadoop.fs.FileStatus<\/code><\/pre>\n<p>\u89e3\u51b3\u65b9\u6848\uff1a<\/p>\n<ul>\n<li>\u67e5\u770b\u5c0f\u6587\u4ef6\u662f\u4e0d\u662f\u7279\u522b\u591a\uff0c\u8868\u8bbe\u8ba1\u662f\u4e0d\u662f\u5408\u7406\u7684\uff08\u5b58\u5728\u4e0d\u5b58\u5728\u591a\u7ea7\u5206\u533a\u3001\u5f88\u591a\u5206\u6876\u3001delta\u6587\u4ef6\u6ca1\u6709\u5408\u5e76\u7684\u60c5\u51b5\uff09\uff1b\u5982\u679c\u662fmetastore\u7aef\uff0c\u662f\u4e0d\u662f\u5408\u5e76\u4efb\u52a1hand\u4f4f\u4e86\uff08warp-32079\uff09<\/li>\n<\/ul>\n<\/li>\n<li>\n<p>\u7279\u5b9asql parse \u4ea7\u751f\u5927\u91cf\u4e2d\u95f4\u6570\u636e\uff0c\u5bfc\u81f4 server gc<\/p>\n<pre><code class=\"language-java\">org.apache.hadoop.hive.ql.parse.ASTNode\norg.antlr.runtime.CommonToken<\/code><\/pre>\n<p>\u89e3\u51b3\u65b9\u6848\uff1a<\/p>\n<ul>\n<li>\u4e00\u822c\u662f\u56e0\u4e3a\u6709\u5927\u91cf\u7684 sql parse \u5bfc\u81f4\u7684\uff0c\u6bd4\u5982 batch insert\uff0c\u6216\u8005\u662f\u5f00\u4e86 MBO \u65f6\u67e5\u8be2\u8bed\u53e5\u5728\u5927\u91cf cube \u4e2d\u9009\u62e9\u65f6\u4f1a\u89e6\u53d1\u8fd9\u79cd\u95ee\u9898\uff1b\u53ef\u4ee5\u5c1d\u8bd5\u68c0\u67e5\u5f53\u65f6\u63d0\u4ea4\u7684 sql \u5927\u5c0f\uff1b<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n<hr \/>\n<ol start=\"8\">\n<li>holodesk \u8868 accumulable \u6cc4\u6f0f\n<pre><code class=\"language-java\">io.transwarp.holodesk.storage.ds.column.block.filter.MinMaxBlockFilter<\/code><\/pre>\n<p>\u89e3\u51b3\u65b9\u6848\uff1a<\/p>\n<ul>\n<li>\u5347\u7ea7\u7248\u672c\uff08\u51fa\u5305\uff09\u6216\u8005\u91cd\u542f\u7f13\u89e3<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n<hr \/>\n<ol start=\"9\">\n<li>with as &#8230;select join \u8bed\u53e5\uff0c\u5728\u5916\u5c42\u67e5\u8be2\u5bf9\u8be5\u4e34\u65f6\u8868\u505a join\uff0c\u800c\u5728 Driver \u7684\u903b\u8f91\u91cc\uff0c\u4f1a\u628a\u67e5\u8be2\u7684\u7ed3\u679c\u5168\u90e8\u5806\u79ef\u5728cachedPlanResult \u8fd9\u4e2a ArrayList \u91cc\u9762\u4ece\u800c\u5bfc\u81f4 GC\n<pre><code class=\"language-java\">org.apache.hadoop.hive.common.type.HiveVarchar\norg.apache.hadoop.hive.common.type.HiveData\norg.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector<\/code><\/pre>\n<p>\u89e3\u51b3\u65b9\u6848\uff1a<\/p>\n<ul>\n<li>\u4fee\u6539 SQL\uff0c\u53c2\u8003 <a href=\"https:\/\/jira.transwarp.io\/browse\/WARP-29073\">https:\/\/jira.transwarp.io\/browse\/WARP-29073<\/a><\/li>\n<\/ul>\n<\/li>\n<\/ol>\n<hr \/>\n<ol start=\"10\">\n<li>\n<p>stellerDB \u56fe\u6570\u636e\u5e93\u76f8\u5173 zk \u8fde\u63a5\u6cc4\u9732\u5bfc\u81f4<\/p>\n<pre><code class=\"language-java\">org.apache.curator.framework.imps.NamespaceWatcher<\/code><\/pre>\n<p>\u89e3\u51b3\u65b9\u6848\uff1a<\/p>\n<ul>\n<li>\u8fdb\u5165 zkcli \u521b\u5efa\u51e0\u4e2a\u7a7a\u7684 node\n<pre><code class=\"language-shell\">export CLIENT_JVMFLAGS=\"-Djava.security.auth.login.config=\/etc\/zookeeper1\/conf\/jaas.conf\"\n# \u6b64\u5904\u5fc5\u987b\u4f7f\u7528 hostname \uff0c\u4e0d\u80fd\u7528 ip\nzookeeper-client -server zk-address-host:2181\ncreate \/graph \"\"\ncreate \/graph\/master \"\", create \/graph\/worker \"\", create \/graph\/database \"\"\n\u7136\u540e\u91cd\u542f inceptor-server<\/code><\/pre>\n<hr \/>\n<\/li>\n<\/ul>\n<\/li>\n<li>\n<p>\u4efb\u52a1\u5931\u8d25\u4f1a\u4fdd\u7559\u6700\u5c0f\u7ebf\u7a0b\u6570\u7684 session \u4e0d\u65ad\u5f00\uff0c\u5982\u679c\u521a\u597d\u8fd9\u4e9b\u6ca1\u65ad\u5f00\u7684session \u4e2d partition\u76f8\u5173\u5185\u5b58\u975e\u5e38\u591a\uff0c\u5c31\u4f1a\u9020\u6210 server \u5185\u5b58\u538b\u529b\u3002<\/p>\n<pre><code class=\"language-java\">org.apache.hadoop.mapred.SplitLocationInfo\norg.apache.hadoop.mapreduce.lib.input.FileSplit\norg.apache.hadoop.hive.metastore.api.FieldSchema\norg.apache.spark.rdd.HadoopValueOnlyPartition<\/code><\/pre>\n<p>\u89e3\u51b3\u65b9\u6848\uff1a<\/p>\n<ul>\n<li>\u91cd\u542f\u9700\u8981\u6362\u5305\u4fee\u590d\uff1b\u53c2\u8003<a href=\"https:\/\/jira.transwarp.io\/browse\/WARP-30965\" title=\"threadToSessions\u5728closeSession\u65f6\u6ca1\u6709\u6e05\u7406\u4fe1\u606f\u5bfc\u81f4session\u6cc4\u9732\">threadToSessions\u5728closeSession\u65f6\u6ca1\u6709\u6e05\u7406\u4fe1\u606f\u5bfc\u81f4session\u6cc4\u9732<\/a><\/li>\n<li>\u4e5f\u53ef\u4ee5\u91cd\u542fInceptor\uff0c\u7136\u540e\u91cd\u5efa\u8868\u7ed3\u6784\uff0c\u5c06\u5206\u533a\u6570\u91cf\u51cf\u5c0f\u5230\u53ef\u63a7\u8303\u56f4\u5185\u6765\u89c4\u907f\u95ee\u9898<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n<hr \/>\n","protected":false},"excerpt":{"rendered":"<p>\u6982\u8981\u63cf\u8ff0 \u65e9\u671f\u7248\u672c\u7684 inceptor server \u4f1a\u6709FGC \u5f02\u5e38\u7684\u60c5\u51b5\uff0c\u672c\u6587 \u8bb0\u5f55\u4e00\u4e0b jmap \u4e2d\u7279\u5b9a ..<\/p>\n<div class=\"clear-fix\"><\/div>\n<p><a href=\"https:\/\/kbwp.transwarp.cn\/?p=11962\" title=\"read more...\">Read more<\/a><\/p>\n","protected":false},"author":11,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-11962","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"acf":[],"_links":{"self":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/11962","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/users\/11"}],"replies":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=11962"}],"version-history":[{"count":4,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/11962\/revisions"}],"predecessor-version":[{"id":11968,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/11962\/revisions\/11968"}],"wp:attachment":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=11962"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=11962"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=11962"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}