{"id":9540,"date":"2024-07-30T14:53:16","date_gmt":"2024-07-30T06:53:16","guid":{"rendered":"https:\/\/nj.transwarp.cn:8180\/?p=9540"},"modified":"2024-07-30T14:53:15","modified_gmt":"2024-07-30T06:53:15","slug":"tdc-client%e4%b8%ad%e4%bd%bf%e7%94%a8kafka%e5%af%b9%e6%8e%a5slipstream%ef%bc%88%e5%ae%89%e5%85%a8%ef%bc%89","status":"publish","type":"post","link":"https:\/\/kbwp.transwarp.cn\/?p=9540","title":{"rendered":"TDC-client\u4e2d\u4f7f\u7528kafka\u5bf9\u63a5slipstream\uff08\u5b89\u5168\uff09"},"content":{"rendered":"<h3>\u6982\u8981\u63cf\u8ff0<\/h3>\n<p>\u672c\u6587\u63cf\u8ff0\u96c6\u7fa4\u5916\u4f7f\u7528tdc-client\uff0c\u5982\u4f55\u8fbe\u5230\uff1b\u94fe\u63a5kafka\uff0c\u4e14\u914d\u7f6e\u5bf9\u63a5slipstream\u7684\u6b65\u9aa4<\/p>\n<h3>\u8be6\u7ec6\u63cf\u8ff0<\/h3>\n<h4>Kafka \u90e8\u5206<\/h4>\n<h5>1 tcc\u9875\u9762\u4fee\u6539\u5bf9\u5e94kafka\u7684\u9ad8\u7ea7\u53c2\u6570<\/h5>\n<p>\u5c06Advance_Config.Access_Type\u6539\u4e3aprivate\uff0c\u7136\u540e\u91cd\u542fkafka<br \/>\n<img decoding=\"async\" src=\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2023\/02\/image-1675931781637.png\" alt=\"file\" \/><\/p>\n<h5>2 \u786e\u8ba4kafka\u7684controller\u8282\u70b9\u548cnodeport\u7aef\u53e3<\/h5>\n<p>\u5728tdc\u96c6\u7fa4\u8282\u70b9\u8fdb\u5165\u5bf9\u63a5\u5b9e\u4f8b\u7684zookeeper\u7684pod\uff0c\u6267\u884c<\/p>\n<pre><code class=\"language-shell\">\/usr\/lib\/zookeeper\/bin\/zkCli.sh<\/code><\/pre>\n<p><img decoding=\"async\" src=\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2023\/02\/image-1675932428786.png\" alt=\"file\" \/><\/p>\n<p>\u7136\u540e\u6267\u884cget \/controller\uff0c\u5f97\u5230brokerid<br \/>\n<img decoding=\"async\" src=\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2023\/02\/image-1675932498284.png\" alt=\"file\" \/><br \/>\n\u77e5\u9053id\u6570\u5b57\u540e\u53ef\u4ee5\u6267\u884c\uff0c\u53ef\u4ee5\u5f97\u5230kafka\u7684controller\u8282\u70b9\u548cnodeport\u7aef\u53e3<\/p>\n<pre><code class=\"language-shell\">get \/brokers\/ids\/{brokerid}<\/code><\/pre>\n<p><img decoding=\"async\" src=\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2023\/02\/image-1675932528790.png\" alt=\"file\" \/><\/p>\n<h5>3 \u8c03\u6574\u96c6\u7fa4\u5916\u7684\u8282\u70b9\u65f6\u95f4\u548ctdc\u96c6\u7fa4\u7684\u65f6\u95f4\u4e00\u81f4<\/h5>\n<h5>4 \u96c6\u7fa4\u5916\u8282\u70b9\u9996\u5148kinit kafka<\/h5>\n<p>\u4f8b\u5982<\/p>\n<pre><code class=\"language-shell\">cd \/root\/tdc-client\/conf\/..\/.env\/cqb39ed_clus-19\nkinit kafka\/tos_cqb39ed@CQB39ED.TDH -kt keytabs\/keytab<\/code><\/pre>\n<h5>5 \u6839\u636enodeport\u7aef\u53e3\uff0c\u521b\u5efatopic\u3001producer\u3001consumer<\/h5>\n<p>\u521b\u5efatopic\uff0ckafka\u76f8\u5173\u811a\u672c\u5df2\u96c6\u6210\u73af\u5883\u53d8\u91cf<\/p>\n<pre><code class=\"language-shell\">kafka-broker-topics.sh  --bootstrap-server 172.22.28.1:32192 --create --topic test1 --partitions 1 --replication-factor 1 --consumer.config .\/etc\/kafka\/conf\/consumer.properties\n<\/code><\/pre>\n<p>\u5176\u4e2d 172.22.28.1 \u662ftdc\u96c6\u7fa4\u7684master \u670d\u52a1\u5668\u4efb\u610fip\u5730\u5740\u5373\u53ef<br \/>\n32192 \u7aef\u53e3\u4e3a\u6b65\u9aa42 \u91cc\u5728zk\u91cc\u5f97\u5230\u7684\u7aef\u53e3\u4fe1\u606f<\/p>\n<p>PS\uff1a\u5982\u679c\u662f7.0\u7248\u672c\u4f7f\u7528kafka-topics.sh\u811a\u672c<\/p>\n<p>\u521b\u5efaproducer<\/p>\n<pre><code class=\"language-shell\">kafka-console-producer.sh --topic test1 --broker-list 172.22.28.1:32192  --producer.config .\/etc\/kafka\/conf\/consumer.properties<\/code><\/pre>\n<p>\u521b\u5efaconsumer<br \/>\n\u65b0\u5f00\u4e00\u4e2aterminal\uff0c\u7136\u540esource tdc-client\u7684\u73af\u5883\uff0ckinit kafka\u7528\u6237<br \/>\ncd \/root\/tdc-client\/conf\/..\/.env\/cqb39ed_clus-19<\/p>\n<pre><code class=\"language-shell\">kafka-console-consumer.sh --topic test1 --bootstrap-server\u00a0172.22.28.1:32192 --consumer.config .\/etc\/kafka\/conf\/consumer.properties --from-beginning\n<\/code><\/pre>\n<h4>Slipstream \u5bf9\u63a5kafka<\/h4>\n<h5>1 beeline\u94fe\u63a5slipstream<\/h5>\n<p>\u901a\u8fc7svc\u786e\u8ba4\u76f8\u5173slipstream\u7684nodeport  \uff0c\u901a\u8fc7beeline\u8fdb\u884c\u94fe\u63a5<\/p>\n<pre><code class=\"language-shell\">kubectl get svc -n xxx | grep 10010<\/code><\/pre>\n<h5>2 \u521b\u5efastream\u6d41<\/h5>\n<p>\u6a21\u677f<br \/>\n<img decoding=\"async\" src=\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2023\/02\/image-1675933353840.png\" alt=\"file\" \/><\/p>\n<pre><code class=\"language-sql\">create stream if not exists demo(\nid INT,name STRING)\nrow format delimited fields terminated by ','\ntblproperties (\n  'topic'='{topic}',\n  'transwarp.producer.sasl.kerberos.service.name'='kafka',\n  'transwarp.consumer.sasl.kerberos.service.name'='kafka',\n  'transwarp.producer.security.protocol'='SASL_PLAINTEXT',\n  'transwarp.consumer.security.protocol'='SASL_PLAINTEXT',\n  'source'='kafka',\n\"kafka.zookeeper\"=\"{zk_pod_ip-1}:2181,{zk_pod_ip-2}:2181,{zk_pod_ip-3}:2181\",\n  'transwarp.producer.sasl.jaas.config'='com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true keyTab=\\\"\/etc\/keytabs\/keytab\\\" principal=\\\"{principal}\\\"',\n  'transwarp.consumer.sasl.jaas.config'='com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true keyTab=\\\"\/etc\/keytabs\/keytab\\\" principal=\\\"{principal}\\\"',\n  'transwarp.consumer.sasl.kerberos.service.principal.instance'='tos_{ns}',\n\"kafka.broker.list\"=\"{kafka_podname}-0.{kafka_podname}.{ns}.svc.transwarp.local:9092,{kafka_podname}-1.{kafka_podname}.{ns}.cqb39ed.svc.transwarp.local:9092,{kafka_podname}-2.{kafka_podname}.{ns}.svc.transwarp.local:9092\");\n<\/code><\/pre>\n<p>\u5176\u4e2d\u9700\u8981\u66ff\u6362\u7684\u53d8\u91cf\u5982\u4e0b<br \/>\n{topic}\u5bf9\u63a5kafka\u7684topic<br \/>\n{zk_pod_ip}\u662fzookeeper\u7684ip\u5730\u5740\uff0c\u9700\u8981\u586b\u51993\u4e2a\uff0c\u53ef\u4ee5\u4ece\u3010\u7ec4\u4ef6\u4fe1\u606f\u3011-\u3010pod\u4fe1\u606f\u3011\u4e2d\u83b7\u53d6<br \/>\n<img decoding=\"async\" src=\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2023\/02\/image-1675933522582.png\" alt=\"file\" \/><\/p>\n<p>{principal}\u662fkafka\u7684principal\uff0c\u901a\u5e38\u4e3akafka\/tos_{namespace}@{NAMESPACE}.TDH<br \/>\n{ns}\u79df\u6237\u7684namespace<br \/>\n{kafka_podname}kafka\u7684pod\u540d\u5b57\uff0c\u53ef\u4ee5\u4ece\u3010\u7ec4\u4ef6\u4fe1\u606f\u3011-\u3010pod\u4fe1\u606f\u3011\u4e2d\u83b7\u53d6<\/p>\n<p><img decoding=\"async\" src=\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2023\/02\/image-1675933622126.png\" alt=\"file\" \/><\/p>\n<p>\u793a\u4f8b\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-sql\">create stream if not exists input_stream_demo1(\nid INT,name STRING)\nrow format delimited fields terminated by ','\ntblproperties (\n  'topic'='test1',\n  'transwarp.producer.sasl.kerberos.service.name'='kafka',\n  'transwarp.consumer.sasl.kerberos.service.name'='kafka',\n  'transwarp.producer.security.protocol'='SASL_PLAINTEXT',\n  'transwarp.consumer.security.protocol'='SASL_PLAINTEXT',\n  'source'='kafka',\n\"kafka.zookeeper\"=\"10.16.49.69:2181,10.16.53.55:2181,10.16.29.45:2181\",\n  'transwarp.producer.sasl.jaas.config'='com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true keyTab=\\\"\/etc\/keytabs\/keytab\\\" principal=\\\"kafka\/tos_cqb39ed@CQB39ED.TDH\\\"',\n  'transwarp.consumer.sasl.jaas.config'='com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true keyTab=\\\"\/etc\/keytabs\/keytab\\\" principal=\\\"kafka\/tos_cqb39ed@CQB39ED.TDH\\\"',\n  'transwarp.consumer.sasl.kerberos.service.principal.instance'='tos_cqb39ed',\n\"kafka.broker.list\"=\"kafka-ljb6l-0.kafka-ljb6l.cqb39ed.svc.transwarp.local:9092,kafka-ljb6l-1.kafka-ljb6l.cqb39ed.svc.transwarp.local:9092,kafka-ljb6l-2.kafka-ljb6l.cqb39ed.svc.transwarp.local:9092\");<\/code><\/pre>\n<h5>3 \u521b\u5efa\u8868\u53ca\u6d41\u4efb\u52a1<\/h5>\n<pre><code class=\"language-sql\">#\u521b\u5efa\u8868\nCREATE TABLE demo1_table(id INT,name STRING);\n#\u521b\u5efa\u6301\u4e45\u5316\u7684\u6d41\u4efb\u52a1\nCREATE STREAMJOB s2 AS(\"insert into demo1_table select * from input_stream_demo1\");\n#\u5f00\u59cb\u6d41\u4efb\u52a1\nSTART STREAMJOB s2;\n#kafka\u7684producer\u53d1\u9001\u6570\u636e\uff0c\u67e5\u770b\u843d\u5165\u6570\u636e\nSELECT * FROM demo1_table;\n#\u505c\u6b62\u6d41\u4efb\u52a1\nstop STREAMJOB s2;\n#\u9ed8\u8ba4\u843d\u8868\u65f6\u95f460s\uff0c\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u53ef\u4ee5\u7f29\u77ed\u4e3a1\u79d2\uff0c\u9700\u8981\u5148stop\u518dstart\nALTER STREAMJOB s2 SET JOBPROPERTIES(\"morphling.hdfs.flush.interval.ms\"=\"1000\");<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"<p>\u6982\u8981\u63cf\u8ff0 \u672c\u6587\u63cf\u8ff0\u96c6\u7fa4\u5916\u4f7f\u7528tdc-client\uff0c\u5982\u4f55\u8fbe\u5230\uff1b\u94fe\u63a5kafka\uff0c\u4e14\u914d\u7f6e\u5bf9\u63a5slipstream\u7684\u6b65 ..<\/p>\n<div class=\"clear-fix\"><\/div>\n<p><a href=\"https:\/\/kbwp.transwarp.cn\/?p=9540\" title=\"read more...\">Read more<\/a><\/p>\n","protected":false},"author":15,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-9540","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"acf":[],"_links":{"self":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/9540","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/users\/15"}],"replies":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=9540"}],"version-history":[{"count":1,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/9540\/revisions"}],"predecessor-version":[{"id":9554,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/9540\/revisions\/9554"}],"wp:attachment":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=9540"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=9540"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=9540"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}