https:\/\/kb.transwarp.cn\/posts\/11406<\/a><\/p>\n2\u3001 argo-slipstream\u521b\u5efastream\/streamjob<\/p>\n
\u8fde\u63a5slipstream\u6570\u636e\u5e93\uff1a\nbeeline -u \"jdbc:hive2:\/\/172.22.37.181:10010\/default\" -n hive -p 123456\n\n\u521b\u5efastream\uff1a\nCREATE stream lj.stream1<\/code> (\ncolum1 string DEFAULT NULL COMMENT '', colum2 bigint DEFAULT NULL COMMENT '', op_type string) \nROW FORMAT DELIMITED FIELDS TERMINATED BY '\\001'\ntblproperties ('kafka.zookeeper'='172.22.37.181:2181,172.22.37.182:2181,172.22.37.183:2181','kafka.broker.list'='172.22.37.181:9092,172.22.37.182:9092,172.22.37.183:9092',\n'kafka.decoder'='io.transwarp.slipstream.api.decoder.dbz.DBZBinlogDecoder', -- cdc\u573a\u666f\u4e0b\u4f7f\u7528\u7684\u89e3\u6790\u51fd\u6570\u4e4b\u4e00\n'kafka.decoder.columns.type'='string\/bigint\/string',\n'kafka.decoder.columns'='colum1\/colum2\/op_type',\n'transwarp.consumer.security.protocol'='SASL_PLAINTEXT',\n'transwarp.consumer.sasl.kerberos.service.name'='kafka',\n'transwarp.consumer.sasl.jaas.config'='com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true \nstoreKey=true keyTab=\\\"\/etc\/slipstream1\/conf\/test.keytab\\\"\n principal=\\\"test@argodb6x\\\"',\n 'topic'='demo1');\n\n \u521b\u5efa\u6210\u529f\u540e\u53ef\u4ee5 show streams;\u67e5\u770b\u3002\n\n \u521b\u5efa\u6d41\u4efb\u52a1streamjob\uff1a\ndrop streamjob test_streamjob1;\ncreate streamjob test_streamjob1 as(\"insert \/*+USE_DELETE_MARK*\/ into out_t1 select colum1,colum2,case when op_type='D' then true else false end as delete_mark from lj.stream1<\/code> \") jobproperties (\"morphling.hdfs.flush.interval.ms\"=\"1000\",\"morphling.job.enable.checkpoint\"=\"true\");\n\n\u521b\u5efa\u76ee\u6807holodesk\u8868\uff1a\nCREATE table lj.out_t1 ( colum1 string DEFAULT NULL COMMENT '', colum2 bigint DEFAULT NULL COMMENT '') CLUSTERED BY (colum2) INTO 9 BUCKETS STORED AS HOLODESK TBLPROPERTIES ('holodesk.rowkey'='colum2') ;\n\n\u542f\u52a8streamjob\uff1a\nstart streamjob test_streamjob1\uff1b\n\n\u53ef\u4ee5\u901a\u8fc7list streamjob\u67e5\u770b\u6b63\u5728\u8fd0\u884c\u7684streamjob<\/code><\/pre>\n3\u3001 Kafka\u751f\u4ea7-\u6d88\u8d39\u6570\u636e<\/p>\n
export KAFKA_OPTS=\"-Djava.security.auth.login.config=\/etc\/eventstore1\/conf\/jaas.conf -Djava.security.krb5.conf=\/etc\/eventstore1\/conf\/krb5.conf\"\n\n\u751f\u4ea7\u6570\u636e\uff1a\n.\/kafka-console-producer.sh --broker-list 172.22.37.181:9092,172.22.37.182:9092,172.22.37.183:9092 --topic demo1 --producer.config \/root\/TDH-Client\/kafka\/config\/security-client.properties\n\n\u53e6\u5916\u5f00\u4e2ashell\u7a97\u53e3\n\u6d88\u8d39\u6570\u636e\uff1a\n.\/kafka-console-consumer.sh --bootstrap-server 172.22.37.181:9092,172.22.37.182:9092,172.22.37.183:9092 --topic demo1 --consumer.config \/root\/TDH-Client\/kafka\/config\/security-client.properties<\/code><\/pre>\n\u5728\u751f\u4ea7\u6570\u636eshell\u4e2d\u624b\u52a8\u8f93\u5165\u4e00\u4e9b\u6570\u636e\uff1a
\n\u793a\u4f8b\uff1a<\/p>\n
{\"before\":null,\"after\":{\"colum1\":\"3\",\"colum2\":5656},\"source\":{\"version\":\"1.9.0-SNAPSHOT\",\"connector\":\"mysql\",\"name\":\"9982bb3099ec459fbf53775194cb98cf\",\"ts_ms\":1730269165000,\"snapshot\":\"false\",\"db\":\"test_rt\",\"sequence\":null,\"positionInfo\":\"{\\n \\\"filename\\\" : \\\"mysql-bin.000007\\\",\\n \\\"position\\\" : 217432819\\n}\",\"schemaVersion\":\"1\",\"tableIdentifier\":\"7A7FE622BA3B2082CC612771B915EA89\",\"table\":\"realtime_table\",\"server_id\":1,\"gtid\":null,\"file\":\"mysql-bin.000007\",\"pos\":217432957,\"nextPos\":217433004,\"row\":0,\"thread\":23539,\"query\":null},\"op\":\"c\",\"syncLabel\":\"2024103014159982bb3099ec459fbf53775194cb98cf\",\"ts_ms\":1730269165411,\"transaction\":null}<\/code><\/pre>\n![\"file\"](\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2024\/10\/image-1730283145203.png\")
\n\u6d88\u8d39\u7aefshell\u53ef\u4ee5\u5b9e\u65f6\u63a5\u6536\u5230\u6570\u636e\uff1a
\n![\"file\"](\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2024\/10\/image-1730283202128.png\")
<\/p>\n
4\u3001\u67e5\u770bholo\u8868\uff0c\u53ef\u67e5\u770b\u5230\u6d41\u6570\u636e
\n![\"file\"](\"https:\/\/nj.transwarp.cn:8180\/wp-content\/uploads\/2024\/10\/image-1730367536895.png\")
<\/p>\n","protected":false},"excerpt":{"rendered":"
\u3010\u6d4b\u8bd5\u80cc\u666f\u3011 \u4f7f\u7528kafka\uff0c\u5c06\u5b9e\u65f6\u6570\u636e\u6d41\u5165ArgoSlipstream holodesk\u8868\u4e2d\u3002CDC\u573a\u666f\u7528\u4e8e ..<\/p>\n
<\/div>\n
Read more<\/a><\/p>\n","protected":false},"author":155,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-14592","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"acf":[],"_links":{"self":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/14592","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/users\/155"}],"replies":[{"embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=14592"}],"version-history":[{"count":3,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/14592\/revisions"}],"predecessor-version":[{"id":17091,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=\/wp\/v2\/posts\/14592\/revisions\/17091"}],"wp:attachment":[{"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=14592"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=14592"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/kbwp.transwarp.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=14592"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}