diff --git "a/app/.vitepress/public/file/blog/deployment\346\226\207\344\273\266.pdf" "b/app/.vitepress/public/file/blog/deployment\346\226\207\344\273\266.pdf"
deleted file mode 100644
index 71ecd20eac3f12546cfa8d0b06b5ac1ba4cb9c3f..0000000000000000000000000000000000000000
Binary files "a/app/.vitepress/public/file/blog/deployment\346\226\207\344\273\266.pdf" and /dev/null differ
diff --git a/app/.vitepress/public/file/blog/jmeter-test.jmx.pdf b/app/.vitepress/public/file/blog/jmeter-test.jmx.pdf
deleted file mode 100644
index e4fee5805902cc7196f986a3af6febd9cd861893..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/file/blog/jmeter-test.jmx.pdf and /dev/null differ
diff --git a/app/.vitepress/public/file/blog/openLooKeng-pushdown.pdf b/app/.vitepress/public/file/blog/openLooKeng-pushdown.pdf
deleted file mode 100644
index d2f8adb5354f825bf910a3d39e45d02643ddb628..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/file/blog/openLooKeng-pushdown.pdf and /dev/null differ
diff --git "a/app/.vitepress/public/file/blog/openLooKeng\345\244\232\345\210\206\347\211\207\347\256\241\347\220\206\347\211\271\346\200\247\344\273\213\347\273\215.pdf" "b/app/.vitepress/public/file/blog/openLooKeng\345\244\232\345\210\206\347\211\207\347\256\241\347\220\206\347\211\271\346\200\247\344\273\213\347\273\215.pdf"
deleted file mode 100644
index 93945e6f6b11008de366d3791343a1c6d40d3686..0000000000000000000000000000000000000000
Binary files "a/app/.vitepress/public/file/blog/openLooKeng\345\244\232\345\210\206\347\211\207\347\256\241\347\220\206\347\211\271\346\200\247\344\273\213\347\273\215.pdf" and /dev/null differ
diff --git a/app/.vitepress/public/file/blog/oracle-connector.pdf b/app/.vitepress/public/file/blog/oracle-connector.pdf
deleted file mode 100644
index bf43bbf5ce8fe564d926da0abfe46e6343088d7a..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/file/blog/oracle-connector.pdf and /dev/null differ
diff --git a/app/.vitepress/public/file/blog/text-tpdcs.csv.xlsx b/app/.vitepress/public/file/blog/text-tpdcs.csv.xlsx
deleted file mode 100644
index cd84f8690833cc23e00e79ded7b321547b65039a..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/file/blog/text-tpdcs.csv.xlsx and /dev/null differ
diff --git a/app/.vitepress/public/file/evangelist/openLooKeng Evangelist Application Form.xlsx b/app/.vitepress/public/file/evangelist/openLooKeng Evangelist Application Form.xlsx
deleted file mode 100644
index d9c68e60a8a40d1a1f2277e45d9b1b2ddfca8fff..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/file/evangelist/openLooKeng Evangelist Application Form.xlsx and /dev/null differ
diff --git a/app/.vitepress/public/file/evangelist/openLooKeng Senior Developer Application Form.xlsx b/app/.vitepress/public/file/evangelist/openLooKeng Senior Developer Application Form.xlsx
deleted file mode 100644
index 9360ef413b2a729d66b9c021d044d05fd1619f90..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/file/evangelist/openLooKeng Senior Developer Application Form.xlsx and /dev/null differ
diff --git "a/app/.vitepress/public/file/evangelist/openLooKeng\347\244\276\345\214\272\344\274\230\347\247\200\345\274\200\345\217\221\350\200\205\346\212\245\345\220\215\347\224\263\350\257\267\350\241\250.xlsx" "b/app/.vitepress/public/file/evangelist/openLooKeng\347\244\276\345\214\272\344\274\230\347\247\200\345\274\200\345\217\221\350\200\205\346\212\245\345\220\215\347\224\263\350\257\267\350\241\250.xlsx"
deleted file mode 100644
index 98d9eca839cd3bce23ca087323b138125b4ad23a..0000000000000000000000000000000000000000
Binary files "a/app/.vitepress/public/file/evangelist/openLooKeng\347\244\276\345\214\272\344\274\230\347\247\200\345\274\200\345\217\221\350\200\205\346\212\245\345\220\215\347\224\263\350\257\267\350\241\250.xlsx" and /dev/null differ
diff --git "a/app/.vitepress/public/file/evangelist/openLooKeng\347\244\276\345\214\272\345\270\203\351\201\223\345\270\210\346\212\245\345\220\215\347\224\263\350\257\267\350\241\250.xlsx" "b/app/.vitepress/public/file/evangelist/openLooKeng\347\244\276\345\214\272\345\270\203\351\201\223\345\270\210\346\212\245\345\220\215\347\224\263\350\257\267\350\241\250.xlsx"
deleted file mode 100644
index 0a7c73a008879c9d0454d4a09b840b5bf5b51d6c..0000000000000000000000000000000000000000
Binary files "a/app/.vitepress/public/file/evangelist/openLooKeng\347\244\276\345\214\272\345\270\203\351\201\223\345\270\210\346\212\245\345\220\215\347\224\263\350\257\267\350\241\250.xlsx" and /dev/null differ
diff --git a/app/.vitepress/public/img/activity/bugs.png b/app/.vitepress/public/img/activity/bugs.png
deleted file mode 100644
index 138bc71930f9fa2195f7ef596aad38fc7a4f7bdc..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/activity/bugs.png and /dev/null differ
diff --git a/app/.vitepress/public/img/activity/internship-en.png b/app/.vitepress/public/img/activity/internship-en.png
deleted file mode 100644
index 15ece028cce41800e970ff1ee7ee0732ba49593c..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/activity/internship-en.png and /dev/null differ
diff --git a/app/.vitepress/public/img/activity/internship.png b/app/.vitepress/public/img/activity/internship.png
deleted file mode 100644
index 797cdfb985055ee68e840c26260faa76d84b44e6..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/activity/internship.png and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/logo/jpg/openLooKengLogo1.jpg b/app/.vitepress/public/img/brand/logo/jpg/openLooKengLogo1.jpg
deleted file mode 100644
index 5df580880f68f1a71a62b253b35b90942f629622..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/logo/jpg/openLooKengLogo1.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/logo/jpg/openLooKengLogo2.jpg b/app/.vitepress/public/img/brand/logo/jpg/openLooKengLogo2.jpg
deleted file mode 100644
index a62ecc0610d36515083b0d87d2852612155a93cf..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/logo/jpg/openLooKengLogo2.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/logo/jpg/openLooKengLogo3.jpg b/app/.vitepress/public/img/brand/logo/jpg/openLooKengLogo3.jpg
deleted file mode 100644
index 7e2c2d33208bbaad8d75a85e6de9517223e68259..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/logo/jpg/openLooKengLogo3.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/logo/jpg/openLooKengLogo4.jpg b/app/.vitepress/public/img/brand/logo/jpg/openLooKengLogo4.jpg
deleted file mode 100644
index e27ab1cb089b8a1f82115af48b85838355a30451..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/logo/jpg/openLooKengLogo4.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/logo/jpg/openLooKengLogo5.jpg b/app/.vitepress/public/img/brand/logo/jpg/openLooKengLogo5.jpg
deleted file mode 100644
index 444a0201044add6e436e12c3b4870c29fa65ae1b..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/logo/jpg/openLooKengLogo5.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/logo/jpg/openLooKengLogo6.jpg b/app/.vitepress/public/img/brand/logo/jpg/openLooKengLogo6.jpg
deleted file mode 100644
index 7f02b60e09e21aec7b72389017a70eb5b5a88da9..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/logo/jpg/openLooKengLogo6.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/logo/png/openLooKengLogo1.png b/app/.vitepress/public/img/brand/logo/png/openLooKengLogo1.png
deleted file mode 100644
index 03064b6daaba0e42772366774daeca58835113fb..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/logo/png/openLooKengLogo1.png and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/logo/png/openLooKengLogo2.png b/app/.vitepress/public/img/brand/logo/png/openLooKengLogo2.png
deleted file mode 100644
index 4037a05d4a76c97a277a805c8cd614a9e1841028..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/logo/png/openLooKengLogo2.png and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/logo/png/openLooKengLogo3.png b/app/.vitepress/public/img/brand/logo/png/openLooKengLogo3.png
deleted file mode 100644
index 5e800f783b6ac203a71c88ddd32baa09b3748677..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/logo/png/openLooKengLogo3.png and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/logo/png/openLooKengLogo4.png b/app/.vitepress/public/img/brand/logo/png/openLooKengLogo4.png
deleted file mode 100644
index 55ec5437fdb2177b74d763ab16712f2ca45a382d..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/logo/png/openLooKengLogo4.png and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/logo/png/openLooKengLogo5.png b/app/.vitepress/public/img/brand/logo/png/openLooKengLogo5.png
deleted file mode 100644
index 12229c9cd2953cc2c35d0df189e5717c417ffda8..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/logo/png/openLooKengLogo5.png and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/logo/png/openLooKengLogo6.png b/app/.vitepress/public/img/brand/logo/png/openLooKengLogo6.png
deleted file mode 100644
index 805706e2d0e6e0d971659a64df3fe93eb6a9597a..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/logo/png/openLooKengLogo6.png and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/logo/svg/openLooKengLogo1.svg b/app/.vitepress/public/img/brand/logo/svg/openLooKengLogo1.svg
deleted file mode 100644
index 18cafcf934bc0c7852b87c9b02c23ab6945b5940..0000000000000000000000000000000000000000
--- a/app/.vitepress/public/img/brand/logo/svg/openLooKengLogo1.svg
+++ /dev/null
@@ -1 +0,0 @@
-
\ No newline at end of file
diff --git a/app/.vitepress/public/img/brand/logo/svg/openLooKengLogo2.svg b/app/.vitepress/public/img/brand/logo/svg/openLooKengLogo2.svg
deleted file mode 100644
index 6aa0e803b9a3579705ef2ea55253680dd1a5e3d1..0000000000000000000000000000000000000000
--- a/app/.vitepress/public/img/brand/logo/svg/openLooKengLogo2.svg
+++ /dev/null
@@ -1 +0,0 @@
-
\ No newline at end of file
diff --git a/app/.vitepress/public/img/brand/logo/svg/openLooKengLogo3.svg b/app/.vitepress/public/img/brand/logo/svg/openLooKengLogo3.svg
deleted file mode 100644
index 21fbfd8dc3a2865bb95f0671caeca8abaeb0ac73..0000000000000000000000000000000000000000
--- a/app/.vitepress/public/img/brand/logo/svg/openLooKengLogo3.svg
+++ /dev/null
@@ -1 +0,0 @@
-
\ No newline at end of file
diff --git a/app/.vitepress/public/img/brand/logo/svg/openLooKengLogo4.svg b/app/.vitepress/public/img/brand/logo/svg/openLooKengLogo4.svg
deleted file mode 100644
index 83abeed8eee561113ed570085d6e710432206c37..0000000000000000000000000000000000000000
--- a/app/.vitepress/public/img/brand/logo/svg/openLooKengLogo4.svg
+++ /dev/null
@@ -1 +0,0 @@
-
\ No newline at end of file
diff --git a/app/.vitepress/public/img/brand/logo/svg/openLooKengLogo5.svg b/app/.vitepress/public/img/brand/logo/svg/openLooKengLogo5.svg
deleted file mode 100644
index 64b3d17cf0725276149d7838d13855bced33e6a2..0000000000000000000000000000000000000000
--- a/app/.vitepress/public/img/brand/logo/svg/openLooKengLogo5.svg
+++ /dev/null
@@ -1 +0,0 @@
-
\ No newline at end of file
diff --git a/app/.vitepress/public/img/brand/logo/svg/openLooKengLogo6.svg b/app/.vitepress/public/img/brand/logo/svg/openLooKengLogo6.svg
deleted file mode 100644
index 1dd84034ad1f13fe0bdcc6e6006a58e38b78789b..0000000000000000000000000000000000000000
--- a/app/.vitepress/public/img/brand/logo/svg/openLooKengLogo6.svg
+++ /dev/null
@@ -1 +0,0 @@
-
\ No newline at end of file
diff --git a/app/.vitepress/public/img/brand/logo/view/logo1.svg b/app/.vitepress/public/img/brand/logo/view/logo1.svg
deleted file mode 100644
index 32a57519f85284c37acb3c5e674b04020ce18a28..0000000000000000000000000000000000000000
--- a/app/.vitepress/public/img/brand/logo/view/logo1.svg
+++ /dev/null
@@ -1,27 +0,0 @@
-
-
\ No newline at end of file
diff --git a/app/.vitepress/public/img/brand/logo/view/logo2.svg b/app/.vitepress/public/img/brand/logo/view/logo2.svg
deleted file mode 100644
index 64755d52d06025ab908b9bc7e51e10c01a4e6f59..0000000000000000000000000000000000000000
--- a/app/.vitepress/public/img/brand/logo/view/logo2.svg
+++ /dev/null
@@ -1,27 +0,0 @@
-
-
\ No newline at end of file
diff --git a/app/.vitepress/public/img/brand/logo/view/logo3.svg b/app/.vitepress/public/img/brand/logo/view/logo3.svg
deleted file mode 100644
index 61687a38503346e6f9973d910a4f33807d94a4fe..0000000000000000000000000000000000000000
--- a/app/.vitepress/public/img/brand/logo/view/logo3.svg
+++ /dev/null
@@ -1,27 +0,0 @@
-
-
\ No newline at end of file
diff --git a/app/.vitepress/public/img/brand/logo/view/logo4.svg b/app/.vitepress/public/img/brand/logo/view/logo4.svg
deleted file mode 100644
index 2ac5f013c527f08f1fc104293d8b910592ee17d8..0000000000000000000000000000000000000000
--- a/app/.vitepress/public/img/brand/logo/view/logo4.svg
+++ /dev/null
@@ -1,27 +0,0 @@
-
-
\ No newline at end of file
diff --git a/app/.vitepress/public/img/brand/logo/view/logo5.svg b/app/.vitepress/public/img/brand/logo/view/logo5.svg
deleted file mode 100644
index 216eaf6ced31208bedd156625763c68b7a9a7580..0000000000000000000000000000000000000000
--- a/app/.vitepress/public/img/brand/logo/view/logo5.svg
+++ /dev/null
@@ -1,27 +0,0 @@
-
-
\ No newline at end of file
diff --git a/app/.vitepress/public/img/brand/logo/view/logo6.svg b/app/.vitepress/public/img/brand/logo/view/logo6.svg
deleted file mode 100644
index 885a56af0a6736c70a44110c96e77bf7c4960608..0000000000000000000000000000000000000000
--- a/app/.vitepress/public/img/brand/logo/view/logo6.svg
+++ /dev/null
@@ -1,27 +0,0 @@
-
-
\ No newline at end of file
diff --git a/app/.vitepress/public/img/brand/ppt/openLooKeng_PPT_Dark_cn.pptx b/app/.vitepress/public/img/brand/ppt/openLooKeng_PPT_Dark_cn.pptx
deleted file mode 100644
index 604eb4cde8431ddcd6b4699f072d708d603e3309..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/ppt/openLooKeng_PPT_Dark_cn.pptx and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/ppt/openLooKeng_PPT_Dark_en.pptx b/app/.vitepress/public/img/brand/ppt/openLooKeng_PPT_Dark_en.pptx
deleted file mode 100644
index b5d2c2129287e90e8b17f5cb2dafefe44d222388..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/ppt/openLooKeng_PPT_Dark_en.pptx and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/ppt/openLooKeng_PPT_Light_cn.pptx b/app/.vitepress/public/img/brand/ppt/openLooKeng_PPT_Light_cn.pptx
deleted file mode 100644
index 7cac89994aa4c1294067cc609b01447cac001947..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/ppt/openLooKeng_PPT_Light_cn.pptx and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/ppt/openLooKeng_PPT_Light_en.pptx b/app/.vitepress/public/img/brand/ppt/openLooKeng_PPT_Light_en.pptx
deleted file mode 100644
index 162051f950cc4ed360a0da568424e63e1d4c401f..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/ppt/openLooKeng_PPT_Light_en.pptx and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/ppt/pic/openLooKeng_PPT_Dark_cn.png b/app/.vitepress/public/img/brand/ppt/pic/openLooKeng_PPT_Dark_cn.png
deleted file mode 100644
index fc51c1088b9493a9b26b8aa348402204539fb306..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/ppt/pic/openLooKeng_PPT_Dark_cn.png and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/ppt/pic/openLooKeng_PPT_Dark_en.png b/app/.vitepress/public/img/brand/ppt/pic/openLooKeng_PPT_Dark_en.png
deleted file mode 100644
index ac9eb349460692cd594e4434c689881ee5f30d94..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/ppt/pic/openLooKeng_PPT_Dark_en.png and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/ppt/pic/openLooKeng_PPT_Light_cn.png b/app/.vitepress/public/img/brand/ppt/pic/openLooKeng_PPT_Light_cn.png
deleted file mode 100644
index 763a091aa1288b119d9ad1a5c9456ae6c4c9226f..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/ppt/pic/openLooKeng_PPT_Light_cn.png and /dev/null differ
diff --git a/app/.vitepress/public/img/brand/ppt/pic/openLooKeng_PPT_Light_en.png b/app/.vitepress/public/img/brand/ppt/pic/openLooKeng_PPT_Light_en.png
deleted file mode 100644
index bb1d72692dac56daa280388ca7bb71c8b7bd9aa4..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/brand/ppt/pic/openLooKeng_PPT_Light_en.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/ArvenDing.png b/app/.vitepress/public/img/community/who-we-are/ArvenDing.png
deleted file mode 100644
index ba465a1663e092d3e4214dc77495251820dcd5b1..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/ArvenDing.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/DenghongLiao.png b/app/.vitepress/public/img/community/who-we-are/DenghongLiao.png
deleted file mode 100644
index 800e022a6f20fbcf1e5024c535abde6793485ebc..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/DenghongLiao.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/DezhiXu.png b/app/.vitepress/public/img/community/who-we-are/DezhiXu.png
deleted file mode 100644
index ad405aca95a68fa96539fedb3580ba11ab8585c9..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/DezhiXu.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/Gobinath.png b/app/.vitepress/public/img/community/who-we-are/Gobinath.png
deleted file mode 100644
index 44ad987a72e236d7d0b2969f8dff49b9ab1941b8..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/Gobinath.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/GuoYang.png b/app/.vitepress/public/img/community/who-we-are/GuoYang.png
deleted file mode 100644
index 148bfc38a42c6709c7c17d0fd14df80631ae7a54..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/GuoYang.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/JingfangZhang.png b/app/.vitepress/public/img/community/who-we-are/JingfangZhang.png
deleted file mode 100644
index cea31af29fdc920e04074fba5ed80d422d98a0c0..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/JingfangZhang.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/KarmveerSingh.png b/app/.vitepress/public/img/community/who-we-are/KarmveerSingh.png
deleted file mode 100644
index e27973cf2f9d9252e5ce05d7148b0b9fe2af9022..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/KarmveerSingh.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/KenZhang.png b/app/.vitepress/public/img/community/who-we-are/KenZhang.png
deleted file mode 100644
index e637c7339f7008199ab363669c6f7eb7b457de8b..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/KenZhang.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/MichaelLi.png b/app/.vitepress/public/img/community/who-we-are/MichaelLi.png
deleted file mode 100644
index ba56719991c30198c741e9b23c0cfb61f9edeb8e..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/MichaelLi.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/Raghunandan.png b/app/.vitepress/public/img/community/who-we-are/Raghunandan.png
deleted file mode 100644
index f8831f3238cf66b12c65d4e544b9b8d9e837dd66..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/Raghunandan.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/WeiMao.png b/app/.vitepress/public/img/community/who-we-are/WeiMao.png
deleted file mode 100644
index 89607bfdf398bf7374652db1d330879a1b5cc889..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/WeiMao.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/WeiZhang.png b/app/.vitepress/public/img/community/who-we-are/WeiZhang.png
deleted file mode 100644
index bea74aff962ca122100c5bc3c90b439949091365..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/WeiZhang.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/ZhengLi.png b/app/.vitepress/public/img/community/who-we-are/ZhengLi.png
deleted file mode 100644
index 31a5f60bd7baf5ddc4c07476af87c45b1735aec2..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/ZhengLi.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/base.png b/app/.vitepress/public/img/community/who-we-are/base.png
deleted file mode 100644
index 2d53be40adfe42907435a20e1466eccda1dcae84..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/base.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/chengyijian.png b/app/.vitepress/public/img/community/who-we-are/chengyijian.png
deleted file mode 100644
index d5858c27b6fcfb2bd1062aa05b72c880549f9459..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/chengyijian.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/luodan.png b/app/.vitepress/public/img/community/who-we-are/luodan.png
deleted file mode 100644
index 4ce1b09791f6b896f16b5deac8973cd609a30e82..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/luodan.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/pmcMember.png b/app/.vitepress/public/img/community/who-we-are/pmcMember.png
deleted file mode 100644
index ddc42e2ee42f571ec7742c0face7f6bba9ae0fdd..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/pmcMember.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/tushengxia.png b/app/.vitepress/public/img/community/who-we-are/tushengxia.png
deleted file mode 100644
index a647dbe1e416481c3da97cfd0d83b740380876fd..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/tushengxia.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/wuqiuming.png b/app/.vitepress/public/img/community/who-we-are/wuqiuming.png
deleted file mode 100644
index d4e32a581bf4355b943be812fc856aa640713611..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/wuqiuming.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/wurunchao.png b/app/.vitepress/public/img/community/who-we-are/wurunchao.png
deleted file mode 100644
index fdbb7e4adac85c9e2973fcecb4b4c099f15f6fe8..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/wurunchao.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/xulifeng.png b/app/.vitepress/public/img/community/who-we-are/xulifeng.png
deleted file mode 100644
index 66b7d6b162df6ceacb655ad1c218aecc65ee9eb5..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/xulifeng.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/xutianli.png b/app/.vitepress/public/img/community/who-we-are/xutianli.png
deleted file mode 100644
index 4a0172bc36dc1a9dbac113ee884849812af0a963..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/xutianli.png and /dev/null differ
diff --git a/app/.vitepress/public/img/community/who-we-are/yujiwen.png b/app/.vitepress/public/img/community/who-we-are/yujiwen.png
deleted file mode 100644
index 5fe3ab78d4f616816eab2064da046b74f9df5fd8..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/community/who-we-are/yujiwen.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/doucumentation/moon.png b/app/.vitepress/public/img/learning/doucumentation/moon.png
deleted file mode 100644
index 3d2445d75bf5de974e496d8b6da577ef42713728..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/doucumentation/moon.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/doucumentation/openLooKeng_white.png b/app/.vitepress/public/img/learning/doucumentation/openLooKeng_white.png
deleted file mode 100644
index 563bd531fa7233418fb3c07b90e10e92de7196cf..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/doucumentation/openLooKeng_white.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/doucumentation/search.png b/app/.vitepress/public/img/learning/doucumentation/search.png
deleted file mode 100644
index 0adc7af961e085b662e853e0678a86e8284c307b..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/doucumentation/search.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/knowledge/base.png b/app/.vitepress/public/img/learning/knowledge/base.png
deleted file mode 100644
index 75e661d71e09029def795ab3e5edddf17270b86d..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/knowledge/base.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/Virtual-Meetup1.png b/app/.vitepress/public/img/learning/mooc/Virtual-Meetup1.png
deleted file mode 100644
index 730b7c31f18abe9858d9383b09b26eb43f3abeca..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/Virtual-Meetup1.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/banner1-en-mo@2x.jpg b/app/.vitepress/public/img/learning/mooc/banner1-en-mo@2x.jpg
deleted file mode 100644
index ede647ee4ede313e4ef3067226626e79811c0519..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/banner1-en-mo@2x.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/banner1-en@2x.jpg b/app/.vitepress/public/img/learning/mooc/banner1-en@2x.jpg
deleted file mode 100644
index 15424c555cdf861e46acd57360e90ab5fa49159d..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/banner1-en@2x.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/banner1-zh-mo@2x.jpg b/app/.vitepress/public/img/learning/mooc/banner1-zh-mo@2x.jpg
deleted file mode 100644
index 7695ab9b195ee76948d4ad0ee61962a8434c47d6..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/banner1-zh-mo@2x.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/banner1-zh@2x.jpg b/app/.vitepress/public/img/learning/mooc/banner1-zh@2x.jpg
deleted file mode 100644
index 6b84e32d42e6b01b072feef8c403bc3949f2258b..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/banner1-zh@2x.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/banner2-en-mo@2x.jpg b/app/.vitepress/public/img/learning/mooc/banner2-en-mo@2x.jpg
deleted file mode 100644
index 162f603b67aff424fb64f0617e9a40e5695cdfa7..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/banner2-en-mo@2x.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/banner2-en@2x.jpg b/app/.vitepress/public/img/learning/mooc/banner2-en@2x.jpg
deleted file mode 100644
index 22c469f16c5caac6ddefab836efcd5aa92845949..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/banner2-en@2x.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/banner2-zh-mo@2x.jpg b/app/.vitepress/public/img/learning/mooc/banner2-zh-mo@2x.jpg
deleted file mode 100644
index 3148de8385b59e554be0125c552b23c8c43defea..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/banner2-zh-mo@2x.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/banner2-zh@2x.jpg b/app/.vitepress/public/img/learning/mooc/banner2-zh@2x.jpg
deleted file mode 100644
index 2dd4ac949ce663d9f8dfbaa2b261baf97e474509..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/banner2-zh@2x.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/banner3-en-mo@2x.jpg b/app/.vitepress/public/img/learning/mooc/banner3-en-mo@2x.jpg
deleted file mode 100644
index 54df0c9ce6c81c6500b50fb04e648f4b78e08cdf..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/banner3-en-mo@2x.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/banner3-en@2x.jpg b/app/.vitepress/public/img/learning/mooc/banner3-en@2x.jpg
deleted file mode 100644
index b9f5a22ca6487045ef38eec646dcdd9cc4606057..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/banner3-en@2x.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/banner3-zh-mo@2x.jpg b/app/.vitepress/public/img/learning/mooc/banner3-zh-mo@2x.jpg
deleted file mode 100644
index 5862ae12b3e9478b9431a601871c98fbf89d9507..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/banner3-zh-mo@2x.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/banner3-zh@2x.jpg b/app/.vitepress/public/img/learning/mooc/banner3-zh@2x.jpg
deleted file mode 100644
index f0065eb8f3169ec4b341334bf96d998c1f2f273f..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/banner3-zh@2x.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/base-bg.png b/app/.vitepress/public/img/learning/mooc/base-bg.png
deleted file mode 100644
index abfe0a4b568d017725c35f7cc441ef8eaf2d238f..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/base-bg.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/base-dark.png b/app/.vitepress/public/img/learning/mooc/base-dark.png
deleted file mode 100644
index fdddf84f47ead7d93a2baea27ed5358c1fb72e6b..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/base-dark.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/base-list-dark.png b/app/.vitepress/public/img/learning/mooc/base-list-dark.png
deleted file mode 100644
index 43010f58ef91609455d76ac6e3acc08abadad75f..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/base-list-dark.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/bg-base-list.png b/app/.vitepress/public/img/learning/mooc/bg-base-list.png
deleted file mode 100644
index 7f6e6807321dd5edc70fa0cd89b3c44db7e196bc..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/bg-base-list.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/bg-community-list.png b/app/.vitepress/public/img/learning/mooc/bg-community-list.png
deleted file mode 100644
index 5881e0ca6ab3dde3068ecce93450b6b17e7eadcd..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/bg-community-list.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/bg-mooc-dark.png b/app/.vitepress/public/img/learning/mooc/bg-mooc-dark.png
deleted file mode 100644
index 9265652bde9a03491f3a48ad15338a2353aec92a..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/bg-mooc-dark.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/bg-practice-list.png b/app/.vitepress/public/img/learning/mooc/bg-practice-list.png
deleted file mode 100644
index 799d00a1604a3305dc3615c547e052cfb1063bc8..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/bg-practice-list.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/bg-theory-list.png b/app/.vitepress/public/img/learning/mooc/bg-theory-list.png
deleted file mode 100644
index 8d2ecbd98452c8c498a9002fa4505cfc29725fd1..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/bg-theory-list.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/practice-bg.png b/app/.vitepress/public/img/learning/mooc/practice-bg.png
deleted file mode 100644
index e66338ea033f5aa625ec6aa3ad6614e3c59d8160..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/practice-bg.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/practice-dark.png b/app/.vitepress/public/img/learning/mooc/practice-dark.png
deleted file mode 100644
index 2352011691ca639ad3d048354df2028d87d95ea4..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/practice-dark.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/practice-list-dark.png b/app/.vitepress/public/img/learning/mooc/practice-list-dark.png
deleted file mode 100644
index 0fdd695ef3cf94e520c2fc40823835815caa8cbb..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/practice-list-dark.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/theory-bg.png b/app/.vitepress/public/img/learning/mooc/theory-bg.png
deleted file mode 100644
index d7fbd77cd14e1e3132a7c4406640e31ce7e63bd1..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/theory-bg.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/theory-dark.png b/app/.vitepress/public/img/learning/mooc/theory-dark.png
deleted file mode 100644
index 5a48f674c85f295fb7710fe3b2c530fdf160f55a..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/theory-dark.png and /dev/null differ
diff --git a/app/.vitepress/public/img/learning/mooc/theory-list-dark.png b/app/.vitepress/public/img/learning/mooc/theory-list-dark.png
deleted file mode 100644
index 3826259a5f4ea2f1941a9646ea043982a63ebb19..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/learning/mooc/theory-list-dark.png and /dev/null differ
diff --git a/app/.vitepress/public/img/powered/powered-card-dark.png b/app/.vitepress/public/img/powered/powered-card-dark.png
deleted file mode 100644
index 28ac4851a2189f42f8808d8f841838f58d0c52c7..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/powered/powered-card-dark.png and /dev/null differ
diff --git a/app/.vitepress/public/img/powered/powered-card.png b/app/.vitepress/public/img/powered/powered-card.png
deleted file mode 100644
index 8a0ba7cd307b8bb0844646b5fd233bab3f47e3c0..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/powered/powered-card.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/developer/en/2021-ccf-bdci.png b/app/.vitepress/public/img/videos/developer/en/2021-ccf-bdci.png
deleted file mode 100644
index e41fb4868ca625819c433bc51d78a6d42b6e4a84..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/developer/en/2021-ccf-bdci.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/developer/en/CCF.png b/app/.vitepress/public/img/videos/developer/en/CCF.png
deleted file mode 100644
index b46fb12d9c423c72b56f714f24e92b5705e5bb2f..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/developer/en/CCF.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/developer/en/dev1.png b/app/.vitepress/public/img/videos/developer/en/dev1.png
deleted file mode 100644
index 9254dde878fb9a1be408cf4dd3935fa852e34f62..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/developer/en/dev1.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/developer/en/dev2.png b/app/.vitepress/public/img/videos/developer/en/dev2.png
deleted file mode 100644
index 9ece51ae1dc990f926abbc005d6008674c2950b6..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/developer/en/dev2.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/developer/en/dev3.png b/app/.vitepress/public/img/videos/developer/en/dev3.png
deleted file mode 100644
index a81ed557c1c582a810288931f7c073b3e8b53bdf..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/developer/en/dev3.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/developer/en/en-2021summer.png b/app/.vitepress/public/img/videos/developer/en/en-2021summer.png
deleted file mode 100644
index 45fde10bc1d828d6ea6d43897baa4d3cad733378..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/developer/en/en-2021summer.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/developer/en/en-xinchuang.png b/app/.vitepress/public/img/videos/developer/en/en-xinchuang.png
deleted file mode 100644
index fdfa231165e73aa8c0c3e480c0b0c57fe0b439ba..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/developer/en/en-xinchuang.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/developer/zh/2021-ccf-bdci.png b/app/.vitepress/public/img/videos/developer/zh/2021-ccf-bdci.png
deleted file mode 100644
index e41fb4868ca625819c433bc51d78a6d42b6e4a84..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/developer/zh/2021-ccf-bdci.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/developer/zh/CCF.png b/app/.vitepress/public/img/videos/developer/zh/CCF.png
deleted file mode 100644
index 2bf7ea0ee6caa1ab29d9c32341e9977bcf43c7cc..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/developer/zh/CCF.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/developer/zh/dev1.png b/app/.vitepress/public/img/videos/developer/zh/dev1.png
deleted file mode 100644
index bc8be093cc6c05d99562463e1b5b550cf21e91aa..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/developer/zh/dev1.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/developer/zh/dev2.png b/app/.vitepress/public/img/videos/developer/zh/dev2.png
deleted file mode 100644
index 8748738fab954d7602ec36eb90a75c1cffc0b103..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/developer/zh/dev2.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/developer/zh/dev3.png b/app/.vitepress/public/img/videos/developer/zh/dev3.png
deleted file mode 100644
index 94efbc51e99fd12f18c2cef6eaa7d306c2184fe9..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/developer/zh/dev3.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/developer/zh/zh-2021summer.png b/app/.vitepress/public/img/videos/developer/zh/zh-2021summer.png
deleted file mode 100644
index 35eaf67d4540b2e9160052547458c734853e10df..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/developer/zh/zh-2021summer.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/developer/zh/zh-2022internship.png b/app/.vitepress/public/img/videos/developer/zh/zh-2022internship.png
deleted file mode 100644
index 9bd60f125bcbbef2c4a705d53158a4b141d33c25..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/developer/zh/zh-2022internship.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/developer/zh/zh-xinchuang.png b/app/.vitepress/public/img/videos/developer/zh/zh-xinchuang.png
deleted file mode 100644
index 829077bf1611302ec066a0ba08ede1180419b00f..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/developer/zh/zh-xinchuang.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/feature/feature1.png b/app/.vitepress/public/img/videos/feature/feature1.png
deleted file mode 100644
index 560d59e7efd7064ee51707f81ef2051a0b2f5615..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/feature/feature1.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/feature/feature2.png b/app/.vitepress/public/img/videos/feature/feature2.png
deleted file mode 100644
index 0eba4d361ad47389ac52bbf6288ac55556df789d..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/feature/feature2.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/feature/feature3.png b/app/.vitepress/public/img/videos/feature/feature3.png
deleted file mode 100644
index 3c32dd65f1fe087636c9d5eda1d0d2c3f92ccaf8..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/feature/feature3.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/0121-pmc-en.png b/app/.vitepress/public/img/videos/live/en/0121-pmc-en.png
deleted file mode 100644
index 492fd4171ba580da3b79b1e05f13b72a6c64dad0..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/0121-pmc-en.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/0318pmc-en.png b/app/.vitepress/public/img/videos/live/en/0318pmc-en.png
deleted file mode 100644
index 353b1a910a1ce413778a82fd76d6a4fbd0d4fa61..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/0318pmc-en.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/0402.png b/app/.vitepress/public/img/videos/live/en/0402.png
deleted file mode 100644
index 7a56fb06cdfb3ab99990e888a5557377f73f3887..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/0402.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/0416.png b/app/.vitepress/public/img/videos/live/en/0416.png
deleted file mode 100644
index 8ffa06ac958aafd10a39cdee09129fb057fddaa1..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/0416.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/0430.png b/app/.vitepress/public/img/videos/live/en/0430.png
deleted file mode 100644
index 408b3c6d80fc3631482bf7bdb410627fdd7e3c12..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/0430.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/0514.png b/app/.vitepress/public/img/videos/live/en/0514.png
deleted file mode 100644
index 7a1ecaad4943f3cb7852b33c92eae8a90d307b36..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/0514.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/0528.png b/app/.vitepress/public/img/videos/live/en/0528.png
deleted file mode 100644
index e575e933fe9e6041a76ffaed151dc52aec8d6dee..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/0528.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/0611.png b/app/.vitepress/public/img/videos/live/en/0611.png
deleted file mode 100644
index 811c538492731f73b9620f935030b425d26715ba..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/0611.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/1210pmc_en.png b/app/.vitepress/public/img/videos/live/en/1210pmc_en.png
deleted file mode 100644
index 4b86f4cac79a15d295034c547a14a80788ce0fac..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/1210pmc_en.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/20210625-pmc.png b/app/.vitepress/public/img/videos/live/en/20210625-pmc.png
deleted file mode 100644
index 594c7be5061daf7ca6cc8413dde61cf12b2e1b20..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/20210625-pmc.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/20210709-pmc.png b/app/.vitepress/public/img/videos/live/en/20210709-pmc.png
deleted file mode 100644
index 6647ae5e681dad072b1b5b368ea185a795b682e0..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/20210709-pmc.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/20210723-pmc.png b/app/.vitepress/public/img/videos/live/en/20210723-pmc.png
deleted file mode 100644
index fe7e9b68163f3587e8dfa368758474e2cdab9cea..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/20210723-pmc.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/20210806-pmc.png b/app/.vitepress/public/img/videos/live/en/20210806-pmc.png
deleted file mode 100644
index bb647ca85b5140436b4e51e641515e3192c36f36..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/20210806-pmc.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/20210820-pmc.png b/app/.vitepress/public/img/videos/live/en/20210820-pmc.png
deleted file mode 100644
index 24846a91ee1068584e29561103d591830549bbe4..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/20210820-pmc.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/20210903-pmc.png b/app/.vitepress/public/img/videos/live/en/20210903-pmc.png
deleted file mode 100644
index bb3751ab91a758fff9a68290549ab5d95c069a8f..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/20210903-pmc.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/20210917-pmc.png b/app/.vitepress/public/img/videos/live/en/20210917-pmc.png
deleted file mode 100644
index 5e845057506a41645448c17305558fcc79197219..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/20210917-pmc.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/20220107-pmc-en.png b/app/.vitepress/public/img/videos/live/en/20220107-pmc-en.png
deleted file mode 100644
index 130a11c3b8e6ddad33ad43d5a2fed61e4d672736..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/20220107-pmc-en.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/20220401pmc-en.png b/app/.vitepress/public/img/videos/live/en/20220401pmc-en.png
deleted file mode 100644
index 66d3dc2452e0182d97a90dce8c5125f21a4e8f5b..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/20220401pmc-en.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/20pmc_en.png b/app/.vitepress/public/img/videos/live/en/20pmc_en.png
deleted file mode 100644
index af3f66c5ad15fc400b26e43e1adebda967b73027..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/20pmc_en.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/2112-pmc-en.png b/app/.vitepress/public/img/videos/live/en/2112-pmc-en.png
deleted file mode 100644
index 0c33282e51fdd5070adffaca645743e33378779e..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/2112-pmc-en.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/21pmc-en.png b/app/.vitepress/public/img/videos/live/en/21pmc-en.png
deleted file mode 100644
index 34e90470ba088dc59a1f085184144131ec15a19b..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/21pmc-en.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/22pmc_en.png b/app/.vitepress/public/img/videos/live/en/22pmc_en.png
deleted file mode 100644
index c4011c261811b21b5cce005b7566166787f81269..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/22pmc_en.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/add.png b/app/.vitepress/public/img/videos/live/en/add.png
deleted file mode 100644
index 0416ef856abd59228606669af3219498af1eb576..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/add.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/en-0429-pmc.png b/app/.vitepress/public/img/videos/live/en/en-0429-pmc.png
deleted file mode 100644
index 869cbd795cd19aa9b9221f0978b067db2e711f3b..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/en-0429-pmc.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/en-0513-pmc.png b/app/.vitepress/public/img/videos/live/en/en-0513-pmc.png
deleted file mode 100644
index 8b858b93bda1349a30e12420afce4fb60a3dd509..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/en-0513-pmc.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/hand.png b/app/.vitepress/public/img/videos/live/en/hand.png
deleted file mode 100644
index 7b93daa879970e68b39b0391097afbb8a642b4a5..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/hand.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/live1.png b/app/.vitepress/public/img/videos/live/en/live1.png
deleted file mode 100644
index 180c074e6bcc3c55a6d1d3f1d5c1d6fd321d3ae3..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/live1.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/live2.png b/app/.vitepress/public/img/videos/live/en/live2.png
deleted file mode 100644
index bfd5de6d3eb91631904df0848ce83ec2e8ae0852..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/live2.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/live3.png b/app/.vitepress/public/img/videos/live/en/live3.png
deleted file mode 100644
index 92292ce32db9e8fc0b6bd1b6738b2b72a38b77be..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/live3.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/live4.png b/app/.vitepress/public/img/videos/live/en/live4.png
deleted file mode 100644
index bb6a663955a28f11f22fbae29d6b99bc797df4f4..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/live4.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/live5.png b/app/.vitepress/public/img/videos/live/en/live5.png
deleted file mode 100644
index c63040049d3f243de073e5f46e50e48465a180c5..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/live5.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/live6.png b/app/.vitepress/public/img/videos/live/en/live6.png
deleted file mode 100644
index b01d64c956593881d201ef4f8bfd28c4465cc385..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/live6.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/tryit.png b/app/.vitepress/public/img/videos/live/en/tryit.png
deleted file mode 100644
index 46720995bc66c815fac6086b2e429daafe22d2d5..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/tryit.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/en/v130-en.png b/app/.vitepress/public/img/videos/live/en/v130-en.png
deleted file mode 100644
index 510de8cf8bc968a61c789f7cfe1a29c2425fbf44..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/en/v130-en.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/0121-pmc-zh.png b/app/.vitepress/public/img/videos/live/zh/0121-pmc-zh.png
deleted file mode 100644
index 8cce59a638989d9b514817adaa4b9d50e928ea7b..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/0121-pmc-zh.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/0318pmc-zh.png b/app/.vitepress/public/img/videos/live/zh/0318pmc-zh.png
deleted file mode 100644
index e95fb6ab6393da603e9a39f845701ba8438a7dbd..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/0318pmc-zh.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/0402.png b/app/.vitepress/public/img/videos/live/zh/0402.png
deleted file mode 100644
index f590514b4312b8934cfb1ead54c92a788f5de26c..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/0402.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/0415_1.png b/app/.vitepress/public/img/videos/live/zh/0415_1.png
deleted file mode 100644
index 8b21571cceea6603155639f85049a22c3aaddf9c..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/0415_1.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/0416.png b/app/.vitepress/public/img/videos/live/zh/0416.png
deleted file mode 100644
index 2caf3505756b57c990ec06baf94cc93423029a42..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/0416.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/0422.png b/app/.vitepress/public/img/videos/live/zh/0422.png
deleted file mode 100644
index e767e1b22d2bb74562114dc2d98751ba51937dc0..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/0422.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/0429_1.png b/app/.vitepress/public/img/videos/live/zh/0429_1.png
deleted file mode 100644
index 4ca9fe362c658741d0a6930f497643166d5c18e9..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/0429_1.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/0429_2.png b/app/.vitepress/public/img/videos/live/zh/0429_2.png
deleted file mode 100644
index ae406f8e5a11f7366a7df8a3a4974c8c14589cc3..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/0429_2.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/0430.png b/app/.vitepress/public/img/videos/live/zh/0430.png
deleted file mode 100644
index c6cc41705c3befbd6682c69ee31462d1a2981b93..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/0430.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/0514.png b/app/.vitepress/public/img/videos/live/zh/0514.png
deleted file mode 100644
index 757952b4fd8484c7dfb40ad1a7406aec2fe7dfd4..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/0514.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/0528.png b/app/.vitepress/public/img/videos/live/zh/0528.png
deleted file mode 100644
index 6863170df7822ded18626e83d96d6c11dbf10bc0..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/0528.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/0611.png b/app/.vitepress/public/img/videos/live/zh/0611.png
deleted file mode 100644
index f8b9bf61441becd05cf77c12d97d8f5ac32c25f2..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/0611.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/1210pmc_zh.png b/app/.vitepress/public/img/videos/live/zh/1210pmc_zh.png
deleted file mode 100644
index 5a3611a213ff8b18b1b84442b05c8e4d4ffc9279..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/1210pmc_zh.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/20210625-pmc.png b/app/.vitepress/public/img/videos/live/zh/20210625-pmc.png
deleted file mode 100644
index 6a52b8968fc23b033a21cbe367ab05d78a4457c4..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/20210625-pmc.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/20210709-pmc.png b/app/.vitepress/public/img/videos/live/zh/20210709-pmc.png
deleted file mode 100644
index b3bdd30567673a84b7ed1333a9298832f3a26bf6..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/20210709-pmc.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/20210723-pmc.png b/app/.vitepress/public/img/videos/live/zh/20210723-pmc.png
deleted file mode 100644
index cea852eb881e103be5569ac91ff2040a708d680f..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/20210723-pmc.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/20210806-pmc.png b/app/.vitepress/public/img/videos/live/zh/20210806-pmc.png
deleted file mode 100644
index 7b447e37680c6cb89d9590ca51d86e71e9889706..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/20210806-pmc.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/20210820-pmc.png b/app/.vitepress/public/img/videos/live/zh/20210820-pmc.png
deleted file mode 100644
index c4a07dddbb5cd532053ca0de3dc015fbc588914c..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/20210820-pmc.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/20210903-pmc.png b/app/.vitepress/public/img/videos/live/zh/20210903-pmc.png
deleted file mode 100644
index c33307a24c156da96eee24987bb2030a0628ff77..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/20210903-pmc.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/20210917-pmc.png b/app/.vitepress/public/img/videos/live/zh/20210917-pmc.png
deleted file mode 100644
index 80ec07306fcfaa5c057ea032e2aa64b50a0bf983..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/20210917-pmc.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/20220107-pmc-zh.png b/app/.vitepress/public/img/videos/live/zh/20220107-pmc-zh.png
deleted file mode 100644
index 9ca2bdf4404e2ad56a6043188b3a9e2b12233dd1..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/20220107-pmc-zh.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/20220401pmc-zh.png b/app/.vitepress/public/img/videos/live/zh/20220401pmc-zh.png
deleted file mode 100644
index d2b5b6506042bfc02971b9446218a223a81d4b01..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/20220401pmc-zh.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/20pmc_zh.png b/app/.vitepress/public/img/videos/live/zh/20pmc_zh.png
deleted file mode 100644
index 3c0dd2815f68d9c44374fb2e1b9ab120feabf28d..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/20pmc_zh.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/2112-pmc-zh.png b/app/.vitepress/public/img/videos/live/zh/2112-pmc-zh.png
deleted file mode 100644
index 21d8f8f3bd01a6710d238707d3ae889360fd676b..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/2112-pmc-zh.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/21pmc-zh.png b/app/.vitepress/public/img/videos/live/zh/21pmc-zh.png
deleted file mode 100644
index 33f135d59c20d869ff00668a75d644b5a9302cbf..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/21pmc-zh.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/22pmc_zh.png b/app/.vitepress/public/img/videos/live/zh/22pmc_zh.png
deleted file mode 100644
index eb52147b478142def96c9a4bd56347ff1d96a716..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/22pmc_zh.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/add.png b/app/.vitepress/public/img/videos/live/zh/add.png
deleted file mode 100644
index 6dc3aef265a805cfe1e472571377f5265192e750..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/add.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/hand.png b/app/.vitepress/public/img/videos/live/zh/hand.png
deleted file mode 100644
index fb0dda5d886f04636d474e55b07b5fc49e325681..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/hand.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/live1.png b/app/.vitepress/public/img/videos/live/zh/live1.png
deleted file mode 100644
index 2e6f01f3daa0de6de44b40c8480c6b3a5d28d595..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/live1.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/live2.png b/app/.vitepress/public/img/videos/live/zh/live2.png
deleted file mode 100644
index 6c330b4b8f5dfe881db34e447c6f17dc720acf8b..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/live2.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/live3.png b/app/.vitepress/public/img/videos/live/zh/live3.png
deleted file mode 100644
index ec155b080a6f8b80f151ecdf95801f2e7e7ed4a0..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/live3.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/live4.png b/app/.vitepress/public/img/videos/live/zh/live4.png
deleted file mode 100644
index 66afc0de48ea2aeee0ee79af132b3ff82f948f1b..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/live4.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/live5.png b/app/.vitepress/public/img/videos/live/zh/live5.png
deleted file mode 100644
index 20e8ac0deb80e35c22f8f8dd9964a9036b750bf0..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/live5.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/live6.png b/app/.vitepress/public/img/videos/live/zh/live6.png
deleted file mode 100644
index 0fa188fe4c838e26bd502d1fb6d7602b41d12d08..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/live6.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/openlookeng_01.jpg b/app/.vitepress/public/img/videos/live/zh/openlookeng_01.jpg
deleted file mode 100644
index 1bb933adfb26162f29adeb4289b41c7fe5d45c5d..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/openlookeng_01.jpg and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/tryit.png b/app/.vitepress/public/img/videos/live/zh/tryit.png
deleted file mode 100644
index 044bb1104212dde1d98c9f75aa390a9396961154..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/tryit.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/v130-zh.png b/app/.vitepress/public/img/videos/live/zh/v130-zh.png
deleted file mode 100644
index 2bb258e0b0609564a20623e49e93e2f4d45c0936..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/v130-zh.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/zh-0429-pmc.png b/app/.vitepress/public/img/videos/live/zh/zh-0429-pmc.png
deleted file mode 100644
index a37362ef78d98edfc78f427e248ea14ec738bd3c..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/zh-0429-pmc.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/live/zh/zh-0513-pmc.png b/app/.vitepress/public/img/videos/live/zh/zh-0513-pmc.png
deleted file mode 100644
index 7edddd5602ce9d318232097f374a10e03f036bf5..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/live/zh/zh-0513-pmc.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/en/HC2020.png b/app/.vitepress/public/img/videos/technical/en/HC2020.png
deleted file mode 100644
index b118fd5b8dca7624a4c57ec00ff10de267286919..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/en/HC2020.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/en/community.png b/app/.vitepress/public/img/videos/technical/en/community.png
deleted file mode 100644
index 21e21207f407f0873558131bd51ade6b359f4542..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/en/community.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/en/intro.png b/app/.vitepress/public/img/videos/technical/en/intro.png
deleted file mode 100644
index 0ba50447d690799e5906420dfc410da2ff6c74d1..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/en/intro.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/en/meetup21-en-01.png b/app/.vitepress/public/img/videos/technical/en/meetup21-en-01.png
deleted file mode 100644
index be0bcc9776d7c65889f5423c7084fcb2721e061f..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/en/meetup21-en-01.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/en/meetup21-en-02.png b/app/.vitepress/public/img/videos/technical/en/meetup21-en-02.png
deleted file mode 100644
index f5391b372bb945f2aff4fa7d584259663d110c7c..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/en/meetup21-en-02.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/en/meetup21-en-03.png b/app/.vitepress/public/img/videos/technical/en/meetup21-en-03.png
deleted file mode 100644
index 21a5163e6a40fb36cc23aafd74d474e04d91c122..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/en/meetup21-en-03.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/en/meetup21-en-04.png b/app/.vitepress/public/img/videos/technical/en/meetup21-en-04.png
deleted file mode 100644
index 01597e4600d0fa5b3bb8cd416cae4d3402e375f6..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/en/meetup21-en-04.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/en/meetup21-en-05.png b/app/.vitepress/public/img/videos/technical/en/meetup21-en-05.png
deleted file mode 100644
index 2dbb031cf6a1cb5d0a22ddd6f7171695930033e7..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/en/meetup21-en-05.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/en/optimization.png b/app/.vitepress/public/img/videos/technical/en/optimization.png
deleted file mode 100644
index 8609e1e567db86df219d82ab9aa31dffc861856a..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/en/optimization.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/en/plan.png b/app/.vitepress/public/img/videos/technical/en/plan.png
deleted file mode 100644
index ff89f30cef0d299ea9616bf02154be62ea8c76d1..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/en/plan.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/en/practice.png b/app/.vitepress/public/img/videos/technical/en/practice.png
deleted file mode 100644
index 6a3de43ec7cb58e6f073472d1f54c9f199036d5c..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/en/practice.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/en/release.png b/app/.vitepress/public/img/videos/technical/en/release.png
deleted file mode 100644
index c4f9a57e7ef74afd9c4d41471d32738d938ac5d7..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/en/release.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/en/speech.png b/app/.vitepress/public/img/videos/technical/en/speech.png
deleted file mode 100644
index f01027f288bec32a5429ebd04910f88538fc35a8..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/en/speech.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/en/tryme.png b/app/.vitepress/public/img/videos/technical/en/tryme.png
deleted file mode 100644
index 8893c2664902221a9ac69ba6b555aa07629dfada..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/en/tryme.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/zh/HC2020.png b/app/.vitepress/public/img/videos/technical/zh/HC2020.png
deleted file mode 100644
index 25a252932b57d0b91def861b33c8659fffcb646b..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/zh/HC2020.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/zh/community.png b/app/.vitepress/public/img/videos/technical/zh/community.png
deleted file mode 100644
index 975e199cf0b4651e4f56d0bcc52ece00ef90d305..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/zh/community.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/zh/intro.png b/app/.vitepress/public/img/videos/technical/zh/intro.png
deleted file mode 100644
index 66242ebec3ca3764c065076f354dd145482b307a..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/zh/intro.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/zh/meetup21-zh-01.png b/app/.vitepress/public/img/videos/technical/zh/meetup21-zh-01.png
deleted file mode 100644
index 202337fd02342efd3b15042750dbcb4498a41788..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/zh/meetup21-zh-01.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/zh/meetup21-zh-02.png b/app/.vitepress/public/img/videos/technical/zh/meetup21-zh-02.png
deleted file mode 100644
index 630ae4ca78b2f734ed37fff8ff878b25fcf2e25a..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/zh/meetup21-zh-02.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/zh/meetup21-zh-03.png b/app/.vitepress/public/img/videos/technical/zh/meetup21-zh-03.png
deleted file mode 100644
index 6e1a9cc15d21dd3f522248eeff9776e9262baa84..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/zh/meetup21-zh-03.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/zh/meetup21-zh-04.png b/app/.vitepress/public/img/videos/technical/zh/meetup21-zh-04.png
deleted file mode 100644
index 8b810fd2420df5720337475580bac2e30207c3d2..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/zh/meetup21-zh-04.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/zh/meetup21-zh-05.png b/app/.vitepress/public/img/videos/technical/zh/meetup21-zh-05.png
deleted file mode 100644
index 3877041f894329df65a59f250f9a637d352d3b11..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/zh/meetup21-zh-05.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/zh/optimization.png b/app/.vitepress/public/img/videos/technical/zh/optimization.png
deleted file mode 100644
index 45ef3026bb2589f31f68c975e603ef5b8684d5c8..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/zh/optimization.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/zh/plan.png b/app/.vitepress/public/img/videos/technical/zh/plan.png
deleted file mode 100644
index e58db5add787ff6142c36bcabf893b2b964d7d88..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/zh/plan.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/zh/practice.png b/app/.vitepress/public/img/videos/technical/zh/practice.png
deleted file mode 100644
index 90520481f44d9af3e2c3d340816a433a0407ac44..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/zh/practice.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/zh/release.png b/app/.vitepress/public/img/videos/technical/zh/release.png
deleted file mode 100644
index 565dbd09962de7d75166c232650311a6c33e22ef..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/zh/release.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/zh/speech.png b/app/.vitepress/public/img/videos/technical/zh/speech.png
deleted file mode 100644
index 387091cfeb47ca3406fb2dea7fd74d798b53c68a..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/zh/speech.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/zh/tryme.png b/app/.vitepress/public/img/videos/technical/zh/tryme.png
deleted file mode 100644
index 7648bd7c534b337ae928f618ce724dbe681a0747..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/zh/tryme.png and /dev/null differ
diff --git a/app/.vitepress/public/img/videos/technical/zh/zh-datafun.png b/app/.vitepress/public/img/videos/technical/zh/zh-datafun.png
deleted file mode 100644
index b0ccf0d03e917943f2827d337f8536b8cd769a15..0000000000000000000000000000000000000000
Binary files a/app/.vitepress/public/img/videos/technical/zh/zh-datafun.png and /dev/null differ
diff --git a/app/.vitepress/src/App.vue b/app/.vitepress/src/App.vue
index c0c826c114b8fbe931d00799522fbb2275789014..498c84d623150949f82a94592d4df489c7aa9d6d 100644
--- a/app/.vitepress/src/App.vue
+++ b/app/.vitepress/src/App.vue
@@ -4,11 +4,8 @@ import { useData } from 'vitepress';
import AppHeader from '@/components/AppHeader.vue';
import AppFooter from '@/components/AppFooter.vue';
-import CookieNotice from '@/components/CookieNotice.vue';
-import AppInfoDetail from '@/components/AppInfoDetail.vue';
const { frontmatter } = useData();
-const isCustomLayout = computed(() => !!frontmatter.value.customLayout);
const bodyHeight = ref('');
onMounted(() => {
@@ -27,15 +24,12 @@ onMounted(() => {
{{ footer.follow
}}
import { computed, onMounted, onUnmounted, ref, watch } from 'vue';
-import { useI18n } from 'vue-i18n';
import { useRouter, useData } from 'vitepress';
import { NavItem } from '@/shared/@type/interface';
import { useCommonStore } from '@/stores/common';
import { windowOpen } from '@/shared/utils';
-import HeaderNav from './ONav.vue';
import AppTheme from './AppTheme.vue';
import AppLanguage from './AppLanguage.vue';
-import NavLangFilter from '@/data/common/navLangFilter';
import IconLightLogo from '@/assets/openlookeng-logo.svg';
import IconDarkLogo from '@/assets/openlookeng-logo-dark.svg';
@@ -18,116 +15,18 @@ import IconMenu from '~icons/app/menu.svg';
import logoMoLight from '@/assets/logo-mo-light.png';
import logoMoDark from '@/assets/logo-mo-dark.png';
-import { GITEE_LINK, GITHUB_LINK, DOCS_LINK } from '@/data/url-config';
-
const router = useRouter();
const useCommon = useCommonStore();
-const { t } = useI18n();
const configData = useData();
const language = configData.lang;
-const navItems = computed(() => {
- return [
- {
- id: 'download',
- label: t('nav.download'),
- path: 'download',
- },
- {
- id: 'learning',
- label: t('nav.learning.title'),
- path: 'learning',
- children: [
- {
- id: 'docs',
- label: t('nav.learning.docs'),
- path: `${DOCS_LINK + configData.lang.value}/docs/docs/overview.html`,
- },
- { id: 'mooc', label: t('nav.learning.mooc'), path: 'learning/mooc' },
- {
- id: 'knowledge',
- label: t('nav.learning.knowledge'),
- path: 'learning/knowledge',
- },
- ],
- },
- {
- id: 'community',
- label: t('nav.community.title'),
- path: '',
- children: [
- {
- id: 'contribution',
- label: t('nav.community.contribution'),
- path: 'community/contribution',
- },
- {
- id: 'who-we-are',
- label: t('nav.community.team'),
- path: 'community/who-we-are',
- },
- {
- id: 'security',
- label: t('nav.community.security'),
- path: 'community/security',
- },
- ],
- },
- {
- id: 'information',
- label: t('nav.information.title'),
- path: '',
- children: [
- {
- id: 'news',
- label: t('nav.information.news'),
- path: 'information/news',
- },
- {
- id: 'blog',
- label: t('nav.information.blogs'),
- path: 'information/blog',
- },
- {
- id: 'events',
- label: t('nav.information.events'),
- path: 'information/activity',
- },
- {
- id: 'video',
- label: t('nav.information.video'),
- path: 'information/video',
- },
- ],
- },
- {
- id: 'code',
- label: t('nav.code.title'),
- path: '',
- children: [
- {
- id: 'gitee',
- label: t('nav.code.gitee'),
- path: `${GITEE_LINK}openlookeng`,
- },
- {
- id: 'github',
- label: t('nav.code.github'),
- path: `${GITHUB_LINK}openlookeng`,
- },
- ],
- },
- ];
-});
const activeNav = ref {{ cookie.title }}
- {{ cookie.desc }}
-
- {{ cookie.link }}
-
- {{ team.pmc.description }}
- {{ item }}
-
- {{ team.pmc.mailTitle
- }}{{ team.pmc.mailAddress }}
- {{ team.pmc.meetinngData }} {{ team.pmc.meetinngTheme }} {{ item.name }} {{ item.job }} {{ item.experience }} {{ team.committers.description }} {{ item.name }} {{ item.job }} {{ item.experience }} {{ team.support.description }}
- {{ team.support.mailTitle }}
- {{
- team.support.mailAddress
- }}
- {{ item.name }} {{ item.job }} {{ item.experience }} {{ team.pmc.description }}
- {{ item }}
-
- {{ team.pmc.mailTitle
- }}{{ team.pmc.mailAddress }}
- {{ team.pmc.meetinngData }} {{ team.pmc.meetinngTheme }} {{ item.name }} {{ item.job }} {{ item.experience }} {{ team.committers.description }} {{ item.name }} {{ item.job }} {{ item.experience }} {{ team.support.description }}
- {{ team.support.mailTitle
- }}{{ team.support.mailAddress }}
- {{ item.name }} {{ item.job }} {{ item.experience }}
- {{ download.descTxt.text1 }}
- {{ download.descTxt.text2 }}
- {{ download.descTxt.text3 }}
- {{ download.descTxt.text4 }}
-
- {{ download.serverDetail1 }}
- {{ item.detail }}
- {{
- item.linkText
- }}
- {{ item.detail2 }}
- {{ activityInfo.description }}
-
-
-
-
- {{ item.date.split('-')[2] }}{{ item.date.substring(0, 7) }}
-
- {{ item.date }}
-
- {{
- authorItem
- }}
- {{ item.description }}
-
- {{ item.date.split('-')[2] }}{{ item.date.substring(0, 7) }}
-
- {{ item.date }}
-
- {{
- authorItem
- }}
- {{ item.description }}
-
- {{ item.label
- }}{{ lang === 'zh' ? `(${item.value})` : ` (${item.value})` }}
- {{ item.title }} {{ item.title }}
-
-
- {{ itemText }}
- {{ item.month }} {{ item.description }}
-
-
- {{ item.month }} {{ item.title }} {{ item.description }}
-
- {{ item.lable }}
- {{ item.description }}
-
-
-
- {{ item.description }}
-
-
- {{ list.title }}
- {{ label }} {{ item.title }} {{ item.title }}
- {{ item.detail }}
- Displays snapshot statistics on the CLI in debug mode: - Number of snapshots being captured currently - Number of snapshots captured already - Number of recovery times (either by resuming or restarting) - Size of all the snapshots captured and time taken to capture the snapshots - Size of snapshots restored and time taken to restore Adds customizable retry policies: - Timeout based retry policy - Max-retry based retry policy (文件采用 ORC 格式) (0~99 个查询) (节点数,核数,内存(只考虑同构情况)) (0~99,分别代表 100 个任务) (包括 openLooKeng 查询的基本配置参数和关于 hive 连接器的相关配置参数) (与集群参数相关) (5%以上) (20s) (简称 CCF BDCI) ▲ CCF BDCI 颁奖现场 ▲ CCF BDCI 颁奖现场 ▲ 中科大星辰队队长:徐宇鸣 ▲ 中科大星辰队答辩现场 (中科大星辰队) (左深连接) (文件使用 ORC 格式) (共 99 个 SQL Query) (图 1) ▲ 图1:给定测试集耗时超过100秒的Query(时间占比) (left-deep tree) (根结点除外) ▲ 图2:Query72的连接操作结构吞吐量展示图 (如图 3) ▲ 图3:openLooKeng将临时中间表内联进主查询 (规模大) (规模小) ▲ 图4:用memory connector暂存临时中间表 ▲ 图5:Query14.SQL预处理示例 ▲ 图6:重排结果展示(部分) ▲ 图7:复用结果展示(部分) ▲ 图8:最终结果 ▲ 大赛颁奖现场 ▲ 左:导师刘松灵、队员余扬、丁海杰 1 功能简介 2 功能实现原理 3 适配开发注意事项 1 安装指南 2 ODBC 连接 3 JDBC 连接 4 Tableau 操作指导 5 Power BI 操作指导 6 Yonghong 操作指导 1 openLooKeng 算子接口 2 openLooKeng 算子执行流程 1.1 多分片特性背景介绍 1.2 多分片原理及交互流程(1) 1.3 多分片原理及交互流程(2)-动态步长调整 1.4 多分片原理及交互流程(3)-业务流程 1.5 多分片原理及交互流程(4) 1.6 多分片配置使用方法介绍: 1.7 多分片特性的效果表现-单表测试 1.8 多分片特性的效果表现-TPC-DS SQL99 测试 1.9 结果分析 1 gaussDB 连接使用方法指导 2 使用 JMeter 执行 tpc-ds 测试 一 openLooKeng 编译和部署 二 Ranger 编译和部署 三 安装 Ranger openLooKeng Plugin 1 概述 2 测试 1 概述 2 测试 (即主 Coordinator) (执行计划缓存、元数据缓存、增量列式缓存) (bitmap/bloomfilter/min-max) (dynamic filters) Q1:林老师是第一次参加开源峰会吗? Q2:参加千人观众、万人在线直播的Keynote,紧张吗? Q3:在这场Keynote Demo中,openLooKeng发挥了什么作用? Q4:那北明软件跟openLooKeng是什么关系呢? Q5:据了解,北明软件与openLooKeng交流互动蛮久了。2020中国开源黑客松活动就见证了你们健康稳定的友好关系,前些天上线的openLooKeng商用版: 北明数据资产管理平台V4.0,更代表双方互动交流的紧密性。我很好奇,对北明软件来说,openLooKeng具备哪些方面的吸引力? Q6:在未来,北明软件希望openLooKeng往哪些方面发力优化? Q7:感谢林老师的采访,期待下次再见 2020 CCF BDCI 大赛颁奖现场 Q1:黎老师,又见面了。我知道老师是第二次参加活动,有没有不一样的体验跟我们分享? Q2:那其他成员对这次活动感觉怎么样? Q3:我看了下,openLooKeng的任务还是蛮重的,要在2天内完成2个特性的适配开发,这个时间充裕吗? Q4:看来团队的氛围很不错呀。 Q5:那作为学生,经过这次活动,刘超同学是怎么理解openLooKeng? Q6:据了解,招商银行其实也跟openLooKeng互动蛮久的,梁老师您怎么评价openLooKeng? Q7:对openLooKeng的未来会有哪些期望? ▲ 2022“科创中国”年度会议{{ t('brand.title') }}
-
-
-
-
-
diff --git a/app/.vitepress/src/views/community/contribution/TheContribution.vue b/app/.vitepress/src/views/community/contribution/TheContribution.vue
deleted file mode 100644
index 65a698cb147b609e0f1db8b9243cb4573870292c..0000000000000000000000000000000000000000
--- a/app/.vitepress/src/views/community/contribution/TheContribution.vue
+++ /dev/null
@@ -1,987 +0,0 @@
-
-
-
-
-
-
-
diff --git a/app/.vitepress/src/views/community/security/TheSecurity.vue b/app/.vitepress/src/views/community/security/TheSecurity.vue
deleted file mode 100644
index caa33269832cc5342d6b141553281f16a9301edc..0000000000000000000000000000000000000000
--- a/app/.vitepress/src/views/community/security/TheSecurity.vue
+++ /dev/null
@@ -1,144 +0,0 @@
-
-
-
-
-
-
-
diff --git a/app/.vitepress/src/views/community/who-we-are/WhoWeAre.vue b/app/.vitepress/src/views/community/who-we-are/WhoWeAre.vue
deleted file mode 100644
index 79949a5d11448b37a2df53ba050f4e6f2ed50272..0000000000000000000000000000000000000000
--- a/app/.vitepress/src/views/community/who-we-are/WhoWeAre.vue
+++ /dev/null
@@ -1,540 +0,0 @@
-
-
-
- {{ team.title }}
- {{ team.pmc.title }}
- {{ team.pmc.memberTitle }}
-
-
-
-
{{ team.committers.title }}
- {{ team.committers.memberTitle }}
-
-
-
-
{{ team.support.title }}
- {{ team.support.memberTitle }}
-
-
-
-
-
- {{ team.pmc.title }}
- {{ team.pmc.memberTitle }}
-
-
-
-
{{ team.committers.title }}
- {{ team.committers.memberTitle }}
-
-
-
-
{{ team.support.title }}
- {{ team.support.memberTitle }}
-
-
-
-
{{ download.pagestTitle }}
- {{ download.title }}
- {{ download.server }}
-
- {{ download.serverDetail2 }}
- {{ download.serverLink }}
- {{ download.serverDetail3 }}
- {{ download.hetuServerGz }}
-
- {{ download.hetuServerRpm }}
-
-
-
- {{ item.title }}
- {{ item.minTile }}
-
-
-
{{ activityInfo.title }}
- openLooKeng{{ t('nav.information.blogs') }}
-
-
- {{ item.title }}
- openLooKeng{{ t('nav.information.news') }}
-
-
- {{ item.title }}
-
-
-
-
-
diff --git a/app/.vitepress/src/views/home/HomeVideo.vue b/app/.vitepress/src/views/home/HomeVideo.vue
deleted file mode 100644
index c753424f851e267634e45ede0af75f8363f60880..0000000000000000000000000000000000000000
--- a/app/.vitepress/src/views/home/HomeVideo.vue
+++ /dev/null
@@ -1,268 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
{{ home.title1 }}
- {{ t('home.mooc_title') }}
-
- {{ t('home.latest_info') }}
- {{ t('home.featured_videos') }}
- {{ t('home.powered_by') }}
- {{ t('related_Links') }}
@@ -134,15 +44,6 @@ const goPath = (path: string) => {
-
-
{{ item.title }}
- {{ powered.title }}
- {{ powered.list[0].title }}
- {{ powered.list[1].title }}
- {{ powered.list[2].title }}
- {{ powered.list[3].title }}
- {{ powered.list[4].title }}
- {{ powered.list[5].title }}
- {{ t('nav.information.events') }}
-
-
- {{ tabsList[0].label }}
-
-
-
-
{{ item.title }}
- {{ tabsList[1].label }}
-
-
-
-
{{ tabsList[2].label }}
- {{ activity.calendarHead.lable }}
-
-
-
-
- {{ t('nav.information.blogs') }}
-
-
- {{ item.title }}
-
- {{ local.alertEmpty }}
- {{ t('nav.information.news') }}
-
- {{ item.title }}
-
- {{ t('nav.information.video') }}
- {{ navItems[0].styles }}
-
-
{{ knowledge.title }}
-
-
- {{ item.title }}
- {{ classify }}
-
-
- {{ mooc.courseTitle }}
- {{ mooc.subhead }}
-
-
-
-
- {{ moocBase.content[0].title }}
-
-
- {{ moocBase.content[1].title }}
-
-
- {{ moocPractice.practiceTitle }}
-
-
- {{ moocTheory.theorylist[0].title }}
-
-
- {{ moocTheory.theorylist[1].title }}
-
-
- {{ moocTheory.theorylist[2].title }}
-
-
- {{ moocTheory.theorylist[3].title }}
-
-
- {{ moocTheory.theorylist[4].title }}
-
-
-
-
-Continue to check the JAR package when openLooKeng finally packs the server. Currently, the logging framework of JDK is used instead of log4j.
-
-
-
-In the ES connector module, the analysis of the lib library shows that the library related to log4j is used.
-
-
-
-**Troubleshooting 2 : The openLooKeng uses the airlift framework, which encapsulates the log framework of the JDK.**
-
-
-
-Also, sort out all JAR packages in the openLooKeng server lib library. You can see that log4j-api and log4j-core do not exist.
-
-
-
-**Troubleshooting 3 : openLooKeng repository ranger plugin.**
-
-Check the repository content and don't find dependency of the log4j.
-
-In the POM file in the ranger, some version declarations about log4j were found, but were not used in subsequent builds.
-
-
-
-
-
-
-
-The openLooKeng ranger plugin depends on Apache Ranger 2.1.0, and the ranger 2.1.0 depends on the log4j2 2.13X version.
-
-In addition, in the current ranger, the module that depends on log4j is agents-audit.
-
-
-
-openLooKeng's ranger plugin does not depend on log4j
-
-### openLooKeng Community Restoration Program
-
-Upgrade to the latest component log4j-2.15.0. The release date of 2.15.0 is 2021-12-09 23:46.
-
-### Scenario 1 - Modify Code
-
-Update the code for the ES connector and Ranger plugin to remove the dependency of log4j.
-
-**Associated PRs are as follows:**
-
-
-
-
-
-### Security Issue Disclosure Process in the openLooKeng Community
-
-The security issues are handled as follows:
-
-- After receiving the security issue reported, the Security Committee (SC) immediately confirms the issue severity and integrity of the information reported.
-- Organize community teams to conduct technical analysis, identify the issue details, and provide analysis reports.
-- If the reported issue is a vulnerability, apply for Common Vulnerabilities and Exposures (CVE), communicate with vulnerability reporter about the issue and the subsequent fixing and release plan, and prepare the security advisory (SA).
-- Develop and verify the patch to fix the vulnerability, and initiate restricted disclosure.
-- Release the patch and SA.
-
-
-
-### Security Committee (SC)
-
-The SC consists of vulnerability management experts in the community. The team is responsible for coordinating the entire vulnerability fixing process, including:
-
-- Vulnerability collection: Potential security vulnerabilities discovered by community members and external researchers can be reported to the VMT via securities@openlookeng.io.
-- Vulnerability tracking and handling: The VMT confirms vulnerabilities, records the confirmed vulnerabilities in the openLooKeng community, fixes the vulnerabilities, and keeps communication with the reporter during the process.
-- Responsible disclosure: After the vulnerability is fixed, the VMT releases the vulnerability information to the community in the form of SA.
-- Organize activities such as project security architecture design, audit and code security review.
-- Provide advice for technical decision-making on community projects.
-
-### Meeting Organization
-
-- Meeting time: Monday morning every two weeks, 11:00 to 12:00, Beijing time
- - JOin us: Community will publish a meeting notice one day in advance. You can subscribe to the openLooKeng community email and join the openLooKeng community to get the notice.
-- Routine topics for SC meetings include:
- - Community vulnerability handling progress tracking
- - Track the progress of community security tools
- - Other community security topics, such as security guides and vulnerability reward programs.
-
-### openLooKeng SA
-
-
-
-
-
-
- Severity Rating
- Score
-
-
- Critical
- 9.0 - 10.0
-
-
- High
- 7.0 - 8.9
-
-
- Medium
- 4.0 - 6.9
-
-
- Low
- 0.1 - 3.9
-
-
- None
- 0.0
-
-
-
-
-
-
-
-
-
- Advisory
- Synopsis
- Severity
- Affected Product
- Affected Component
- Release Date
-
-
-
-
- openLooKeng-SA-2021-1213
-
-
- log4j security update
- Critical
- openLooKeng Engine
- HetuCore;ESConnector;RangerPlugin
- 2021/12/13
-
-
-
-
-
- Advisory
-
- openLooKeng-SA-2021-1213
-
-
-
-
- Synopsis
- log4j security update
-
-
- Severity
- Critical
-
-
- Affected Product
- openLooKeng Engine
-
-
- Affected Component
- HetuCore;ESConnector;RangerPlugin
-
-
-
- Release Date
- 2021/12/13
-
-
-
-
-These optimizations are done in the background after table creation so tables can be
-created quickly. The table can still be queried during this time, although you may not
-see the complete performance boost during this time.
-
-This comparison was done using queries similar to the following, using different predicate values:
-
-```
-SELECT COUNT(*) FROM table WHERE int_id=61;
-
-SELECT COUNT(*) FROM table WHERE varchar_id='APPID00061';
-```
-
-## Comparison With Existing Options
-
-Let’s compare our new solution with other existing options, ClickHouse and the Hive
-Connector:
-
-- ClickHouse is a column-oriented database management system for online
- analytical processing of queries.
-
-- The Hive Connector for openLooKeng allows accessing data from Apache Hive, a data
- warehouse software for reading, writing and managing large datasets residing in
- distributed storage using SQL.
-
-### Sorted and Indexed Data
-
-The new Memory Connector is much faster than the Hive Connector and is as fast as
-Clickhouse on simple equality queries on sorted **INTEGER** and **VARCHAR** columns. The
-results of these tests are shown in this graph:
-
-
-
-
-
-This comparison was done using queries similar to the following, using different predicate values:
-
-```
-SELECT COUNT(*) FROM table WHERE int_id=61;
-
-SELECT COUNT(*) FROM table WHERE varchar_id='APPID000001';
-```
-
-### Aggregation Queries Using JOIN
-
-The new Memory Connector can run complex, memory intensive JOIN and other
-Aggregation queries much faster because it benefits from openLooKeng’s built-in query
-optimization capabilities. Where similar queries fail in ClickHouse due to the lack of a
-JOIN query optimizer and memory limitations [1]. The results are shown in this figure:
-
-
-
-The complex queries used for this comparison were JOIN queries aggregating values
-from two tables on a VARCHAR column:
-
-```
-SELECT COUNT(*) FROM table_dm AS m
-INNER JOIN table_ds AS s
-ON m.varchar_id=s.varchar_id AND m.varchar_id='APPIDTEST000180';
-```
-
-### Unsorted and Unindexed Data
-
-Without sorting and indexing the performance of the new Memory Connector is
-similar to that of the old Memory Connector. So if you don’t take advantage of our
-new changes you can still expect the same performance as before.
-
-The times for the Hive Connector were more than 20 times slower than all of the
-other data sources, because of this it is not shown in this comparison. See this
-comparison in the picture below:
-
-
-
-For the tests without sorting and indexing the queries were similar to:
-
-```
-SELECT COUNT(*) FROM table WHERE int_id=61;
-```
-
-### StarTree Index
-
-As mentioned earlier, integration with StarTree index is in progress. This will allow
-cube tables to be stored in the Memory Connector so queries can gain additional
-performance benefits. Some preliminary testing (see image below) showed that
-StarTree index performance increased by **30%** at 100 concurrency using the Memory
-Connector compared to Hive Connector.
-
-
-
-This is an example of the queries used for this test:
-
-```
-SELECT
- event_id
- ,pt_d
- ,COUNT(DISTINCT did) as uv
- ,COUNT(DISTINCT session_id) as session_cnt
- ,COUNT(1) as pv
-FROM table
- where app_id = 'APPIDTEST00038'
-GROUP BY app_id, event_id, pt_d;
-```
-
-## Set-Up
-
-These tests used a dataset with 240 million rows with a total size of 4.0 GB. They
-were done on a computing cluster of 14 nodes where each node had 330GB of
-memory and 48 CPUs.
-
-### Memory Connector
-
-The tables on Memory Connector were set up differently for the sorted tests and the
-unsorted tests (which was the same as the old Memory Connector).
-
-For the sorted tests the table was created with the **sorted_by** property on the column
-we were querying on:
-
-```
-CREATE TABLE memory.default.table
-WITH (
- sorted_by=array['id']
-)
-AS SELECT * FROM hive.schema.table;
-```
-
-For the unsorted tests the table was created without the **sorted_by** keyword.
-
-```
-CREATE TABLE memory.default.table
-AS SELECT * FROM hive.schema.table;
-```
-
-### Hive Connector
-
-For Hive Connector, first, the table was created with data stored on HDFS:
-
-```
-CREATE TABLE hive.default.table (*)
-WITH (
- external = false,
- format = 'ORC',
- location = 'hdfs://hacluster/user/hive/warehouse/default.db/table'
-);
-```
-
-Next, a Bloom index was created on the Hive table using Heuristic Index to make the
-comparison more fair:
-
-```
-CREATE INDEX idx USING bloom ON hive.default.table;
-```
-
-### ClickHouse
-
-The tables on ClickHouse were created with this query:
-
-```
-CREATE TABLE default.table_distributed
-(
- `app_id` String
- ...
-)
-ENGINE = Distributed('cluster', 'default', 'table', rand())
-
-CREATE TABLE default.table
-(
- `app_id` String
- ...
-)
-ENGINE = MergeTree()
-PARTITION BY day
-ORDER BY app_id
-SETTINGS index_granularity_bytes = 1048576,
-index_granularity = 8192;
-```
-
-Queries were run against the distributed table in order to utilize all of the nodes.
-
-### StarTree Index
-
-First a table was created in the Hive Connector.
-
-```
-CREATE TABLE hive.default.table (*)
-WITH (
- external = false,
- format = 'ORC',
- location = 'hdfs://hacluster/user/hive/warehouse/default.db/table'
-);
-```
-
-Next, cubes are created on the above table and stored in the Hive Connector and the
-Memory Connector.
-
-```
-CREATE CUBE hive.default.cube ON hive.default.table
-WITH (
- aggregations =
- (
- COUNT(DISTINCT session_id),
- COUNT(DISTINCT did),
- COUNT(*)),
- group=(
- app_id,
- event_id,
- pt_d),
- format='orc',
- orc_bloom_filter_columns=array['app_id']);
-
-CREATE CUBE memory.default.cube on hive.default.table
-WITH (
- aggregations =
- (
- COUNT(DISTINCT session_id),
- COUNT(DISTINCT did),
- COUNT(*)),
- group=(
- app_id,
- event_id,
- pt_d),
- sorted_by=array['app_id']);
-```
-
-Then data was inserted into the cubes from the specified table:
-
-```
-INSERT INTO hive.default.cube
-WHERE app_id BETWEEN 'APPIDTEST0001' AND 'APPIDTEST00015';
-
-INSERT INTO memory.default.cube
-WHERE app_id BETWEEN 'APPIDTEST0001' AND 'APPIDTEST00015';
-```
-
-The insert into statements were repeated with different ranges of **app_id** values. The
-final cube has 20M rows.
-
-## Conclusion
-
-If you need to explore and aggregate data across multiple sources use Memory
-Connector with openLooKeng. Our new changes to Memory Connector make it even more
-useful by speeding up query times and allowing tables to persist even when some
-workers are restarted.
-
-## Sources
-
-[1] https://clickhouse.tech/docs/en/sql-reference/statements/select/join/#performance
diff --git a/app/en/information/blog/blogguidance/index.md b/app/en/information/blog/blogguidance/index.md
deleted file mode 100644
index a440d6f39b9c617820fd8832e2ce142d807df838..0000000000000000000000000000000000000000
--- a/app/en/information/blog/blogguidance/index.md
+++ /dev/null
@@ -1,104 +0,0 @@
----
-title: 'Guidance to Post a Blog'
-date: '2021-07-20'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng']
-author: 'openLooKeng'
-description: 'Guidance to Post a Blog'
----
-
-## Preparation
-
-1. Refer to
-
-Three years later, and Canalys has just released its latest figures for the China public cloud market, as of June 2020:
-
-
-
-Several things are noteworthy from a comparison between 2017 and 2020:
-
-- Over 3 years, China’s public cloud infrastructure market has grown about 800%, from $1B over 6 months in H1 2017 to US$3.9 billion over 3 months in Q1 2020.
-
-- China’s public cloud market is coalescing around its larger technology leaders, BAT and Huawei.
-
-- During these 3 years, HUAWEI CLOUD grew and gained share the fastest, going from non-existent to now the #2 public cloud in China.
-
-On this 3-year anniversary of HUAWEI CLOUD, I’d like to take the opportunity to reflect on a few key trends in cloud computing: the effects of COVID-19 on cloud, the evolution of global cloud industries, and the importance of being open.
-
-### COVID-19 and the Acceleration of Digital Trends
-
-COVID-19 has been a terrible blight upon the world, inflicting both human suffering and economic hardship. As people have been staying at home and moving interactions online, the use of digital services and connections has spiked dramatically.
-
-From events to meetings to media to shopping, everything that can be online is moving online. Of course, in China, many of these activities had already moved online and onto smartphones years ago—and at much higher adoption rates than in other countries. Ubiquitous app-based food delivery, restaurants with QR-code-based menus, cashless spending, and many other digital services are so normal here, they are the rule rather than the exception.
-
-Even with all these digital services, though, China’s total spending on public cloud infrastructure is still relatively small both for its economy size and also when compared with other markets. **So, it is remarkable that even in China, COVID-19 has brought such a dramatic increase in spending on cloud computing services.**
-
-As evidenced by a 67% increase in Q1 cloud spending against a massive global slowdown, COVID-19 has demonstrated to Chinese companies that in times of abrupt need, only cloud can truly deliver the necessary scale, agility, and depth of services required. Thus, even after this COVID-19 crisis has passed, the acceleration towards cloud in China will continue.
-
-### The Evolution of Global Cloud Industries
-
-As China and the world accelerate their adoption of cloud, what will this look like? Different regions will evolve differently, but they will share many similar characteristics. For example, consider both China and Europe.
-
-### China and New Infrastructure
-
-To help address the economic fallout from COVID-19, the Chinese government recently announced to accelerate a multi-trillion yuan investment plan towards New Infrastructure, the goal of which is to provide ubiquitous and high quality connectivity and computing. Target technologies and industries for New Infrastructure spending include cloud, AI, 5G, blockchain, and industrial internet.
-
-
-
-So, not only did COVID-19 bring a spike in cloud consumption in China while everyone stayed home, it is also leading to a massive new investment across the country in digital technologies. This in turn will only further accelerate the adoption of cloud services in China.
-
-### Europe and Digital Sovereignty
-
-Like China, the European Commission has recently introduced a massive €750 billion stimulus plan to deal with the COVID-19 fallout, Next Generation EU. And, like with China’s New Infrastructure, Europe’s **Next Generation EU** calls for spending heavily on projects towards strengthening for the digital age:
-
-- Investing in more and better connectivity, especially in the rapid deployment of 5G networks.
-
-- A stronger industrial and technological presence in strategic sectors, including artificial intelligence, cybersecurity, supercomputing and cloud.
-
-- Building a real data economy as a motor for innovation and job creation.
-
-- Increased cyber resilience.
-
-In addition, though, as Europe invests in these technologies, it is also doing so with an eye towards digital sovereignty. And so, also this past month, (GAIA-X)[https://www.data-infrastructure.eu/GAIAX/Navigation/EN/Home/home.html], a project to build a federated digital infrastructure for Europe, officially announced their new European foundation:
-
-_With GAIA-X, representatives from politics, business and science from France and Germany, together with other European partners, create a proposal for the next generation of a data infrastructure for Europe: a secure, federated system that meets the highest standards of digital sovereignty while promoting innovation. This project is the cradle of an open, transparent digital ecosystem, where data and services can be made available, collated and shared in an environment of trust._
-
-[Data Infrastructure EU, GAIA-X](https://www.data-infrastructure.eu/GAIAX/Navigation/EN/Home/home.html)
-
-According to a recent Gartner study, the adoption rate of public cloud for many European countries is similar to that of China and also lags many other markets:
-
-
-
-With projects like Next Generation EU and GAIA-X, however, European adoption of cloud and digital technologies should rapidly accelerate over the next few years. And just as in China and around the world, we look forward to sharing our experience and expertise and to working together openly and securely to build out this next generation of digital services.
-
-### The Importance of Being Open
-
-How was HUAWEI CLOUD able to grow so quickly over such a short time? One of the things that distinguishes Huawei is that we believe in the power of open innovation.
-
-This is how we view our HUAWEI CLOUD business:
-
-
-
-### Open Hardware
-
-First, Huawei believes that as we move towards next generation cloud services, AI, energy efficiency, edge computing, and many additional factors are going to drive cloud towards heterogeneous architectures. To support that, Huawei offers a variety of advanced chipsets optimized for different types of use cases, including x86 CPU, Arm CPU, GPU, NPU etc.
-
-Unlike other cloud providers, though, we share our hardware and chipsets openly with other vendors to implement their own systems and clouds. We do not restrict them only to HUAWEI CLOUD.
-
-### Open Source Software
-
-HUAWEI CLOUD is built on open source software. From OpenStack at the IaaS layer to Kubernetes and other CNCF projects at the container and PaaS layer to projects like our own open source AI framework [MindSpore](https://www.mindspore.cn/) for innovative services, HUAWEI CLOUD is arguably the most open cloud in the industry.
-
-### Open, Collaborative Innovation
-
-Huawei is not just consuming open source software, though. Indeed, we are a top-level member and top contributor across all the major open source foundations and projects, including Linux Foundation, CNCF, OpenStack Foundation, Apache Foundation, and Eclipse Foundation. It is by deeply participating in the open source ecosystem that we are able to bring such fast innovation to market and grow at such a high rate.
-
-It’s not just in open source communities that we collaborate with others. This is why we have such a big partner focus at HUAWEI CLOUD and look to enable them to be successful as well. When they do well, our customers do well, and we all do well together.
-
-### To Shared Recovery and Triumph
-
-I hope that during this period of deep trouble, a spirit of open, collaborative innovation can help us all work together for our shared recovery. Cloud computing works because it shares resources across many different users. Huawei Cloud has succeeded because we share in the open development and benefits of cloud.
-
-Europe, China, and the world will all have different paths forward out of our current situation. But, may the next three years and beyond bring for everyone a hopeful and triumphant future.
diff --git a/app/en/information/blog/gravitys169/2020-06-30-introduction-to-data-virtualization-engine-openlookeng-01.jpg b/app/en/information/blog/gravitys169/2020-06-30-introduction-to-data-virtualization-engine-openlookeng-01.jpg
deleted file mode 100644
index b504c61a006578676f21a24f9f5cf9e9c00f81f5..0000000000000000000000000000000000000000
Binary files a/app/en/information/blog/gravitys169/2020-06-30-introduction-to-data-virtualization-engine-openlookeng-01.jpg and /dev/null differ
diff --git a/app/en/information/blog/gravitys169/2020-06-30-introduction-to-data-virtualization-engine-openlookeng-02.jpg b/app/en/information/blog/gravitys169/2020-06-30-introduction-to-data-virtualization-engine-openlookeng-02.jpg
deleted file mode 100644
index 648d378ae6c583789bbc110dd75bac7159990a6d..0000000000000000000000000000000000000000
Binary files a/app/en/information/blog/gravitys169/2020-06-30-introduction-to-data-virtualization-engine-openlookeng-02.jpg and /dev/null differ
diff --git a/app/en/information/blog/gravitys169/index.md b/app/en/information/blog/gravitys169/index.md
deleted file mode 100644
index 6044a3f7b55c0fcb693c0f730066709413674a94..0000000000000000000000000000000000000000
--- a/app/en/information/blog/gravitys169/index.md
+++ /dev/null
@@ -1,148 +0,0 @@
----
-title: 'OmniRuntime:Powering Next-Generation Analytics'
-date: '2021-08-06'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', 'BigData', 'OmniRuntime']
-archives: '2021-08'
-author: 'gravitys169'
-description: "Introduce what's OmniRuntime and it's benefits。"
----
-
-Modern analytics engines play a crucial role in serving businesses with real-time analysis at low latency and cost amidst data explosion in scale and variety. Moreover, present-day applications expect these systems to deliver high query concurrency and high throughput while supporting queries over both structured and unstructured data.
-
-To satisfy these requirements, modern analytics engines and databases often rely on the compilation of query plans to native code. Hand-written operators such as FILTER, GROUPBY, and JOIN, typically contain control flows required to process different parameters and data types. Native code generation aims at minimizing the number of instructions by eliminating unnecessary code and specializing it for the underlying platform.
-
-These techniques are widely adopted in many analytics engines such as Spark, Flink to optimize similar functionality. However, current solutions tend to have limitations in terms of ease of feature development and support of heterogeneous hardware. We designed **OmniRuntime**, a generic data analytics accelerator, to address this limitation and provide support to further improve popular analytics platforms such as openLooKeng, Spark, and Hive.
-
-To deliver such capabilities, OmniRuntime provides five core components as illustrated in the following diagram:
-
-
-
- Figure 1. OmniRuntime Core Components Schematic
-
-- **OmniJit**: a transparent, easy–to-use, Just-In-Time compilation requiring zero knowledge of any low-level code generation framework such as LLVM or Janino.
-
-- **OmniVec**: a columnar in-memory format providing high-performance memory access, full memory lifecycle management and built-in SIMD operations. It also supports all commonly used data types such as int, double, varchar, and decimal.
-
-- **OmniOperator:** Native operator library, which makes full use of the computing capability of available hardware in today’s heterogeneous computing environment. When combined with **OmniJit**, the **OmniOperators** are capable of dnamically adapting to the workload, the parameters and the data profile to achieve the best performance.
-
-- **OmniCache**: a relational cache that outperforms traditional block oriented cache with extremely high hit ratio.
-- **OmniData**: a fast data access and collaboration layer provides bidirectional communication and data transportation between the data and compute layers.
-
-In the following section we will provide a high level description of each components.
-
-### OmniJIT
-
-OmniJit component is at the Omniruntime stack's core and contributes significantly to the overall performance improvement for analytics workload.
-
-OmniJit aims at democratizing Just-In-Time compilation for regular developers. It offers an easy-to-use framework in a high-level language such as C/C++. OmniJit automatically identifies the most performance-critical code sections in operators and optimizes them. The resulting operators are specialized using query-, runtime, and hardware-specific machine codes. OmniJit relies on runtime information to apply the best optimization based on the query context. This can be the dataset cardinality, column size, datatypes, hardware available (SIMD, accelerators, etc..), and many other information to produce the best operator optimization.
-
-By using OmniJit, analytics engine developers will no longer need to wrestle with the convoluted low-level APIs provided by systems such LLVM or Janino to extract the best performance.
-
-
-
- Figure 2. OmniJit branch cropping and circular unfolding optimization
-
-### OmniOperator
-
-OmniOperator represent the computational code that will process the data for a specific query, potentially written in SQL. A SQL query can be composed of a multitude of different operators and OmniJit is responsible for optimizing them on the fly. OmniJit optimizes the C++ operators to produce executables with minimum executed instruction count.
-
-The structure of the Operator interface is similar to the Volcano model as it follows a similar interface life cycle: Instanciate, AddInput, GetOuput and Close. All OmniOperators offer the same standard interface with the implementation being opaque to the analytics engine.
-
-By leveraging a standard interface, we can easily expose these operators to the upper-layer computing engine. Moreover, it helps us to provide a consistent development life cycle experience while allowing portability across various analytics platforms. The developer can either let OmniJit optimize the operator using the automated approach. Or, he can guides optimization policies such as parameter fixing, cyclical development, and vectorized adaptive execution for specific core methods within the code.
-
-The resulting OmniOperators have less executed instruction count, lower resource consumption, and reduce development overhead than native platform operators. The following figure shows the overall OmniOperator life cycle within a Java Analytics platform. As most analytics engines are coded in Java, we also provide a JNI interface to facilitate the integration.
-
-
-
- Figure 3. Interaction between the OmniOperator engine and the Native Operator
-
-### OmniVec
-
-OmniVector is the next component of OmniRuntime. OmniVector defines a standard columnar memory format. OmniVector is designed to be The portable language-independent column-based memory data format for OmniRuntime. OmniVector supports a rich data type system designed to address the needs of the various analytic data systems. This in-memory data format delivers high-performance and high scalability for data intensive applications.
-
-At its core, OmniVector is implemented in C++ with high-level language binding for cross-compatibility. It offers an asynchronous interface allowing reading and writing operations to be carried by the various components in parallel. This allows us to expose the content of an OmniVector while write or persistency operations are being carried over.
-
-The following figure shows the overall architecture of OmniVec.
-
-
-
- Figure 4. OmniVec Binging and Native Architecture
-
-Scope based full life cycle management of OmniVector not only eliminates the possibility of memory leak, but also provides the high performance memory access. Each OmniVector goes through several steps in its life cycle. Each step is triggered by a OmniVector operation:
-
-1. Scope Creation : In order to have efficient memory management and avoid pagefault overhead, vectors are allocated within a specific execution scope. This allows efficient memory pooling, which minimizes memory management overhead.
-
-2. OmniVector Allocation : By using the scope create above, developers can now allocate a new vector.
-
-3. OmniVector Mutation : the API support Set and Put operations. The former operates a single value at a specific corresponding index position. The latter is the batch put method, in which an array is inserted to a specified start position.
-
-4. OmniVector Read : Like the Mutation, we either support single value retrieval or batch operations.
-
-5. OmniVector Release : Once a vector is not used, it can then be released to the scope memory pool
-
-6. Scope Release: Once all vectors are released, we can release the scope and the associated memory pool.
-
-As a result of this design OmniVector support the following features:
-
-- Zero copy operations
-- Supports data lifecycle management and memory leakage detection.
-- Supports complex data structures, such as MAP, LIST, and Structure.
-- SIMD instruction optimization and hardware acceleration interface
-- Automatic spill over to storage
-- High performance memory allocation and pooling.
-
-### OmniJIT, OmniVector, And OmniOperator performance
-
-We integrated OmniRuntime and OmniJit in popular Big Data systems such as openLooKeng, Spark, and Hive. We then benchmarked the integration using TPC-H to show that the designed framework significantly outperforms commercial analytics engines.
-
-
-
- Figure 5. Operator effect of OmniJit optimization
-
-### OmniCache
-
-The OmniCache is a relational cache within OmniRuntime. The OmniCache not only caches data, but also maintains relation between data with schema information in the cache. OmniCache constructs virtual data sets, caches relational data derived from physical data sets or other virtual data sets, and uses SQL SELECT statements to define relational data for cache.
-
-This approach has the advantage over the traditional file block-based cache system. The entire file block must be cached or discarded, which may cause many data swap-in and swap-out operations, resulting in a very low cache hit ratio.
-
-To maintain relational information, OmniCache exposes its status and schema information to the query optimizer to maximize the hit ratio. The exposed cache metadata helps the analytics engine to optimize the query planand access the data in the cache first rather than through the slow data storage. Moreover, the cache is also able to store intermediate query results. It allows accelerated operation by enabling the query planner to directly access the previously computed data rather than executing the complete query plan as shown in figure 5.
-
-
-
- Figure 6. OmniCache operation schematic
-
-Omnicache expose the following features:
-
-- Cache management: Generate and manage materialized views by using materialized view commands.
-- SQL rewriting: SQL rewriting is performed using relational algebra and cost models to efficiently utilize cache data.
-- Cache Store: The relational cache store of the global memory pool is implemented based on the external memory of the heap, implementing efficient data storage and access.
-
-### OmniData
-
-OmniData is OmniRuntime's fast data access and collaboration layer targetting data transfer reduction between the segregated data and compute layers, which commonly seen in morden data centers.
-
-Rather than directly loading files from the storage side and then processing data, the OmniData offloads specific operations to the storage side for near-data processing. The objective is to cull the amount of required network communication and overall computation.
-
-OmniData achieves this by dividing the query execution into several segments matching the data distribution. Each segment representing sub-processing operations is then sent to the near or on storage node for execution. Affinity scheduling is used to avoid overloading the storage node computational capability and retain a high through overall throughput. The segment operations allow local data loading and processing.
-
-On top of its near data processing capabilities, OmniData also offer the following features:
-
-- Data encryption and compression leveraging native heterogeneous computing devices, the hardware accelerated adedicated encryption and compression capability engine can dramatically increase the data processing throughput.
-
-- Dynamic data filtering: we leverage dynamic code generation to support in-nic operations in order to maximize the processing throughput while reducing the resource used.
-
-- Network stack offloading: TOE and Zero-copy acceleration is further leverage for transferring the results from the local computation
-
-
-
- Figure 7. OmniData Operation Schematic
-
-### Summary
-
-OmniRuntime uses the combination of OmniJIT, OminVec, OmniOperator, OmniCache, and OmniData to provide a common data processing foundation for next-generation analytics platforms. OmniRuntime dramatically reduces the burden of creating custom optimization by offering a generic engine with contextualized optimization capabilities for different analysis engines.
-
-Omniruntime framework supports heterogeneous hardware environments including various processors (x86, ARM), accelerators (GPU, FPGA, ..), and interconnects.
-
-OmniRuntime's powerful capabilities for next-generation data processing enable the data analytics engines to meet current and future business requirements of high concurrency, high throughput, over structured and unstructured queries.
diff --git a/app/en/information/blog/gravitys169/omniruntime1.png b/app/en/information/blog/gravitys169/omniruntime1.png
deleted file mode 100644
index 082ca2836901d77b92c75314d2b7c5c6c06890d1..0000000000000000000000000000000000000000
Binary files a/app/en/information/blog/gravitys169/omniruntime1.png and /dev/null differ
diff --git a/app/en/information/blog/gravitys169/omniruntime2.png b/app/en/information/blog/gravitys169/omniruntime2.png
deleted file mode 100644
index 08770b5ec4c2485da294c678107ffb435ec3137b..0000000000000000000000000000000000000000
Binary files a/app/en/information/blog/gravitys169/omniruntime2.png and /dev/null differ
diff --git a/app/en/information/blog/gravitys169/omniruntime3.png b/app/en/information/blog/gravitys169/omniruntime3.png
deleted file mode 100644
index 1ec06c15aec125bfaac5944f28f4eae754be71ba..0000000000000000000000000000000000000000
Binary files a/app/en/information/blog/gravitys169/omniruntime3.png and /dev/null differ
diff --git a/app/en/information/blog/gravitys169/omniruntime4.png b/app/en/information/blog/gravitys169/omniruntime4.png
deleted file mode 100644
index 0e8a69c18c52e8111c9239b654c4114d25a750cf..0000000000000000000000000000000000000000
Binary files a/app/en/information/blog/gravitys169/omniruntime4.png and /dev/null differ
diff --git a/app/en/information/blog/gravitys169/omniruntime5.png b/app/en/information/blog/gravitys169/omniruntime5.png
deleted file mode 100644
index a7506480d2d1d3f112c840c3e384799787829e6a..0000000000000000000000000000000000000000
Binary files a/app/en/information/blog/gravitys169/omniruntime5.png and /dev/null differ
diff --git a/app/en/information/blog/gravitys169/omniruntime6.png b/app/en/information/blog/gravitys169/omniruntime6.png
deleted file mode 100644
index 03c0741daa9db5a974d35ba05c7401c592fbda17..0000000000000000000000000000000000000000
Binary files a/app/en/information/blog/gravitys169/omniruntime6.png and /dev/null differ
diff --git a/app/en/information/blog/gravitys169/omniruntime7.png b/app/en/information/blog/gravitys169/omniruntime7.png
deleted file mode 100644
index a9720f96605ce661fd0705219e7b630a79493081..0000000000000000000000000000000000000000
Binary files a/app/en/information/blog/gravitys169/omniruntime7.png and /dev/null differ
diff --git a/app/en/information/blog/index.md b/app/en/information/blog/index.md
deleted file mode 100644
index b23c9a85496a336d0f165933760149a4c530d12c..0000000000000000000000000000000000000000
--- a/app/en/information/blog/index.md
+++ /dev/null
@@ -1,5 +0,0 @@
-
-
-
-
-#### Building a Converged, Intelligent, Open Data Infrastructure
-
-With the best-of-breed computing power delivered by the "Kunpeng + Ascend" dual-engine, Huawei will continue to build computing, storage, and AI capabilities through continuous, intensive R&D investments, and technological innovation. Huawei’s data infrastructure includes data storage, processing, management systems and a data virtualization engine. It features the following:
-
-- **Converged**: Breakthroughs in storage, database, and big data technologies help ensure full data convergence at four levels: between storage systems; between databases and storage systems; between big data platform and storage systems; and between databases and the big data platform. This unprecedented level of convergence is expected to help customers slash TCO by over 30% and double data access, processing, and analytics performance.
-
-- **Intelligent**: A three-layer architecture—AI chipsets, storage, and HUAWEI CLOUD—allows efficient collaboration between cloud and on-premises resources, such as the flexible combination of on-cloud training and on-premises inference, delivering ever-faster performance and higher cost-efficiency as it adapts to the workloads. Huawei’s Ascend processors improve cache prefetching hit rate by automatically learning and identifying I/O flows, improving system performance by 20%. Huawei’s Kunpeng processors, with its multi-core architecture, automatically select the optimal data reduction algorithms in real time depending on the data types, reducing TCO by 25%. With HUAWEI CLOUD’s extensive, proven know-how in large system O&M, this intelligent infrastructure is able to predict disk faults 14 days in advance, performance bottlenecks 60 days in advance, and capacity insufficiency 365 days in advance. In addition, it allows self-healing for 30% of the faults.
-
-- **Open**: HetuEngine, Huawei’s data virtualization engine, makes it easier for users to locate, fetch, and utilize data. It makes data governance and use much simpler by offering a unified SQL interface, unified data catalogs, streamlined data, and unified security management, and hiding the differences in data types, localities, and syntaxes. It removes the technical complexity of the data infrastructure, allowing developers access to big data capabilities with the simplicity of an ordinary database. Reusing legacy ecosystems, tools, and skillsets improves development efficiency 2 to 10 times and saves on upfront investments.
-
-At this forum, Huawei also announced the open-sourcing of HetuEngine, its proprietary data virtualization engine designed to help customers and partners better connect data and applications. The open-source HetuEngine is called openHetu, and its launch time is scheduled for June 2020. With an open-source kernel, developers will be able to customize this engine, including data source extensions and SQL execution policies, to allow fast interoperability and development.
-
-
-
-Guided by the "Platform + Ecosystem" strategy and the principle of "open hardware, open-source software, and partner enablement", Huawei is committed to developing an open, rich Kunpeng ecosystem together with its customers and partners for the intelligent world.
-
-From November 18 to 19, 2019, Huawei hosted the 2019 Global Intelligent Data Infrastructure Forum themed "Pioneer Data Infrastructure for the Intelligent Era” in Shenzhen, China. Industry leaders and Huawei customers and partners attended this forum to discuss the future trends for computing in the intelligent era and ways to advance the computing industry, and also showcased the latest data infrastructure products and solutions. For more information, please visit
-
-[Shenzhen, China, July 1, 2020] Yesterday, Huawei officially announced openLooKeng, the data virtualization engine, goes open source at
-
-OmniData pushes the Filter, Aggregation, and Limit operators from the compute nodes to the storage nodes to implement near-data computing, unleashing diverse computing power to alleviate the CPU pressure on the compute nodes. Then it transmits the operator processing result to the compute nodes over the network, reducing invalid data to be transmitted and effectively improving big data computing performance.
-
-
-
-**Application scope of OmniData:**
-
-1. Supports openLooKeng v1.4.0 and Spark 3.0.0 and provides the OmniData plug-in for the engine.
-
-2. Pushes operators down to HDFS and supports access interfaces of Amazon S3.
-
-3. Supports different data formats such as TXT, ORC, and Parquet.
-
-4. Suitable for TaiShan servers and supports the Kunpeng processor architecture.
-
-OmniData makes big data simpler in the scenarios where storage and compute are decoupled or converged at scale, which complies with the vision of openLooKeng.
-
-The OmniData connector will further improve the performance of the openLooKeng engine.
-
-For details about the OmniData connector, visit the following website:
-
-
-
-openLooKeng was granted the Trusted Open Source Community certification at Cloud Native Industry Conference 2021
-
-The openLooKeng open source community has more than 30 partners, including China Merchants Bank (CMB), China Everbright Bank (CEB), AsiaInfo Technologies, Chinese Academy of Sciences (CAS), China Mobile, and China Unicom. They have brought contributions to key features of openLooKeng. The community also works closely with industry solution software partners to jointly launch scenario-specific solutions. For example, with openLooKeng, Beiming Software launched their Data Asset Management Platform v4.0 to offer data exchange and sharing services, and Legion Technology introduced the UIAUTO, a data asset management platform.
-
-
-Partners listed in no particular order
-
-While attracting a number of leading enterprises and scientific research institutes, openLooKeng has drawn attention from many open source big data enthusiasts. We would like to extend our gratitude to Wu Qiumin, a big data expert of CMB, Cheng Xiaojian of CEB, Hu Jiyun, general manager of Hong Hu Yuan Shu, He Zhengjie, a student developer, and other friends from the community for their sharing and contributions, which have brought new vitality to the community.
-
-
-Collaboration between users in the openLooKeng community
-
-Community users make contributions and communicate with each other to build a better openLooKeng
-
-We also have many open source enthusiasts who support the community behind the scenes. They are seen in community organizations, forums, and groups, asking and answering questions and making each type of contribution.
-
-
-openLooKeng community contributors (Gitee IDs)
-
-
-
-- openLooKeng has been downloaded for more than 90,000 times.
-- More than 300 contributors have made contributions to the community.
-- More than 1,900 pull requests have been submitted.
-- More than 1,400 commits have been created.
-- Over 600 members have joined the community communication groups.
-
-### Thank you to all the people who are with openLooKeng. We're connected in our passions for a better future.
-
-### openLooKeng, Make Big Data Simplified
diff --git a/app/en/information/news/2021-12-30/index.md b/app/en/information/news/2021-12-30/index.md
deleted file mode 100644
index 503cfc9da4660157cee58a613eed1b6f2df96347..0000000000000000000000000000000000000000
--- a/app/en/information/news/2021-12-30/index.md
+++ /dev/null
@@ -1,59 +0,0 @@
----
-title: 'openLooKeng v1.5.0 Officially Released'
-date: '2021-12-30'
-module: 'news'
-customLayout: 'true'
-categories: ['Announcement']
-author: 'openLooKeng'
-description: 'the community has attracted over 100,000 users and 2,000 pull requests, received 1,500 issues, and housed 1,500 comments. openLooKeng has been downloaded more than 100,000 times by users from 210 cities across 41 countries and regions. '
----
-
-On June 30, 2020, the data virtualization engine openLooKeng officially went open source. Since then, the community has attracted over 100,000 users and 2,000 pull requests, received 1,500 issues, and housed 1,500 comments. openLooKeng has been downloaded more than 100,000 times by users from 210 cities across 41 countries and regions. More and more people are paying attention to openLooKeng. Today, the neighborhood presents a vibrant, leafy scene.
-
-openLooKeng has provided users with simpler data analytics experience with each iteration. On December 30, 2021, openLooKeng v1.5.0 will be officially released. Based on the earlier versions, the existing features are optimized and new features are provided to improve the engine performance.
-
-### Star Tree Indexing
-
-In openLooKeng v1.5.0, the Star Tree indexing has been enhanced as follows:
-
-1、 Support for optimizing join queries, such as star schema queries.
-
-2、 Optimized the query plan by redirecting the aggregation operation to the Cube, which contains the pre-aggregation results. The performance improvement is obvious when the GROUP BY clause completely matches the Cube group.
-
-3、 Bug fixes to further enhance the usability, and robustness of Cubes.
-
-### Memory Connector
-
-1、Improved the performance of the memory connector by adding support for memory table partitioning to allow data skipping of entire partitions.
-
-2、 Support for openLooKeng cost based optimizers by collecting statistics on the memory tables.
-
-### Task Recovery
-
-Several important bug fixes to address data inconsistency issues, and query hanging issues that occasionally occur during high concurrency, and during worker failures.
-
-### Spill to Disk
-
-Optimized the spill-to-disk mechanism. Serialized pages are directly written to the disk instead of being cached. In this way, the operator uses less memory, boosting the performance by 30%.
-
-### OLK-on-Yarn
-
-Support for the deployment of HA-enabled openLooKeng cluster instance on Yarn. The instance contains a reverse proxy (Nginx by default), and two or more coordinator nodes. The cluster can be scaled in and out manually by adding and removing Yarn containers to the coordinator and worker components.
-
-These are the optimizations of the openLooKeng v1.5.0. As a key project of big data, openLooKeng has been committed to providing users with ultra-fast and simplified data experience. If you are a fan of big data, welcome to join openLooKeng for more big data knowledge and trivia. Together, we will build a prosperous big data ecosystem. Welcome to download and use openLooKeng.
-
-openLooKeng, Make Big Data Simplified!
-
----
-
-Download openLooKeng v1.5.0 at
https://gitee.com/openlookeng/hetu-core/pulls/1443 |
-
-## Hive UDF upgrade:
-
-| Description | PR |
-| --------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------- |
-| Added the adapation to the Hive user-defined function (UDF) namespace to support UDFs (including GenericUDF) based on the Hive UDF framework. | https://gitee.com/openlookeng/hetu-core/pulls/1455 |
-
-You are welcomed to submit issues to openLooKeng Gitee repositories and share your experience and suggestions. Your voice may become the key to improving the performance of the openLooKeng engine.
-
-openLooKeng repositories:
-
-
-
-
-## Task Recovery
-
-
-
-
-
- Allows you to customize the physical execution plans of worker nodes.
-Allows you to replace the native implementation with your own operator pipelines to accelerate operator processing.
-Supports native operator calculation.
-
-
-
-
-## openLooKeng Web UI Enhancement
-
-
-
- 1
- Enables query recovery by restarting the query, without capturing snapshots during query execution.
-
-
- 2
-
-
-
-
- 3
- Resource Manager: Suspends or resumes a query when it consumes a large amount of memory and the memory is insufficient. You can suspend or resume a query (only in CTAS and INSERT INTO SELECT).
-
-
- 4
-
-
-
-
-
-5
- Adds Gossip based failure detection mechanism: The coordinator continuously monitors the worker nodes to detect whether they are alive by receiving periodic heartbeats. The Gossip mechanism enables all worker nodes to monitor each other, and let the coordinator know about their local knowledge on each other's availability. Gossiping can prevent premature query failure in the case of transient network faults in the cluster.
-
-
-
-
-## openLooKeng Cluster Management
-
-
-
- 1
- openLooKeng audit log feature enhancements.
-
-
- 2
- openLooKeng SQL history query feature enhancements.
-
-
- 3
- SQL Editor supports automatic association.
-
-
- 4
- Supports the collection of common SQL statements.
-
-
- 5
- When a data source loads an exception, the front end can be displayed through exception tags and does not affect the normal operation of the openLooKeng service.
-
-
-
-6
- The UI can display specific information about the data source (SHOW CATALOG).
-
-
-
-
-In the past two years, we have witnessed rapid growth of the openLooKeng community. This is all thanks to you, more than 120,000 developers from all over the world. It is your contribution that makes our community active and energetic.
-
-From your first encounter with openLooKeng to your first line of code and PR, we believe those are your special memories with openLooKeng. So what do you want to say to openLooKeng on its second anniversary? Leave your comments here: https://gitee.com/openlookeng-competition/events-issue/issues/I5EKF3?from=project-issue
-
-Download openLooKeng v1.7.0: https://openlookeng.io/en/download/
-
-openLooKeng repositories: https://gitee.com/openlookeng
-
-openLooKeng official website: https://openlookeng.io/en/
-
-WeChat ID of the openLooKeng Assistant: openLooKengoss
-
-Write down wishes for openLooKeng to win rewards: https://gitee.com/openlookeng-competition/events-issue/issues/I5EKF3?from=project-issue
diff --git a/app/en/information/news/2022-08-04/index.md b/app/en/information/news/2022-08-04/index.md
deleted file mode 100644
index a15236f47b882203f1cb7bb16d6a1e547196e383..0000000000000000000000000000000000000000
--- a/app/en/information/news/2022-08-04/index.md
+++ /dev/null
@@ -1,57 +0,0 @@
----
-title: 'Companies shouldn’t think of data as ‘an asset to be mined’'
-date: '2022-08-04'
-module: 'news'
-customLayout: 'true'
-categories: ['']
-author: 'Jenny Darmody'
-description: 'Huawei’s Benoit Hudzia discusses why the data companies collect is only as good as the data management strategy they have in place.'
----
-
-### Huawei’s Benoit Hudzia discusses why the data companies collect is only as good as the data management strategy they have in place.
-
-The phrase ‘data is the new oil’ was first attributed to British mathematician Clive Humby in 2006. Almost two decades later, the sheer volume of data that has been created is nearly insurmountable and harnessing the power of that data has become an art in itself.
-
-Benoit Hudzia, chief AI operations expert at the Huawei Ireland Research Centre, believes we are now entering an era where “data is the new soil for the industrial ecosystem”.
-
-“Today, the internet and smartphones have made data copious, ubiquitous and far more valuable. But companies need to avoid thinking about data as an asset to be mined,” he told SiliconRepublic.com.
-
-“They need to consider it as the raw material you can use to offer value to your customers and build a better relationship. And like how engines are required to change oil into energy, analytics are required to gain insight, extort value and produce business benefits from data.
-
-“We are entering the age where data is not just extracted and processed, but it now needs to be nurtured and enhanced to yield long-term recurring maximum value.”
-
-### The rapid growth of analytics must be balanced with ever-more effective governance and control’– BENOIT HUDZIA
-
-Hudzia first cut his teeth on big data and analytics in 2008 while working in the research department at SAP.
-
-“It was an exciting time to be in the industry because SAP had just demonstrated its new database platform, SAP High Performance Analytic Appliance, which allowed organisations to gain instant insight into business operations, so that they can analyse all the data available and quickly react to rapidly changing business conditions.”
-
-Hudzia went on to help build and run private cloud solutions at an Israeli start-up, and then to run public cloud operations for one of the largest custodian banks in the world, before moving to Huawei in 2020.
-
-In his current role, Hudzia helps develop new technology for Huawei big data solutions, with a particular focus on OpenLooKeng, Huawei’s open-source big data platform.
-
-“openLooKeng is a drop-in engine that enables in-situ analytics on any data anywhere, including geographically remote data sources,” he explained. “It provides a global view of all of data via its SQL 2003 interface with high availability, auto-scaling, built-in caching and indexing support.”
-
-With years of analytics experience under his belt, Hudzia believes data management is now becoming central to the data strategy of any company and that simply collecting data is not enough to extract meaningful insights.
-
-“To leverage data for delivering best practice analytics solutions, organisations must have a coherent strategy and operations for collecting, organising, protecting and storing, so it can be analysed for business decisions,” he said.
-
-“Data plays a critical role in today’s financial, transport and telecommunication system. If these systems are to remain stable and trusted, fair and effective, then the rapid growth of analytics must be balanced with ever-more effective governance and control.
-
-“Mastering these data processing challenges through effective data management in real and virtual worlds will enable organisations to use resources effectively, save time and money, and ultimately be successful in their business.”
-
-### Ongoing data challenges
-
-Hudzia said data must be treated as a corporate asset in order to properly leverage its power. This means allowing different business domains to operationalise their own data, backed by a central and self-service data infrastructure.
-
-“However, this decentralisation of property introduces a couple of challenges. Difficulties with managing the multiple data products and their corresponding metadata may very well lead to a mess of spaghetti data pipelines,” he said.
-
-In addition, regulations are increasingly playing a role in analytics and data management, and companies must ensure the way they manage data complies with relevant regulations, such as GDPR in the EU.
-
-“The EU has been at the forefront of this trend with many of the regulations such as Privacy Shield, Schrems II, GDPR, the legal framework on AI, etc,” said Hudzia.
-
-“Moreover, the EU is also trying to realise that you cannot ensure data sovereignty via regulation only. You need to be able to deploy infrastructure and services on native soil. That’s why it is promoting large-scale projects such as GaiaX to ensure the existence of a native compliant European data service ecosystem.”
-
-He added that other countries are also working on data regulations, with the US having recently announced work on the American Data Privacy and Protection Act.
-
-“This flurry of policy and regulation is forcing the larger players to rethink their data and cloud strategy. The consequence for many of them is the need to offer solutions; to offer hyper-local services satisfying regulatory rules, and to satisfy the growing need and requirements of the customer data ecosystem.”
diff --git a/app/en/information/news/2022-09-30/index.md b/app/en/information/news/2022-09-30/index.md
deleted file mode 100644
index 483dada2c259e3b50bc8ea6910b402b9df98dd13..0000000000000000000000000000000000000000
--- a/app/en/information/news/2022-09-30/index.md
+++ /dev/null
@@ -1,46 +0,0 @@
----
-title: 'openLooKeng v1.8.0 Officially Released'
-date: '2022-09-30'
-module: 'news'
-customLayout: 'true'
-categories: ['Announcement']
-author: 'openLooKeng'
-description: 'After three months of continuous community development, openLooKeng v1.8.0 is now ready for developers. It is an all-round update compared with the earlier version, delivering multiple new features and improvements.'
----
-
-After three months of continuous community development, openLooKeng v1.8.0 is now ready for developers. It is an all-round update compared with the earlier version, delivering multiple new features and improvements.
-
-Now, let's have a quick look at the key features of openLooKeng v1.8.0.
-
-### Task Level Recovery
-
-1.Enables query recovery by retrying only failed task(s) using exchange materialization
-
-2.Supported using HDFS for spooling.
-
-3.Supported Direct Serialization and compression too while materializing the exchange data.
-
-### Query Resource Management
-
-Query level resource monitoring that triggers mitigation actions like suspend/resume query or task, spill revocable memory, throttle scheduling, and kill for problematic queries in a resource group.
-
-### Connector Extension
-
-The singleData connector supplements the OLAP capability for the openGauss, enhances the capability of data analysis.
-Supports adaptor to the ShardingSphere and tidRange modes.
-
-For details, see https://gitee.com/openlookeng/hetu-core/tree/1.8.0/hetu-docs/en
-
-
- 1
- Provides graphical indicator monitoring and customization to obtain key system information.
-
-
- 2
- Provides configuration management of service attributes to meet actual service performance requirements.
-
-
- 3
- Provides operations on clusters, services, and instances to meet one-click start and stop requirements.
-
-
- 4
- Provides the openLooKeng online deployment function and supports online upgrade.
-
-
- 5
- Provides the node scaling service.
-
-
- 6
- Integrates the existing WebUI of openLooKeng.
-
-
-
-7
- The openLooKeng Manager tool supports user login and logout.
-
-
-Do you wish to be the first ones to try out the new features? Welcome to download and experience them, and give us your feedback.
-
-Download link:
-
-
-The open source version of HetuEngine, named openHetu, is scheduled to go live in June next year. After Huawei opens the openhetu kernel code, community developers can carry out personalized and customized secondary development based on openhetu, including data source extension, SQL execution strategy adjustment, etc., which can realize rapid application docking and further improve development efficiency.
-
-Hetuengine has four core competencies:
-
-① A directory – through online metadata awareness, build a global virtual data view of 1000 + different heterogeneous data sources to improve data query efficiency.
-
-② An entry – through the open connection framework and 5000 node SQL Engine, 30 kinds of heterogeneous data sources in different places can be accessed by unified SQL statements to obtain data quickly.
-
-③ A piece of data – through the carbondata technology, a piece of data can be analyzed in multiple scenarios, shared in multiple applications, without data migration and easy to use.
-
-④ Unified security: through fine-grained dynamic authorization and sensitive data auto sensing technology, centralized security configuration and control of heterogeneous data sources in different places are realized, data authorization time is greatly reduced, and data security and compliance are further strengthened.
-
-Huawei has announced the open source of its server operating system, Euler OS and database operating system, Gauss, and has announced that it is based on “Kunpeng”+ Through cloud services, intelligent computing and big data, the Kunpeng computing industry layout of shengteng will build end-to-end data storage, computing and use capabilities, build a converged, intelligent and open data infrastructure, let the data system shift from isolation to integration, from closed to open, and then use unified SQL with the help of data virtualization engine Statement, using big data just like using a database.
-
-Source:
-
-## openLooKeng Projects
-
-### Project 1: Iceberg Connector (Basic Difficulty)
-
-Apache Iceberg is an open table format designed for PB-level tables, one of the three currently popular open source data lake solutions. openLooKeng does not yet support Iceberg. This is where you come in!
-
-What you'll do:
-
-- Iceberg connector design documents
-- Iceberg connector code and test cases
-- Performance test result
-
-Skills you'll need:
-
-- SQL and Java programming
-- openLooKeng connector principles and design
-- Iceberg usage and design
-
-Community mentor: [Zhou Sipei](zhousipei@huawei.com)
-
-Related open source software repositories:
-
--
-
-
-继续排查 openLooKeng 最终打包 server 时的 jar 包,当前使用的是 JDK 自己的 logging 框架,没有使用 log4j。
-
-
-
-在对应的 ES connector 模块下,对 lib 库的分析,发现的确使用了 log4j 相关的库。
-
-
-
-**问题排查二:openLooKeng 使用的是 airlift,封装的 JDK 本身的 log 框架**
-
-
-
-同样对打包后的 openLooKeng server lib 库下所有的 jar 包进行梳理,可以看到没有 log4j-api 以及 log4j-core。
-
-
-
-**问题排查三 :openLooKeng 仓库 ranger plugin 代码**
-
-通过排查仓库的内容,并未发现相关 log4j 相关代码的依赖。
-
-在 ranger 中的 POM 文件中,发现了部分关于 log4j 的版本声明,但是并未在后续的构建中使用。
-
-
-
-
-
-
-
-当前 openLooKeng ranger plugin 依赖于 apache ranger 2.1.0 组件,而 ranger 2.1.0 组件依赖 log4j2 2.13X 版本,为间接依赖关系。
-
-并且,当前 ranger 中,对于 log4j 有依赖的模块是 agents-audit
-
-
-openLooKeng 的 ranger plugin 并不依赖 log4j
-
-### openLooKeng 社区修复方案
-
-升级至最新组件:log4j-2.15.0,其中 2.15.0 的发布时间应该为 2021-12-09 23:46 的版本!
-
-#### 方案 1 - 修改代码
-
-1、 针对 ES connector 和 Ranger plugin,更新代码,移除 log4j 的依赖。**关联 PR 如下** :
-
-
-
-
-
-
-
-### openLooKeng 社区安全问题披露流程
-
-
-
-
-
- 严重等级(Severity Rating)
- CVSS评分(Score)
-
-
- 致命(Critical)
- 9.0 - 10.0
-
-
- 高(High)
- 7.0 - 8.9
-
-
- 中(Medium)
- 4.0 - 6.9
-
-
- 低(Low)
- 0.1 - 3.9
-
-
-
- 无(None)
- 0.0
-
-
-收到问题后,我们将会按照如下流程处理安全问题:
-
-- 收到疑似安全问题后,安全委员会(Security Committee, SC)立即确认上报信息完整性和问题严重性;
-
-- 组织社区团队开展技术分析,确认问题细节,并给出分析报告;
-
-- 确认漏洞并申请 CVE,与漏洞上报者开展问题沟通,对齐后续修复&发布计划,准备安全公告(SA);
-
-- 完成漏洞补丁开发/验证,启动受限披露;
-
-
-
-- 公开发布补丁和安全公告(SA)。
-
-### openLooKeng 安全委员会(Security Committee, SC)
-
-通过接收和响应 openLooKeng 产品安全问题报告、提供社区安全指导,开展安全治理等活动提升社区产品的安全性。
-SC 的使命:为 openLooKeng 用户提供最安全的产品和开发环境。
-
-#### 工作职责
-
-安全委员会(Security Committee, SC)由社区内的漏洞管理专家组成,工作职责包括:
-
-- 漏洞收集:社区成员和外部研究者发现的疑似安全漏洞,都可以通过 securities@openlookeng.io 上报给 SC;
-- 漏洞跟踪处置:VMT 会将确认的漏洞录入 openLooKeng 社区,并负责漏洞的确认/修复,期间会与上报者保持有效沟通;
-- 负责任的披露:漏洞得到妥善的修复后,VMT 将会以 SA 的形式将漏洞信息发布到社区。
-- 组织开展项目安全架构设计审计、代码安全检视等活动。
-- 提供社区项目技术决策的安全意见。
-
-#### 组织会议
-
-- 公开的会议时间:北京时间,每双周周五上午,11:00 点~12:00 点
-- 会议参与方式:提前一天在 openLooKeng 社区发布会议通知,你可以订阅 openLooKeng 社区邮件,加入 slack/ 微信等渠道获得通知。
-- SC 会议的例行议题包括:
- - 社区漏洞处理进展跟踪
- - 社区安全工具进展跟踪
- - 其他社区安全专项议题,如安全指南、漏洞奖励计划等。
-- 欢迎对社区安全感兴趣的开发者参加旁听
-
-### openLooKeng 安全公告(SA)
-
-
-
-
-
-
-
-
-
- 公告
- 概要
- 严重等级
- 影响产品
- 影响组件
- 发布时间
-
-
-
-
- openLooKeng-SA-2021-1213
-
-
- log4j security update
- Critical
- openLooKeng Engine
- HetuCore;ESConnector;RangerPlugin
- 2021/12/13
-
-
-
-
-
- 公告
-
- openLooKeng-SA-2021-1213
-
-
-
-
- 概要
- log4j security update
-
-
- 严重等级
- Critical
-
-
- 影响产品
- openLooKeng Engine
-
-
- 影响组件
- HetuCore;ESConnector;RangerPlugin
-
-
-
- 发布时间
- 2021/12/13
-
-
-接下来在节点中查看配置文件信息。可以看到,localfile.porperties 及其相关的 metadata 并没有存放在本地目录 etc/catalog 下,而是在一个新建的共享目录 etc/catalog/catalog 中:
-
-
-
-### 3.动态目录实现
-
-以创建新的目录为例,其实现类为 `io.prestosql.catalog.DynamicCatalogService`,在 `createCatalog()`方法中,传入的参数包括 catalog 配置信息 CatalogInfo、用于写入配置文件的 `CatalogFileInputStream` 以及 HTTP 请求体 `HttpRequestSessionContext`。
-
-在 CatalogInfo 中,catalogName 和 connectorName 是不允许为空。其他内容包括以 Map 形式存储的 porperties 信息、需要加密的信息使用的 securityKey 等:
-
-
-
-在检查权限以及目录存在情况后,经过逐级调用,最终由 `AbstractCatalogStore` 类中的 `createCatalog()`方法执行。该方法中在设定的 file system 下生成了一个 `propertiesOutputStream` 用于在建立的配置文件中写入信息。先储存相关的元数据信息,最后储存对应的 properties 文件:
-
-
-
-在调用该方法的 `loadCatalogAndCreateShareFiles` 方法中,可以注意到:在将配置文件存储到共享文件系统中后不会立即写入日志,而是先查询所有节点的目录添加情况,当至少有一个节点确认添加了新的目录后,才会退出方法。
-
-
-
-### 4.动态目录更新的特性
-
-讨论 openLooKeng 配置动态目录后,在各种情况下进行更新操作的成功/失败情况。除了信息填错的情况之外,更新动态目录失败主要发生在不同数据源之间的切换中,是由动态目录提供的一种特性引起的。
-
-首先,在集群上通过动态目录配置一个 localfile 数据源:
-
-```
-curl http://coordinator_ip:port/v1/catalog -X POST --header 'X-Presto-User:admin' --form 'catalogInformation={"catalogName":"localfile","connectorName":"localfile"}
-```
-
-1. 数据源配置填写错误
-
- 以下两种更新操作均会直接表现为**更新失败**,而非**更新完成后无法连接到数据源**:
-
- ```
- curl http://coordinator_ip:port/v1/catalog -X PUT --header 'X-Presto-User:admin' --form 'catalogInformation={"catalogName":"localfile","connectorName":"local"}'
- ```
-
- 试图添加一个名为 local 的数据源:
-
- ```
- ex = {IllegalArgumentException@28937} "java.lang.IllegalArgumentException: No factory for connector [local]. Available factories: [memory, jmx, postgresql, elasticsearch, sqlserver, localfile, greenplum, tpch, mysql, hana, mongodb, example-http, tpcds, opengauss, hbase-connector, oracle, system, clickhouse, kafka, atop, hive-hadoop2, vdm, presto-thrift, dc, kylin]"
- ```
-
- ```
- curl http://coordinator_ip:port/v1/catalog -X PUT --header 'X-Presto-User:admin' --form 'catalogInformation={"catalogName":"localfile","connectorName":"localfile","properties":{"test":"test"}}'
- ```
-
- 试图向 localfile 数据源的配置文件中添加一个 test=test 的配置项:
-
- ```
- com.google.inject.CreationException: Unable to create injector, see the following errors:
- 1) Configuration property 'test' was not used
- 1 error
- ```
-
- 这两种情况本质都是在填写更新信息时有错误。运行以下命令将 localfile 目录更新为 MySQL 数据源成功:
-
- ```
- curl http://coordinator_ip:port/v1/catalog -X PUT --header 'X-Presto-User:admin' --form 'catalogInformation={"catalogName":"localfile","connectorName":"mysql","properties":{"connection-url":"jdbc:mysql://**.**.**.**:3306","connection-user":"root","connection-password":"**********","jdbc.pushdown.remotenamespace":"mysqlfun.default"} }'
- ```
-
-2. 改变连接的数据源时出错
-
- 现在,localfile 目录所连接的数据源是 MySQL。尝试将其改回 localfile:
-
- ```
- curl http://coordinator_ip:port/v1/catalog -X PUT --header 'X-Presto-User:admin' --form 'catalogInformation={"catalogName":"localfile","connectorName":"localfile"}'
- ```
-
- 更新报错时
-
- ```
- com.google.inject.CreationException: Unable to create injector, see the following errors:
- 1) Configuration property 'connection-password' was not used
- 2) Configuration property 'connection-url' was not used
- 3) Configuration property 'connection-user' was not used
- 4) Configuration property 'jdbc.pushdown.remotenamespace' was not used
- 4 errors
- ```
-
- 报错信息表明,`localfile.properties` 中 connectorName 改为 localfile 后,属于 MySQL 的其他配置信息没有删除,导致连接 localfile 数据源时报错。
-
- 查看代码,定位至 `io.prestosql.catalog.DynamicCatalogStore` 类中的 `updateCatalogAndShareFiles()`方法:
-
-
-
- 此段代码中,`previousCatalogInfo `是原目录中的信息,而 catalogInfo 中的内容来自于 HTTP 请求,即目标是将配置信息更改为 catalogInfo。先将 `previousCatalogInfo `里面的 properties 复制到新的配置信息 newProperties 中,然后将 catalogInfo 中的 properties 加到 newProperties 中。
-
- 可以注意到,对于 catalogInfo 和 `previousCatalogInfo `中所共有的配置项,在添加进 HashMap 类型的 proerties 时,会被 catalogInfo 覆盖,从而实现信息的更新。而 `previousCatalogInfo `中存在但 catalogInfo 中不存在的配置项,就会在这个过程中保留下来而不是被删除。因此在上面命令执行过程中,MySQL 的配置信息保留了下来。
-
- 这段逻辑实际上是一种特性,即用户在更新动态目录信息时,只需要在 HTTP 请求中填入目录名称和需要更改的配置项,而不需要更改所有的内容。例如,运行如下语句将 MySQL 的数据源的登录用户名由 root 改为 rooot:
-
- ```
- curl http://coordinator_ip:port/v1/catalog -X PUT --header 'X-Presto-User:admin' --form 'catalogInformation={"catalogName":"localfile","connectorName":"mysql","properties":{"connection-user":"rooot"} }'
- ```
-
- 刷新 web ui,localfile 目录无法连接到数据源,说明更新成功:
-
-
-
- 这种特性使用户在不改变数据源的更新中,操作更加方便,无需填入大量繁琐的信息;但当更新前的部分配置信息在更新后不再需要时,这种特性就会发生错误。这种错误常见于从一种数据源切换到另一种数据源的过程中。前段中从 localfile 切换到 MySQL 的更新成功执行,就是由于 `localfile.properties` 文件中,除了 `connector.name=localfile` 以外就没有其他的配置信息,因此在添加 MySQL 配置后也不会产生影响。
-
-**总结:** 用 PUT 命令进行更新主要适用于只改变数据源的部分配置的场合,而在更改目录所连接的数据源时,先 DELETE 后 POST 是更鲁棒的选择。
diff --git "a/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\2422.png" "b/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\2422.png"
deleted file mode 100644
index 7890516d24299315dad6c55faf98918b7730adc8..0000000000000000000000000000000000000000
Binary files "a/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\2422.png" and /dev/null differ
diff --git "a/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\2423.png" "b/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\2423.png"
deleted file mode 100644
index 2fb068287500ff6bf8aeda012a03884c6f4916d6..0000000000000000000000000000000000000000
Binary files "a/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\2423.png" and /dev/null differ
diff --git "a/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\2424.png" "b/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\2424.png"
deleted file mode 100644
index 82e84ebf65521c415311ef20ed14ab2a3e9a7a30..0000000000000000000000000000000000000000
Binary files "a/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\2424.png" and /dev/null differ
diff --git "a/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\2425.png" "b/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\2425.png"
deleted file mode 100644
index 05fcaee34d33fe630f8f9d9c37fc308ec9aea7cc..0000000000000000000000000000000000000000
Binary files "a/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\2425.png" and /dev/null differ
diff --git "a/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\2426.png" "b/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\2426.png"
deleted file mode 100644
index d55d8ee479dca735b32603812066292e34a93c08..0000000000000000000000000000000000000000
Binary files "a/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\2426.png" and /dev/null differ
diff --git "a/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\2427.png" "b/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\2427.png"
deleted file mode 100644
index 49b12c4f16f3a8302f004a26ed9ac1f83aaca639..0000000000000000000000000000000000000000
Binary files "a/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\2427.png" and /dev/null differ
diff --git "a/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\242\346\226\207\346\241\243\345\233\276\347\211\2071.png" "b/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\242\346\226\207\346\241\243\345\233\276\347\211\2071.png"
deleted file mode 100644
index 93754c41103a94af40dab34bfc9af886d81b513b..0000000000000000000000000000000000000000
Binary files "a/app/zh/information/blog/0827-zhangchenlu/\345\215\232\345\256\242\346\226\207\346\241\243\345\233\276\347\211\2071.png" and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/082812.png b/app/zh/information/blog/0829zhangchenlu/082812.png
deleted file mode 100644
index 6a23210d3c48ba22d7519f18addec8e96425c494..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/082812.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/082903.png b/app/zh/information/blog/0829zhangchenlu/082903.png
deleted file mode 100644
index d059d0bd67a75914ef3582124c32acd0ffdfa0f3..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/082903.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/082904.png b/app/zh/information/blog/0829zhangchenlu/082904.png
deleted file mode 100644
index a64f971e1875a82161b3d85b7ad217db5deef530..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/082904.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/082905.png b/app/zh/information/blog/0829zhangchenlu/082905.png
deleted file mode 100644
index 9b0ff1c41290a04303ec6c47f7f5a9fbe27b1b4d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/082905.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/082906.png b/app/zh/information/blog/0829zhangchenlu/082906.png
deleted file mode 100644
index 0821a7ea66f4945ab3dbcd7f71df55470b3cb1de..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/082906.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/082907.png b/app/zh/information/blog/0829zhangchenlu/082907.png
deleted file mode 100644
index 036c9090e1375fe68941445a6ee64aaef68b05bb..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/082907.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/082908.png b/app/zh/information/blog/0829zhangchenlu/082908.png
deleted file mode 100644
index 3de7fd4f19befa50b8357f01c303de3d4d4c6958..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/082908.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/082909.png b/app/zh/information/blog/0829zhangchenlu/082909.png
deleted file mode 100644
index 42a2ad81e764619f9c513066ebf869873818b5a4..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/082909.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/08291.png b/app/zh/information/blog/0829zhangchenlu/08291.png
deleted file mode 100644
index 48fd4dffc87316bd6a88ec87163b8104116b990b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/08291.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/082910.png b/app/zh/information/blog/0829zhangchenlu/082910.png
deleted file mode 100644
index 9e777abc3eef5b55d664557b4fe992c0db1677f7..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/082910.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/082911.png b/app/zh/information/blog/0829zhangchenlu/082911.png
deleted file mode 100644
index f591011b2a52f96444a31a4c1ed5c8937c071986..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/082911.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/082913.png b/app/zh/information/blog/0829zhangchenlu/082913.png
deleted file mode 100644
index 99e198a070317707e49fb0859a1d8e44b1876fdc..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/082913.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/082914.png b/app/zh/information/blog/0829zhangchenlu/082914.png
deleted file mode 100644
index 23e6b075801a390466cd5aac50f13d289db81d7f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/082914.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/082915.png b/app/zh/information/blog/0829zhangchenlu/082915.png
deleted file mode 100644
index e97592eeff0c6202ff2dfc9a140a038bf373ff22..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/082915.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/082916.png b/app/zh/information/blog/0829zhangchenlu/082916.png
deleted file mode 100644
index a34bdc25ec68c6dcac73f26774bcdf033e1b701a..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/082916.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/08292.png b/app/zh/information/blog/0829zhangchenlu/08292.png
deleted file mode 100644
index 25ba700ce25b79b16546a210e8c1456cdda66548..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/08292.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/17.png b/app/zh/information/blog/0829zhangchenlu/17.png
deleted file mode 100644
index 948e2e01a83c936599b8009e4555b58c41372fa6..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/17.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/18.png b/app/zh/information/blog/0829zhangchenlu/18.png
deleted file mode 100644
index 2ed180f3df026b04ddb305731b88bc13861057d9..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/18.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/19.png b/app/zh/information/blog/0829zhangchenlu/19.png
deleted file mode 100644
index 06673585ea486293e7422af0abc2c32a32f9f08f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/19.png and /dev/null differ
diff --git a/app/zh/information/blog/0829zhangchenlu/index.md b/app/zh/information/blog/0829zhangchenlu/index.md
deleted file mode 100644
index c6809296e46ddc88536ac0b878370fdf67770efd..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/0829zhangchenlu/index.md
+++ /dev/null
@@ -1,168 +0,0 @@
----
-title: 'openLooKeng SQL语句查询计划'
-date: '2022-08-29'
-module: 'blog'
-customLayout: 'true'
-tags: ['大数据', 'sql', 'openLooKeng']
-archives: '2022-08'
-author: '隋晓宇'
-description: ''
----
-
-基于源码梳理 openLooKeng 解析并执行传入的 SQL 语句的过程,执行流程:提交查询 -> 生成查询计划 -> 查询调度 -> 查询执行。在解析语句生成查询计划时,主要的流程为语法/语义分析 -> 执行计划生成 -> 执行计划优化。
-
-## 语法/语义分析
-
-### 1.1 语法分析
-
-openLooKeng 的词法与语法分析模块 SQL Parser 是基于 ANTLR 实现的。通过输入词法描述文件来自动构造语法分析树,且生成基于 Visitor 模式的树遍历器。
-
-Visitor 模式的工作方法为:对于由若干节点元素 Node 构成的语法树,每个 Node 都拥有一个 accept 方法来接受访问者对象 Visitor 的访问;Visitor 类是一个拥有 visit 方法的接口,对访问到的不同类型 Node 做出不同的反应。
-
-在遍历语法树时,对遍历到的每个 Node 都调用 accept 方法,在每个元素的 accept 方法中回调 Visitor 的 visit 方法,从而使得 Visitor 得以处理树上的每个 Node。
-
-openLooKeng 的词法分析在 `io.prestosql.sql.parser.SqlParser` 类的 invokeParser 方法中。使用词法分析器 SqlBaseLexer 进行词法分析,生成 token 序列:
-
-
-
-接下来先生成抽象语法树,然后进入语法分析:
-
-
-
-openLooKeng 使用 Visitor 模式进行语法分析。invokeParser 中的 `new AstBuilder().visit(tree)`经过多层调用进入 AstBuilder 中不同的 visitXXX 方法中;以 select 单列为例:
-
-
-
-可以发现,visitSelectSingle 方法会构造一个 SingleColumn 对象,它包含 select 的表名和一个 Expression 对象,该对象也是通过 `visit(context.)`调用 AstBuilder 类中其他 visitXXX 方法得到的。因此语法分析实际上是一个递归调用过程,构造出若干节点并形成抽象语法树。一棵语法树实际上就是一个 Statement。
-
-### 1.2 语义分析
-
-查询语句在 openLooKeng 中经过词法语法分析后,生成抽象语法树 AST。`io.prestosql.sql.analyzer.StatementAnalyzer` 类,对传入的 SQL 语句,针对不同的 Statement 实现类进行语义分析。StatementAnalyzer 使用 Visitor 模式进行语义分析,类中的各个 visitXXX 方法针对不同的 Statement 实现类做了处理。以 visitQuery 方法为例:
-
-
-
-visitQuery 首先分析 With 语句,然后对查询语句主体进行分析,最后对某些没有分析的 order by 语句进行分析,先检验 order by 所需的列是否在 select 语句中,然后考虑 order by 不是简单的列而是更复杂的语句时的情况,对其进行分析。
-
-分析子语句时,每个 analyzeXXX 方法中表达式分析的入口都是 Visitor 子类中的 process 方法。表达式分析器主要获取表达式的类型、函数信息、列名与列编号及 in 语句子查询等。
-
-### 1.3 分析查询主体语句
-
-分析语句主体的入口在该类的 visitQuerySpecifiction 方法中。存储传入的查询语句的实现类是 `io.prestosql.sql.tree.QuerySpecification`,该方法将对各类子句分别进行分析。
-
-```
-select id, name from test where salary > 500 order by id;
-```
-
-获取传入的查询,经过词义与语义分析后每个子语句已经被分离出来:
-
-
-
-以 seclect 子语句为例,词义分析的结果中 select 的 selectitems 即为提取出的列,analyzeSelect 方法返回的结果为 Expression 类型的 ImmutableList 列表 outputExpression,其中的每一个 Expression 表示 select 的一列。该方法遍历 select 中的每一列 item,根据列的类型进行分析。
-
-如果列为\*时,找到列描述符对应的 field,记录其对应的索引下标,存入 outputExpression 中:
-
-
-
-如果列为单列,就使用表达式分析器对其进行分析,并将列表达式存入 outputExpression:
-
-
-
-## 2.执行计划生成
-
-在语义分析后,要根据针对 SQL 语句分析的结果,生成逻辑执行计划。与语义分析树 AST 类似,执行计划也会生成执行计划树,不同类型的节点对应不同种类的语句。
-执行这项工作的是 `io.prestosql.sql.planner.LogicalPlanner` 类,根据 SQL 语句的类型生成不同的执行计划,并选择对应的执行计划优化器进行优化。
-以上述 SQL 语句为例,查看 QueryPlanner 子类中对 QuerySpecifiction 的处理:
-
-
-
-①:planFrom 处理 from 语句,使用 RelationPlanner 处理语句生成 Relation 执行计划。此处如果不存在 from 语句,说明输入的 SQL 语句是属于简单语句,直接构造 ValuesNode 返回。
-
-②:FilterNode 是进行过滤操作的节点。此处 filter 为 where 条件添加一个 FilterNode,然后 aggregation 处理聚合操作,接着后面的 filter 再为 Having 语句添加一个 FilterNode。
-
-③:ProjectNode 是用于列映射的节点,将该节点的下层节点(例如 TableScanNode)等输出的列映射到该节点的上层节点。此处 project 将 order by 等条件分析出的 Expression 列表进行传递。
-
-④ :进行其他操作的处理,例如 SortNode 用于排序操作。
-
-对于生成的初始查询计划,要对其进行优化。当查询计划需要优化时,遍历已有的所有 Optimizer,如果当前待优化的执行计划可以使用这个 Optimizer,就进行优化:
-
-
-
-查看运行信息可发现,在优化过程中多次调用了 `io.prestosql.sql.planner.iterative.IterativeOptimizer`。
-
-IterativeOptimizer 可以看作是在 PlanOptimizer 的基础上改进的。PlanOptimizer 中只有一个接口,输入一个 PlanNode 以及辅助参数,输出优化后的 PlanNode。这个接口实现一般都是实现一个使用 Visitor 设计模式的类,将多个优化策略杂糅到一个类中处理。
-
-IterativeOptimizer 在 PlanOptimizer 上面又包装了一层,将每一条优化规则抽象出单独的类 Rule,这样在查询优化的时候只需要编写 Rule,不需要实现 Visitor 模式。
-
-另一方面,IterativeOptimizer 为了执行计划优化的方便性及提高性能,在对 PlanNode 优化前会将其转化成一个可变的对象 Memo。在这个类中,所有的 PlanNode 被一个新的类 GroupReference 包装一层,这里的 GroupReference 仍然是不可变的,但它可以映射到可变的 Group,PlanNode 优化的过程实际上就是遍历 GroupReference 树,不断修改对应的 Group 里面的 PlanNode。
-
-上述语句优化过程中还使用了 SetFlatteningOptimizer,合并能够合并的 union 节点,减少执行计划树层数。优化前后的执行计划树节点情况对比如下:
-
-
-
-最后,使用 explain 命令查看执行计划。为了观察执行计划分段情况,执行如下语句:
-
-
-
-如图,该条语句的执行计划分为两段,Source 阶段是从数据源的表中读取数据的阶段,一般有 TableScanNode 和 FilterNode 等,此处同时进行了局部聚合;有时局部聚合、局部 Join 等操作也会在 Source 之后的 Fixed 阶段中;最后是 Single 阶段,只在单个节点上执行以汇总所有的处理结果,例如最终聚合、全局排序等。先通过 filter 扫描表的数据处理 where 语句,然后进行排序。排序时采用的是分段排序再合并的思想。
-
-openLooKeng 在生成查询计划后,会将其分段,拆分成多个有层级关系的 stage。在查询计划的调度与执行中,每个 stage 都会被进一步分解成若干个 task,分配到各个计算节点中执行。
-
-## 3.查询调度
-
-对于每一个 stage,openLooKeng 都生成一个对应的 SqlStageExecution 实例,称为调度执行器。对于 Single 和 Fixed 类型的 stage,均为调用 NodeSelector.selectRandomNodes。因此运行如下语句,主要观察 Source stage:
-`select count(\*) from hive.sxy.test where salary >400`;
-
-
-
-在 `io.prestosql.execution.SqlStageExecution` 中的 NodeSelector 是查询调度的核心,维护了一个 nodeMap,由 nodeManager 获取存活的节点列表;同时维护一个保存当前分配的 task 和节点的映射的列表 nodeTaskMap。
-在查询请求中,一个 table 会对应一个 Source stage,而 Source Stage 的节点选择策略是由 table 中 spilt 的数目决定的,尽量使每个节点处理的 split 相对平均。
-在进行调度前,先由 selectExactNodes 方法选择出一批候选节点 candidateNodes:
-
-
-
-接下来针对每个 source split 从候选节点中选择出满足条件的节点进行调度。
-遍历所有的 split,对于当前的 split,在 candidateNodes 范围内,选择出[`历史已分配 split 个数 splitCountByNode`]与[`当前批次已分配 split 个数 assignmentCount`]较小的节点。此二者之和定义为 TotalSplitCount,同时 TotalSplitCount 不能超过每个节点允许的最大数目 maxSplitsPerNode,如下图:
-
-
-
-通过查看 split 计数 assignmentCount 或选择出的节点 chosenNode 均可发现,此查询计划的 splits 被轮流分配到 candidateNodes 中的两个节点上,以保证这两个节点基本上是对 splits 平均分配的。
-
-## 4.查询执行
-
-在制定与调度查询计划的过程中,openLooKeng 中的查询会依次转化为 `SqlQueryExecution`、`SqlStageExecution`、`SqlTaskExecution`。因此所有的查询最终都转化为若干 Task 在每一个 worker 节点上执行。
-
-在创建 Task 时,调用 HttpRemoteTask 类的方法创建一个 RemoteTask 对象并启动其 start 方法,这个方法又会调用 httpClient 向 worker 节点发起 RESTful 请求,从而启动 SqlTaskExecution 进行数据处理。
-
-Client 端
-
-新 task 的创建是由 `io.prestosql.execution.SqlStageExecution` 类中的 scheduleTask 方法完成的,它的作用就是在指定的 Node 上启动一个 Task 并执行。以 SourceTask 为例,由于分批调度 Task,因此可能会多次调用这个方法。先判断对应的 Node 上是否已经创建 Task,若已经存在则直接 `addNewTask(taskId)`。
-
-
-
-①:收集所有的 SourceSplit。将 Source Stage 传入参数中的 split 与上游 Stage 中每个 Task 的输出组合起来,作为整体的 split 集合 initialSplits。从代码可以看出,planNodeId 代表一个 Stage,遍历每个上游 Stage 中的每个 task,将其组装成一个 split 添加进 initialSplits 中。
-
-②:更新当前 stage 的所有 outputBuffers,它用于将当前 stage 的数据传给下游 stage。
-
-③:根据已有的 initialSplits、Node、taskId 等信息,在 Node 上创建一个 Task,并调用 `task.start()`启动 initialSplits 的处理。
-
-Resource 端
-
-收到 RESTful 请求后,对其的处理主要是 `io.prestosql.server.TaskResource` 类中的 createOrUpdateTask 方法完成。该方法会调用 SqlTaskManager 类中的 `updateTask()`方法:
-
-
-
-该方法是通过全局缓存 tasks 的 `getUnchecked(taskId)`获取 SqlTask 对象的,获取到 SqlTask 对象后就会调用 `SqlTask.UpdateTask()`启动 Task 进行计算,此方法会更新 TaskId 所对应的 OutputBuffers 和待处理的 Split 列表。
-
-从前面对 Task 创建过程的梳理可以发现,openLooKeng 中每个 Task 在 worker 端(即 TaskResource 端)是以 SqlTaskExecution 的形式运行的。
-
-`io.prestosql.execution.executor.TaskExecutor` 类在每个 worker 节点启动的时候调用 start 方法运行,在该类中启动多个线程并加入线程池,每个 split 的处理都由一个内部类 TaskRunner 中的 `split.process()`完成:
-
-
-
-该方法逐级调用,经过 `PrioritizedSplitRunner.process()`最终调用 `split.processFor()`。此处代码中 split 的类型为 SplitRunner,而查看 SqlTaskExecution 类中的设置可得其实现类为 DriverSplitRunner,作用在 split 上的一系列操作的封装类也为 Driver 类:
-
-
-
-对 Split 的实际处理工作是由 `Driver.processInternal()`方法完成的。在该方法中,遍历所有的 operators,每次从 operators 中取出相邻的两个 operator,计算得到前一个 operator 的输出数据,然后将该数据作为后一个 operator 的输入数据:
-
-
diff --git a/app/zh/information/blog/0829zhangchenlu/x1.png b/app/zh/information/blog/0829zhangchenlu/x1.png
deleted file mode 100644
index 761f02f8cc2c41eb4af98f48b41cbdb5b276030b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/0829zhangchenlu/x1.png and /dev/null differ
diff --git a/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-01.jpg b/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-01.jpg
deleted file mode 100644
index ba3085f8ad717ca4d69f7a0b2e6fbe35e1f6c724..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-01.jpg and /dev/null differ
diff --git a/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-02.jpg b/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-02.jpg
deleted file mode 100644
index 1a1fe9222881a5dc8cc46fa647b4268d1e53f884..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-02.jpg and /dev/null differ
diff --git a/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-03.jpg b/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-03.jpg
deleted file mode 100644
index ee16e1534a7519ddc52e19f3fb964ac167825a80..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-03.jpg and /dev/null differ
diff --git a/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-04.jpg b/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-04.jpg
deleted file mode 100644
index 257fd8fd4df7e277fd14d3125672e40fbe0c966e..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-04.jpg and /dev/null differ
diff --git a/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-05.jpg b/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-05.jpg
deleted file mode 100644
index 1090651054440b09ce39283a960f0e78e4bfff84..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-05.jpg and /dev/null differ
diff --git a/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-06.jpg b/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-06.jpg
deleted file mode 100644
index 9f234835e5f9c13d00dc33066ff093c787b60e36..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-06.jpg and /dev/null differ
diff --git a/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-07.jpg b/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-07.jpg
deleted file mode 100644
index c6a42b2498ce76c5a12a73b78f4f09046081ee40..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/2020-ccf-bdci-02/2021-02-01-ccf-07.jpg and /dev/null differ
diff --git a/app/zh/information/blog/2020-ccf-bdci-02/index.md b/app/zh/information/blog/2020-ccf-bdci-02/index.md
deleted file mode 100644
index 9b7a9ada2b2411ca9ee0a7c3e1e89e7ab61ec8c6..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/2020-ccf-bdci-02/index.md
+++ /dev/null
@@ -1,70 +0,0 @@
----
-title: '黑盒调优,我们用它优化openLooKeng对接Hive数据源的执行效率'
-date: '2021-02-01'
-module: 'blog'
-customLayout: 'true'
-tags: ['大数据', '数据虚拟化', 'openLooKeng']
-archives: '2021-01'
-author: '进击的巨人'
-description: '2020 CCF大数据与计算智能大赛单赛题奖项中,参与openLooKeng赛题的3支队伍荣获大赛的一、二等奖。本期,荣获二等奖的团队将带来关于赛题「openLooKeng性能优化」的分享。让我们一睹这支队伍的风采。'
----
-
-### 前言
-
-2020 CCF 大数据与计算智能大赛单赛题奖项中,参与 openLooKeng 赛题的 3 支队伍荣获大赛的一、二等奖。本期,荣获二等奖的团队将带来关于赛题「openLooKeng 性能优化」的分享。让我们一睹这支队伍的风采。
-
-### 团队简介
-
-团队名称:进击的巨人
-团队成员:
-刘 祥 中国科学技术大学计算机学院,研究生一年级
-邱明璞 中国科学技术大学信息科学技术学院,研究生一年级
-秦杰杰 东北大学计算机科学与技术专业,研究生一年级
-
-
-
-### 2020 CCF BDCI 大数据与计算智能大赛
-
-### 赛题 | openLooKeng 性能优化
-
-#### BY 进击的巨人
-
-
-
-### 方案思路
-
-我们本次设计主要是面向 openLooKeng 对接 Hive 数据源
-
-首先建立 openLooKeng 查询执行时间与任务类型
-
-其中,T 代表任务的运行时间,c 为集群参数,包括节点数,每个节点的核数和内存数,t 为任务类型
-
-在集群参数 c 未知的情况下,我们可以使用以下策略求得相对较优的解:集群参数在合理的范围内均匀取值,每个集群参数都可以在参数的搜索空间中采用多起点爬山算法去搜索到一个最优的 p,然后对所有的 p 取均值,最后得到最优的 openLooKeng 参数作为最后的配置参数。
-
-若测试集群的配置已知,则在集群参数代入后,我们可将目标函数简化为:
-
-
-
-可以直接使用搜索算法求取最优的 p,使得目标函数最小。我们采用的搜索算法是带限制条件的多起点爬山算法,可以在可行的时间内搜索到质量较优的解。
-
-
-
-算法是在爬山算法的基础上运行多次,直到连续三次收敛到的局部最优解都不比之前搜索到的最优解更优,则结束搜索,并认为当前最优解即是我们需要的最优解。
-
-### 总结
-
-总的来说,结合性能建模和最优化搜索可以得到基本上最优的参数组合,在没有改动源码的基础上也能针对特定的运行环境和特定的测试集产生不错的优化效果
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-### 关键特性
-
-#### 1. 毫秒级查询性能
-
-DLI 使用的 openLooKeng 引擎在内存计算框架的基础上,还利用许多查询优化技术来满足高性能毫秒级的交互式分析的需要。
-
-1.1 索引
-
-openLooKeng 提供基于 Bitmap Index、Bloom Filter 以及 Min-max Index 等索引。通过在现有数据上创建索引,并且把索引结果存储在数据源外部,在查询计划编排时便利用索引信息过滤掉不匹配的文件,减少需要读取的数据规模,从而加速查询过程。
-
-1.2 Cache
-
-openLooKeng 提供丰富多样的 Cache,包括元数据 cache、执行计划 cache、ORC 行数据 cache 等。通过这些多样的 cache,可加速用户多次对同一 SQL 或者同一类型 SQL 的查询时延响应。
-
-1.3 动态过滤
-
-所谓的动态过滤是指是在运行时(run time)将 join 一侧表的过滤信息的结果应用到另一侧表的过滤器的优化方法,openLooKeng 不仅提供了多种数据源的动态过滤优化特性,还将这一优化特性应用到了 DataCenter Connector,从而加速不同场景关联查询的性能。
-
-1.4 算子下推
-
-openLooKeng 通过 Connector 框架连接到 RDBMS 等数据源时,由于 RDBMS 具有较强的计算能力,一般情况下将算子下推到数据源进行计算可以获取到更好的性能。openLooKeng 目前支持多种数据源的算子下推,包括 Oracle、HANA 等,特别地,针对 DC Connector 也实现了算子下推,从而实现了更快的查询时延响应。
-
-#### 2. 高可用
-
-2.1 HA AA 双活
-
-openLooKeng 引入了高可用的 AA 特性,支持 coordinator AA 双活机制,能够保持多个 coordinator 之间的负载均衡,同时也保证了 openLooKeng 在高并发下的可用性。
-
-2.2 Auto-scaling
-
-openLooKeng 的弹性伸缩特性支持将正在执行任务的服务节点平稳退服,同时也能将处于不活跃状态的节点拉起并接受新的任务。openLooKeng 通过提供“已隔离”与“隔离中”等状态接口供外部资源管理者(如 Yarn、Kubernetes 等)调用,从而实现对 coordinator 和 worker 节点的弹性扩缩容。
-
-#### 3. 融合场景
-
-实时分析、离线分析、交互式分析这三种场景中在很多实际业务中都是同时存在的,DLI 引入 openLooKeng 引擎之初就考虑了如何跟已有的 Spark 引擎进行元数据层面的互通,从而实现离线分析结果,免数据搬迁直接就可以用 openLooKeng 引擎进行交互式分析。Spark 和 openLooKeng 都支持 Hive 的建表方式,通过这种方式,实现了元数据层面的互通。
-
-#### 4. 统一目录,跨域跨 DC 查询
-
-
-
-DLI 老用户使用比较多的功能是跨多种数据源的联合查询,用于更全面地对数据进行关联分析,释放数据价值。这次引入 openLooKeng 引擎将跨源查询的能力进一步延伸,开发了跨域跨 DC 查询的 DataCenter Connector。通过这个新 Connector 可以连接到远端另外的 openLooKeng 集群,从而提供在不同数据中心间协同计算的能力。 其中的关键技术如下:
-
-4.1 并行数据访问
-
-worker 可以并发访问数据源以提高访问效率, 客户端也可以并发从服务端获取数据以加快数据获取速度。
-
-4.2 数据压缩
-
-在数据传输期间进行序列化之前,先使用 GZIP 压缩算法对数据进行压缩,以减少通过网络传输的数据量。
-
-4.3 跨 DC 动态过滤
-
-过滤数据以减少从远端提取的数据量,从而确保网络稳定性并提高查询效率。
-
-### 总结展望
-
-这次加入交互式查询能力,弥补了数据湖探索 DLI 在毫秒级场景下的短板,构建起从实时分析、到离线分析再到交互式分析整个链路完整的技术栈。未来,DLI 还将探索如何根据业务场景自动识别计算引擎,用户只需要下发 SQL,无需关心最终的计算引擎,让大数据真正变成“像使用数据库一样”,“会 SQL 就会大数据分析”。
diff --git a/app/zh/information/blog/20210416/hbase-01.jpg b/app/zh/information/blog/20210416/hbase-01.jpg
deleted file mode 100644
index 321d736ab0ae571d0a4706d03958c339d716f547..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210416/hbase-01.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20210416/hbase-02.jpg b/app/zh/information/blog/20210416/hbase-02.jpg
deleted file mode 100644
index 8594a7ea67367a8756943b97dac44d7f45db8afd..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210416/hbase-02.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20210416/hbase-03.jpg b/app/zh/information/blog/20210416/hbase-03.jpg
deleted file mode 100644
index a073157198cd4d02006a15fe36cd6a4133399bc9..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210416/hbase-03.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20210416/hbase-04.jpg b/app/zh/information/blog/20210416/hbase-04.jpg
deleted file mode 100644
index 481d24679326912827cd1ea49c1100c2315f2149..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210416/hbase-04.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20210416/hbase-05.jpg b/app/zh/information/blog/20210416/hbase-05.jpg
deleted file mode 100644
index 78a337fc55e3e514b9c16ca2c0f236489f564cb4..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210416/hbase-05.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20210416/hbase-06.jpg b/app/zh/information/blog/20210416/hbase-06.jpg
deleted file mode 100644
index 78a337fc55e3e514b9c16ca2c0f236489f564cb4..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210416/hbase-06.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20210416/hbase-07.jpg b/app/zh/information/blog/20210416/hbase-07.jpg
deleted file mode 100644
index d6964c99216a1547a006d3cf008fe831277c7675..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210416/hbase-07.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20210416/hbase-08.jpg b/app/zh/information/blog/20210416/hbase-08.jpg
deleted file mode 100644
index dd06939b6bec35bca5fc3a9625e6fc915d71d131..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210416/hbase-08.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20210416/hbase-09.jpg b/app/zh/information/blog/20210416/hbase-09.jpg
deleted file mode 100644
index cab57827cc25c88ab0116cb78016c9b294c1bb7b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210416/hbase-09.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20210416/hbase-10.jpg b/app/zh/information/blog/20210416/hbase-10.jpg
deleted file mode 100644
index 35e760d11fdf25784b8fb7fe7cfedeeb3a943bf3..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210416/hbase-10.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20210416/index.md b/app/zh/information/blog/20210416/index.md
deleted file mode 100644
index 4feea59ae77c7910d76f7b2187c943f19ba4f8f6..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/20210416/index.md
+++ /dev/null
@@ -1,146 +0,0 @@
----
-title: '如何通过openLooKeng更高效访问HBase'
-date: '2021-04-16'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', 'Hbase']
-archives: '2021-04'
-author: '涂盛霞'
-description: 'openLooKeng中的HBase Connector支持访问Apache HBase集群并进行查询和创建表的操作。用户可以在openLooKeng中创建表,并映射到HBase Cluster中已有的表,支持INSERT、SELECT和DELETE操作。'
----
-
-### 1. HBase Connector 介绍
-
-openLooKeng 中的`HBase Connector`支持访问`Apache HBase集群`并进行查询和创建表的操作。用户可以在 openLooKeng 中创建表,并映射到`HBase Cluster`中已有的表,支持`INSERT`、`SELECT`和`DELETE`操作。
-
-**一个简单的全表扫描的 sql 的执行,会经历哪些阶段:**
-
-首先该 Sql 将要访问的数据,一定是属于某一个数据源的,那么一个通用的`Connector`需要做哪些事情。Sql 的解析是由 openLooKeng 本身完成的;接下来是生成执行计划,在这个阶段需要验证用户所访问的表的合法性,那么`Connector`则需要提供该功能(即元数据管理);然后就到了任务调度阶段,openLooKeng 会将一个大任务划分为多个小任务,由多个`Worker`分工完成,那么`Connector`会提供`split`分割的接口,即`SplitManager`;`Worker`在收到任务之后,以分片为最小单元进行数据加载,此时需要用到`Connector`中的`PageSource/PageSink`来完成数据的读写操作。所以在 HBase Connector 中我们实现了这些关键模块(`SplitManager`,`HBaseClient`,`HetuMetastore`)。
-
-HBase Cluster 的主要组件:
-
-`ZooKeeper`用来记录一些元数据信息,
-
-`Master`用来处理用户发过来的请求,
-
-`RegionServer`用来执行用户请求并管理 Region 的分裂和合并。
-
-
-
-**HBase Connector 数据流:**
-
-- 建表(HBase Connector 支持两种模式的建表)。
-
- ① 直接关联远端 HBase 数据源上的表(即外表的形式)
-
- ② 在 openLooKeng 上创建一张 HBase 数据源不存在的新表
-
-- 用户发送一条查询 HBase 数据的 Sql 请求给`Coordinator`
-
-- `Coordinator`收到请求后,从`hetuMetastore`中获取`table`信息,以验证用户 Sql 所访问表和数据列的合法性
-
-- `Coordinator`通过`SplitManager`获取所有的分片信息,并生成执行计划和任务,将任务下发到各个 Worker 上
-
-- 每个 Worker 会处理一部分的数据。Worker 通过`HBase Client`来实现对`HBase Cluster`的数据读写交互
-
-**使用 openLooKeng 访问 HBase 集群**
-
-配置说明:使用 openLooKeng 来访问 HBase 集群,我们需要配置 HBase 的相关信息在 Catalog 中,主要是 ZooKeeper 的信息。创建并编辑`etc/catalog/hbase.properties`:
-
-具体操作可参考:
-
-**HBase Connector 所支持的语法:**
-
-HBase Connector 基本上支持所有的 SQL 语句,包括创建、查询、删除模式,添加、删除、修改表,插入数据,删除行等。以下是一些示例:
-
-
-
-**算子下推支持:**
-
-HBase 连接器支持下推大部分运算符,如基于`RowKey`的点查询、基于`RowKey`的范围查询等。此外,还支持这些谓词条件以进行下推:=、>=、>、<、<=、!=、in、not in、between and。
-
-### 2.HBase Connector 性能分析
-
-openLooKeng1.1.0 版本并未对 HBase Connector 做过全方面的性能优化。
-
-我们先了解一下 HBase 读取数据的机制。实际上,`HBase Client`首先会从`ZooKeeper`中获取 HBase 表元数据所在的`RegionServer`,然后根据`RowKey`,找到数据所在的`RegionServer`,然后发送读数据请求给`RegionServer`。
-
-
-
-每个`RegionServer`由多个`Region`构成,`Region`是存储数据的最小单元。每个`Region`里面会维护一定范围的`key值`。
-
-
-
-**那么先介绍一下我们之前切片的做法:`HBase Client`调用 API 获取数据所在的`region`信息。每个`region`的`start key`和`end key`组成一个`split`。`Split`的个数即数据所分布在的`region`的个数。**
-
-这种情况下,我们没有利用到读取`Region`的并发能力。我们知道,分片数决定了任务的并发度,影响性能。所以从这个角度出发,需要提高数据读取的并发度。那么在 openLooKeng 1.2.0 版本中,我们引入了一种新的数据切片方式以及支持访问快照的模式。
-
-### 3.HBase Connector 性能优化
-
-**优化点 1(新的分片规则)**
-
-- 建表时指定分片切割规则,提升单表全表扫描性能
-
-```java
-
-create table xxx() with(split_by_char='0~9,a~z,A~Z')
-
-```
-
-`split_by_char`表示`rowKey`的第一个字符的范围,为分片切割的依据。
-
-若`RowKey`的第一个字符由数字构成,则可以根据不同的数字进行分片切割,提高查询并发度。不同类型的符号用逗号隔开。如果设置不当,会导致查询数据结果不完整,请根据`RowKey`的实际情况进行配置。无特殊要求时,无需修改。默认情况已包含所有的数字和字母。若`RowKey`为汉字,则`create table xxx() with(split_by_char='一~锯')`;另外,建表时会根据`split_by_char`指定预分区的`splitKey`,尽量将数据分散到各个`region`,那样在进行`HBase`读写时,对性能会有很好的改善。
-
-- `HBase Server`支持使用`startRow`和`endRow`来获取`Scanner`
-
-比如,`splitByChar`为 0~2,我们会生成一些键值对。键值对的个数将会小于一个常量(如 20),所以需要首先计算每个键值对的`gap`大小。
-
-```java
-
-(startKey = 0, endKey = 0|),(startKey = 1, endKey = 1|),(startKey = 2, endKey = 2|)
-splitByChar为0~9, a~z
-(startKey = 0, endKey = 1|),(startKey = 2, endKey = 3|)……(startKey = y, endKey = z|)
-
-```
-
-
-
-**优化点 2(支持访问快照模式)**
-
-
-
-- 可配置`ClientSide`模式来读取数据,提升多并发查询性能
-
-`ClientSide`的工作机制是在`HDFS`上创建`HBase`表的`Snapshot`,记录各个数据文件所在的`Region`地址,在读取数据时,不需要经过`HBase Region Server`,而是直接访问`Region`,这样可以在高并发下降低`Region Server`的压力。
-
-
-
-**性能测试**
-
-`HBase 3`节点,openLooKeng 3 节点,`e_mp_day_read_52:10138492行,64列`
-
-
-
-
-
-`HBase Shell`对于操作千万行的表做`count`操作时,性能会很差;`HBase`也有提供计算行数的`jar包`,这里没有进行测试。因为 openLooKeng 1.2.0 优化了`count`操作,只会去加载第一列,所以`sql1`的情况下,`Normal Scan`和`ClientSide`方式性能差异不大。`Sql2`会获取多列数据,当`HBase Server`成为瓶颈时,`ClientSide`的优势就凸显出来了。
-
-当然,HBase 的应用场景并非是全表扫描,而应该是根据`RowKey`进行点查询的场景。该场景下,openLooKeng HBase Connector 会直接根据`RowKey`调用对应的`API`,高效获取数据即可。openLooKeng 1.2.0 在具备访问 HBase 基本功能的前提下,优化了对于全表扫描的场景下的性能,openLooKeng 1.2.0 HBase Connector 的全表扫描相比 1.1.0 版本,性能提升多倍。
-
-**参考资料**
-
-视频回顾:[如何通过 openLooKeng 高效访问 HBase?](https://www.bilibili.com/video/BV1254y1j725?spm_id_from=333.999.0.0)
-
----
-
-如果您有任何疑问或建议,欢迎在社区代码仓内提 Issue;也欢迎加小助手微信(openLooKengoss),进入专属技术交流群。
-
-社区代码仓
-
-
-
-如上图所示,接收到用户 SQL 语句之后,SQL 被转换成一个 Abstract syntax tree (AST)树。AST 树再被转换成逻辑执行计划树。然后,也就是执行计划优化过程中最重要的一步,使用规则(Rule)或者优化器(Optimizers)进行执行计划优化,每一个 PlanOptimizer 可以操作一个子执行计划树,PlanOptimizer 基于统计或者经验,用一个更优的子执行计划替换当前的子树,达到优化的目的。PlanOptimizers 通常是长期的经验累积得出来的一些优化规则,比如谓词下推、join reorder 等等。PlanOptimizers 可以存储一些物理执行信息在 ConnectorHandle 中。新下推框架则工作在这一层。
-
-得到最优的执行计划之后,逻辑执行计划被转换成物理执行计划,然后被分片,分割成按照 stage 执行的一个个子树,最终调度到 worker 上执行。
-
-### PrestoSql 下推方案介绍
-
-最初,openLooKeng 是从 PrestoSql 演进而来,在其基础上增加了很多功能特性,以及大量的性能优化。在讲述 openLooKeng 的新下推框架之前,先大概介绍其原下推方案。
-
-根据社区的讨论,PrestoSql 原下推框架有一些目标:
-
-1. 使用现有的 Rule 或 Optimizer 的框架,而且不是使用基于 visitor 模式的 PlanOptimizers 来实现下推,同时能够让 connector 提供转换 rules 来实现下推;
-
-2. 并不是建立一个原生的机制来支持所有操作的下推。
-
-先来看看 prestoSql 的执行过程:
-
-1. 首先,引入一些列的下推规则,每一个规则负责下推相应的操作到 TableScan 操作中,比如 PushFilterIntoConnector, PushProjectionIntoConnector, PushAggregationIntoConnector 等等;
-
-2. 上述的这些 rules 通过指定的 metadata 调用与 connectors 交互,如果 connector 支持这个操作下推,操作则被下推到 TableScan 操作,同时在 connectorTableHadle 中记录相关信息。
- 下面以 PushFilterIntoConnector 为例说明。
-
-
-
-在上述的例子中,假设 filter 中有两个过滤条件,一个是 like,一个是 f 函数,其中 connector 能处理 like 表达式。
-PushFilterIntoConnector 会调用 Metadata.pushFilter,实际上是调用 connector 的 pushFilter 函数,这个函数会返回一个新的 tableHandle,新的 tableHandle 中记录了 like 表达式的相关信息,同时返回一个 remaining filter,即 connector 不能处理的表达式。最终,PushFilterIntoConnector 就把原来的执行计划(上图中第一个框)转换成一个新的执行计划(上图中最后一个框)。
-
-上述的基于 Rule 的下推方案存在以下的几个问题:
-
-1. 不能 JoinNode,WindowNode 等 Nodes 的下推,特别是 join 的情况,join node 不仅仅需要 visit 当前的 join node,还需要 visit 他的左右节点,同时,还需要保存 join 的上下文信息,基于 rule 的下推方案难以处理这种情况;
-
-2. 下推逻辑复杂,下推上下文信息无法保存,对于 Join 的情况,join 的下推信息不知道存储在哪;
-
-基于上述原因,当前的下推方案不能把 sql 语句下推到数据源操作中,这样对于那些执行速度相当快的数据源就不能充分发挥数据源本身的能力,所以引入了新的下推框架。
-
-### openLooKeng 新下推框架
-
-#### 思想
-
-openLooKeng 新下推框架的主要思想是把执行计划子树暴露给 connector,让 connector 提供 PlanOptimizers(基于 visitor 模式的)给执行优化引擎,这样可以让 connector 引入任意的优化。
-为了防止一个 connector 的 PlanOptimizers 修改其他 connector 的执行计划子树,openLooKeng 对于暴露给 Connector 的 PlanNode 做了两个限制:
-
-1. 暴露出来的 PlanNodes 须移动到 presto-spi 模块;
-
-2. 仅仅暴露属于 connector 的子执行计划树给相应的 connector。如下图所示,左子树只会暴露给 Hive Connector,右子树只会暴露给 Mysql Connector。然后会应用他们各自的 PlanOptimizers。
-
-
-
-#### 实现
-
-
-
-openLooKeng 的下推框架如上图所示,新下推框架的工作原理很简单,主要分为两步:
-
-1. Connector 在启动的时候会告诉执行优化引擎其提供的 ConnectorPlanOptimizer,如下图的 HiveFpuPushdownOptimizer,其需要实现上图的 optimize 接口,optimize 函数以子执行计划为入口,返回优化后的执行计划;
-
-2. 在执行优化引擎中引入 ApplyConnectorOptimization 优化器,该 Optimizer 会把根据子执行计划所在的 connector,调用其 connectorPlanOptimizer。如下图所示,经过 HiveFpuPushdownOptimizer 优化之后,Aggregation 和 Filter 操作都下推到了数据源中。
-
-
-
-#### 修改
-
-新框架特性 PR 链接为
-
-不下推的情况
-
-
-
-部分下推的情况,在这个例子中,filter 下推了
-
-
-
-全部下推情况
-
-
-
-### 如何贡献
-
-下面简单介绍一下开发者如何适配新的下推框架,即在新增的 connector 中,如何添加 connectorPlanOptimizer。主要步骤如下:
-
-第一:在 XXXConnector 中复写下面的函数
-
-
-
-第二:实现 XXXPlanOptimizerProvider
-
-
-
-第三:在 XXXConnector 中实现 PlanOptimizer
-
-
-
-第四:实现 PlanOptimizer 里面的 optimize 函数,主要是实现一个 visitor 去 visit 执行计划树
-
-
-
-第五:实现 Visitor,用来生成下推的语句,同时修改执行计划树
-
-
-
-第六:实现 XXXQueryGenerator,在 XXXQueryGenerator 中实现一个 visitor 用来把下推的信息记录到 XXXQueryGeneratorContext,如果存在节点可以下推,则生成对应的 sql
-
-
-
-详细的实现可以参考 openLooKeng 的`baseJdbc`的实现,实现了 JDBC 数据源的下推。
-
-**参考资料**
-
-视频回顾:[想提升查询性能?openLooKeng 新下推框架为您助力!](https://www.bilibili.com/video/BV1of4y1p7sc?spm_id_from=333.999.0.0)
-
-资料下载:openLooKeng 新下推框架的介绍与实践
-
----
-
-欢迎关注 openLooKeng,加小助手微信(openLooKengoss)进入专属技术交流群。
-
-社区代码仓
-
-
-
-7 月 10 日,经过一天的开发挑战,「万有引力」团队获得了「openLooKeng 机器学习库支持算法扩展挑战赛」的冠军。据了解,该团队虽然仅有 2 人,但开发思路与逻辑非常清晰,配合上也十分默契。本期,小助手邀请到该团队队长余杨,来跟我们聊聊此次的参赛经历与体会。
-
-
-
-
-
-
-
-2. AccumulatorState:AccumulatorState 用于充当一个中间变量的角色,负责保存读取和聚合之后的数据。通过实现此接口来创建符合自己功能的 state,然后实现创建 state 的对应的 Factory 类(工厂类,用于创建这个 state)以及对应的序列化类(表示序列化与反序列化此 state 的方法,用于在聚合过程中不同节点之间的传输);
-
-
-
-3. 在 presto-ml 这个模块里面,可以发现已经一些造好的轮子了,例如 features 函数,这个自定义函数是实现将 sql 读取的每一行数据封装为一个 Block;还有用于存放特征向量的 FeatureVector,这些已经实现好的都可以利用起来。
-
-#### 四、我们的实现
-
-总的来说分为如下所示三步:数据从数据库中查出来聚合成数据集,然后用于模型训练,之后将训练好的模型保存起来,后面的预测和分类函数从保存的地方加载模型进行预测和分类;
-
-
-
-以 knn 分类算法为例:
-
-> **对于用于训练的聚合函数主要分为以下三步,详细的可以参考 openLooKeng 官方文档的函数一章:**
-
-1. input(AccumulatorState state, Block featureData):用于获取每行的输入,将输入的特征数据存放到 state 里面,而在 openLooKeng 集群里,input 读取数据阶段和下面的 combine 聚合阶段很有可能在不同节点上进行,因此就需要将 state 里面的数据序列化传输;
-2. combine(AccumulatorState state1, AccumulatorState state2):用于聚合上一步骤得到的 state,将结果存放到第一个 state1 中;
-3. output(AccumulatorState state, BlockBuilder out):这里面的 state 就是最终聚合得到的数据,也是在这一步从 state 里面取出所有用于训练的数据,然后对模型进行训练,然后将训练好的模型序列化到 BlockBuilder 里面;
-
-> **模型,以 knn 模型为例**
-
-knn 算法很简单,在模型训练层面,需要把只需要训练的数据经过预处理之后保存起来即可,KnnModel 模型的类图如下所示,其中我们建立了 ParameterNode 用于保存模型参数:
-
-
-
-> **模型加载使用**
-
-在模型保存方面由于 openLooKeng 目前貌似不支持统一的保存训练好的模型,因此我们选择的是将模型序列化之后传送给 classifier,然后分类器再反序列化出来即可;
-
-#### 五、最后
-
-感谢赛事主办方和 openLooKeng 社区提供这个更近一步接触开源的机会,也祝愿 openLooKeng 越来越好。
diff --git a/app/zh/information/blog/20210724-waic-hackday/waic-02.jpg b/app/zh/information/blog/20210724-waic-hackday/waic-02.jpg
deleted file mode 100644
index 2210009f4e2091d8ecfd31443dcf2003c799a1d0..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210724-waic-hackday/waic-02.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20210724-waic-hackday/waic-03.jpg b/app/zh/information/blog/20210724-waic-hackday/waic-03.jpg
deleted file mode 100644
index 0c244dad4ab307903c00abf92a0d1e0197f5178c..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210724-waic-hackday/waic-03.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20210724-waic-hackday/waic-04.jpg b/app/zh/information/blog/20210724-waic-hackday/waic-04.jpg
deleted file mode 100644
index f087d79a192c1106727471f93702eaee8f66873b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210724-waic-hackday/waic-04.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20210724-waic-hackday/waic-05.png b/app/zh/information/blog/20210724-waic-hackday/waic-05.png
deleted file mode 100644
index b36f93a00ce3fa3c5c85a94814169c1d889861f8..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210724-waic-hackday/waic-05.png and /dev/null differ
diff --git a/app/zh/information/blog/20210724-waic-hackday/waic-06.png b/app/zh/information/blog/20210724-waic-hackday/waic-06.png
deleted file mode 100644
index cd11b5e43727c1c9c09cdba05b4d8748099762f8..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210724-waic-hackday/waic-06.png and /dev/null differ
diff --git a/app/zh/information/blog/20210724-waic-hackday/waic-07.png b/app/zh/information/blog/20210724-waic-hackday/waic-07.png
deleted file mode 100644
index 8607a88f17ddda80c880c611a25633e6f6b13a10..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210724-waic-hackday/waic-07.png and /dev/null differ
diff --git a/app/zh/information/blog/20210724-waic-hackday/waic-08.jpg b/app/zh/information/blog/20210724-waic-hackday/waic-08.jpg
deleted file mode 100644
index 9e19da88be3747f57cedd8eb80c1498fed546abd..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210724-waic-hackday/waic-08.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20210903/cover.jpg b/app/zh/information/blog/20210903/cover.jpg
deleted file mode 100644
index 322d1b6db904ed6a8be4a0e9b71933b59283ceb6..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210903/cover.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20210903/index.md b/app/zh/information/blog/20210903/index.md
deleted file mode 100644
index 5f0bc1413087356875a2687ee58f77ecc24b9162..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/20210903/index.md
+++ /dev/null
@@ -1,982 +0,0 @@
----
-title: 'openLooKeng安装指导'
-date: '2021-09-03'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', '安装部署']
-categories: ['Announcement']
-author: 'openLooKeng'
-description: '如果您想自行配置安装openLooKeng,或者在安装过程中有一些困难,可以参考这篇博客。这是一篇来自社区小伙伴的分享,希望对您有帮助。'
----
-
-**社区代码仓:
-
-## 1 软件安装
-
-说明:需要提前手动部署 jdk
-
-### 1.1 快速部署集群脚本使用
-
-下载安装脚本:
-
-部署时只需配置 config.txt 文件
-
-
-
-修改内容如下:
-
-
-
-修改完 config.txt 后执行 main.sh 开始部署集群
-
-
-
-main.sh 执行后,openlookeng 会部署在如下路径:
-
-
-
-在安装路径里存在以下几个文件:
-
-
-
-## 2 客户端使用
-
-### 2.1 CLI 使用
-
-登录客户端命令:
-`java -jar /opt/hetu-install/etc/hetu-cli-316-executable.jar --server ip:port --catalog CATALOGNAME `
-
-`ip`:服务的节点 IP
-
-`port`:服务的端口号,默认为 8090
-
-`CATALOGNAME`:待访问的数据源,对应`/etc/hetu/catalog/`目录下的文件名
-
-其它参数:
-`--schema` :指定`catalog`中的`schema`,后跟`schema`名称
-`--user`:指定访问数据源的用户,后跟用户名
-`--execute`:指定待执行的 sql,后跟 sql 语句(sql 需用双引号括起来,要带分号)
-`-f`:指定 sql 文件,后跟 sql 文件名。用于不进入客户端执行 sql,同-e。
-
-### 2.2 JDBC 使用
-
-1. 获取`jdbc`的 jar 包`hetu-jdbc-316.jar`并放在所需要的目录;
-
-2. 使用 jdbc 工具连接时, URL 为:`jdbc:hetu://ip:8090`,driver class 为:`io.hetu.core.jdbc.HetuDriver`,其中 ip 为 hetu 服务 ip。
-
-### 2.3 UDF 使用
-
-#### 2.3.1 CBG UDF 的集成
-
-1. /etc/hetu/目录下上传 udf 函数注册文件`udf.properties`。文件格式为`function_name class_path`,如下:
-
- ```java
- booleanudf io.hetu.core.hive.dynamicfunctions.examples.udf.BooleanUDF
- shortudf io.hetu.core.hive.dynamicfunctions.examples.udf.ShortUDF
- byteudf io.hetu.core.hive.dynamicfunctions.examples.udf.ByteUDF
- intudf io.hetu.core.hive.dynamicfunctions.examples.udf.IntUDF
- ```
-
-2. `${node.data-dir}`下创建`externalFunctions`文件夹,并将 udf 函数和依赖的类上传到该文件夹下;
-
-3. 上传 udf 函数依赖的配置文件至`${node.data-dir}`中;
-
-4. openLooKeng 服务的各个节点均执行以上步骤,完成后重启服务。
-
-#### 2.3.2 UDAF 的集成
-
-1. `${node.data-dir}`下创建 externalFunctionsPlugin 文件夹;
-
-2. 将 UDAF 的 zip 包如`cbg-hive-functions-1.0-SNAPSHOT.zip`解压至`externalFunctionsPlugin`下;
-
-3. openLooKeng 服务的各个节点均执行以上步骤,完成后重启服务
-
-### 2.4 SQL migration tool 使用
-
-1. 交互模式: `java -jar hetu-sql-migration-tool-316.jar`
-
-
-
-2. 执行模式:`java -jar hetu-sql-migration-tool-316.jar --execute` " 待转换的`hive-sql`"
-
-
-
-3. 文件模式:`java -jar hetu-sql-migration-tool-316.jar --file` 待转换的 hive sql 文件 `--output` 转换后的文件目录
-
-
-
-4. 将转换后的 sql 在 Hetu 上面执行。
-
-### 2.5 Web UI 使用
-
-在`coordinator`节点`config.properties`里面配置如下参数:
-
-```java
-hetu.queryeditor-ui.allow-insecure-over-http=true --打开Web Interface,默认关闭(无法使用web 管理界面)
-openlookeng.admins=lk --指定Web 管理节点admin用户
-```
-
-## 3 数据源配置
-
-### 3.1 配置文件添加数据源
-
-#### 3.1.1 配置 FI Hive 数据源
-
-可通过以下两种方法进行配置:
-
-**方法一:服务安装前通过脚本自动配置**
-
-1. FI 集群上下载 FI 集群配置文件及用户凭证:
- 配置文件:登录 FI 页面,在【集群】页面点击【更多】,选择【下载客户端】。弹框中选择【仅配置文件】,点击【确定】。下载完成后解压。
- 用户凭证:登录 FI 页面,在【系统】页面选择用户,点击【更多】,点击【下载认证凭据】。弹框中点击【确定】。下载完成后解压文件。
-
-
-2. 将步骤 1 获取到的 HDFS 文件`core-site.xml、hdfs-site.xml`和用户认证凭据`user.keytab、 krb5.conf `以及`hosts`传至`hetu`安装包解压目录中的`client_dependencies`下;
-
-3. 修改解压目录下的`client_dependencies/clientMainConfig`文件中的配置项:
- `KRB_PRINCIPAL`:默认值修改为认证凭据对应的用户名;
- `HIVE_METASTORE_URI`:默认值修改为步骤 1 获取的 hive 配置文件`hive-site.xml`中`hive.metastore.uris`的值。
-
-之后执行 install.sh 脚本安装服务时,FI hive 数据源会自动配置到服务各个节点上。
-
-**方法二:服务安装后手动配置**
-
-1. 获取 FI hive 配置文件及用户认证凭据,同方法一的步骤 1;
-2. 将步骤 1 获取到的 HDFS 文件`core-site.xml、hdfs-site.xml`和用户认证凭据`user.keytab、 krb5.conf`传至服务各个节点上;
-3. `/etc/hetu/catalog`目录下创建`hive.properties`文件,写入以下内容:
-
- ````java
- connector.name=hive-hadoop2
- hive.metastore.uri=thrift://IP:21088,thrift://IP:21088 --改为步骤1获取的hive配置文件hive-site.xml中hive.metastore.uris的值
- hive.config.resources=/opt/hetu/conf/core-site.xml,/opt/hetu/conf/hdfs-site.xml -- core-site.xml和hdfs-site.xml实际所在路径
- hive.metastore.authentication.type=KERBEROS
- hive.metastore.service.principal=hive/hadoop.hadoop.com@HADOOP.COM
- hive.metastore.client.principal=USER@HADOOP.COM --USER改为认证凭据对应的用户名
- hive.metastore.client.keytab=/opt/hetu/conf/user.keytab -- user.keytab文件所在路径
- hive.metastore.krb5.conf.path=/opt/hetu/conf -- krb5.conf文件所在目录
- hive.hdfs.wire-encryption.enabled=true
- hive.collect-column-statistics-on-write=true
- hive.hdfs.authentication.type=KERBEROS
- hive.hdfs.impersonation.enabled=false
- hive.hdfs.hetu.principal=USER@HADOOP.COM --USER改为认证凭据对应的用户名
- hive.hdfs.hetu.keytab=/opt/hetu/conf/user.keytab -- user.keytab文件所在路径
- hive.allow-drop-table=true ```
-
- ````
-
-4. 将步骤 1 获取到的 hosts 文件中的内容添加到/ets/hosts 文件中;
-
-5. 步骤 2~4 每个节点都要执行,完成后重启服务
-
-#### 3.1.2 配置开源 Hive 数据源
-
-1. `HetuServer`部署完成后,进入`“/etc/hetu/catalog”`目录,编辑`“hive.properties”`文件
-
- ```java
- connector.name=hive-hadoop2
- hive.metastore.uri=thrift://ip:9083 --ip改为开源hive服务节点IP
- hive.allow-drop-table=true
- hive.allow-rename-table=true
- hive.allow-add-column=true
- hive.allow-drop-column=true
- hive.allow-rename-column=true
- hive.allow-comment-table=true
- ```
-
-2. 重启 hetu 服务。
-
-#### 3.1.3 配置 DC Connector
-
-```java
-connector.name=dc
-connection-url=http://IP:8090 --IP改为对端hetu服务coordinate节点的IP
-connection-user=root
-dc.query.pushdown.enabled=false
-dc.split-number=10
-dc.http.commpression=false
-dc.metadata.cache.ttl=10s
-dc.ssl=false
-```
-
-1. Hetu 服务每个节点`/etc/hetu/catalog`目录下创建`dc.properties`文件,写入以下内容:
-
-2. 重启 hetu 服务。
-
-#### 3.1.4 配置 Hbase 数据源
-
-1. 将 Hbase 服务所在的 Hadoop 服务的配置文件`core-site.xml、hdfs-site.xml`传至 hetu 服务的各个节点上;
-
-2. `/etc/hetu/catalog`目录下创建`hbase.properties`文件,写入以下内容:
- ```java
- connector.name=hbase-connector
- hbase.zookeeper.quorum=***.***.***.*** --hbase对应的zookeeper服务器IP
- hbase.zookeeper.property.clientPort=2181 --hbase对应的zookeeper服务的端口号
- hbase.authentication.type=root
- hbase.core.site.path=/opt/hbase/core-site.xml --core-site.xml文件所在路径
- hbase.hdfs.site.path=/opt/hbase/hdfs-site.xml --hdfs-site.xml文件所在路径
- hbase.metastore.uri=/opt/hbase/hbasemetastore.ini --hbase元数据存放位置,访问hbase数据源时自动生成ini文件
- ```
-3. hetu 服务每个节点执行完步骤 1 和 2 后重启服务。
-
-#### 3.1.5 配置 Oracle 数据源
-
-1. Hetu 服务每个节点`/etc/hetu/catalog`目录下创建`oracle.properties`文件,写入以下内容:
-
- ```java
- connector.name=oracle
- connection-url=jdbc:oracle:thin:@ip:1521:orcl --ip: oracle数据库IP
- connection-user=cicd_alpha_tpc --连接oracle服务的用户名
- connection-password=****** --连接oracle服务用户对应的密码
- ```
-
-2. 重启 hetu 服务。
-
-#### 3.1.6 配置 Oracle 数据源
-
-1. Hetu 服务每个节点`/etc/hetu/catalog`目录下创建`mysql.properties`文件,写入以下内容:
-
- ```java
- connector.name= mysql
- connection-url= jdbc:mysql://ip: 3306 --ip: mysql数据库IP
- connection-user=DATAHUB --连接mysql服务的用户名
- connection-password=****** --连接mysql服务用户对应的密码
- ```
-
-2. 重启 hetu 服务。
-
-#### 3.1.7 配置 Hana 数据源
-
-1. Hetu 服务每个节点`/etc/hetu/catalog`目录下创建`hana.properties`文件,写入以下内容:
-
- ```java
- connector.name=hana
- connection-url=jdbc:sap://ip: 34215/datahub --ip: hana数据库IP,datahub: hana上已存在的database
- connection-user=DATAHUB --连接hana服务的用户名
- connection-password=****** --连接hana服务用户对应的密码
- ```
-
-2. 重启 hetu 服务。
-
-#### 3.1.8 配置 VDM Connector
-
-1. Hetu 服务每个节点/etc/hetu/目录下创建`hetu-metastore.properties`文件,写入以下内
- 容:
- `java hetu.metastore.type=jdbc hetu.metastore.db.url=jdbc:mysql://ip:3306/vdmmeta --配置vdm存储数据的库,这里使用mysql数据库,其中vdmmeta需要在mysql中提前创建 hetu.metastore.db.user=username -- 登陆mysql服务器的用户名 hetu.metastore.db.password=****** --登陆mysql服务器的密码 `
-
-2. Hetu 服务每个节点/etc/hetu/catalog 目录下创建 vdm.properties 文件,写入以下内容:
-
- ```java
- connector.name=vdm
- ```
-
-3. 重启河图服务。
-
-#### 3.1.9 配置 Carbondata Connector
-
-1. Hetu 服务每个节点将`mapred-site,yarn-site.xml,core-site.xml`和`hdfs-site.xml`添加到安装到`/opt/ hetu/conf`下。
-
-2. Hetu 服务每个节点`/etc /hetu /catalog`中添加`carbondata.properties`文件,写入以下内容:
-
- ```java
- connector.name=carbondata
- hive.metastore.uri=thrift://ip:21088 -- ip:hive数据源ip
- hive.config.resources=/opt/hetu/conf/core-site.xml,/opt/hetu/conf/hdfs-site.xml,/opt/hetu/conf/yarn-site.xml,/opt/hetu/conf/mapred-site.xml -- 4个xml文件的存放路径
- hive.metastore.authentication.type=KERBEROS --认证类型为KERBEROS
- hive.metastore.service.principal=hive/hadoop.hadoop.com@HADOOP.COM
- hive.metastore.client.principal=prestouser@HADOOP.COM -- prestouser为对接的用户
- hive.metastore.client.keytab=/opt/hetu/conf/user.keytab -- 用户凭证存放位置
- hive.metastore.krb5.conf.path=/opt/hetu/conf -- krb5凭证存放位置
- hive.hdfs.wire-encryption.enabled=true
- hive.collect-column-statistics-on-write=true
- hive.hdfs.authentication.type=KERBEROS
- hive.hdfs.impersonation.enabled=false
- hive.hdfs.hetu.principal=prestouser@HADOOP.COM -- prestouser为对接的用户
- hive.hdfs.hetu.keytab=/opt/hetu/conf/user.keytab -- 用户凭证存放位置
- hive.allow-drop-table=true -- 以下为允许的操作权限,默认为false
- hive.allow-rename-table=true
- hive.allow-add-column=true
- hive.allow-drop-column=true
- hive.allow-rename-column=true
- hive.non-managed-table-writes-enabled=true
- ```
-
-3. 重启河图服务。
-
-### 3.2 动态添加数据源
-
-#### 3.2.1 动态目录配置
-
-1. 在所有节点`node.properties` 中添加如下内容:
-
- ```java
- # Root directory for storing configuration files in local disk.
- catalog.config-dir=/etc/hetu/
- # Root directory for storing configuration files in the shared file system.
- catalog.share.config-dir=/hetu/dynamiccatalog/
- ```
-
-2. 在所有节点`config.properties`中添加如下内容:
-
- ```java
- # Whether to enable dynamic catalog
- catalog.dynamic-enabled=true
- # Interval for scanning catalogs in the shared file system, default value is 5s.
- catalog.scanner-interval=5s
- # Maximum file size, default value is 128k.
- catalog.max-file-size=128k
- ```
-
-3. 参考 4.4.2 章节配置`local-config-catalog.properties`
-
-#### 3.2.2 配置 FI Hive 数据源
-
-```java
-curl --location --request POST 'http://ip:port/v1/catalog/' \--IP 为hetu coordinator ip
---header 'X-Hetu-User: admin' \
---form 'catalogInformation={
-"catalogName" : "hive",
-"connectorName" : "hive-hadoop2",
-"properties" : {
- "hive.hdfs.impersonation.enabled" : "false",
- "hive.hdfs.authentication.type" : "KERBEROS",
- "hive.collect-column-statistics-on-write" : "true",
- "hive.metastore.service.principal" : "hive/hadoop.hadoop.com@HADOOP.COM",
- "hive.metastore.authentication.type" : "KERBEROS",
- "hive.metastore.uri" : "thrift://***.***.***.***:21088,thrift://***.***.***.***:21088",//hive配置文件hive-site.xml中hive.metastore.uris的值
- "hive.allow-drop-table" : "true",
- "hive.config.resources" : "core-site.xml,hdfs-site.xml",
- "hive.hdfs.hetu.keytab" : "user.keytab",
- "hive.metastore.krb5.conf.path" : "krb5.conf",
- "hive.metastore.client.keytab" : "user.keytab",
- "hive.metastore.client.principal" : "test@HADOOP.COM",
- "hive.hdfs.wire-encryption.enabled" : "true",
- "hive.hdfs.hetu.principal" : "test@HADOOP.COM"
- }
- }'
---form 'catalogConfigurationFiles=@/E:/catalog/core-site.xml'
---form 'catalogConfigurationFiles=@/E:/catalog/hdfs-site.xml'
---form 'catalogConfigurationFiles=@/E:/catalog/user.keytab'
---form 'globalConfigurationFiles=@/E:/catalog/krb5.conf'
-```
-
-1. Curl 命令添加:
- **四个凭证在本地计算机中路径要正确,该方式参数与本地参数一致,用户凭证需放在本地,请求才能发送成功。**
-
-2.Postman 添加
-
-```java
- Header: X-Hetu-User:admin
- Body:catalogInformation: {
- "catalogName" : "hive",
- "connectorName" : "hive-hadoop2",
- "properties" : {
- "hive.hdfs.impersonation.enabled" : "false",
- "hive.hdfs.authentication.type" : "KERBEROS",
- "hive.collect-column-statistics-on-write" : "true",
- "hive.metastore.service.principal" : "hive/hadoop.hadoop***.***.***.***.com@HADOOP***.***.***.***.COM",
- "hive.metastore.authentication.type" : "KERBEROS",
- "hive.metastore.uri" : "thrift://***.***.***.***:21088,thrift://***.***.***.***:21088",
- "hive.allow-drop-table" : "true",
- "hive.config.resources" : "core-site.xml,hdfs-site.xml",
- "hive.hdfs.hetu.keytab" : "user.keytab",
- "hive.metastore.krb5.conf.path" : "krb5.conf",
- "hive.metastore.client.keytab" : "user.keytab",
- "hive.metastore.client.principal" : "testdr@HADOOP***.***.***.***.COM",
- "hive.hdfs.wire-encryption.enabled" : "true",
- "hive.hdfs.hetu.principal" : "testdr@HADOOP***.***.***.***.COM",
- "connector.name" : "hive-hadoop2"
- }
- }
- catalogConfigurationFiles:core-site.xml/ hdfs-site.xml/ user.keytab
-globalConfigurationFiles: krb5.conf
-```
-
-#### 3.2.3 配置开源 Hive 数据源
-
-```java
-curl -X POST http://ip:8090/v1/catalog ---hetu coordinator 地址
- -H 'Cache-Control: no-cache'
- -H 'Postman-Token: ********'
- -H 'X-Hetu-User: admin'
- -H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW'
- -F 'catalogInformation={
- "catalogName" : "hivekaiyuan", //***.***.***.***:9083", \ip为开源hiveip
- "connector.name" : "hive-hadoop2" ,
- "hive.allow-drop-table":"true",
- "hive.allow-rename-table":"true" ,
- "hive.allow-add-column":"true" ,
- "hive.allow-drop-column":"true",
- "hive.allow-rename-column":"true",
- "hive.allow-comment-table":"true"
- }'
- -F 'catalogConfigurationFiles=@path/core-site.xml'
- -F 'catalogConfigurationFiles=@path/ core-site.xml ' //地址为开源hive地址
-```
-
-1. Curl 命令添加:
-
- 用`postman`添加开源`hive`需要`core-site.xml、core-site.xml`这两个文件,文件需存放在本地,且本地路径填写正确
-
- ```java
- Header: X-Hetu-User:admin
- Body:catalogInformation:{
- "catalogName" : "hivekaiyuan",
- "connectorName" : "hive-hadoop2",
- "properties" : {
- "hive.metastore.uri" : "thrift://***.***.***.***:9083",
- "connector.name" : "hive-hadoop2" ,
- "hive.allow-drop-table":"true",
- "hive.allow-rename-table":"true" ,
- "hive.allow-add-column":"true" ,
- "hive.allow-drop-column":"true",
- "hive.allow-rename-column":"true",
- "hive.allow-comment-table":"true"
- }
- }
- catalogConfigurationFiles: core-site.xml、core-site.xml
- ```
-
-2. Postman 添加
-
-#### 3.2.4 配置 DC Connector
-
-1. Curl 命令添加:
-
- ```java
- curl -X POST \
- http://ip:8090/v1/catalog \---hetu coordinator 地址
- -H 'Cache-Control: no-cache' \
- -H 'Postman-Token: ********' \
- -H 'X-Hetu-User: admin' \
- -H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW' \
- -F 'catalogInformation={
- "catalogName" : "dc",
- "connectorName" : "dc",
- "properties" : {
- "connection-url" : "http://***.***.***.***:8090",
- "connection-user" : "root",
- "dc.metadata.cache.ttl" : "10s",
- "dc.ssl" : "false"
- }
- }'
- --地址为dc地址
- ```
-
-2. Postman 添加
- ```java
- Header: X-Hetu-User:admin
- Body:catalogInformation:{
- "catalogName" : "dc",
- "connectorName" : "dc",
- "properties" : {
- "connection-url" : "http://***.***.***.***:8090",
- "connection-user" : "root",
- "dc.metadata.cache.ttl" : "10s",
- "dc.ssl" : "false"
- }
- }
- ```
-
-#### 3.2.5 配置 Hbase 数据源
-
-1. Curl 命令添加:
- ```java
- curl -X POST \
- http://ip:8090/v1/catalog \---hetu coordinator 地址
- -H 'Cache-Control: no-cache' \
- -H 'Postman-Token: ********' \
- -H 'X-Hetu-User: admin' \
- -H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW' \
- -F 'catalogInformation={
- "catalogName" : "hbase",
- "connectorName" : "hbase-connector",
- "properties" : {
- "hbase.zookeeper.quorum" : "***.***.***.***,
- "hbase.zookeeper.property.clientPort" : "2181",
- "hbase.authentication.type" : "root",
- "hbase.core.site.path" : "core-site.xml",
- "hbase.hdfs.site.path" : "hdfs-site.xml",
- "hbase.metastore.uri" : "hbasemetastore.ini"
- }
- }'
- ----地址为hbase地址----
- ```
- ```java
- Header: X-Hetu-User:admin
- Body:catalogInformation:{
- "catalogName" : "hbase",
- "connectorName" : "hbase-connector",
- "properties" : {
- "hbase.zookeeper.quorum" : "***.***.***.***",
- "hbase.zookeeper.property.clientPort" : "2181",
- "hbase.authentication.type" : "root",
- "hbase.core.site.path" : "core-site.xml",
- "hbase.hdfs.site.path" : "hdfs-site.xml",
- "hbase.metastore.uri" : "hbasemetastore.ini"
- }
- }
- ```
-2. Postman 添加
-
-#### 3.2.6 配置 Oracle 数据源
-
-1. Curl 命令添加:
-
- ```java
- curl -X POST \
- http://ip:8090/v1/catalog \---hetu coordinator 地址
- -H 'Cache-Control: no-cache' \
- -H 'Postman-Token: ********' \
- -H 'X-Hetu-User: admin' \
- -H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW' \
- -F 'catalogInformation={
- "catalogName" : "Oracle",
- "connectorName" : "oracle",
- "properties" : {
- "connection-url" : "jdbc:oracle://***.***.***.***:3306",
- "connection-user" : "root",
- "connection-password" : "******"
- }
- }'
- 地址为Oracle地址
- ```
-
-2. Postman 添加
- ```java
- Header: X-Hetu-User:admin
- Body:catalogInformation:{
- "catalogName" : "Oracle",
- "connectorName" : "oracle",
- "properties" : {
- "connection-url" : "jdbc:oracle://***.***.***.***:3306",
- "connection-user" : "root",
- "connection-password" : "******"
- }
- }'
- ```
-
-#### 3.2.7 配置 Mysql 数据源
-
-1. Curl 命令添加:
-
- ```java
- curl -X POST \
- http://ip:8090/v1/catalog \---hetu coordinator 地址
- -H 'Cache-Control: no-cache' \
- -H 'Postman-Token: ********' \
- -H 'X-Hetu-User: admin' \
- -H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW' \
- -F 'catalogInformation={
- "catalogName" : "mysql",
- "connectorName" : "mysql",
- "properties" : {
- "connection-url" : "jdbc:mysql://***.***.***.***:3306",
- "connection-user" : "root",
- "connection-password" : "******"
- }
- }'
- 地址为MySQL ip
- ```
-
- ```java
- Header: X-Hetu-User:admin
- Body:catalogInformation:{
- "catalogName" : "mysql",
- "connectorName" : "mysql",
- "properties" : {
- "connection-url" : "jdbc:mysql://***.***.***.***:3306",
- "connection-user" : "root",
- "connection-password" : "******"
- }
- }'
-
- ```
-
-2. Postman 添加
-
-#### 3.2.8 配置 Hana 数据源
-
-1. Curl 命令添加:
- ```java
- curl -X POST \
- http://ip:8090/v1/catalog \---hetu coordinator 地址
- -H 'Cache-Control: no-cache' \
- -H 'Postman-Token: ********' \
- -H 'X-Hetu-User: admin' \
- -H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW' \
- -F 'catalogInformation={
- "catalogName" : "hana",
- "connectorName" : "hana",
- "properties" : {
- "connection-url" : " jdbc:sap://***.***.***.***:34215/datahub",
- "connection-user" : "root",
- "connection-password" : "******"
- }
- }'
- 地址为hana地址
- ```
- ```java
- Header: X-Hetu-User:admin
- Body:catalogInformation:{
- "catalogName" : "hana",
- "connectorName" : "hana",
- "properties" : {
- "connection-url" : " jdbc:sap://***.***.***.***:34215/datahub",
- "connection-user" : "root",
- "connection-password" : "******"
- }
- }'
- ```
-2. Postman 添加
-
-## 4 特性参数配置
-
-### 4.1 State-store 参数配置
-
-1. 所有 Hetu 节点的`/etc/hetu/`目录下创建`state-store.properties`,按照如下模板配置:
-2. 重启 hetu 服务。
-
- ```java
- # For now only support hazelcast
- state-store.type=hazelcast
- # User defined state store name
- state-store.name=query
- # User defined cluster name
- # must be unique if sharing the seed store with other clusters
- state-store.cluster=cluster1
-
- # Hazelcast Config
- hazelcast.discovery.mode=tcp-ip
- hazelcast.discovery.port=5701
- hazelcast.discovery.tcp-ip.seeds=
-
-
-
-解决方法:
-`if table is large, during drop it tries to delete partitions and all, which might be taking more than 10s , if that time is expire, then timeout happens .. in the backend table is dropped but due to timeout failure propagated error becomes different "table not found". though its deleted in the same request`
-
-`hive.properties` 添加如下参数:
-`hive.metastore-timeout=60s`
-
-三. `Query 20210306_101831_00184_wx7j4 failed: Unable to create input format org.apache.hadoop.mapred.TextInputFormat`
-
-解决方法:
-将缺少的`hadoop-plugins-1.0.jar`打包到`hive connector plugin`中。
diff --git a/app/zh/information/blog/20210903/install01.png b/app/zh/information/blog/20210903/install01.png
deleted file mode 100644
index 725cdbca952c4c564a8f1464b65fa3326cc3e183..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210903/install01.png and /dev/null differ
diff --git a/app/zh/information/blog/20210903/install02.png b/app/zh/information/blog/20210903/install02.png
deleted file mode 100644
index 9bb9c4f1c3bceef2814c2fb808a704e603d90ffc..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210903/install02.png and /dev/null differ
diff --git a/app/zh/information/blog/20210903/install03.png b/app/zh/information/blog/20210903/install03.png
deleted file mode 100644
index 60ac7246bc65b85508866bae4d00fd3c6e91862d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210903/install03.png and /dev/null differ
diff --git a/app/zh/information/blog/20210903/install04.png b/app/zh/information/blog/20210903/install04.png
deleted file mode 100644
index 2ae0cf09e82f26880e4d9c5210cbda4f8013d9d9..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210903/install04.png and /dev/null differ
diff --git a/app/zh/information/blog/20210903/install05.png b/app/zh/information/blog/20210903/install05.png
deleted file mode 100644
index 2ae0cf09e82f26880e4d9c5210cbda4f8013d9d9..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210903/install05.png and /dev/null differ
diff --git a/app/zh/information/blog/20210903/install06.png b/app/zh/information/blog/20210903/install06.png
deleted file mode 100644
index c103faafcdf466bd001dd1b4fb7dbf67b4d15ab3..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210903/install06.png and /dev/null differ
diff --git a/app/zh/information/blog/20210903/install07.png b/app/zh/information/blog/20210903/install07.png
deleted file mode 100644
index 0f2fea53182008f67f4dec883ea0f11c37a25cb7..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210903/install07.png and /dev/null differ
diff --git a/app/zh/information/blog/20210903/install08.png b/app/zh/information/blog/20210903/install08.png
deleted file mode 100644
index 7b62dcc381b4db02328ebc98883a2b7539cfe9af..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210903/install08.png and /dev/null differ
diff --git a/app/zh/information/blog/20210903/install09.png b/app/zh/information/blog/20210903/install09.png
deleted file mode 100644
index fb33b8d740594437126f67708d26ff7cbc60040f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210903/install09.png and /dev/null differ
diff --git a/app/zh/information/blog/20210903/install10.png b/app/zh/information/blog/20210903/install10.png
deleted file mode 100644
index 2c159a4b4f50f0d49839b908890baf7c6060618c..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210903/install10.png and /dev/null differ
diff --git a/app/zh/information/blog/20210903/install11.png b/app/zh/information/blog/20210903/install11.png
deleted file mode 100644
index cf663a17c9f9a5afef3975531803bc2ae0818043..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210903/install11.png and /dev/null differ
diff --git a/app/zh/information/blog/20210903/install12.png b/app/zh/information/blog/20210903/install12.png
deleted file mode 100644
index 7d05339dfedcc656a03ec183ed6120a4fcee4385..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210903/install12.png and /dev/null differ
diff --git a/app/zh/information/blog/20210908/001.png b/app/zh/information/blog/20210908/001.png
deleted file mode 100644
index cc187f158a039d11e5e211a4c2fd58f183fda131..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210908/001.png and /dev/null differ
diff --git a/app/zh/information/blog/20210908/002.jpg b/app/zh/information/blog/20210908/002.jpg
deleted file mode 100644
index d1a8f50bdf80854ca290bd407ec626f1240fee16..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210908/002.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20210908/003.jpg b/app/zh/information/blog/20210908/003.jpg
deleted file mode 100644
index d6935d456e14b9e49730590e0ec060a90fa2c428..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210908/003.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20210908/004.jpg b/app/zh/information/blog/20210908/004.jpg
deleted file mode 100644
index b93d8e3892d8364d97c0485382c70729c03eba42..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210908/004.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20210908/005.png b/app/zh/information/blog/20210908/005.png
deleted file mode 100644
index 357c8bd206684c41d31ed9773ecf1d34a46e2adb..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210908/005.png and /dev/null differ
diff --git a/app/zh/information/blog/20210908/006.png b/app/zh/information/blog/20210908/006.png
deleted file mode 100644
index 94cc978c10be87c401259e7704ea3d483d8e460a..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210908/006.png and /dev/null differ
diff --git a/app/zh/information/blog/20210908/007.png b/app/zh/information/blog/20210908/007.png
deleted file mode 100644
index d6f07e62f2eb19c45de7a60c52d1cfe5a73c9e44..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210908/007.png and /dev/null differ
diff --git a/app/zh/information/blog/20210908/008.png b/app/zh/information/blog/20210908/008.png
deleted file mode 100644
index 25b29ef68d14b849697e544c17eb6b4f550d7269..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210908/008.png and /dev/null differ
diff --git a/app/zh/information/blog/20210908/009.png b/app/zh/information/blog/20210908/009.png
deleted file mode 100644
index 6e90a7dcbfb69304e547d73f0d19955217129b46..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210908/009.png and /dev/null differ
diff --git a/app/zh/information/blog/20210908/010.png b/app/zh/information/blog/20210908/010.png
deleted file mode 100644
index 5c68956ae7c8ab89f27742a53b9be140c9587cc7..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210908/010.png and /dev/null differ
diff --git a/app/zh/information/blog/20210908/011.png b/app/zh/information/blog/20210908/011.png
deleted file mode 100644
index 330bfca48cd31c97b6b79c3b597f04932269dbde..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210908/011.png and /dev/null differ
diff --git a/app/zh/information/blog/20210908/012.png b/app/zh/information/blog/20210908/012.png
deleted file mode 100644
index 1feb0672e8240000c9b40f859c425c0c1a0c5481..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210908/012.png and /dev/null differ
diff --git a/app/zh/information/blog/20210908/013.png b/app/zh/information/blog/20210908/013.png
deleted file mode 100644
index 3bd0eb39fa66aa35edc8f0cf1bbba38306fc9813..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210908/013.png and /dev/null differ
diff --git a/app/zh/information/blog/20210908/014.png b/app/zh/information/blog/20210908/014.png
deleted file mode 100644
index 3623746439eba3308f3814bb46549a0617f2d65f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20210908/014.png and /dev/null differ
diff --git a/app/zh/information/blog/20210908/index.md b/app/zh/information/blog/20210908/index.md
deleted file mode 100644
index 82906731b6f0b58c291e7931631b3e538a87e05d..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/20210908/index.md
+++ /dev/null
@@ -1,255 +0,0 @@
----
-title: 'openLooKeng+Ranger+LDAP 认证鉴权能力演示'
-date: '2021-09-08'
-module: 'blog'
-customLayout: 'true'
-tags: ['Ranger', 'openLooKeng', '认证鉴权', 'LDAP']
-archives: '2021-09'
-author: 'openLooKeng'
-description: 'openLooKeng可以对接LDAP完成认证,同时对接Ranger完成权限控制。本次演示使用的是我们的实验openLooKeng版本(开源openLooKeng的三层结构catalog-schema-table,在实验版本中扩展为catalog-vdb-schema-table4层结构),你也可以采用开源openLooKeng达到完全相同的认证和鉴权的安全能力。'
----
-
-### 前言
-
-openLooKeng 可以对接 LDAP 完成认证,同时对接 Ranger 完成权限控制。本次演示使用的是我们的实验 openLooKeng 版本(开源 openLooKeng 的三层结构 catalog-schema-table,在实验版本中扩展为 catalog-vdb-schema-table4 层结构),你也可以采用开源 openLooKeng 达到完全相同的认证和鉴权的安全能力。
-
-### 总体演示步骤:
-
-1. LDAP 上已经配置好用户 tom,密码为:******
-
-2. 在 Ranger 已经配置好用户 tom 的访问权限
-
-3. 通过 openLooKeng Client 访问相应资源,检查是否符合策略配置
-
-### 环境说明:
-
-
-
-### Testcases:
-
-**1.未认证用户 kobe 访问失败**
-
-root@slave2:/home/xdz# ./openLooKeng_cli --server https://xdz3:9090 --catalog mysql --keystore-path /home/xdz/key138/openLooKeng-public.store --keystore-password ****** --user kobe --password
-Password:
-lk> select \* from view."vdb02:schema02".view02;
-Error running command: Authentication failed: Access Denied: Invalid credentials
-
-**2.在 LDAP 上创建用户 tom(密码:******)**
-
-
-
-
-
-Ranger-usersync 即刻能同步到 LDAP 上新建用户的信息,可以在 Ranger-admin 上查询到:
-
-
-
-**3.未在 Ranger 上配置 tom 用户可访问的资源时**,查询
-root@slave2:/home/xdz# ./openLooKeng_cli --server https://xdz3:9090 --catalog mysql --keystore-path /home/xdz/key138/openLooKeng-public.store --keystore-password ****** --user tom --password
-
-```
-Password:
-lk >
-lk > show catalogs;
- Catalog
----------
- mysql
- system
- view
-(3 rows)
-
-lk > show schemas from view;
-Query 20190723_024326_00005_frwmt failed: Access Denied: Cannot access catalog view
-
-```
-
-**4.在 Ranger 上配置 tom 访问 cataloge view 的权限**,查询
-
-
-
-
-
-```
-
-lk > show schemas from view;
- Schema
---------------------
- information_schema
- qqvdb
- testschema
- testvdb
-(4 rows)
-
-Query 20190723_024637_00021_frwmt, FINISHED, 1 node
-Splits: 19 total, 19 done (100.00%)
-0:00 [4 rows, 60B] [20 rows/s, 310B/s]
-
-```
-
-**5.在 Ranger 上配置 tom 访问 cataloge mysql 的权限**
-
-
-
-
-
-**6.tom 创建 view**
-
-- a. 3 层结构
-
-```
-lk > create schema view.vdb01;
-CREATE SCHEMA
-
-lk > create view view.vdb01.view01 as select * from mysql.testdb.testtb;
-CREATE VIEW
-
-lk > select * from view.vdb01.view01;
- id | name | score | comments
-----+----------+-------+-----------
- 1 | zhangsan | 80 | normal
- 2 | lisi | 85 | normal
- 3 | wangwu | 99 | very good
- 4 | zhaoliu | 55 | stupid
-(4 rows)
-
-Query 20190723_031647_00029_frwmt, FINISHED, 1 node
-Splits: 17 total, 17 done (100.00%)
-0:00 [4 rows, 0B] [21 rows/s, 0B/s]
-
-```
-
-- b. 4 层结构
-
-```
-lk > create schema view.vdb02;
-CREATE SCHEMA
-lk > create schema view."vdb02:schema02";
-CREATE SCHEMA
-lk > create view view."vdb02:schema02".view02 as select * from mysql.testdb.testtb;
-CREATE VIEW
-lk > select * from view."vdb02:schema02".view02;
- id | name | score | comments
-----+----------+-------+-----------
- 1 | zhangsan | 80 | normal
- 2 | lisi | 85 | normal
- 3 | wangwu | 99 | very good
- 4 | zhaoliu | 55 | stupid
-(4 rows)
-
-Query 20190723_031827_00035_frwmt, FINISHED, 1 node
-Splits: 17 total, 17 done (100.00%)
-0:00 [4 rows, 0B] [20 rows/s, 0B/s]
-
-lk > create view view.vdb02.view03 as select * from mysql.testdb.testtb;
-CREATE VIEW
-
-```
-
-**7.授权 view 给另一个用户 jack**(jack 已经在 LDAP 上创建好,密码为:******)
-
-- a. 授权 view.vdb01.view01 给 jack
-
-
-
-
-
-```
-
-root@slave2:/home/xdz# ./openLooKeng_cli --server https://xdz3:9090 --catalog mysql --keystore-path /home/xdz/key138/openLooKeng-public.store --keystore-password ****** --user jack --password
-Password:
-lk > select * from view.vdb01.view01;
- id | name | score | comments
-----+----------+-------+-----------
- 1 | zhangsan | 80 | normal
- 2 | lisi | 85 | normal
- 3 | wangwu | 99 | very good
- 4 | zhaoliu | 55 | stupid
-(4 rows)
-
-Query 20190723_033521_00044_frwmt, FINISHED, 1 node
-Splits: 17 total, 17 done (100.00%)
-0:00 [4 rows, 0B] [22 rows/s, 0B/s]
-
-lk > select * from view."vdb02:schema02".view02;
-Query 20190723_033821_00049_frwmt failed: Access Denied: Cannot select from columns [score, comments, name, id] in table or view view02
-
-lk > select * from view.vdb02.view03;
-Query 20190723_055918_00064_frwmt failed: Access Denied: Cannot select from columns [score, comments, name, id] in table or view view03
-
-```
-
-- b. 授权 view."vdb02:schema02".view02 给 jack
-
-
-
-
-
-```
-
-lk > select * from view."vdb02:schema02".view02;
- id | name | score | comments
-----+----------+-------+-----------
- 1 | zhangsan | 80 | normal
- 2 | lisi | 85 | normal
- 3 | wangwu | 99 | very good
- 4 | zhaoliu | 55 | stupid
-(4 rows)
-
-Query 20190723_060319_00066_frwmt, FINISHED, 1 node
-Splits: 17 total, 17 done (100.00%)
-0:00 [4 rows, 0B] [27 rows/s, 0B/s]
-
-lk > select * from view.vdb01.view01;
-Query 20190723_060322_00067_frwmt failed: Access Denied: Cannot select from columns [score, comments, name, id] in table or view view01
-
-lk > select * from view.vdb02.view03;
-Query 20190723_060316_00065_frwmt failed: Access Denied: Cannot select from columns [score, comments, name, id] in table or view view03
-
-```
-
-- c. 授权 view.vdb02 给 jack,即包含 view.vdb02.view03 和 view."vdb02:schema02".view02
-
-
-
-
-
-```
-
-lk > select * from view."vdb02:schema02".view02;
- id | name | score | comments
-----+----------+-------+-----------
- 1 | zhangsan | 80 | normal
- 2 | lisi | 85 | normal
- 3 | wangwu | 99 | very good
- 4 | zhaoliu | 55 | stupid
-(4 rows)
-
-Query 20190723_061024_00088_frwmt, FINISHED, 1 node
-Splits: 17 total, 17 done (100.00%)
-0:00 [4 rows, 0B] [45 rows/s, 0B/s]
-
-lk > select * from view.vdb02.view03;
- id | name | score | comments
-----+----------+-------+-----------
- 1 | zhangsan | 80 | normal
- 2 | lisi | 85 | normal
- 3 | wangwu | 99 | very good
- 4 | zhaoliu | 55 | stupid
-(4 rows)
-
-Query 20190723_061022_00087_frwmt, FINISHED, 1 node
-Splits: 17 total, 17 done (100.00%)
-0:00 [4 rows, 0B] [32 rows/s, 0B/s]
-
-lk > select * from view.vdb01.view01;
-Query 20190723_061020_00086_frwmt failed: Access Denied: Cannot select from columns [score, comments, name, id] in table or view view01
-
-```
-
----
-
-如果您有任何想要交流的,欢迎在社区代码仓内提 Issue;也欢迎加小助手微信(openLooKengoss),进入专属技术交流群。
-
-欢迎访问官网
-
-```java
-org.codehaus.plexus.component.repository.exception.ComponentLifecycleException: Error constructing component role: 'org.apache.maven.repository.RepositorySystem', implementation: 'org.apache.maven.repository.legacy.LegacyRepositorySystem', role hint: 'default'
-```
-
-6. **presto-main/etc/config.properties**
-
-删除`plugin.bundles`中 hive 和 oracle 插件:
-
-(在 win 中运行 hive 和 oracal 需要别的插件,暂且用不到,先屏蔽掉。后期有时间解决这个问题)。
-
-```java
-- ../presto-hive-hadoop2/pom.xml,\
-- ../hetu-oracle/pom.xml, \
-```
-
-7. **presto-main/etc/catalog/**
-
- hive 及 oracle 需要在 windows 上安装相关程序的 windows 版本才能使用,这里直接删除删除 hive 及 oracle 的配置文件
-
- ```java
- # 在GitBash中
- cd presto-main/etc/catalog/
- mv hive.properties hive.properties.bak
- mv oracle.properties oracle.properties.bak
- ```
-
-### 2.2 **编译构建**
-
-首先需要配置antlr4,运行命令
-
-```javascript
-mvn antlr4:antlr4
-```
-
-在出现BUILD SUCCESS标识后,进入File>Project Structure>Modules,选择presto-parser/target/generated-sources,将该目录下的antlr4文件夹标记为Sources,如图所示:
-
-
-
-由于openLooKeng使用antlr4解析SQL,不进行配置会在编译时报错“程序包SqlBaseParser不存在”。 接下来就可以进行打包与编译构建:
-
-```javascript
- mvn clean package -DskipTests
-```
-
-这里提供另外一种编译构建的方法,如果遇到如下问题:
-
-问题: 打包时候出现 check 错误还有其他检查错误。并且 mvn clean install -DskipTests 并不能跳过 check,需要在 hetu-core 的 pom 的 properties 里面增加
-
-```javascript
-
-
-## (三)运行
-
-### 3.1 运行前准备
-
-1)打开 idea
-
-2)Crtl+Alt+S 进入设置页面(file-settings)
-
-- _Build,Excution,Deployment > Complier > JavaComplier_
-
-- 取消勾选 Use ‘–release’ option for corss-compilation
-
-- 设置 project bytecode version 为 8
-
-
-
-3. 运行参数设置
-
-run->edit confurations,创建一个Application类型的运行配置,设置参数如下:
-
-```
-Main class: io.prestosql.server.PrestoServer
-
-VM options: -ea -XX:+UseG1GC -XX:G1HeapRegionSize=32M -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent -Xmx2G -Dconfig=etc/config.properties -Dlog.levels-file=etc/log.properties -Duser.home=D:\document\Maven
-
-Working directory: $MODULE_DIR$
-
-Use classpath of module: presto-main
-
-注意-Duser.home=D:/Marven是maven仓库.m2文件夹的父目录,且最后两级目录名必须是/.m2/repositor
-```
-
-`-Duser.home=D:\document\Maven`未添加这一项的,会出现如下错误:
-
-```
-Caused by: org.apache.maven.project.ProjectBuildingException: 1 problem was encountered while building the effective model for io.hetu.core:presto-resource-group-managers:1.2.0-SNAPSHOT
-[FATAL] Non-resolvable parent POM for io.hetu.core:presto-root:1.2.0-SNAPSHOT: Could not transfer artifact io.airlift:airbase:pom:97 from/to central (http://repo.maven.apache.org/maven2): connection timed out to http://repo.maven.apache.org/maven2/io/airlift/airbase/97/airbase-97.pom and 'parent.relativePath' points at wrong local POM @ io.hetu.core:presto-root:1.2.0-SNAPSHOT, D:\document\hetu-core\pom.xml, line 6, column 13
- for project io.hetu.core:presto-resource-group-managers:1.2.0-SNAPSHOT at D:\document\hetu-core\presto-main\..\presto-resource-group-managers\pom.xml
-```
-
-
-
-4)**presto-main\etc\config.properties**
-
-添加:
-
-```java
-hetu.queryeditor-ui.allow-insecure-over-http=true
-```
-
-添加原因是:不添加会报 http 不安全的错误:
-
-
-
-接下来删除文件中的plugin.bundles,将其替换为plugin.dir。此处应当指向plugin目录:
-
-```java
-plugin.dir=../hetu-server/target/hetu-server-{VERSION}/plugin
-```
-
-其中{VERSION}自行替换为对应版本号。否则,后面运行时会在io.prestosql.server.PluginManager中调用pom时出现```java.lang.NullPointerExecption```空对象异常。
-
-5)**presto-main\etc\hetu-metastore.properties**
-
-设置metastore类型,以jdbc为例:
-
-```java
-hetu.metastore.type=jdbc
-hetu.metastore.db.url=jdbc:mysql://**.**.**.**:3306/hetumetastore?useSSL=false
-hetu.metastore.db.user=****
-hetu.metastore.db.password=***********
-```
-
-### 3.2 运行结果
-
-点击 IDEA 上的运行,然后在浏览器中输入`http://localhost:8080/`
-
-
-
-
-
-## (四)进入 openLooKeng
-
-```
-# 进入源码根目录
-cd hetu-core
-# 运行cli客户端连接OpenLooKeng
-java -jar presto-cli/target/hetu-cli-*-executable.jar --server localhost:8080
-SHOW CATALOGS;
-SHOW SCHEMAS FROM jmx;
-use jmx.current;
-show tables;
-```
-
-
-
-图形化界面查询正常:
-
-
-
-## 其余问题解决方法:
-
-问题 1:
-
-```
-Failed to execute goal org.skife.maven:really-executable-jar-maven-plugin:1.0.5:really-executable-jar (default) on project presto-benchmark-driver: FAILURE!
-
-```
-
-直接注掉所有的依赖引用。涉及到的模块比较多,因此就不一一截图了。搜下,引用到的都注掉。
-
-问题 2:
-
-```
-(imports) UnusedImports: 无用导入 - javax.management.MBeanServer 。
-```
-
-进入指定的类,将这个 import 删除
-
-问题 3:
-
-遇到一些类无法加载的,重新 reload 一下 maven
-
-
-
-或者运行命令```mvn idea:idea```然后通过File>Invalidate Caches重启IDEA。
-
-问题 4:
-
-```
-Failed to execute goal org.apache.maven.plugins:maven-enforcer-plugin:3.0.0-M3:enforce (default) on project hetu-state-store: Some Enforcer rules have failed. Look above for specific messages explaining why the rule failed.
-
-[WARNING] Rule 2: org.apache.maven.plugins.enforcer.BanDuplicatePomDependencyVersions failed with message:
-Found 1 duplicate dependency declaration in this project:
- - dependencies.dependency[com.hazelcast:hazelcast:jar] ( 2 times )
-
-```
-
-重复定义的配置 duplicate dependency,找到直接删除。
-
-问题 5:
-
-```
-1)org.codehaus.plexus.component.repository.exception.ComponentLifecycleException: Error constructing component role: 'org.apache.maven.repository.RepositorySystem', implementation: 'org.apache.maven.repository.legacy.LegacyRepositorySystem', role hint: 'default'
-
-2)Error loading pom......
-```
-
-
-
-通过定位:
-
-```
-line356 pomFile ..\presto-resource-group-managers\pom.xml
- PlexusContainer container = container();
-```
-
-
-
-
-
-无法加载一些 jar 包:
-
-修改方法:
-
-**presto-main/pom.xml**
-
-注释掉一下文件:
-
-```javascript
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-```
-
-问题6:
-
-```Could not transfer aitifact xxx from/to xxx and 501 HTTPS Required```
-
-Maven2之后使用了https协议,需要添加配置忽略SSL错误。在IDEA的设置中进入Build,Execution,Deployment > Build Tools > Maven > Runner,添加VM options:
-
-```javascript
--Dmaven.wagon.http.ssl.insecure=true -Dmaven.wagon.http.ssl.allowall=true
-```
-
-问题7:
-
-配置MySQL数据源后,执行查询计划时```could not create connection to database server```
-
-这是由机器上安装的MySQL版本和项目中pom.xml配置的mysql-connector版本号不同引起的。将pom.xml的版本号修改为本地MySQL版本号后刷新maven即可。
-
-问题8:
-
-```the pom of xxx is missing```
-
-可能之前由于网络或配置原因,jar包下载失败产生了.lastUpdated文件,下次下载时如果不删除这些文件,jar包就不会正确下载,也不会被识别。解决方案是在maven的repository仓库目录下删除所有的.lastUpdated文件。
-
-参考资料:
-
-https://blog.csdn.net/ren88320036/article/details/106916338
-
-https://blog.csdn.net/ren88320036/article/details/106916338
-
----
-
-如果您有任何想要交流的,欢迎在社区代码仓内提 Issue;也欢迎加小助手微信(openLooKengoss),进入专属技术交流群。
-
-欢迎访问官网
-
-读写策略模式如下:
-
-
-
-Cache Aside Pattern,Delete the existing Cache when writing the database。Cache Aside Pattern 能有效避免并发问题。
-
-Cache Aside 的优点:
-
-当写操作发生时,假设淘汰缓存作为对缓存通用的处理方式,又面临两种抉择:
-
-(1)先写数据库,再淘汰缓存
-
-(2)先淘汰缓存,再写数据库
-
-我们假设:两个并发操作,一个是更新操作,另一个是查询操作,更新操作删除缓存后,查询操作没有命中缓存,先把老数据读出来后放到缓存中,然后更新操作更新了数据库。于是,在缓存中的数据还是老的数据,导致缓存中的数据是脏的,而且还一直这样脏下去了。所以这个设计是错误的,不建议使用。
-
-一个是查询操作,一个是更新操作的并发,首先,没有了删除 Cache 数据的操作了,而是先更新了数据库中的数据,此时,缓存依然有效,所以,并发的查询操作拿的是没有更新的数据,但是,更新操作马上让缓存的失效了,后续的查询操作再把数据从数据库中拉出来。
-
-### **Hazelcast 学习**
-
-Hazelcast 作为一个分布式机制,可以用 Hazelcast 的 Imap 作为分布式缓存。
-
-这里需要注意的是由于 Hetumetastore 存储有六个缓存,需要对每个缓存实例化。不能用一套。
-
-```java
-IMap
-
-**重构难点 1:**
-
-两套 Cache,用 HetuMetastoreCache 接口来解决。解决办法:用泛型,用泛型,用泛型。
-
-**重构难点 2 :**
-
-对外暴露接口要统一 HetuMetastoreCache ,而且需要对声明变量进行初始化。解决办法:善于应用继承 Extends;Guava Cache 和 Hazelcast Cache 继承 HetuMetastoreCache 这个接口。这里面用到 Cache 变量又要统一使用 Hetucache,Hetucache 有两种模式 Guava 的 Cache 和 Hazelcast 的 Map。
-
-实际实现结构如下:
-
-
-
-现在结构的优点,对外只暴露 HetumetastoreCache。这样易扩充,后期如果需要实现 redis 的 Cache,只需要在后面实现即可。
-
-### **一些问题的解决方法**
-
-1. 由于对 Inject 的用法存在误解,报错 Explicit bindings are required。Java 的 Guice,如果你需要使用@Inject,那你的构造函数里面所有参数都需要实现的 binding。
-
-2. Hazelcast 版本 4.0.3 由于对序列化有依赖,需要自己定义序列化,这里面涉及到对 Optional 的序列化,Option 序列化存在一定问题。幸运的是查到官方文档 4.2 的版本提升了对 Optional 的支持,查看源码的实现,我们可以借签过来。
-
-### **总结**
-
-看源码,多向优秀的朋友和同事咨询,解决问题的角度会让人茅塞顿开。重构果然对语言的掌握能力要求较高。这个项目中,对我这薄弱的 Java 知识进行了一次深化。
-
----
-
-欢迎关注 openLooKeng,加小助手微信(openLooKengoss)进入专属技术交流群。
-
-社区代码仓
-
-在 openLooKeng 的代码中认证相关的 filter 会在 ServerSecurityModule 类中进行注册,AuthenticationFilter 和 PaswordAuthenticatorManager 就是跟认证相关的。
-
-
-
-当然从该模块中我们还知道除了支持 Kerberos 和 Password 认证之外还支持 CERTIFICATE 和 JWT。当请求来到时会先经过前面注册的 AuthenticationFilter,该类实现了 Filter 的 doFilter 方法,在其中我们可以看到可以支持内部的认证、webUI 认证和外部认证,这里主要讲解外部内证,
-
-
-
-这里会根据配置认证方式进行遍历,并调用对应的认证模式进行认证,认证通过后则进入下一下 Filter 进行处理,最后请求到达服务器端的 rest 接口内部,进行相应 服务处理。
-
-### 支持认证方式
-
-前面讲解到会在 doFilter 方法进行认证,其中 Authenticator 是一个接口,看到其中有 4 种实现,下面主要以 Kerberos 和 Password 为例进行讲解。
-
-
-
-**· Kerberos 认证**
-
-Kerberos 认证主要在 KerberosAuthenticator 类中完成的,在类初始化的时候会根据配置的信息如 krb5.conf、keytab 和 principal 等信息初始化好 loginContext。当有请求带来时,会从 request 中取出用户信息或是 token 进行认证,如下:
-
-
-
-当成功认证后会返回 KerberosPrincipal 继续传递给下一个 Filter。具体的 Kerberos 认证原理可以查看 Kerberos 官网,这里不做详细描述。
-
-**· Password 认证**
-
-除了上面说的支持 Kerberos 的认证外,还支持密码认证,默认支持的是基于 LDAP 的密码认证。密码的认证是主要在 PasswordAuthenticator 类中的 authenticate 完成的,同样的也是从 request 中取出用户名和密码交给 authenticatorManager 进行认证。
-
-
-
-目前我们看到 PasswordAuthenticator 的接口只有 LDAP 的实现。
-
-
-
-在 LdapAuthenticator 类中认证主要由 authenticate 方法完成的。里面会调用 LDAP 相关的 API 跟 LDAP 服务器进行交互完成认证。
-
-
-
-**· Password 认证框架**
-
-其实 openLooKeng 密码认证是以 plugin 方式进加载的,可以解压编译好的包在 plugin 路径下可以看到 password 相关的 plugin。
-
-
-
-从代码上看到 PasswordAuthenticatorPlugin 是实现了 Plugin 接口的,其中里面添加了 LdapAutehnticatorFactory。
-
-
-
-而 LdapAutehnticatorFactory 类则是实现了 PasswordAuthenticatorFactory 接口,该接口主要提供了两个方法一个 getName 和 create,其中在 LdapAuthenticatorFactory 中 getName 返回的是 ldap,以区别其他密码认证模式,create 方法中则是通过 airlift 框架初始化 LdapAuthenticator 实例,保存在 PasswordAuthenticatorManager 类中,这样当进行密码认证的时候从该类中取出对应的 Authenticator 实例即可。
-
-另外在系统启动的时候,会由 PasswordAuthenticatorManager 调用 addPasswordAuthenticatorFactory 接口加载多个 PasswordAuthenticatorFactory 实现类,如下:
-
-
-
-当 plugin 加载完成后再由 PasswordAuthenticatorManager 类调用 loadPasswordAuthenticator 方法完成具体 PasswordAuthenticator 实现的创建:
-
-
-
-这个便是 openLooKeng 整个密码认证框架的运行原理。
-
-**· 扩展认证**
-
-根据前面的介绍,我们可以看出如果实现自己的认证方式的话,笔者认为可以有两种主要方式:
-
-第一种是基于 persto-main 模块的接口 Authenticator 直接实现自己认证和配置即可。
-
-第二种如果是基于密码认证,那么可以这样实现:
-
-1. 基本 presto-spi 模块的 PasswordAuthenticator 接口实现 MyPasswordAuthenticator 类
-
-2. 基于 PasswordAuthenticatorFactory 实现 MyPasswordAuthenticatorFactory,然后实现 getName 和 create 方法,其中 create 方法中参考 LDAP 密码认证实现创建基于前面实现的 MyPasswordAuthenticator 类的实例。
-
-3. 最后在 PasswordAuthenticatorPlugin 中加上实现的 MyPasswordAuthenticatorFactory 类即可。
-
-
-
-通过上面的步骤后那么 openLooKeng 启动时也会加载自己实现的 Factory。只要在配置文件中配置成自己实现的认证即可使用。
-
----
-
-欢迎关注 openLooKeng,加小助手微信(openLooKengoss)进入专属技术交流群。
-
-社区代码仓
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-下面我们举例来说明。
-
-
-
-如图所示,join key 只有 1 个,page 只有 1 个,其值如 ① 所示。对这些行进行地址编码,则编码后地址如 ② 所示。Hash table 构建步骤: 1) 对原始数据进行 hash 计算,结果如 ③ 所示; 2) 逐行处理 addresses:
-
-
-
-最终得到的 key 数组如 ④ 所示,得到的 positionLinks 如 ⑤ 所示。
-
-### 2.4 Hash Table 使用
-
-HashBuilderOperator 构建完 hash table 后,LookupJoinOperator 才能开始处理数据进行探测。而 LookupJoinOperator 使用 hash table 的核心代码片段如下:
-
-
-
-Hash table 使用步骤:
-
-1) 对原始数据进行 hash 计算得到 rawHash;
-
-2) 对 rawHash 再进行 hash 计算得到其在 hash table 的 offset,即 pos;
-
-3) 若 key[pos]为-1,则没有匹配;
-
-4) 若 key[pos]不是-1,则 hash 值匹配,若原始数据是否相等,相等则完全匹配上,返回 key[pos],即原始数据的地址 address;若原始数据不相等则 pos 加 1 再循环判断。
-
-## 3 总结
-
-本文介绍了 openLooKeng 支持的 join 类型,并展开介绍了 Lookup join 的 partition,然后重点介绍了 hash table 的构建和使用过程,但其实 Lookup Join 的内容不止这些,比如 HashBuilderOperator 和 LookupJoinOperator 如何实现同步,LookupJoinOperator 的 probe 后的输出数据如何构造,非等值的 join 又是如何实现的,请期待后续的文章!
-
----
-
-如果您有任何疑问或建议,欢迎在社区代码仓内提 Issue;也欢迎加小助手微信(openLooKengoss),进入专属技术交流群。
-
-社区代码仓
-
-
-
-配置 Rowfilter 策略:
-
-
-
-查询效果:
-
-
-
-### Rowfilter 实现方式
-
-回顾一下 openLooKeng 的 coordinator 接收到一个 SQL 的处理过程:终端用户通过 ODBC 或者 JDBC driver 将一个 SQL 语句发送到 coordinator,这时 SQL 是以文本的形式被接收的。coordinator 会解析 parse 和分析 analysis,然后生成 query plan。
-
-
-
-openLooKeng analysis 过程分为两部分:StatementRewrite 和 StatementAnalyzer。这两部分中都有权限相关的检查。
-
-考虑到:
-
-1. Rangeradmin 侧仅存储鉴权策略,而鉴权逻辑在 openLooKeng 侧的 ranger-plugin 中完成。
-
-2. Rowfilter 策略实际就是 where 条件子句。
-
-所以,在 StatementRewrite 中将 Rowfilter 条件子句重写到 sql 中。
-
-主要的代码修改涉及:
-
-1. Ranger 侧:修改 openLooKeng 数据源的策略定义,增加行权限策略的配置界面。
-
-2. openLooKeng 侧:增加一个新的 rewrite 类:RowFilteRewrite,将条件子句加到 sql 中。
-
-
-
-
-
-
-
----
-
-如果您有任何疑问或建议,欢迎在社区代码仓内提 Issue;也欢迎加小助手微信(openLooKengoss),进入专属技术交流群。
-
-社区代码仓
-
-
-
-以上是基于 hive connector 中的一张 student 表创建了一个视图存储在 vdm 中。
-
-### View in Hive
-
-除了上面的提到的 VDM 连接器外,openLooKeng 视图也可以存储在 Hive connector 中,我们可以按照官网的指导:
-
-以上是基于 hive connector 创建的 openLooKeng 视图然后存储在 hive connector 中。
-
-### Hive View
-
-所谓的 Hive view 其实就是在 hive 中创建的视图,openLooKeng 也支持访问 hive 中创建的视图,如下我们在 hive 中创建一个视图然后在 openLooKeng 中访问:
-
-先在 beeline 中创建一个 hive view
-
-
-
-然后我们在 openLooKeng 侧对这个视图进行查询,可以看到查询结果和 hive 中的一致的。
-
-
-
-### 原理解析
-
-这里主要介绍视图的创建和查询流程,如下图示为视图的创建流程:
-
-
-
-当 openLooKeng 接收到创建视图请求时,会调用 DataDefinitionExecution 进行处理,然后再调用对应 MetaDataManager 处理,如果是 Hive 连接器则会调用 HiveMetadata 处理,VDM 连接器则会调用 VdmMetadata 处理,将 view 中相关的 sql 语句编码成 viewData,最后存储在 connector 中。
-
-以下是视图的查询流程:
-
-
-
-这个跟视图创建流程不太一样,当一个请求到达服务端时,由 SqlQueryExectuion 进行处理做语法解析,在做 visitTable 时,先通过 MetadataManager 通过对应连接器的 metadata 获取视图,hive 连接器则为 HiveMetadata, VDM 则为 VdmMetadata,获取实际的 table,如果获取的 table 为视图,则递归再进行 StatementAnalyzer#paserView 和#analyzeView 进行视图的解析,直到解析出最终的物理表为止。在 HiveMetadata#getView 时,如果是 Hive View 则由 HiveMetadata 转成 openLooKeng 的 view,再交给 StatementAnalyzer 做语法解析,这样处理流程就像 openLooKeng 的 view 一样处理了。
-
-以上就是 openLooKeng 中视图的创建和查询流程。
-
----
-
-如果您有任何疑问或建议,欢迎在社区代码仓内提 Issue;也欢迎加小助手微信(openLooKengoss),进入专属技术交流群。
-
-社区代码仓
-
-
-
-### 二、 使用 helm 安装 openLooKeng
-
-Helm 是一个命令行下的客户端工具。使用 helm 命令启动 openLooKeng。
-
-```
-1. 下载helm包(因下的包不完整,故从别人机器上直接取的helm)
-
-2. 解压
-
-3. mv linux-amd64/helm /usr/local/bin/helm
-helm
-
-4.[root@51-38-77-146 k8s]# helm version
-version.BuildInfo{Version:"v3.3.4", GitCommit:"a61ce5633af99708171414353ed49547cf05013d", GitTreeState:"clean", GoVersion:"go1.14.9"}
-```
-
-#### **安装 openLooKeng**
-
-1. 在 registry 节点,load openLooKeng 镜像:
-
-```
-docker load –i openlookeng-x86-docker20210511144940.tar
-```
-
-2. 将镜像包打 tag 并 push 到 registry:
-
-```
-docker tag openlookeng:fc981cce ***.***.***.***:5000/hetu/openlookeng:fc981cce
-docker push ***.***.***.***:5000/hetu/openlookeng:fc981cce
-```
-
-3. openLooKeng 的配置文件:
-
-1)在 registry 节点,创建目录(名字随意):openlk
-
-2)cd openlk
-
-3)mkdir templates
-
-4)mkdir config-catalog
-
-
-
-放入需要添加的外部数据源。
-
-5)mkdir config- filesystem
-
-
-
-6)mkdir config- coordinator
-
-
-
-7)mkdir config- worker
-
-
-
-8)cd templates
-
-9)创建 deployment.yaml。具体参考:deployment.yaml
-
-10)cd ..
-
-11)创建 Chart.yaml
-
-```
-[root@51-38-77-146 openlk]# cat Chart.yaml
-apiVersion: v2
-type: application
-
-name: openLK-cluster
-description: A Helm chart for openLooKeng cluster
-version: 0.1.0
-```
-
-12)创建 values.yaml
-
-```
-[root@51-38-77-146 openlk]# cat Chart.yaml
-apiVersion: v2
-type: application
-
-name: openLK-cluster
-description: A Helm chart for openLooKeng cluster
-version: 0.1.0
-[root@51-38-77-146 openlk]# cat values.yaml
-# One of "local", "cce-test", and "cce-prod"
-env: local
-
-# Whether this is a shared cluster for DC connector
-shared: false
-
-docker:
- registry: ***.***.***.***:5000/hetu/-----registry ip和端口号
- imageTag: unknown
-
-service:
- type: NodePort
- port:
- # Port used natively by application. Must match "http-server.http.port"
- app: 8080
- # Port used by service in K8s. Must match port in "discovery.uri" if specified
- service: 8080
- # Port exposed externally, e.g. to connect CLI or Web UI
- node: 8889----登录openLooKeng UI界面的端口号
-
-configMap:
- coordinator:
- name: config-coordinator
- srcPath: config-coordinator
- worker:
- name: config-worker
- srcPath: config-worker
- catalog:
- name: config-catalog
- srcPath: config-catalog
- filesystem:
- name: config-filesystem
- srcPath: config-filesystem
-
-pod:
- coordinator:
- replica: 1-----coordinator个数
-
- volume:
- base:
- mountPath: /customConfig
- name: config-coordinator
- optional: true
- catalog:
- mountPath: /customConfig/catalog
- name: config-catalog
- optional: true
- filesystem:
- mountPath: /customConfig/filesystem
- name: config-filesystem
- optional: true
-
- worker:
- replica: 50---worker个数
-
- volume:
- base:
- mountPath: /customConfig
- name: config-worker
- optional: true
- catalog:
- mountPath: /customConfig/catalog
- name: config-catalog
- optional: true
- filesystem:
- mountPath: /customConfig/filesystem
- name: config-filesystem
- optional: true
-```
-
-4. 启动 openLooKeng:
-
-```
-helm upgrade --install openlk . --set docker.imageTag=fc981cce
-```
-
-也可以在后面加其他参数:
-
-```
-helm upgrade --install openlk-ray /root/raymond/openlk --set pod.worker.replica=3 --set service.port.node=30602 --set docker.imageTag=$tag
-```
-
-5. 查看拉起的 cn 和 workers:
-
-```
-kubectl get pods -n openlk -o wide
-```
-
-
-
-6. 执行命令进入 pod
-
-```
-kubectl exec -ti coordinator-7867b8548c-2lbjn -n openlk -- /bin/bash
-```
-
-7. 可以查看到本地 config-\*中的配置文件在 customConfig 路径下:
-
-
-
-8. 进入 cli
-
-```
-openlk(后面可以跟--catalog xx --schema xx这些)
-```
-
----
-
-以上便是在 openLooKeng 部署 K8s 集群的操作指导,欢迎有需要的朋友们采用。有任何疑问或建议,可以在社区仓库提 issue 或交流群沟通。再次欢迎大家使用
-
----
-
-加小助手微信(openLooKengoss)进入专属技术交流群。
-
-社区代码仓
-
-
-
-
-
-> pathToRoot,
- @JsonProperty("progress") Optional
-
-### 3 openLooKeng 1.3.0 AA 特性优化
-
-#### 1. 减少 hazelcast 的拉取数据量
-
-当前对于每一个语句的提交,都会从 hazelcast 里拉取所有 running 语句状态信息。但是语句的状态信息拿取之后,主要是进行聚合的计算,比如计算正在运行的 SQL 总数以及当前所有的语句状态的内存和 CPU 总体使用量。
-由于最终的目的是进行计算,当前对于处理优化的思路是,在 hazelcast 里面放置的并非所有语句的状态信息,而是已经经过预计算之后的 partial 结果。如放置的已经是已经预计算之后的内存以及 CPU 的使用量,正在运行的 SQL 总数等信息,这样,每次 SQL 提交时,只需要拿到已经部分预计算的数据信息。在 hazelcast 里面存放的数据量因此可以减少,对于同步时所消耗的延迟因此降低。如下图所示,与之前版本相比,可以减少同步带来的延迟,减少开销,提升并发能力。
-
-
-
-#### 2. 优化当前锁的开销
-
-对于拉取状态的更改,只能解决部分问题,由于锁的机制存在,所有的 SQL 处理必须串行执行。对于锁修改,减少锁的范围,可以极大地提升并行处理的能力,同样可以减少语句提交带来的开销,提升并发度。
-
-**参考文档:**
-
-
-
-通常场景下,所有的 join node 都会生成 dynamic filter,然后依靠优化器规则 PredicatePushDown 将生成的 dynamic filter 下推给 TableScanNode。然而,默认的动态过滤生成与应用缺乏对于元数据信息的感知,查询分析引擎 probe 侧表的 filter 需要等待 build 侧表读取完成之后生成,默认 join 结点的 probe 表的读取会首先等待一个时间(如 1000ms),如果没有等待到 build 侧的 filter 传入则无法利用。如果 build 过滤后的数据量相对大时,也会导致大量的 filter 在网络中传输。总的来说,直接利用 join 进行动态过滤生成与应用,缺乏利用统计信息以及元数据信息对于动态过滤条件生成及应用进行预估,导致选择率很低的动态过滤条件生成与使用,造成额外的系统开销。
-
-### 2 动态过滤优化
-
-#### 2.1 基于选择率优化
-
-当前,优化器规则中 RemoveUnsupportedDynamicFilters 仅仅删除那些不应该存在的动态过滤条件,比如 build 侧存在的动态过滤条件或者是 probe 侧存在动态过滤条件但是对应的 JoinNode 中并不存在。因此,这个规则是无法支撑我们基于元数据和 CBO 进行动态过滤生成与应用调整的。因此,我们需要扩展 RemoveUnsupportedDynamicFilters 规则:
-
-1. 将 Metadata 以及 cost provider 提供给 RemoveUnsupportedDynamicFilters
-
-2. 删除我们不需要的动态过滤条件,例如,如果表很小且动态过滤无法帮助过滤,那么表应该被过滤得足够多,我们不需要启用动态过滤。
-
-#### 2.2 动态过滤条件生成优化
-
-此外,openLooKeng 早期版本,对于动态过滤条件的合并与生成,是为每一个 DynamicFilterSourceOperator 都注册一个 driver id。如果 DynamicFilterSourceOperator 快速的完成,那么刚注册了他的 driver id 就很快完成,那么在 coordinator 上的 DynamicFilterService 可能会错误的以为所有 worker 已经完成了自身的读取任务,可以合并动态过滤条件生成最终的数据,这可能会导致错误的结果。此外,对于每个查询,与 coordinator 中 hazelcast 的交互次数为
-
-```
-num(Dynamic Filter) * num(Worker) * num(DynamicFilterSourceOperator per Worker) * 4(register, finish, worker, partial result)
-```
-
-当部署到具有高并发查询的大群集时,这可能会意味着 coordinator 需要为每个 DynamicFilterSourceOperator 执行合并,可能造成极大的网络传输数据量以及计算开销。为了解决这个问题,openLooKeng 1.1.0 中在每个任务中由各个 worker 完成自己持有的动态过滤条件的合并。因此,DynamicFilterService 只需要合并每个 worker 提供的过滤条件。
-
-### 2.3 基本实现原理
-
-在 RemoveUnsupportedDynamicFilters 的优化器中,在进行动态过滤条件检验时:
-
-1. 检查 build 侧表的预估输出行大小,并且丢弃可能会比较大的动态过滤条件(handleTooLargePredicate()方法仍将保留在 DynamicFilterSourceOperator 中作为保护措施)。
-
-2. 计算 build 侧的选择率,如果发现 build 侧的表的本身自带的过滤条件无法很好的过滤数据,即基本是全表扫描,那我们将对该 build 侧的表参与的 JoinNode 中的动态过滤条件删除。
-
-如下图所示,红色框标记的参与 join 计算的 build 侧的表,其本身行数过大或者是全表扫描,那么就不会针对该 JoinNode 生成并应用动态过滤。
-
-
-
-另外一种场景,对于 tpcds q2,执行计划如下图所示。实际上,DF-3 可以过滤顶部 JoinNode 的 probe 侧的表。但是由于 pipeline 调度的关系,因为我们配置了 hivePageSouce 等待时间,所以整个 subplan 的执行必须等待右侧的 subplan 完成。因此,我们可以删除 DF-3,对于底部表 A,可以尽快获得动态过滤条件,对于表 B 扫描,它可以使用表 A 中的动态过滤条件。由于整个左子树可以更早地利用动态过滤,语句整体的执行时间因此缩短。
-
-
-
-对于动态过滤条件生成,我们会将每个 worker 负责的 partial 动态过滤条件在 LocalDynamicFilter:addOperatorResult()中进行合并,当所有分区的读取完成之后,worker 将把合并后的 partial filters 放入到 hazelcast 当中。相对应的,`DynamicFilterService`中的逻辑也可以简化,我们只需要检查所有参与的 worker 是否完成了他们自己的任务,并将最终合并后的 partial filters 放入到了 hazelcast 中。因此,我们不再需要维护复杂的状态信息,因为 coordinator 参与处理的 worker 总数,其处理逻辑也得到很大的简化。
-
-**参考文档:**
-
-
-
-2. 命令行输入命令
-
-```java
-
-(bash <(wget -qO- https://download.openlookeng.io/install.sh) -m)
-
-```
-
-输入命令后,看打印的信息貌似是安装了一些依赖的软件,之后便会提示输入 coordinator 的 ip
-
-
-
-输入 coordinator ip 之后,接着又会让你输入 worker 节点,并用逗号分隔,因为我现在之后两个节点,所以我只输入了一个 worker 节点,这里还会打印一次确认输入的正确性
-
-
-
-紧接着询问集群的密码,这里要求所有节点的密码必须一致,不然往其他节点分发包的时候会出现错误。
-
-
-
-通过控制台的打印可以看到,安装程序会自动做节点之间的免密,会校验 java 的安装情况,如果没有 java 应该要自动安装。
-
-
-
-接下来就是远程下载安装包,做配置。
-
-
-
-安装成功以后会自动启动集群。
-
-
-
-启动集群后使用客户端,使用简单的 sql 查询试探功能
-
-
-
-至此 openLooKeng 多节点一键部署安装成功。
-
-
-
-Ps:
-自动安装时配置会自动生成,他会保存在/home/openlkadmin/.openlkadmin/cluster_node_info文件中,同时集群节点信息也会保存在/home/openlkadmin/.openlkadmin/cluster_node_info文件中。如果文件存在则不会进行自动配置,因此如果有特殊需求配置可以在文件中更改后再安装。
-
----
-
-如果您有任何疑问或建议,欢迎在社区代码仓内提 Issue;也欢迎加小助手微信(openLooKengoss),进入专属技术交流群。
-
-社区代码仓
-
-
-
-### 二、openLookeng 引擎的 Connector 下推
-
-下面是一条 SQL 语句执行的各个环节,查询语句经过解析之后形成抽象语法树,然后经过 Analyze 生成逻辑计划,逻辑计划再通过一系列的优化规则来生成优化后的更高效的逻辑计划进而转成物理计划。而 Connector 下推就处于逻辑计划优化阶段的某一环节。
-
-
-
-我们来看一下下面这张图,openLooKeng 新下推框架的主要思想是把执行计划子树暴露给 connector,让 connector 提供 PlanOptimizers(基于 visitor 模式的)给执行优化引擎,这样可以让 connector 引入任意的优化。这张图展示了 Connector 下推在整个查询优化缓环节的位置,其实不难看出,真正触发这个过程的就是 ApplyConnectorOptimization 这条优化规则,它也仅仅是众多优化规则中的一条规则。只是这条规则会把关于该 Connector 相关的最大子查询计划(maxsubplan)推给 Connector 去优化,该 Connector 针对本数据源进行一系列的定制优化。
-
-
-
-### 三、openGauss Connector 下推优化实践
-
-上面说了这么多,到底怎么来具体实现一个 Connector 的下推优化呢,我们接下来以 openGauss Connector 为例。这个 Connector 顾名思义,就是用来连接 GaussDB 数据库的,它本身继承或复用了 postgresql 和 basejdbc 的一些类,所以在进行下推实现的时候,我们也可以继续去复用一些类。
-
-首先,我们从逻辑计划开始,LogicalPlanner 类是对刚刚解析出来的抽象语法树(AST)进行逻辑计划生成的类,在生成逻辑计划的同时,他还会做一件很重要的事就是对逻辑计划进行优化,我们可以从 222 行看到,这里的 planOptimizers 包含了几十条优化规则,而我们上面提到的 ApplyConnectorOptimization 就在第 55 条规则中,当循环遍历到这条规则的时候,其实也就是 Connector 逻辑优化的开始了。
-
-
-
-然后,就如上面我所介绍的,当遍历到 ApplyConnectorOptimization 规则的时候,就会调用对应的 Connector 的 Optimizer,在这里我们可以清晰地看到,因为我查询的 catalog 是一个 GaussDB 表,所以这个 Optimizer 就是 JdbcPlanOptimizer(按理说应该是 opengaussPlanOptimizer,但是上面说到过,因为 openGauss Connector 很多功能都是复用了 Jdbc,这里也不意外),这个优化器中包含的一个比较重要的成员变量就是 queryGenerator,因为正是通过他来进行后续的 sql 语句的生成。这里我们可以看到这个具体的 Generator 是 opengaussQueryGenerator,为了方便,在具体实现这个类的时候,里面也是复用了 BaseJdbcQueryGenerator 类中的内容。
-
-
-
-那接下来我们就看看 JdbcPlanOptimizer 做了哪些具体的优化。首先我们看到它会调用自己的 optimize 方法,来对推下来的 maxSubPlan 进行优化,具体的执行就通过调用 accept 方法来调用 Visitor 这个类来进行具体的节点遍历。
-
-
-
-然后就是通过调用 visitPlan 来进行算子的转换,如下图就是主备对聚合算子 AggregationNode 进行相关操作。
-
-
-
-tryCreatingNewScanNode 会调用 queryGenerator 对象来进行算子的重写(其实就是把能推下去的通过重写 SQL 的方式把该算子加进去)。
-
-
-
-我们看到这里开始准备重写,就进入到了 BaseJdbcQueryGenerator 类中来了(其实是进入到了 openGaussQueryGenerator 类,只是我复用了 BaseJdbc,所以最后实在这里来做的),这里是调用的 visitAggregation 方法,主要就是来进行聚合算子的提取工作,
-
-
-
-进一步,buildSql 方法顾名思义,将提取出来的算子进行推到重写的 SQL 中,如下所示
-
-
-
-最后重写完成,又回到了 JdbcPlanOptimizer 进行下一步的操作。毕竟优化器只是将子计划进行重新优化,所以最后还是要返回一个 PlanNode 的,所以我们看到接下来我们在上面重写的 sql 会被用来进行封装,最后封装成了 TableScanNode 里被返回。
-
-
-
-其实到了这里就已经完成了具体的 opengauss Connector 的下推了,但是我们如果跳出来,看看它在整个执行过程中的位置,回想一下前面我们提到的,这也仅仅是我们完成了 ApplyConnectorOptimization 这一条优化规则的任务,如果你忘了我再重新贴一下图。
-
-
-
-所以接下来,还会把我们刚刚返回的封装好的子查询计划继续应用其他规则。
-
-再往后,引擎其实还会在全局的角度对整个查询进行一个重写,也就是在 BaseJdbcClient 这里所做的。
-
-
-
-这里会通过 QueryBuilder 来重新梳理出一条 sql 语句,最终推给数据源。如下图所示,其实我们前面做的那么多,在全局看来只是一个 table,别名为 pushdown。
-
-
-
-这里再次重写完的 SQL,其实就是最终我们推给数据源执行的 SQL 了。
-
-### 四、优化效果
-
-通过查看执行计划,我们来看(下推)优化与不优化的效果对比。
-
-
-
-
-
-即使不标明,我相信你也应该能看出哪一个是进行优化的效果了。第一张图片,我们可以看到正常情况下会从数据源读取数据,然后进行过滤、聚合、shuffle 再聚合,而第二张图片我们看到 Connector 直接将条件和聚合算子推给了数据源,最后只接收一个聚合结果,从而大大解放了 Connector,减少了数据的传输等效率损耗,从而提高查询性能。
-
-### 五、总结
-
-通过本篇文章,我相信你已经大致对 Connecor 下推以及查询下推的原理有了一个比较形象的了解了,如果你想继续深入了解,可以再次按照这个思路去捋一捋源码,其他情况的话,相信这篇文章已经足够能够解答你的疑惑了。opengauss connector 下推的 PR 我已经提到社区了https://gitee.com/openlookeng/hetu-core/pulls/1354,欢迎交流。
diff --git a/app/zh/information/blog/20220108/index.md b/app/zh/information/blog/20220108/index.md
deleted file mode 100644
index 72a0fbe5c9cda0106533005cf893bf02e73dc5cf..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/20220108/index.md
+++ /dev/null
@@ -1,120 +0,0 @@
----
-title: 'Oracle update和delete支持简介及实现'
-date: '2022-01-08'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng']
-archives: '2022-01'
-author: '王明月'
-description: 'Oracle update和delete是openLookeng在Oracle connector上提供的对Oracle数据源进行更新和删除操作的特性,此特性除实现对Oracle数据源update和delete操作的直接支持,还提供了适配开发接口,基于这些接口可以开发某些数据源的update和delete操作。'
----
-
-Oracle update 和 delete 是 openLooKeng 在 Oracle connector 上提供的对 Oracle 数据源进行更新和删除操作的特性,此特性除实现对 Oracle 数据源 update 和 delete 操作的直接支持,还提供了适配开发接口,基于这些接口可以开发某些数据源的 update 和 delete 操作。
-
-##
-
-二个操作都要经过上面的不同阶段的处理, 语法解析阶段对 SQL 语句进行解析并转化为内部的 AST 树,语法&语义分析阶段对 AST 树中的各种标识进行识别(如表,列,表达式等)转化为逻辑计划树,优化阶段对这个逻辑树进行优化,然后转化为一个物理执行树,执行阶段框架按照这个执行树进行执行,其中有些树节点的算子会与数据源进行交互。
-
-交互的接口大致分 2 类:
-
-1. 涉及元数据的接口: 如在语法&语义阶段,对 rowid 列信息的获取;
-
-2. 涉及业务流程处理接口:
-
- a) 第一类: 主要是预和后处理接口,beginXXX, applyXXX, finishXXX
-
- b) 第二类:数据处理接口:XXXRows, executeXXX 前者是记录级的操作,后者是“表”级操作(直接将业务下推到数据源中执行),这里操作跟具体的物理执行计划节点类型相关,目前有 Delete 节点,TableDelete 节点和 Update 节点,由这些节点来调用相应的接口完成功能。
-
-功能实现上在接口上分 2 层,第一层 SPI 层通用处理接口,全部数据源都遵循的接口,第二层 BaseJDBC 层针对基于 JDBC 协议实现的数据源通用处理,方便 JDBC 数据源来实现业务。
-
-
-
-下文将重点对与数据源的数据处理各个接口结合业务进行讲解。
-
-### 2.1 delete 业务流程
-
-#### 2.1.1 主要接口及实现
-
-- **rowid 列**
-
-在分析阶段,框架通过 ConnectorMetadata#getDeleteRowIdColumnHandle 获取 rowid 列的处理句柄,在 Oracle Connector 中直绑定 Oracle 数据源中每个表的隐藏列 ROWID 列,这个列会在查询数据的时候自动添加到 TableScan 中。
-
-
-
-- **记录级删除**
-
-Delete 算子在执行时从下游的算子获取 rowid 的列数据,然后调用 UpdatablePageSource#deleteRows 接口,通知数据源删除这些列。在 BaseJDBC 层,通过二个 jdbcClient 接口完成,一个生成删除预处理语句,一个执行预处理命令的参数设置(rowid 列值的设置)
-
-
-
-- **下推及表级删除**
-
-一个 delete 业务在优化阶段前都是走记录级删除流程,优化阶段对某些场景可优化成“表”级删除流程(下推 TableScan 优化器和下推 Delete 优化器共同作用的结果),对应的算子由 Delete 转为 TableDelete 算子执行。
-
-下推:
-
-
-
-
-
-
-
-从上表可以看出,表级删除大大优化的处理的流程,与数据源的交互算子缩减到一个。
-
-**表删除处理:**
-
- TableDelete算子在执行删除操作的时候不在关注到底是哪些行,只要针对有下推和无下推场景,来组装delete语句即可。
-
-
-
-#### 2.1.2 涉及接口汇总:
-
-
-
-### 2.2 update 业务流程:
-
-#### 2.2.1 主要接口及实现
-
-- rowid 列:同 delete
-- 记录级更新:
- Update 算子在执行时从下层的算子获取 rowid 的列和参与 Set 子句的列,然后调用 UpdatablePageSource#updateRows 接口,通知数据源对这些列进行更新操作。在 BaseJDBC 层,通过二个 jdbcClient 接口完成,一个生成更新预处理语句,一个执行预处理命令参数设置(rowid 列和待更新列值的设置),注意这里设置的值,都是从上游获取的,由上游计算好的,比如待更新列的值可能是通过表达式算出来的,这个表达式的运算在执行 Update 算子前计算好。
-
-
-
-#### 2.2.2 涉及接口汇总:
-
-
-
-##
-
-- 配置 ODBC,方法如下:
-
-1. 点击添加
-
-
-
-2. 选择安装的数据源驱动为 OneQuery ODBC 1.1 Driver
-
-
-
-3. 填写创建数据源信息
-
-
-
-4. 添加完成后,用户数据源处会出现新创建的数据源
-
-
-
-### 2.2 Tableau 连接 ODBC
-
-首次登录 Tableaau 在选择 ODBC 连接数据源时点击“更多”
-
-
-
-选择“其他数据库 ODBC”
-
-
-
-确认 ODBC 连接方式后,会弹框选择之前配置 ODBC 时的名称,然后点击连接
-
-
-
-### 2.3 PowerBI 连接 ODBC
-
-登录 PowerBI -- 获取数据源
-
-
-
-搜索 ODBC
-
-
-
-确认 ODBC 连接方式后,会弹框选择之前配置 ODBC 时的名称 Hetu_test,然后点击确定
-
-
-
-输入 Hetu server 服务器的用户名密码,点击连接
-
-
-
-##
-
-### 3.2 Tableau 连接 JDBC
-
-和 ODBC 一样,首次使用 JDBC 方式连接数据源点击左边菜单中“更多”,再点击“其他数据库(JDBC)”
-
-
-
-选择 JDBC 连接方式后会有弹框如下:
-
-
-
-信息填写完毕点击“登录”即可。
-
-### 3.3 Yonghong 连接 JDBC
-
-#### 3.3.1 添加数据源连接
-
-启动 Yonghong 进入首页,点击左侧的“添加数据源”按钮,之后选择要连接数据源的类型,这里选择“Presto”,选择完成后会要求填写对接的数据源信息,填写信息如下:
-
-
-
-
-
-信息填写完毕,点击“连接测试”,返回测试成功后,点击菜单栏->保存。之后创建数据集和制作报告模块都可以用该数据源。
-
-
-
-#### 3.3.2 刷新数据源
-
-刷新数据源操作如下:
-
-
-
-##
-
-- 选择架构时支持三种匹配查找,如果要查询所有架构则直接敲回车
-
-
-
-- 选择表和选择架构一样,如果需要查询架构对应的所有表直接回车,然后点击“加载”即可
-
-
-
-#### 4.1.2 JDBC 新建数据源
-
-- 登录 Tableau 后,在左菜单栏选择数据库
-
-
-
-- 选择架构
-
-
-
-- 选择架构后,会自动弹出对应的表
-
-
-
-### 4.2 单表操作
-
-#### 4.2.1 预览表数据
-
-- 找到需要预览的表点击后方的“查看数据”按钮即可预览表中数据
-
-
-
-#### 4.2.2 操作表数据
-
-- 对表数据进行操作时,需要将表拖至界面中间空白位置
-
-
-
-- 将表拖至空白区域后,点击“立即更新”或者“自动更新”即可看到表数据
-
-
-
-- 表数据加载后可对数据进行一系列操作如下
-
-
-
-
-
-### 4.3 多表操作
-
-#### 4.3.1 同一架构下不同表的联接操作
-
-将同一架构下的两张表拖到中间空白区域
-
-
-
-#### 4.3.2 不同架构下不同表的联接操作
-
-选择一个架构将一张表拖至中间空白区域,然后选择另一个架构将一张与前表有关联的表也拖至中间的空白区域,之后操作两表联接与 4.3.1 中的方式一致。
-
-#### 4.3.3 不同数据库的不同架构下不同表的联接操作
-
-- 该操作需要新建一个数据源连接具体操作如下:
-
-
-
-- 新建数据连接后选择一张有关联的表拖至中间空白区域,之后联接操作与前面 4.3.1 的操作一致
-
-
-
-### 4.4 自定义 SQL 语句查询
-
-自定义 sql 语句查询方法如下:
-
-
-
-### 4.5 工作表操作
-
-#### 4.5.1 数据可视化操作
-
-- 先加载一张表,然后点击“工作表”,转到工作表界面
-
-
-
-- 进入工作表后操作如下
-
-
-
-#### 4.5.2 保存制作好的图表
-
-保存图表方法如下
-
-
-
-##
-
-### 5.2 操作表
-
-选择 catalog—>schema—>table,勾选预览
-
-
-
-可以对表进行加载或编辑。加载后可以对表进行操作:
-
-
-
-加载后的表可以以 1 报表 2 数据 3 模型的方式展现。点击编辑,可以对表进行各项操作(复制、删除、更改类型、拆分、重命名、筛选等)
-
-
-
-### 5.3 自定义 SQL 语句查询
-
-获取数据 -- 搜索 ODBC -- 点击高级选项 -- 输入 sql 语句 -- 点击确定
-
-
-
-##
-
-#### 6.1.2 SQL 数据集
-
-在新建数据集中选择 SQL 数据集,进入 SQL 数据集界面,如下:
-
-
-
-
-
-#### 6.1.3 自定义 SQL 查询
-
-自定义 SQL 查询方法如下:
-
-
-
-#### 6.1.4 自服务数据集
-
-点击“创建数据集”,选择“自服务数据集”,进入自服务数据集操作界面
-
-
-
-
-
-
-
-连线完成后点击“刷新元数据”就可以看到规则后的数据了。
-
-#### 6.1.5 组合数据集
-
-点击“创建数据集”,选择“组合数据集”,进入组合数据集操作界面
-
-
-
-选择需要关联的不同表,拖至空白区域
-
-
-
-### 6.2 数据治理
-
-创建数据集之后,刷新元数据,可对元数据进行一系列数据清洗、治理操作。
-
-- 转换为数字列
-
-- 转换为日期列
-
-- 转换为维度列
-
-- 转换为度量列
-
-- 设置数据量
-
-- 查看细节数据
-
-- 设置文件夹
-
-- 设置数据层次
-
-- 设置表达式
-
-- 拆分数据
-
-- 缺失数据填充
-
-- 去除数据空格
-
-- 值映射
-
-- 查看数据特征值
-
-具体操作入口如下:
-
-
-
-### 6.3 制作报告
-
-#### 6.3.1 新建报告
-
-- 从首页引导区或左侧导航栏,点击“制作报告”,进入制作报告模块
-
-
-
-- 在主题页面选择不同的主题来创建仪表盘
-
-
-
-- 选择主题后,打开仪表盘编辑页面,选择所需的数据集,如下所示
-
-
-
-#### 6.3.2 保存报告
-
-报告制作完成后点击保存按钮即可保存。
-
-
-
-### 6.4 查看报告
-
-#### 6.4.1 查看已保存报告
-
-从首页引导区域或左侧引导栏,点击“查看报告”进入该模块,查看已保存的报告。
-
-
-
-
-
-#### 6.4.2 导出报告
-
-在查看报告时,可以将报告以 PDF,Excel,Word 等方式进行导出并保存在本地,以便日后查看。具体操作如下:
-
-
-
-### 6.5 调度任务
-
-#### 6.5.1 添加调度任务
-
-- 在左侧导航栏点击“调度任务”按钮,进入调度界面
-
-
-
-- 点击“新建作业”,打开新建作业界面
-
-
-
-- 设置作业基本信息,包括作业名称、存储路径和作业描述
-
-
-
-这里以同步数据集为例:
-
-
-
-
-
-#### 6.5.2 运行调度任务
-
-
-
-### 6.6 深度分析
-
-- 点击左侧导航栏上的深度分析按钮,进入该模块。
-
-
-
-- 这里提供 5 中分析模型,也可以自定义新建一个模型,点击“新建实验模型”。
-
-- 进入编辑页面,用户从左侧操作列表中拖拽相应节点到编辑区,通过连线形式连接各点,在右侧页面探索数据特征,构建分析模型,查看预测结果。
-
-
-
-下面以“采样为例”:
-
-首先拖拽一个数据集和一个采样节点到编辑区,连接数据集和采样节点。
-
-
-
-然后设置采样规则:
-
-
-
-最后点击“元数据”或者“探索数据”查看采样结果。
-
----
-
-如果您有任何疑问或建议,欢迎在社区代码仓内提 Issue;也欢迎加小助手微信(openLooKengoss),进入专属技术交流群。
-
-社区代码仓
-
-
-
-选择性能差距最大的`query72`在当前环境和之前环境跑,发现两个问题:
-
-1. 两个环境数据传输差异比较大
-
-2. 两个环境的执行计划差别比较大
-
-
-
-#### **2 问题定位**
-
-执行对`hive`表的`analyze`命令,可以收集给定表的表和列的统计信息,包括`numFiles`、`numRows`、`rawDataSize`、`totalSize`等。在执行复杂和包括 join 操作的大型数据集查询时,`analyze`可以通过统计信息来提高查询性能。由于 11 节点执行 stage 个数大大多于之前环境 stage 的数量,因此怀疑执行计划没有优化。也就是说执行后没有统计信息。
-
-针对这个疑问,在`hive client`端找一个表,以表`customer_demographics`为例,在 openLooKeng 客户端执行`show stats for customer_demographics`;在查看对应的统计信息发现,统计信息全部为`null`。如下图:
-
-
-
-
-
-基于此有两个怀疑,一是统计信息没有生成,另一个就是生成了,但 openLooKeng 没有读到。第二个原因基本可以排除,因为有一些表,比如`store`是可以读到统计信息的。所以怀疑是统计信息没有生成。
-
-在执行`analyze`命令,openLooKeng 会将表级的统计信息存储到`TABLE_PARAMS`表中,列级信息存储到`TBL_COL_STATS`表中,登录`dbservice客户端`,查询`customer_demographics`表对应的`TABLE_PARAMS`表中生成的数据,发现生成了`numFiles`,`totalSize`等字段但是并没有生成`numRows`字段,因此统计字段并非完全没有生成。
-
-
-
-因此尝试跟踪代码确认 openLooKeng 读取数据并生成统计信息的流程。
-定位代码`MetastoreHiveStatisticsProvider`类方法`getTableStatistics`中发现,`calculateAverageRowsPerPartition`方法计算得到的`optionalAverageRowsPerPartition`的值,其中会判断`rowCount`的值,当`rowCount`不存在时会直接返回空的统计信息。经上述查询可知,数据库中没有生成`rowCount`字段因此在查询`customer_demographics`表统计信息时均返回 null。
-
-
-
-#### **3 解决方式**
-
-由上分析,尝试在**openLooKeng 客户端执行 analyze 操作**。
-
-
-
-查询`dbserver`中`TABLE_PARAMS`表中字段,此时生成了`numRows`字段。
-
-
-
-openLooKeng 客户端查询统计信息也可以查到。
-
-
-
-此时重新执行上述 sql,stage 数量由 21 个降到 13 个,查询耗时与之前环境的测试数据差别不大,且传输速率也提高了。
-
-
-
-#### **4 总结**
-
-为提高查询效率,在执行查询之前还是要执行`analyze`,生成统计信息,进而提升 openLooKeng 的执行性能。
-
----
-
-如果您有任何疑问或建议,欢迎在社区代码仓内提 Issue;也欢迎加小助手微信(openLooKengoss),进入专属技术交流群。
-
-社区代码仓
-
-
-
-
-
-
-
-
-
-翻译如下:
-
-```java
-
-如果 !current.isFinished() && next.isBlocked()空值 && next.needsInput() :
- // 上游算子没有结束,下游算子没有阻塞且需要数据输入
- current.getOutput()
- // 上游算子输出
- next.addInput()
- // 下游算子输入
-如果current.isFinished() :
- // 上游算子已结束(不再有输出),通知下游算子finish(不再有输入)
- next.finish()
- // 通知下游算子结束
-
-```
-
-
-
-
-
-对于只读数据表,分片划分时直接按用户的设置将表分割为 N 个分片。
-
-对于非只读数据表,分割规则在只读数据表的基础上增加(-∞,minValue)和(maxValue,+ ∞)这两段,分割为 N+2 个分片。
-
-##
-
-核心思想:对所有分片按它们的取值范围进行排序,合并记录数小于平均值的连续分片,按比例拆分记录数大于平均值的分片,在调整前、后总的分片个数保持不变。
-
-##
-
-##
-
-##
-
-- 详细配置说明:
-
-
-
-多分片特性的效果表现
-
-
-
-
-
-多分片特性的效果表现-部署图
-
-
-
-##
-
-##
-
-ps:`port=25308`为 CN 的端口号
-
-#### **1.1.1 本地连接数据库的方法:**
-
-1. 使用用户`omm`登录集群主机
-
-2. 执行`source ${BIGDATA_HOME}/mppdb/.mppdbgs_profile`
-
-3. 登录命令行客户端:`gsql -d postgres -p 25308`
-
-#### **1.1.2 远程连接数据库的方法:**
-
-默认情况下 gaussDB 200 只能在集群主机上通过自带的客户端登录数据库,通过配置 IP 规则后方能从远程登录数据库。
-
-1. 从浏览器登录(`URL`一般为
-
-
-
-
-
-2. 允许用户`testuser`从`xx.xx.xx.xx`(openLooKeng 服务器 server ip)访问 gaussDB
- 以 omm 登录集群主机执行以下 shell 命令:
-
-```java
-
-gs_guc set -Z coordinator -N all -I all -h "host all testuser xx.xx.xx.xx/16 sha256" ---假设ip掩码取16位
-可通过cat /srv/BigData/mppdb/data1/coordinator/pg_hba.conf | grep testuser 检查是否添加成功
-
-```
-
-
-
-对于 openLooKeng 服务器为集群且 IP 网段不同的场景,由于每个节点都有可能需要连接 gaussDB server,那么需要执行者对每个网段的 server 都按上述方法添加到列表。
-
-#### **1.1.3 重启 gaussDB**
-
-如需要重启 gaussDB,请按以下步骤执行:
-
-以`omm账户`登录集群主机执行以下命令:
-
-```java
-
-source ${BIGDATA_HOME}/mppdb/.mppdbgs_profile
-cm_ctl stop && cm_ctl start
-
-```
-
-### 1.2 gaussDB 作为数据源接入 openLooKeng:
-
-在 openLooKeng 服务器的`vi etc/catalogs/gaussdb.properties` 内容如下:
-
-```java
-
-connector.name=opengauss
-connection-url=jdbc:postgresql://xx.xx.xx.xx:portId/postgres?useSSL=false
-connection-user=testuser
-connection-password=******
-use-connection-pool=true
-
-```
-
-ps:设置`use-connection-pool`非常重要,由于我们所使用的环境从`openLooKeng server`建立到 gaussDB 的`jdbc connection`耗时相对很长,不使用连接池会导致`query`的相当大比例的时间消耗在频繁建立`connection`这环节上,进而导致多分片的效果打折扣。
-
-##
-
-5. 对生成数据做格式加工处理
-
-在`/srv/tpcds-data0`创建`handled`目录,并把应用下面`shell`语句放入脚本执行数据格式加工处理。
-
-
-
-6. 拷贝 handled 目录内已处理过的数据到安装了 JMETER 的主机上
-
-### 2.2 测试数据导入 GaussDB 数据库
-
-推荐使用通过·Data Studio·工具(请从以下网址下载:
-
-- “主机”一栏请填写`GaussB server`的真实 ip
-
-- “数据库”一栏请根据实际数据库名填写
-
-2. 创建`tpc-ds schema`及`tpc-ds表`:导入数据前先创表(`create schema tpcds`; 然后拷贝 tpcds 工具的`/tools/tpcds.sql`里的语句完成创表工作)
-
-3. 将`tpcds-kit`生成的测试数据文件导入 gaussDB,如下图:
-
-
-
-
-
-### 2.3 JMeter 测试指引
-
-#### **2.3.1 JMeter 启动内存设置**
-
-修改`JMeter`启动内存避免测试过程发生`OOM`:
-
-`apache-jmeter-5.2.1\bin\`
-
-
-
-#### **2.3.2 创建 JMeter 测试工程**
-
-测试工程参考模板:jmeter-test.jmx
-
-将模板放入`jmeter`的`projects`目录下,从`JMeterUI`上选择打开该文件
-
-### **2.3.3 配置 openLooKeng 连接信息**
-
-指定 openLooKeng 的`URL`(需包含`catalogName、schemaName`),如:
-
-`jdbc:lk://xx.xx.xx.xx:8096/gaussdb/testuser`
-
-`jdbc driver class`填:`io.hetu.core.jdbc.OpenLooKengDriver`
-
-前置条件是已经把`hetu-jdbc-XXX-SNAPSHOT.jar`包放入对应的目录(见下图)
-
-
-
-
-
-设置用例文件,如下图:
-
-
-
-模板参考(在每行用例的第一列加”//”,测试时可跳过该用例):
-
-text-tpdcs.csv
-
-设置每个用例的循环次数:
-
-
-
-
-
-经过以上操作,现在可以开始启动测试了。
-
-测试进行的同时可以查收已完成的语句的执行结果,如下图所示(根据需要可通过`“Save Table Data”`可以保存测试结果,方便后期分析):
-
-
-
----
-
-如果您有任何疑问或建议,欢迎在社区代码仓内提 Issue;也欢迎加小助手微信(openLooKengoss),进入专属技术交流群。
-
-社区代码仓
-
-
-
-这里面主要涉及到两种类型的节点,分别为 Coordinator 和 Worker 节点。
-
-- Coordinator 节点是用来解析语句,执行计划分析和管理 openLooKeng 的 Worker 节点,同时 Coordinator 跟踪每个 Work 的活动情况并协调查询语句的执行。Coordinator 为每个查询建立模型,模型包含多个 Stage,每个 Stage 再转为 Task 分发到不同的 Worker 上执行。
-
-- Worker 是负责执行任务和处理数据。Worker 从 Connector 获取数据。Worker 之间会交换中间数据。Coordinator 是负责从 Worker 获取结果并返回最终结果给 Client。
-
-具体步骤为:
-
-1. 客户端通过协议发送一条查询语句给 openLooKeng 集群的 Coordinator;
-
-2. Coordinator 接手到客户端传送到的查询语句,对语句进行解析、生成查询执行计划,根据查询执行计划一次生成 sqlQueryExecution->sqlStageExecution->HttpRemoteTask;
-
-3. Coordinator 将每个 task 分发到所需要处理的数据所在的 Worker 节点上执行;
-
-4. Task 通过 Connector 从数据源中读取数据,并执行 Source Stage 的 task;
-
-5. 处于下游的 Stage 中的 task 会读取上游 Stage 产生的输出结果,并在该 Stage 的每个 task 所处的 Worker 的内存中进行后续处理;
-
-6. Coordinator 从发布的 task 后,持续地从 Singe Stage 中的 task 获取结果,并将结果缓存到 buffer 中,知道查询结束;
-
-7. Client 从提交查询之后,会持续地从 Coordinator 获取本次查询的计算结果,知道获得所有的计算结果。
-
-### 1.3 openLooKeng 内核调用
-
-openLooKeng 的多源查询能力是通过 Connector 机制来实现的。其中 MySQL、postgreSQL 等 Connector 是主要是通过 presto-base-jdbc 中的代码来实现对 SQL 等数据源的读写。
-
-
-
-其中**openLooKeng-spi**中主要定义了一些公共接口,供 openLooKeng-main 中的代码进行调用。
-
-**openLooKeng-base-jdbc**是数据库连接器的公共模块,对 openLooKeng-main 进行了实现和补充。其代码经过编译后,会对应加载到 mysql-plugin 等插件中,实现 mysql 对数据源的访问功能。base-jdbc 本身自己不会编译单独的插件。
-
-通过断点调试,对 openLooKeng 接受一条 update SQL 语句的函数栈调用流程进行梳理总结,如下图所示。
-
-
-
-结合源码分析,发现一条 update 语句在 connector jdbc 中的执行流程为**beginUpdate()->getConnection()->buildUpdateSql()->setUpdateSql()->finishUpdate()**。在这个过程中需要一个字段作为行标识符来表示数据行。
-
-其中**buildUpdateSql**将 openLooKeng 传入的 update SQL 语句转换为 connector 更新时的预编译语句,这里用到获取的行标识符作为 where 的筛选条件;**setUpdateSql**作用是以 where 条件对数据行进行更新。
-
-因此若想要实现 postgreSQL conncetor 的 update/delete table 功能,我们需要重写上述函数即可完成。
-
-## 2 代码实现
-
-### 2.1 方案设计及论证
-
-通过前期的调研,共形成了以主键为标识符和以 ctid 字段为标识符的两种开发方案。
-
-**方案一:以主键为标识符**
-
-主键的定义为:表中经常有一个列或列的组合,其值能唯一地标识表中的每一行。由定义可知,主键对应到一张数据表中会设计到多种场景,分别为:没有主键、单字段主键以及多字段组合成的复合主键,因此需要考虑的情况较多。
-
-**方案二:以 ctid 字段为标识符**
-
-ctid 表示数据行在它所处的表内的物理位置,ctid 字段的类型是 tid。尽管 ctid 可以快速定位数据行,每次 vacuum full 之后,数据行在块内的物理位置就会移动,即 ctid 会发生变化,所以 ctid 不能作为长期的行标识符。因此如果要以 ctid 字段为标识符进行代码开发,需首先验证 ctid 在同一事务中进行 update/delete 操作是否会发生变化。
-
-### 2.2 ctid 在同一事务中的变化情况
-
-为了验证 ctid 字段能否用于 postgreSQL 的 update/delete 操作,我们分别在单线程和多线程的情况下去验证 ctid 在同一事务和不同事务的变化情况。
-
-**单线程**
-
-通过 jdbc 模拟了单线程,在线程中对数据表进行 update 操作,分别查询 update 前后的信息,观察 ctid 的变化情况。
-
-
-
-autocommit=false 时,结果显示 ctid 未发生变化:
-
-
-
-autocommit=true 时,结果显示 ctid 发生变化:
-
-
-
-**多线程**
-
-同理,通过 jdbc 模拟了两个线程,在两个线程中分别对数据表进行 update 操作,分别查询 update 前后的信息,观察 ctid 的变化情况。
-
-autocommit=false 时,结果显示 ctid 未发生变化:
-
-
-
-autocommit=true 时,结果显示 ctid 发生变化:
-
-
-
-由验证实验结果来看,在同一事务中,update 未提交前或者回滚之后 ctid 是不变的。
-
-### 2.3 开发方案选择
-
-综合对比两种方案,其优缺点对比如下:
-
-| 方案 | 优点 | 缺点 |
-| :-------: | :----------------------------------------------- | :---------------------------------------------------------------------------------------------------------------- |
-| 主键 | 1、每一数据行的标识符能够始终保持不变。 | 1、场景复杂,需考虑数据表无主键、单字段主键及多字段复合主键等多种情况;
2、不太适合现有 openLooKeng 下推框架。 |
-| ctid 字段 | 1、ctid 性质类似与 rowid,代码开发可借鉴性较高。 | 1、需考虑在同一事务中对数据行进行 update/delete。 |
-
-综上所述,本文拟采用以 ctid 字段为行标识符进行代码开发。
-
-### 2.4 ctid 字段为行标识符的代码开发
-
-autocommit 设置为 false,即保证能在同一事务中进行操作,ctid 是可以保证唯一性的。而 PostgreSql 的继承关系如下图,同时 OpenGaussClient 也继承于 BasePostgreSqlClient
-
-
-
-因此我们的主要开发重点在于重写**beginUpdate()-> getConnection()-> buildUpdateSql-> setUpdateSql ->finishUpdate()**,其代码实现主要放在 BasePostgreSqlClient
-
-**getUpdateRowIdColumnHandle**用于获取行标识符
-
-
-
-**beginUpdate()**
-
-
-
-getConnection 重写 BaseJdbcClient 该方法,用于将只读模式设为 false,以便于进行更新操作
-
-
-
-**buildUpdateSql**将 openLooKeng 传入的 update SQL 语句转换为 oracle 更新时的预编译语句,其内部调用关系为
-
- buildUpdateSql()
- buildRemoteSchemaTableName()
- getColumnNameFromDataSource()
- getColumnNameMap()
-
-其中 buildRemoteSchemaTableName()获取数据表名,getColumnNameFromDataSource()用于从源数据中获取要更新的字段名,getColumnNameMap()用于构建字段名的映射关系,将原始字段统一转换为小写字母。具体实现如下图所示
-
-
-
-**setUpdateSql**是以 where 条件对数据行进行更新,其内部调用关系为
-
- setUpdateSql()
- setStatement()
-
-其中 setStatement()用于对需要更新的字段根据数据类型进行赋值。具体实现如下图所示:
-
-
-
-到此,我们完成了 postgreSQL connector 的 update 功能代码,而 delete 功能代码与此具有很大的相似性,此处不做展开讲述,具体可参考https://gitee.com/openlookeng/hetu-core/pulls/1380 。
-
-## 3 功能展示
-
-此处我们简单演示 postgreSQL/openGauss connector 的 update/delete 特性。首先需对代码进行编译,运行\*\*mvn clean install -DskipTests -T 1C 即可,编译完成之后,拷贝 hetu-server-1.5.0-SNAPSHOT 文件夹及 hetu-cli-1.50-SNAPSHOT-executable.jar 包。紧接着相关配置可参考社区文档。
-
-在 web 界面输入 ip 及端口号,显示
-
-
-
-数据表已经建好,具体实例为
-
-
-
-针对 update,执行语句及结果包括
-
-| 功能点 | SQL | 结果 |
-| :------------: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------ |
-| boolean 表达式 | update infomation set name = ‘Kitty’, age = 26, birthday = date’1995-08-16’ where id = 1;
update infomation set name = ‘Jane’, sex = ‘2’ where score = ‘good’;
update infomation set score = ‘qualified’ where salary = 10000.00;
update infomationa set age = age + 1, birthday = date’1998-08-16’ where age = 22;
update infomation setT salary = 36000 where birthday = date’1996-08-14’;
update infomation set score = ‘bad’, birthday = date’1995-10-16’ where name = ‘Jane’; | success |
-| 子查询 | update infomation set age=age+1 where salary=(select avg(salary) from data);
update infomation set salary=salary+1000 where age in (select age from data where birthday between date’1993-09-01’ and date’1997-08-31’); | success |
-
-针对 delete,执行语句及结果包括
-
-| 功能点 | SQL | 结果 |
-| :------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------ |
-| boolean 表达式 | delete from infomation where name = ‘Bob’;
delete from infomation where birthday = date’1996-08-14’;
delete from infomation where salary = 15000; | success |
-| 子查询 | delete from infomation where salary<(select avg(salary) from data);
delete from infomation set salary=salary+1000 where age in (select age from data where birthday between date’1993-09-01’ and date’1997-08-31’); | success |
diff --git a/app/zh/information/blog/20220408-ranger/index.md b/app/zh/information/blog/20220408-ranger/index.md
deleted file mode 100644
index 840c22691731ffae6ce2918a5a923e16c4247bd6..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/20220408-ranger/index.md
+++ /dev/null
@@ -1,410 +0,0 @@
----
-title: 'openLooKeng+Ranger:使用指南'
-date: '2022-04-08'
-module: 'blog'
-customLayout: 'true'
-tags: ['Ranger']
-archives: '2022-04'
-author: '马晓琦'
-description: '希望本篇安装指南能够帮助到有需要的同学们。'
----
-
-##
-
-新建测试数据库:`create database db_test`;插入数据:
-
-```java
-use db_test;
-CREATE TABLE table_test (id int(10),name char(20),column_test varchar(20), date_test DATETIME);
-INSERT INTO table_test (id,name,column_test,date_test) VALUES(01,'Tom','11AAGada110110','2011-11-11 11:11:33');
-INSERT INTO table_test (id,name,column_test,date_test) VALUES(02,'Jack','119asf19119','2012-11-12 11:22:33');
-INSERT INTO table_test (id,name,column_test,date_test) VALUES(03,'Rose','112Gfaf112112','2013-11-13 11:33:33');
-INSERT INTO table_test (id,name,column_test,date_test) VALUES(04,'Adam','1111kkk1HG11','2014-11-14 11:44:33');
-INSERT INTO table_test (id,name,column_test,date_test) VALUES(05,'Haya','222afaf222222','2015-11-15 11:55:33');
-
-```
-
-
-
-查看新建的表数据:`select * from table_test`;
-
-
-
-#### 2)新增 HBase 数据源
-
-HBase 需要使用`hetuMetastore`,并手动建立和 HBase 表的关联映射,示例:在`etc目录`新增`mysql的元数据`配置`hetu-metastore.properties`
-
-
-
-HBae 连接器配置文件:
-
-
-
-
-
-
-
-
-
-成功建立表关联,正常查询数据:
-
-
-
-### 4. 登录 openLooKeng 管理页面
-
-(默认使用 lk 用户):
-
-##
-
-**【注意】:使用代理时,编译时无法下载`node`和`npm`,可以手动下载放到本地仓库对应位置。**
-
-```java
-mkdir -p /root/.m2/repository/com/github/eirslett/node/8.12.0/
-cd /root/.m2/repository/com/github/eirslett/node/8.12.0/
-wget https://nodejs.org/dist/v8.12.0/node-v8.12.0-linux-x64.tar.gz --no-check-certificate
-mv node-v8.12.0-linux-x64.tar.gz node-8.12.0-linux-x64.tar.gz
-
-mkdir -p /root/.m2/repository/com/github/eirslett/npm/6.4.1/
-cd /root/.m2/repository/com/github/eirslett/npm/6.4.1/
-wget https://registry.npmjs.org/npm/-/npm-6.4.1.tgz --no-check-certificate
-mv npm-6.4.1.tgz npm-6.4.1.tar.gz
-
-```
-
-### 2. 安装 MySQL
-
-安装可参考网上教程:
-
-### 3. 安装 Ranger Admin
-
-安装可参考官方教程:
-
-
-
-如果出现上述报错,需要在`mysql客户端`中进行如下设置:
-
-```java
-SET GLOBAL innodb_file_format=Barracuda;
-SET GLOBAL innodb_file_per_table=1;
-SET GLOBAL innodb_large_prefix=1;
-
-```
-
-启动`Ranger Admin`服务: `service ranger-admin start`
-
-
-
-进入`ews目录`,使用命令:`sh ranger-admin-services.sh start`
-
-
-
-### 4. 检查是否成功
-
-访问`Ranger`控制台:
-
-##
-
-`Username`填写:lk
-
-openLooKeng 控制台(http://`openLooKeng-IP`:8080)默认使用`lk用户`进行数据查询和展示,使用`lk用户`创建服务,ranger 默认会给创建服务用户全部权限。
-
-Passworld 不写,这边只验证 openLooKeng 使用 Ranger 实现权限控制的功能,暂不考虑用户登录认证等。
-
-jdbc.url 填写:jdbc:lk://`openLooKeng-IP`:8080/catalog
-
-
-
-### 4. 在 openLooKeng 中安装 Ranger openLooKeng 的`客户端`插件
-
-```java
-# 代码openlookeng-ranger-plugin根目录
-# 进入Ranger代码编译生成的target目录下
-cd target
-tar -zxf ranger-2.1.0-openlookeng-plugin.tar.gz
-cd ranger-2.1.0-openlookeng-plugin/
-vi install.properties
-
-```
-
-`install.properties`修改如下部分
-
-```java
-# Location of Policy Manager URL
-# Example: POLICY_MGR_URL=http://policymanager.xasecure.net:6080
-POLICY_MGR_URL=http://ranger-IP:6080
-
-# This is the repository name created within policy manager
-# Example: REPOSITORY_NAME=openlookengdev
-# Ranger 控制台创建的Service名
-REPOSITORY_NAME=openlookengdev
-
-#Presto component installed directory
-#COMPONENT_INSTALL_DIR_NAME=../openlookeng
-COMPONENT_INSTALL_DIR_NAME=/root/hetu-server-1.0.0-SNAPSHOT
-
-#为了简单,此处不开启审计功能
-XAAUDIT.SOLR.ENABLE=false
-
-#虽然文档中没有提及,不设置的话,enable-presto-plugin.sh脚本执行出错
-XAAUDIT.SUMMARY.ENABLE=false
-
-```
-
-以`root`用户执行脚本: `./enable-presto-plugin.sh`
-
-
-
-检查 openLooKeng 配置文件目录是否生成`access-control.properties`
-
-
-
-检查 openLooKeng 的`plugin目录`下是否生成 ranger 目录,其中 jar 包连接到`ranger-2.1`.`0-openlookeng-plugin/lib`
-
-
-
-【重启 openLooKeng】
-
-进入 openLookeng 安装目录下的`bin子目录`,新建`lk用户`并授权,使用`lk用户`执行:`./launcher restart`
-
-### 5. 添加权限控制,验证是否成功
-
-访问 Ranger 控制台:http://`ranger-ip`:6080,点击 openlookengdev 服务,进行权限控制:
-
-
-
-添加资源权限控制:
-
-
-
-`show schemas/tables/columns`等显示元数据信息操作,对应的`catalog/schema/table/column`需要授权`select权限`,还必须给对应`Catalog/Schame`(information_schema)/Table(schemata,tables,columns)/Column(\*)授予 select 权限)
-
-
-
-示例如下:
-
-【配置前】userA 没有配置权限
-
-
-
-#### 1)在 Ranger 上授权`userA`访问 Catalog(hbase)的 use 权限
-
-**注意:管理表权限必须先设置 catalog 的 use 权限**
-
-
-
-#### 2)在 Ranger 上授权`userA`访问`Catalog(hbase)/Schame(default)/Table(user)/Column(id,info_age)`数据的权限
-
-【配置权限】给 id、info_ge 两个列访问权限
-
-
-
-【配置后】
-
-
-
-完成以上配置,我们就可以在 openLooKeng `Web UI`上使用`Ranger`。希望本篇安装指南能够帮助到有需要的同学们。
-
-**参考资料**
-
-视频回顾:[如何使用 Ranger 增强权限管理](https://www.bilibili.com/video/BV1kb4y1D7GK)
-
----
-
-如果您有任何疑问或建议,欢迎在社区代码仓内提 Issue;也欢迎加小助手微信(openLooKengoss),进入专属技术交流群。
-
-社区代码仓
-
-
-
-然而,SQL 中用 WITH … AS …所声明的 SQL 表达式并不是 C/C++/java/python 中所声明的变量。后者在每次被使用的时候可以直接调用,但前者即便被声明了,在之后被使用的时候仍然需要重新用具体的语句去选取并计算相应的数据。这就消耗了极大的机器资源。至此,我们便很容易思考一个问题,能否将被声明的 SQL 表达式存储在一个“表格”中以避免反复查询和生成,从而极大地节约运算资源,提高运算效率?
-
-### 1.2 原理
-
-CTE 机制就是为了解决上述问题,应运而生的。当第一次声明某个 SQL 表达式作为临时表时,我们将该 SQL 表达式从硬盘读取、计算并储存至缓存,此时这张表被称为“生产者”(producer);而当这张表被反复从缓存中读取以便使用时,我们称其为“消费者”(consumer)。由于消费者是被储存在缓存中的,我们不需要反复生成“生产者”。而从缓存读取数据又比从硬盘读取要快,且不需要二次计算,这样同一段 SQL 代码,在有了 CTE 机制后,执行速度会大大提高。比如,下图中,若不开起 CTE 机制,同一个 SQL 表达式在声明后仍需要被反复生成。但开启了 CTE 机制,从计算流程图上,我们都能很轻易地看出流程被大大简化。
-
-
-
-用户执行一段启动了 CTE 机制的 SQL 代码的流程图如下图所示。首先解析是否存在相似的子查询,若存在,则将其查询结果储存在外部的缓存中以便之后使用。
-
-
-
-##
-
-这两张图分别是 CTE 机制开启前后,在同样的节点上运行同样的程序(Q95.sql)的计算图。可见,就像原理部分中所叙述的那样,计算图本身已经被简化。左图中标红的两段 SQL 语句,在右图中,已经被 CTE Reuse 机制“融合”成了一个部分,精简了计算流程。
-最后,这张表是 5 段不同的代码在 CTE 机制开启前后的运行效率对比。
-不开 CTE 运行时间(s) 开启 CTE 运行时间(s)
-
-
-
----
-
-如果您有任何疑问或建议,欢迎在社区代码仓内提 Issue;也欢迎加小助手微信(openLooKengoss),进入专属技术交流群。
-
-社区代码仓
-
-
-
-除此以外,主要是 Hash join 和 Grace Hash join。Sort-Merge join 由于和动态过滤并不相关,本文在此不予以讨论。Hash join 分为两个阶段,构建(build)和探测(probe)。我们将 join 两端中较小的那个集合 R 作为构建集(build set),另一个 S 作为探测集(probe set)。在构建阶段,我们将 R 的所有数据按照主键进行哈希散列,构成一张哈希表,而哈希表的值便是 R 原来的行数据。而在探测阶段,我们扫描探测集 S,取得 S 的主键的哈希值并判断其是否在哈希表之中,输出结果。此二阶段的过程的示意图如下所示。易知,其时间复杂度为 O(|R|+|S|)。
-
-
-
-而 Grace Hash join 的原理,只是改进版,并不影响我们对于动态过滤机制的理解,因而在此处略去不表。
-
-### 1.2 原理
-
-上文中已经介绍了 join 本身的原理,接下来我们介绍动态过滤。
-在筛选机制比较苛刻的场景中,绝大多数探测端的行是一经筛选,不匹配则直接丢弃的。但如果我们将这个谓词从计划阶段下沉到执行阶段,在构建端已经有严格筛选条件的情况下,直接减少构建集的计算,从而不去读取探测端那些不匹配的行而不是先读取再丢弃,就节约了大量的筛选时间,极大地提高了运行效率。而这个过程,就叫做动态过滤。
-
-
-
-如上图所示,item 作为一个已经被严格筛选过的构建集,我们用过滤器 F 将筛选机制直接传递给探测集。如此,探测端的工作效率便会提升。而动态过滤机制的主要难点就在于如何把构建端的值从 inner-join 操作符传递到探测端,因为操作符很可能是在不同的机器上运行的。
-而在实施的时候,我们主要依靠“基于代价的优化器”(CBO, cost-based optimizer),这个优化器让我们可以使用“广播合并”(broadcast join)。在我们的案例中,构建端远小于探测端,探测端的扫描和 inner-join 操作符运行在同一进程中,这样信息的通信机制会容易很多。
-在确保了广播合并被使用且构建端的信息也能被传送到探测端的时候,我们加入“收集操作符”(collection operator),就放在哈希构建操作符之前。
-
-
-
-如图,收集操作符收集了构建端的值,而当构建端的值输入完毕后,我们将动态过滤机制放到探测端一边。由于探测端的哈希映射和收集操作符的哈希映射是并行执行的,这里并不需要花费多余的时间。然后哈希匹配的时候,探测端就可以直接去被筛选过后的构建端的哈希表进行查找。
-
-##
-
-
-
-这两张图分别是动态过滤机制开启前后,在同样的节点上运行同样的程序的计算图。可见,就像原理部分中所叙述的那样,计算图中出现了动态过滤机制,右图中标红的 SQL 语句就是体现。
-
-最后,这张表是 6 段不同的代码在动态过滤机制开启前后的运行效率对比。
-
-
-
----
-
-如果您有任何疑问或建议,欢迎在社区代码仓内提 Issue;也欢迎加小助手微信(openLooKengoss),进入专属技术交流群。
-
-社区代码仓
-
-
-
-## 2 问题
-
-那么在 pipeline 的模式下,TableScanOperator 的处理速度加快后,是否其他算子的处理速度也会加快呢?
-
-为了验证这个猜想,我们构造了以下两种场景:
-
-1. 正常的`table scan`,读取 ORC 文件;
-
-2. 使用`ORC Cache`,将读取后生成的 block 缓存在内存中,下次读取无需读取原始文件。
-
-以下为测试数据:
-
-
-
-## 3 结论
-
-结论:从以上数据可知,通过 ORC Cache 后,Scan 的 CPU 时间减少了约一半,E2E 的时间也因为 Scan 算子的减少而相应减少,但是其他算子的处理速度却不会因此而提升。
-
-理论分析:A 算子从上游算子获取的数据是一定的,无论上游算子是在 1s 内还是 10s 内给算子 A,那么算子 A 执行`addinput`和`getoutput`进行数据处理的时间就是一定的,因为假设 10s 某些时间上游算子没有喂给 A 数据,那么 A 是不参与运算的,即既不记录 CPU 时间也不计`wall time`。
-
-以下为`scan`和`agg`处理速度的截图,从图中直观看到`scan`处理速度加快,而下游的`hashagg`几乎没有变化:
-
-
-
-## 几个发散的思考
-
-### ORC cache
-
-开启`hetu.split-cache-map.enabled=true`后并执行`cache table`,那么`table cache`后的`split`会`cache`,会使用`cache`的`split`调度策略,而非默认的调度策略,此时会突破`node-scheduler.max-splits-per-node`的限制,这看起来更像是个 bug。
-
-`hive.orc.row-data.block.cache.enabled`开启后,`split`和`row data`都会`cache`,且`cache`的`data`是解压后的`block`,同时`filetail`、`footer`等元数据信息没有开启,因此仍然会去读元数据信息,但是感觉已经没有必要了,因为读这些信息并不能过滤数据,仍然会使用 cache 的数据?----这是一个 bug?
-
-### 调度影响
-
-当`node-scheduler.max-splits-per-node`使用默认值 100 时,由于单节点处理的数据量很小,因而`queued`和`running`的`split`最大为 100,需要分多批次调度,而`hetu.split-cache-map.enabled=true`后`queue`和`running`的`splits`无限大,**为何分批次调度的影响这么大,值得进一步分析。**
-
-### 文件系统 Cache
-
-`row data cache`后减少的这一半时间主要是 ORC 读取上来后的解压缩时间,因为实际数据已经缓存在文件系统`cache`中,因而数据从内核态拷贝到用户态是很快的。
-
-在最早的分析中,我们发现算子级别的`wall`和`cpu time`一样,而端到端时间差异几倍,怀疑是`IO wait`,但是实际上通过`IOstat`等命令看,没有`io wait`,甚至没有`io`读写,发现是因为文件只有 700M,基本全缓存到文件系统`cache`了,从而进一步理解了文件系统`cache`。
-
-同时我们怀疑是否有异步线程的时间没有统计到`driver`的`process`中,实际上也没有这种情况发生,因为哪怕是异步线程处理,那么在 cpu 图上一定能看到痕迹。
-
-通过以下两篇文章,能较好的了解异步 IO:
-
-[译]
-
-Linux 异步 I/O 框架 io_uring:基本原理、程序示例与性能压测(2020)
-来自
-
-# 4. 重新生成配置文件信息
-
-```shell
-# 删除之前配置
-rm -rf /etc/openldap/slapd.d/*
-
-# 生成新配置
-slapadd -F /etc/openldap/slapd.d -n 0 -l /root/ldap_confs/slapd.ldif
-
-# 测试配置
-slaptest -u -F /etc/openldap/slapd.d
-# 注意:完成后查看下/etc/openldap/slapd.d/下生成的配置信息是否完整!
-
-# 授权
-chown -R ldap. /etc/openldap/slapd.d/
-chown -R ldap. /var/lib/ldap/
-
-# 启动服务
-systemctl start slapd
-systemctl enable slapd
-systemctl status slapd
-```
-
-# 5. 导入根域及管理员账号
-
-生成 root.ldif
-
-```properties
-# cd /root/ldap_confs
-# vim root.ldif
-dn: dc=tcjf,dc=com
-dc: tcjf
-objectClass: top
-objectClass: domain
-
-dn: cn=Manager,dc=tcjf,dc=com
-objectClass: organizationalRole
-cn: Manage
-```
-
-```shell
-# 导入根域及管理员信息到OpenLDAP服务中
-ldapadd -D "cn=Manager,dc=tcjf,dc=com" -W -x -f root.ldif
-## 导入密码是:slapd.ldif中配置的对应明文密码******
-```
-
-```shell
-ldapsearch -h ***.***.***.*** -b "dc=tcjf,dc=com" -D "cn=Manager,dc=tcjf,dc=com" -W
-##查看是否导入成功(-h后为ldap服务ip)
-```
-
-# 6. 导入用户和用户组
-
-1. 添加基础信息
-
-```properties
-# cd /root/ldap_confs
-# vim base.ldif
-dn: ou=People,dc=tcjf,dc=com
-ou: People
-objectClass: top
-objectClass: organizationalUnit
-
-dn: ou=Group,dc=tcjf,dc=com
-ou: Group
-objectClass: top
-objectClass: organizationalUnit
-```
-
-2.添加组信息
-
-```properties
-# vim group.ldif
-dn: cn=root,ou=Group,dc=tcjf,dc=com
-objectClass: posixGroup
-objectClass: top
-cn: root
-userPassword: {crypt}x
-gidNumber: 0
-
-dn: cn=lk,ou=Group,dc=tcjf,dc=com
-objectClass: posixGroup
-objectClass: top
-cn: lk
-userPassword: {crypt}x
-gidNumber: 1501
-memberUid: lk
-```
-
-3.添加用户信息
-
-```shell
-# /usr/share/migrationtools/migrate_passwd.pl /etc/passwd >user.ldif
-# mv user.ldif user.ldif.bak
-# vim user.ldif.bak #复制里面的 dn: uid=root,ou=People,dc=xxx,dc=com 的条目内容,粘贴到 user.ldif
-# vim user.ldfi #粘贴到此,再复制粘贴下面 dn: uid=lk,ou=People,dc=tcjf,dc=com 条目内容
-# rm user.ldif.bak
-```
-
-```properties
-# vim user.ldif
-dn: uid=root,ou=People,dc=tcjf,dc=com
-uid: root
-cn: root
-objectClass: account
-objectClass: posixAccount
-objectClass: top
-objectClass: shadowAccount
-userPassword: {crypt}$6$TC.3Iv9i$n8MVv4Eh4E7cVGP8nhiATKVJBgkmt4Zz7aapaW8/rvuwuWqP7.xwcpyFvQ3gN9Ha0LfzlCMAdIlfqhBtz/30e.
-shadowLastChange: 19113
-shadowMin: 0
-shadowMax: 99999
-shadowWarning: 7
-loginShell: /bin/bash
-uidNumber: 0
-gidNumber: 0
-homeDirectory: /root
-gecos: root
-
-dn: uid=lk,ou=People,dc=tcjf,dc=com
-uid: lk
-cn: lk
-objectClass: account
-objectClass: posixAccount
-objectClass: top
-objectClass: shadowAccount
-userPassword: {crypt}!!
-shadowLastChange: 17988
-loginShell: /bin/bash
-uidNumber: 1501
-gidNumber: 1501
-homeDirectory: /home/lk
-gecos: lk
-```
-
-使用 slapadd 命令将基础文件及用户和组导入 OpenLDAP:
-
-```shell
-ldapadd -D "cn=Manager,dc=tcjf,dc=com" -W -x -f base.ldif
-ldapadd -D "cn=Manager,dc=tcjf,dc=com" -W -x -f group.ldif
-ldapadd -D "cn=Manager,dc=tcjf,dc=com" -W -x -f user.ldif
-```
-
-4.查看是否导入成功:
-
-```shell
-ldapsearch -h ***.***.***.*** -b "dc=tcjf,dc=com" -D "cn=Manager,dc=tcjf,dc=com" -W|grep dn
-```
-
-
-
-# 7. 生成 OpenLDAP 证书
-
-## 7.1 修改 openssl 配置文件
-
-1. 复制 openssl 配置文件到 /tmp 下:
-
- ```shell
- cp /etc/pki/tls/openssl.cnf /tmp/openssl.cnf
- ```
-
-2. 编辑 /tmp/openssl.cnf
-
- 此文件的格式是类似 ini 的配置文件格式,找到 [ req ] 段落,加上下面的配置:
-
- ```shell
- req_extensions = v3_req
- ```
-
- 
-
- 这段配置表示在生成 CSR 文件时读取名叫 v3_req 的段落的配置信息,因此我们再在此配置文件中加入一段名
- 为 v3_req 的配置:
-
- ```shell
- [ v3_req ]
- # Extensions to add to a certificate request
- basicConstraints = CA:FALSE
- keyUsage = nonRepudiation, digitalSignature, keyEncipherment
- subjectAltName = @alt_names
- ```
-
- 
-
- 这段配置中最重要的是在最后导入名为 alt_names 的配置段,因此我们还需要添加一个名为 [ alt_names ] 的
- 配置段:
-
- ```shell
- [ alt_names ]
- IP = ***.***.***.***
- ```
-
- 注:ip 为 OpenLDAP 服务器 ip
-
-## 7.2 生成 OpenLDAP 证书(crt 文件)
-
-```shell
-cd /etc/openldap/certs/
-openssl genrsa -out ldap.key 1024
-openssl req -new -key ldap.key -out ldap.csr -config /tmp/openssl.cnf
-```
-
-```shell
-[root@olk0 certs]# openssl req -new -key ldap.key -out ldap.csr -config /tmp/openssl.cnf
-You are about to be asked to enter information that will be incorporated
-into your certificate request.
-What you are about to enter is what is called a Distinguished Name or a DN.
-There are quite a few fields but you can leave some blank
-For some fields there will be a default value,
-If you enter '.', the field will be left blank.
------
-Country Name (2 letter code) [XX]:CN
-State or Province Name (full name) []:ZJ
-Locality Name (eg, city) [Default City]:HZ
-Organization Name (eg, company) [Default Company Ltd]:synway
-Organizational Unit Name (eg, section) []:bigdata
-Common Name (eg, your name or your server's hostname) []:***.***.***.***
-Email Address []:aaabbb@foxmail.com
-
-Please enter the following 'extra' attributes
-to be sent with your certificate request
-A challenge password []:
-An optional company name []:
-```
-
-> your name or your server's hostname:输入 ldap server ip 地址。
-
-查看证书请求文件的内容:
-
-```shell
-openssl req -text -noout -in ldap.csr
-```
-
-生成签名:
-
-```shell
-openssl x509 -req -days 3650 -in ldap.csr -signkey ldap.key -out ldap.crt -extensions v3_req -extfile /tmp/openssl.cnf
-```
-
-# 8. 配置 ldap 开启 ssl
-
-## 8.1 编辑 certs.ldif
-
-编写 ldif 文件,导入证书。
-
-```shell
-# cd /root/ldap_confs
-# vim certs.ldif
-```
-
-**该配置为 tls 相关配置,配置文件必须为此格式,此顺序,此格式导入,不然会报错!**
-
-```properties
-dn: cn=config
-changetype: modify
-replace: olcTLSCACertificatePath
-olcTLSCACertificatePath: /etc/openldap/certs
--
-replace: olcTLSCertificateFile
-olcTLSCertificateFile: /etc/openldap/certs/ldap.crt
--
-replace: olcTLSCertificateKeyFile
-olcTLSCertificateKeyFile: /etc/openldap/certs/ldap.key
--
-replace: olcTLSVerifyClient
-olcTLSVerifyClient: never
-```
-
-执行命令导入配置到 /etc/openldap/slapd.d/cn=config.ldif 文件中:
-
-```shell
-ldapmodify -Y EXTERNAL -H ldapi:/// -f certs.ldif
-```
-
-## 8.2 编辑/etc/openldap/ldap.conf
-
-打开 /etc/openldap/ldap.conf 文件,在 `TLS_CACERTDIR /etc/openldap/certs` 的下面添加如下内容:
-
-```shell
-# vim /etc/openldap/ldap.conf
-TLS_REQCERT allow
-TLS_CERT /etc/openldap/certs/ldap.crt
-TLS_KEY /etc/openldap/certs/ldap.key
-URI ldaps://***.***.***.***
-BASE dc=tcjf,dc=com
-```
-
-其中,URI ldaps 为 ldap server 地址。
-
-
-
-## 8.3 配置同时启动 ldap 和 ldaps
-
-编辑 /etc/sysconfig/ldap (没有就手动创建):
-
-```shell
-# vim /etc/sysconfig/ldap
-SLAPD_LDAP=yes
-SLAPD_LDAPI=yes
-SLAPD_LDAPS=yes
-```
-
-编辑 /etc/sysconfig/slapd:
-
-```shell
-# 配置slapd启动监听ldaps
-# vim /etc/sysconfig/slapd
-SLAPD_URLS="ldapi:/// ldap:/// ldaps:///”
-```
-
-## 8.4 启动并查看是否已经成功
-
-```shell
-service slapd restart
-
-netstat -tnlp |grep 389
-
-netstat -tnlp |grep 636
-```
-
-# 9. 使用 ldap client 连接
-
-使用 LDAP Client 连接。Windows 可以下载 LDAP Admin,它是一款很棒的 LDAP 管理工具,界面简洁直观,连接到 LDAP 服务器后,即可快速执行管理操作。下载地址:http://www.xitongzhijia.net/soft/208026.html
-
-下载完成,双击打开软件,依次点击「Start」->「Connection」->「New Connection」,再按如下步骤操作,最后点击 OK。
-
-
-
-# 10. 开启 ldap 日志
-
-1.在 lapd.conf 中加一行
-
-```properties
-# vim /etc/openldap/ldap.conf
-loglevel 4095
-```
-
-2.在 /etc/rsyslog.conf 中加入 ldap 日志文档,增加该条配置:
-
-```properties
-# vim /etc/rsyslog.conf
-local4.* /var/log/ldap.log
-```
-
-3.在终端用命令重启 syslog 服务和 ldap 服务
-
-```shell
-service rsyslog restart
-service slapd restart
-```
-
-4.在 /var/log/ 下可以看到 ldap.log 文件
-
-5.查看日志
-
-```shell
-tailf /var/log/ladp.log
-```
diff --git a/app/zh/information/blog/20220625-02/index.md b/app/zh/information/blog/20220625-02/index.md
deleted file mode 100644
index e2db506c3a60e4cfdb8a1b0f7de202980882dac7..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/20220625-02/index.md
+++ /dev/null
@@ -1,317 +0,0 @@
----
-title: 'openLooKeng安全配置操作指南(二)----集成OpenLDAP'
-date: '2022-06-25'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', 'OpenLDAP']
-archives: '2022-06'
-author: 'jaybo'
-description: 'openLooKeng集成OpenLDAP'
----
-
-# 0. 环境
-
-| Role | IP | Hostname | Hostname+domain name |
-| :---------------------- | :-------------- | :------- | :------------------- |
-| openldap、ranger、mysql | **_._**.**_._** | olk0 | olk0.example.com |
-| kerberos | **_._**.**_._** | olk1 | olk1.example.com |
-| coordinator | **_._**.**_._** | olk2 | olk2.example.com |
-| worker | **_._**.**_._** | olk3 | olk3.example.com |
-
-# 1. 导入证书(crt 文件)到 jdk
-
-拷贝 openldap 的 /etc/openldap/cert 文件夹下的 ldap.crt 证书到 openLooKeng 集群各个节点。
-
-```shell
-scp /etc/openldap/cert/ldap.crt ***.***.***.***:/root/hetu-server-1.4.0
-scp /etc/openldap/cert/ldap.crt ***.***.***.***:/root/hetu-server-1.4.0
-```
-
-在 openLooKeng 集群各个节点使用 keytool 导入证书到 jdk。
-
-(密码:**\*\***)
-
-```shell
-# 导入证书
-keytool -import -trustcacerts -file ldap.crt -alias ldapserver -keystore /usr/local/java/jdk1.8.0_261/jre/lib/security/cacerts
-
-# 移除证书(无需运行此命令)
-# -delete 删除密钥库中某条目
-# 格式:keytool -delete -alias 指定需删除的别名 -keystore 指定keystore -storepass 密码
-keytool -delete -alias ldapserver -keystore /usr/local/java/jdk1.8.0_261/jre/lib/security/cacerts
-```
-
-**如果这步跳过,后面当输入账号和密码登录 openLooKeng,ldap 会报 TLS negotiation failure:**
-
-```xml
-May 3 03:27:57 olk0 slapd[11426]: conn=1001 fd=19 ACCEPT from IP=***.***.***.***:48838 (IP=***.***.***.***:636)
-May 3 03:27:57 olk0 slapd[11426]: conn=1001 fd=19 closed (TLS negotiation failure)
-```
-
-# 2. 生成 openLooKeng 的 keystore
-
-> 使用 Kerberos 和 LDAP 身份验证时,必须通过 HTTPS 访问 openLooKeng 协调节点。openLooKeng 协调节点使用`JavaKeystore
-
-## 6.2 Hetu CLI
-
-```shell
-./hetu-cli \
---server https://coordinator-ip:port \
---keystore-path /root/hetu-server-1.4.0/keystore.jks \
---keystore-password ****** \
---user
-
-## 6.4 存在的问题: 基于 HTTP 访问(无需密码)依然可用
-
-(1) Web UI
-
-HTTP 访问地址:http://**_._**.**_._**:9999/ui/login.html
-
-(2) Hetu CLI
-
-```shell
-./hetu-cli --server http://olk2.example.com:9999 --user test
-```
-
-(3) JDBC
-
-```shell
-String url = "jdbc:lk://olk2.example.com:9999/hive/";
-Properties properties = new Properties();
-properties.setProperty("user", "test");
-properties.setProperty("password", "");
-```
-
-**以上三种基于 http 访问的方式(无需密码)依然可使用。**
-
-# 7. 集成 ldap 成功,可先备份 openLooKeng,再去集成 Ranger
-
-```shell
-cp -r hetu-server-1.4.0 hetu-server-1.4.0-ldap
-```
-
-# 参考资料
-
-[1] [openLooKeng 集成 OpenLDAP(二)](https://mp.weixin.qq.com/s/daC1Rs3fy2iXBnktgB4sow)
-
-[2] [openLooKeng AA 安全配置指导(一)----对接 Kerberos](https://openlookeng.io/zh/information/blog/configguide-01/)
-
-[3] [openLooKeng 官网文档\_Java 密钥库和信任库](https://docs.openlookeng.io/zh/docs/docs/security/tls.html)
diff --git a/app/zh/information/blog/20220625-03/index.md b/app/zh/information/blog/20220625-03/index.md
deleted file mode 100644
index 791664e55a3eda984bb308827bdc9e261b40f6f4..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/20220625-03/index.md
+++ /dev/null
@@ -1,367 +0,0 @@
----
-title: 'openLooKeng安全配置操作指南(三)----集成Ranger'
-date: '2022-06-25'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', 'Ranger']
-archives: '2022-06'
-author: 'jaybo'
-description: 'openLooKeng集成Ranger'
----
-
-# 0. 环境
-
-| Role | IP | Hostname | Hostname+domain name |
-| :---------------------- | :-------------- | :------- | :------------------- |
-| openldap、ranger、mysql | **_._**.**_._** | olk0 | olk0.example.com |
-| kerberos | **_._**.**_._** | olk1 | olk1.example.com |
-| coordinator | **_._**.**_._** | olk2 | olk2.example.com |
-| worker | **_._**.**_._** | olk3 | olk3.example.com |
-
-# 1. 编译 Ranger
-
-由于 Ranger 不提供二进制安装包,故需要 maven 编译,如何编译 Ranger 不做本文重点。
-
-反正最后编译成功后会得到如下一些 tar.gz 包(图片来源网络):
-
-
-
-当前我们只要 ranger admin 包:
-
-```xml
-ranger-2.1.0-admin.tar.gz
-```
-
-若需要同步 ldap 用户,还需安装 usersync,这个可以等以后再安装:
-
-```shell
-ranger-2.1.0-usersync.tar.gz
-```
-
-# 2. 编译 openLooKeng-ranger-plugin
-
-[openlookeng-ranger-plugin](https://gitee.com/openlookeng/openlookeng-ranger-plugin) 是为 openLooKeng 开发的 Ranger 插件,用于全面的数据安全监控和权限管理。
-
-- 代码仓库:https://gitee.com/openlookeng/openlookeng-ranger-plugin
-
-编译过程:
-
-1. 从 Git 仓库检出 openlookeng-ranger-plugin 代码
-
- ```shell
- git clone -b 1.4.0 https://gitee.com/openlookeng/openlookeng-ranger-plugin.git
- ```
-
- **本文以 openLooKeng 1.4.0 版本为例。**
-
-2. 进入代码根目录,执行 maven 命令:
-
- ```shell
- mvn clean package
- ```
-
-3. 在上述 maven 命令执行完成后,可以在 target 目录发现以下 tar.gz 文件:
-
- ```xml
- ranger-2.1.0-admin-openlookeng-1.4.0-plugin.tar.gz
- ranger-2.1.0-openlookeng-1.4.0-plugin.tar.gz
- ```
-
-# 3. 所要用到的安装包
-
-这是这次所要用到的安装包:
-
-```xml
-ranger-2.1.0-admin.tar.gz
-ranger-2.1.0-admin-openlookeng-1.4.0-plugin.tar.gz
-ranger-2.1.0-openlookeng-1.4.0-plugin.tar.gz
-```
-
-
-
-接下来按照步骤安装即可。
-
-# 4. 安装 Ranger Admin
-
-## 4.1 解压 Ranger Admin
-
-```shell
-tar -zxvf ranger-2.1.0-admin.tar.gz
-```
-
-## 4.2 安装 Ranger Admin
-
-需要使用 root 用户,**ranger admin 依赖 MySQL、JDK**(如何安装 MySQL 和 JDK,这里略过)。
-
-进入 ranger admin 根目录,打开 install.properties:
-
-```shell
-cd ranger-2.1.0-admin
-vim install.properties
-```
-
-修改配置:
-
-```properties
-# 指定一个mysql的驱动,需要自己指定一个jar包
-SQL_CONNECTOR_JAR=/usr/local/java/mysql-connector-java-5.1.47.jar
-
-# 配置mysql数据库的信息, mysql账号、密码、地址
-db_root_user=root
-db_root_password=******
-db_host=localhost
-
-# 安装时会使用上面的root用户创建如下数据库
-db_name=ranger
-db_user=ranger
-db_password=****** #数据库密码
-
-#登录ranger-admin的用户名密码:admin/******
-rangerAdmin_password=******
-rangerTagsync_password=******
-rangerUsersync_password=******
-keyadmin_password=******
-
-#Source for Audit Store. Currently solr and elasticsearch are supported.
-# * audit_store is solr
-# 不使用solr做审计存储,置为空即可
-audit_stoe=
-
-# * audit_solr_url Elasticsearch Host(s). E.g. ***.***.***.***
-audit_elasticsearch_urls=
-audit_elasticsearch_port=
-audit_elasticsearch_protocol=
-audit_elasticsearch_user=
-audit_elasticsearch_password=
-audit_elasticsearch_index=
-audit_elasticsearch_bootstrap_enabled=true
-
-#localhost或本机ip
-policymgr_external_url=http://localhost:6080
-policymgr_http_enabled=true
-policymgr_https_keystore_file=
-policymgr_https_keystore_keyalias=rangeradmin policymgr_https_keystore_password=******
-
-unix_user=ranger
-unix_user_pwd=******
-unix_group=ranger
-```
-
-使用 root 用户执行如下脚本,安装 RangerAdmin:
-
-```shell
-./setup.sh
-```
-
-## 4.3 启动 Ranger Admin
-
-```shell
-# stop关闭 restar重启
-ews/ranger-admin-services.sh start
-#或 ews/ranger-admin start
-```
-
-**注:服务器关机后,需重启 ranger admin 服务。**
-
-> 可以按如下步骤设置 ranger admin 服务为开机自启:
->
-> 1. 将 `/etc/rc.d/rc.local` 文标记为可执行文件(因为在 CentOS 7 中,`/etc/rc.d/rc.local` 文件的权限被降低了——是没有执行权限的)
->
-> ```shell
-> chmod +x /etc/rc.d/rc.local
-> ```
->
-> 2. 打开 `/etc/rc.d/rc.local` 文件,在文件末尾加入启动命令即可
->
-> ```shell
-> /root/ranger-2.1.0-admin/ews/ranger-admin-services.sh start
-> ```
->
-> 3. 可以重启服务器看看是不是开机自启了:
->
-> ```shell
-> reboot
-> lsof -i:6080
-> ```
-
-## 4.4 web 登录验证是否安装
-
-访问网址:http://**_._**.**_._**:6080/login.jsp
-
-- 用户名:admin
-- 密码:**\*\***(install.properties 中设置的)
-
-如下登录成功,表示安装成功。
-
-
-
-# 5. 安装 Ranger Admin 插件
-
-**(1) 解压 ranger-2.1.0-admin-openlookeng-1.4.0-plugin.tar.gz**
-
-```shell
-tar -zxvf ranger-2.1.0-admin-openlookeng-1.4.0-plugin.tar.gz
-```
-
-可以发现以下目录:
-
-```xml
-openlookeng
-service-defs
-```
-
-**(2) Ranger 服务类型定义的注册**
-
-使用 Ranger Admin 提供的 REST API 向 Ranger 注册服务类型定义。注册后,Ranger Admin 将提供 UI 以创建服务实例(在以前的版本中称为存储库)和服务类型策略。Ranger 插件使用服务类型定义和策略来确定请求是否有访问权限以进行授权。如下示例所示,可以使用 curl 命令调用 REST API 接口注册服务:
-
-```shell
-curl -u admin:password -X POST -H "Accept: application/json" -H "Content-Type: application/json" -d @service-defs/ranger-servicedef-openlookeng.json http://ranger-admin-host:port/service/plugins/definitions
-```
-
-eg:
-
-```shell
-curl -u admin:****** -X POST -H "Accept: application/json" -H "Content-Type: application/json" -d @service-defs/ranger-servicedef-openlookeng.json http://***.***.***.***:6080/service/plugins/definitions
-```
-
-**(3) 复制 openlookeng 目录到 Ranger Admin 安装目录下的 ranger-plugins 目录**
-
-```shell
-scp -r openlookeng/ /root/softwares/ranger-2.1.0-admin/ews/webapp/WEB-INF/classes/ranger-plugins/
-```
-
-这时打开 Ranger Admin 可以看到多了“OPENLOOKENG”服务。
-
-
-
-# 6. 安装 openLooKeng 插件
-
-**(1) 解压 ranger-2.1.0-openlookeng-1.4.0-plugin.tar.gz**
-
-```shell
-tar -zxvf ranger-2.1.0-openlookeng-1.4.0-plugin.tar.gz
-```
-
-**(2) 适当的修改 install.properties 文件**
-
-如下示例所示,修改了部分参数值:
-
-```properties
-# Location of Policy Manager URL
-# ranger admin 地址
-POLICY_MGR_URL=http://***.***.***.***:6080
-
-# This is the repository name created within policy manager
-# 这个随便取,回头在ranger admin中新建jdbc连接的时候需要填写这个
-REPOSITORY_NAME=openlookengdev
-
-# openLooKeng component installed directory
-# openLooKeng 安装目录
-COMPONENT_INSTALL_DIR_NAME=/root/hetu-server-1.4.0
-
-XAAUDIT.SOLR.ENABLE=false
-
-# 添加该条配置,不然后面执行会报错
-XAAUDIT.SUMMARY.ENABLE=false
-```
-
-**(3) 执行 ./enable-openlookeng-plugin.sh**
-
-```shell
-./enable-openlookeng-plugin.sh
-```
-
-重启服务:
-
-```shell
-重启 Ranger Admin 服务:ews/ranger-admin-services.sh restart
-重启 openLooKeng 服务:sh launcher restart
-```
-
-
-
-# 7. 添加 openLookeng Service
-
-## 7.1 添加 openLooKeng Service
-
-打开 Admin Ranger:http://**_._**.**_._**:6080/index.html,找到「OPENLOOKENG」点击「+」添加 Service。
-
-输入 Service Name、Username、Password、jdbc.driverClassName、jdbc.url 这些信息:
-
-- Username 填写:
-
- - 未对接 openLDAP,用户任意
- - 对接了 openLDAP,填写 ldap 用户
-
-- Passworld:对接了 openLDAP 需填写;未对接 openLDAP 不用填写。
-
-- jdbc 填写(openLooKeng 关闭了 http 选择 ②,否则选择 ①):
-
- ① 非安全 openLooKeng 集群:jdbc:lk://openLooKeng-IP:http-port/catalog
-
- ```properties
- eg:
- jdbc:lk://***.***.***.***:9999/hive
- ```
-
-
-
- ② 安全 openLooKeng 集群:jdbc:lk://openLooKeng-IP:https-port/hive?SSL=true
-
- ```properties
- eg:
- jdbc:lk://***.***.***.***:8443/hive?SSL=true
- ```
-
-
-
-点击「Test Connection」可能会报如下这样错误:
-
-```xm
-Connection Failed.
-Unable to retrieve any files using given parameters, You can still save the repository and start creating policies, but you would not be able to use autocomplete for resource names. Check ranger_admin.log for more info.
-
-org.apache.ranger.plugin.client.HadoopException: Unable to execute SQL [SHOW CATALOGS]. .
-Error executing query.
-javax.net.ssl.SSLHandshakeException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target.
-PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target.
-unable to find valid certification path to requested target.
-```
-
-忽略,点击「Save」保存。
-
-## 7.1 验证是否添加成功
-
-重启 openLooKeng,再打开 Ranger Admin Web 页面,点击「Adult」->「Plugins」查看,出现如下表明 openLooKeng 插件安装成功。
-
-
-
-点击「OPENLOOKENG」下的 openlookengdev,可以看到生成了 jdbc 连接信息中填写的 lk 用户的 Policy,该默认权限配置,包含了执行如下 sql 权限:
-
-```sql
-select * from system.information_schema.tables;
-```
-
-后面就以该条 sql 进行验证。注:没有进行配置的用户,没有任何权限,比如 lk2、test 用户没配置,则没有任何权限。
-
-
-
-# 8. 验证权限控制
-
-(1) HTTP
-
-访问地址:http://**_._**.**_._**:9999/ui/login.html
-
-- 分别使用 lk,test 用户进入执行 sql,可以发现 test 用户没有权限
-
-(2) HTTPS
-
-访问地址:https://**_._**.**_._**:8443/ui/
-
-- 分别使用 lk,test 用户并输入正确的密码进入执行 sql,可以发现 test 用户没有权限
-
-# 参考资料
-
-[1] [openLooKeng 官网文档\_Ranger 访问控制](https://docs.openlookeng.io/zh/docs/docs/security/ranger-access-control.html)
-
-[2] [大数据权限管理工具 Ranger2.0.0 的安装部署到使用测试(踩坑详情)](https://juejin.cn/post/6844904159930482696)
diff --git a/app/zh/information/blog/20220625-04/index.md b/app/zh/information/blog/20220625-04/index.md
deleted file mode 100644
index 09a1844dd4860dcfe93755327ded2d1156e44af9..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/20220625-04/index.md
+++ /dev/null
@@ -1,426 +0,0 @@
----
-title: 'openLooKeng安全配置操作指南(三)----关闭HTTP(含Kerberos安装)'
-date: '2022-06-25'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', 'Kerberos']
-archives: '2022-06'
-author: 'jaybo'
-description: 'openLooKeng关闭HTTP,含Kerberos安装'
----
-
-## 0. 环境
-
-| Role | IP | Hostname | Hostname+domain name |
-| :---------------------- | :------------ | :------- | :------------------- |
-| openldap、ranger、mysql | ***.***.***.*** | olk0 | olk0.example.com |
-| kerberos | ***.***.***.*** | olk1 | olk1.example.com |
-| coordinator | ***.***.***.*** | olk2 | olk2.example.com |
-| worker | ***.***.***.*** | olk3 | olk3.example.com |
-
-# 1. 安装 Kerberos Server
-
-## 1.1 安装
-
-```shell
-yum install -y krb5-server krb5-lib krb5-workstation
-```
-
-## 1.2 修改配置文件
-
-**配置/etc/krb5.conf**:vim /etc/krb5.conf
-
-```properties
-# Configuration snippets may be placed in this directory as well
-
-includedir /etc/krb5.conf.d/
-[logging]
- default = FILE:/var/log/krb5libs.log
- kdc = FILE:/var/log/krb5kdc.log
- admin_server = FILE:/var/log/kadmind.log
-
-[libdefaults]
- dns_lookup_realm = false
- ticket_lifetime = 24h
- renew_lifetime = 7d
- forwardable = true
- rdns = false
- pkinit_anchors = FILE:/etc/pki/tls/certs/ca-bundle.crt
- default_realm = EXAMPLE.COM
- default_ccache_name = /tmp/krb5cc_%{uid}
-
-[realms]
- EXAMPLE.COM = {
- kdc = host4 //kerberos服务器主机名
- admin_server = host4
- }
-
-[domain_realm]
- .example.com = EXAMPLE.COM
- example.com = EXAMPLE.COM
-```
-
-注:
-
-1. 修改 default_realm 与 [realms] 中命名相同;
-2. kdc、admin_server 配置为本地的 hostname;
-3. 配置 default_ccache_name。
-
-**拷贝 /etc/krb5.conf 文件到 openLooKeng 集群所有节点上,config.properties 认证配置会用到。**
-
-**配置** **/var/kerberos/krb5kdc/kdc.conf**:vim /var/kerberos/krb5kdc/kdc.conf
-
-```properties
-[kdcdefaults]
- kdc_ports = 88
- kdc_tcp_ports = 88
-
-[realms]
- EXAMPLE.COM = {
- \#master_key_type = aes256-cts
- acl_file = /var/kerberos/krb5kdc/kadm5.acl
- dict_file = /usr/share/dict/words
- admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab
- supported_enctypes = aes256-cts:normal aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal camellia256-cts:normal camellia128-cts:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal
- }
-```
-
-**注:** kdc.conf 中 realm 名与 krb5.conf 相同
-
-## 1.3 创建数据库
-
-```shell
-/usr/sbin/kdb5_util create -r EXAMPLE.COM -s
-```
-
-其中需要设置管理员密码,创建完成会在 /var/kerberos/krb5kdc/ 下面生成 principal 文件,若重建数据库则需先删除 /var/kerberos/krb5kdc 下面 principal 相关文件,附删除命令:
-
-```shell
-rm -rf /var/kerberos/krb5kdc/principal*
-```
-
-## 1.4 创建管理员账号
-
-编辑 /var/kerberos/krb5kdc/kadm5.acl,vim /var/kerberos/krb5kdc/kadm5.acl,增加:
-
-```shell
-*/admin@EXAMPLE.COM *
-```
-
-**注:** \* 代表全部权限
-
-## 1.5 重启 Kerberos 服务器
-
-```shell
-systemctl restart krb5kdc
-systemctl restart kadmin
-```
-
-服务器关机后,kerberos 服务也会关闭。可以将 kerberos 加入开机自启:
-
-```shell
-systemctl enable krb5kdc
-systemctl enable kadmin
-```
-
-## 1.6 Kerberos Client 安装(若 client 和 server 部署在同台服务器,不用安装)
-
-```shell
-yum install krb5-workstation krb5-libs krb5-auth-dialog
-```
-
-复制 kerberos server 的 /etc/krb5.conf 到 kerberos client 的 /etc/krb5.conf。
-
-## 1.7 在客户端添加用户
-
-为 Kerberos database 添加 administrative principals:
-
-```shell
-kadmin.local -q "addprinc admin/admin@EXAMPLE.COM"
-```
-
-## 1.8 验证客户端可以访问 KDC
-
-```shell
-kinit admin/admin
-kadmin -p 'admin/admin' -w '******' -s 'ip4' -q 'list_principals'
-```
-
-其中,ip4 为 kerberos server 地址。
-
-# 2. openLooKeng 对接 Kerberos
-
-## 2.1 生成 openLooKeng 的 Keytab
-
-在装有 Kerberos 客户端的机器上执行下面语句(**生成服务名为 HTTP**)
-
-创建 principal:
-
-```shell
-kadmin -p admin/admin -q "addprinc -randkey HTTP@EXAMPLE.COM"
-kadmin -p admin/admin -q "addprinc -randkey HTTP/olk2@EXAMPLE.COM"
-kadmin -p admin/admin -q "addprinc -randkey HTTP/olk3@EXAMPLE.COM"
-```
-
-生成 keytab:
-
-```shell
-kadmin -p admin/admin -q "ktadd -k ./lk.keytab HTTP@EXAMPLE.COM"
-kadmin -p admin/admin -q "ktadd -k ./lk.keytab HTTP/olk2@EXAMPLE.COM"
-kadmin -p admin/admin -q "ktadd -k ./lk.keytab HTTP/olk3@EXAMPLE.COM"
-```
-
-**注:**openLooKeng 主机名必须为小写,如 olk2、olk3。所有节点主机名的 principal 都要创建。
-
-**拷贝 lk.keytab 文件到 openLooKeng 集群所有节点上,config.properties 认证配置会用到。**
-
-```shell
-scp lk.keytab olk2:/root/hetu-server-1.4.0/
-scp lk.keytab olk3:/root/hetu-server-1.4.0/
-```
-
-## 2.2 配置 openLooKeng 的 config.properties
-
-所有节点进行如下修改:
-
-1. 禁用 http 端点
-
- ```shell
- http-server.http.enabled=false
- ```
-
-2. 配置集群使用集群节点的 FQDN(全量域名)进行通信:
-
- ```shell
- node.internal-address=
-
-最后在 jvm.config 中添加或修改如下配置:
-
-```properties
--Djava.security.krb5.conf=/root/hetu-server-1.4.0/krb5.conf
-```
-
-# 3. 验证
-
-## 3.1 Web UI
-
-**(1) HTTP**
-
-基于 HTTP 访问:http://***.***.***.***:9999/ui/login.html,访问不了。[达到目的]
-
-**(2) HTTPS**
-
-基于 HTTPS 访问:https://***.***.***.***:8443/ui/ 或 https://olk2.example.com:8443/ui/(可访问)
-
-- 输入正确账户和密码才能进入。
-
-执行 sql:
-
-```sql
-select * from system.information_schema.tables;
-```
-
-报错如下:
-
-```xml
-javax.net.ssl.SSLPeerUnverifiedException: Hostname localhost not verified: certificate: sha256/uU1H9VEjdwlWBym1uNWTRGr3O7hNKYXslIphe2Hv1D0= DN: CN=*.example.com, OU=Unknown, O=Unknown, L=Unknown, ST=Unknown, C=Unknown subjectAltNames: []
-```
-
-
-
-> 问官方伙伴,貌似意思是一个 bug。
-
-## 3.2 Hetu CLI
-
-**(1) HTTPS**
-
-- 输入 test 用户密码,执行 sql,可以看到 test 用户没有执行该条 sql 权限
-
- ```shell
- ./hetu-cli --server https://olk2.example.com:8443 --keystore-path /root/hetu-server-1.4.0/keystore.jks --keystore-password ****** --user test --password
- ```
-
- 
-
-- 输入 lk 用户密码,执行 sql,因为 lk 具有该条 sql 权限,可以看到执行结果
-
- ```shell
- ./hetu-cli --server https://olk2.example.com:8443 --keystore-path /root/hetu-server-1.4.0/keystore.jks --keystore-password ****** --user lk --password
- ```
-
- 
-
-**(2) HTTP**
-
-http 访问,可以看到无法连接对应端口:[达到目的]
-
-```shell
-./hetu-cli --server http://olk2.example.com:9999 --user test
-```
-
-
-
-## 8.3 JDBC
-
-**(1) HTTPS**
-
-如下基于 https 的连接方式,可用:
-
-```xshell
-String url = "jdbc:lk://olk2.example.com:8443/hive/";
-Properties properties = new Properties();
-properties.setProperty("user", "test");
-properties.setProperty("password", "******");
-properties.setProperty("SSL", "true");
-properties.setProperty("SSLKeyStorePath", "D:\\keystore.jks");
-properties.setProperty("SSLKeyStorePassword", "******");
-```
-
-
-
-可以看到 test 用户没有该权限。改为 lk 用户可以。
-
-
-
-**(2) HTTP**
-
-基于 http 方式,可以看到无法连接对应接口:[达到目的]
-
-```shell
-String url = "jdbc:lk://olk2.example.com:9999/hive/";
-Properties properties = new Properties();
-properties.setProperty("user", "test");
-properties.setProperty("password", "");
-```
-
-
-
-# 参考资料
-
-[1] [openLooKeng AA 安全配置指导(一)----对接 Kerberos](https://openlookeng.io/zh/information/blog/configguide-01/)
-
-[2] [openLooKeng 官网文档\_内部通信安全](https://openlookeng.io/zh-cn/docs/docs/security/internal-communication.html)
diff --git a/app/zh/information/blog/20230129-01/img/image10.png b/app/zh/information/blog/20230129-01/img/image10.png
deleted file mode 100644
index 63c0d4909f64d4030cf1310460351e5d1ab4af85..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-01/img/image10.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-01/img/image11.png b/app/zh/information/blog/20230129-01/img/image11.png
deleted file mode 100644
index 86da6b88fdbbc851528701b3e0f0383e6b665bef..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-01/img/image11.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-01/img/image12.png b/app/zh/information/blog/20230129-01/img/image12.png
deleted file mode 100644
index c94a4e345dba2416777510537337f407bde37a1f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-01/img/image12.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-01/img/image13.png b/app/zh/information/blog/20230129-01/img/image13.png
deleted file mode 100644
index 08adc91c3a2c27dae1c69f4dedeec221979c3ca2..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-01/img/image13.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-01/img/image14.png b/app/zh/information/blog/20230129-01/img/image14.png
deleted file mode 100644
index dbdc0d2b5739040a3680e44c5287b5cb2e9e9ac6..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-01/img/image14.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-01/img/image3.png b/app/zh/information/blog/20230129-01/img/image3.png
deleted file mode 100644
index 0e56a31d0d41bc334292684338d8966ada8fb0d9..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-01/img/image3.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-01/img/image4.png b/app/zh/information/blog/20230129-01/img/image4.png
deleted file mode 100644
index 45763b0a2c79829c14d4f346ca668f39ee773801..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-01/img/image4.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-01/img/image5.png b/app/zh/information/blog/20230129-01/img/image5.png
deleted file mode 100644
index e6e380c11376b1c2b2fe27955b1a3850f1d40e6d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-01/img/image5.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-01/img/image6.png b/app/zh/information/blog/20230129-01/img/image6.png
deleted file mode 100644
index 8e397430fe2fa60636c652322134b995fc9229ec..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-01/img/image6.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-01/img/image7.png b/app/zh/information/blog/20230129-01/img/image7.png
deleted file mode 100644
index 480cd1497433beac67ac04c852e928bdb042662f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-01/img/image7.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-01/img/image8.png b/app/zh/information/blog/20230129-01/img/image8.png
deleted file mode 100644
index 2f83ec6f17d3920c7e6f751fe3ab41a64a8ec947..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-01/img/image8.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-01/img/image9.png b/app/zh/information/blog/20230129-01/img/image9.png
deleted file mode 100644
index ae313d0ed8d61a00af1c85c89d51fa7317436672..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-01/img/image9.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-01/index.md b/app/zh/information/blog/20230129-01/index.md
deleted file mode 100644
index fb4868150af43132bc7f5e4e7fcb4de135a99b00..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/20230129-01/index.md
+++ /dev/null
@@ -1,293 +0,0 @@
----
-title: 'Ranger安装文档'
-date: '2022-12-25'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', 'Ranger', '认证鉴权', '安全认证']
-archives: '2022-12'
-author: 'tushengxia'
-description: 'Ranger安装文档'
----
-
-环境要求
-========
-
-安装Maven
----------
-
-### 下载Maven压缩包
-
-(1) 访问Maven官网:https://maven.apache.org/download.cgi
-
-(2) 找到对应的压缩包下载,这里以apache-maven-3.8.6-bin.tar.gz为例:
-
-
-
-### 上传
-
-(1) 上传至合适位置,这里上传路径为:/usr/local/packages:
-
-
-
-### 安装
-
-(1) 解压至合适位置,这里解压至/usr/local/soft目录下
-
-(2) tar -zxvf apache-maven-3.8.6-bin.tar.gz -C /usr/local/soft/:
-
-
-
-(1) 重命名:
-
-\#重命名maven
-
-mv apache-maven-3.8.6/ maven-3.8.6
-
-
-
-(1) 配置环境变量:
-
-> vim /etc/profile
->
-> \#maven
->
-> export MAVEN\_HOME=/usr/local/soft/maven-3.8.6
->
-> export PATH=\$MAVEN\_HOME/bin:\$PATH
-
-
-
-(1) 使环境变量生效
-
-> source /etc/profile
-
-(1) 验证maven:
-
-> mvn -version
->
-> 
->
-> (6)创建maven本地仓库:
->
-> cd /usr/local/soft/maven-3.8.6
->
-> mkdir repository
->
-> (7)修改settings.xml文件:
->
-> ①修改本地仓库路径:
->
-> <localRepository>/usr/local/soft/maven-3.8.6/repository</localRepository>
->
-> 
->
-> ②修改国内镜像地址:
->
-> <mirror>
->
-> <id>alimaven</id>
->
-> <name>aliyun maven</name>
->
-> <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
->
-> <mirrorOf>central</mirrorOf>
->
-> </mirror>
->
-> 
-
-安装git
--------
-
-yum install git
-
-安装 c++
---------
-
-yum install gcc g++
-
-*\
-*编译Ranger
-===========
-
-编译Ranger
-----------
-
-### 下载Ranger压缩包
-
-(1) 访问官网:https://ranger.apache.org/download.html
-
-(2) 找到对应的压缩包下载,这里以apache-ranger-2.1.0.tar.gz为例:
-
-
-
-### 上传
-
-(1) 上传至合适位置,这里上传路径为:/usr/local/packages:
-
-
-
-### 编译
-
-(1) 在soft目录下创建ranger-data目录:
-
-> cd /usr/local/soft
->
-> mkdir ranger-data
-
-(1) 解压至合适位置,这里解压至/usr/local/soft/ranger-data目录下:
-
-> tar -zxvf apache-ranger-2.1.0.tar.gz -C /usr/local/soft/ranger-data/
-
-(1) 编译ranger:
-
-> 进入/usr/local/soft/ranger-data/apache-ranger-2.1.0目录,进行编译:
->
-> cd /usr/local/soft/ranger-data/apache-ranger-2.1.0
->
-> mvn clean compile package assembly:assembly install -DskipTests
-> -Drat.skip=true
->
-> \#或
->
-> mvn package -DskipTests -Drat.skip=true
-
-
-
-\
-安装Ranger Admin
-================
-
-安装 Ranger Admin
------------------
-
-> 1、解压Ranger Admin
->
-> tar -zxvf ranger-2.1.0-admin.tar.gz -C /usr/local/soft/
->
-> 2、安装Ranger Admin
->
-> 需要使用root用户,ranger
-> admin依赖mysql、JDK(安装mysql和JDK,这里略过)。
-
-(1) 登录mysql:
-
-> mysql -uroot -******
-
-(1) 更改mysql密码策略,为了可以采用比较简单的密码:
-
-> set global validate\_password\_length=6;
->
-> set global validate\_password\_policy=0;
->
-> set global log\_bin\_trust\_function\_creators = 1;
-
-(1) 进入 ranger admin 根目录,编辑 install.properties:
-
-> cd /usr/local/soft/ranger-2.1.0-admin
->
-> []{#OLE_LINK1 .anchor}vim install.properties
->
-> \# 指定一个mysql的驱动,需要自己指定一个jar包
->
-> SQL\_CONNECTOR\_JAR=/usr/share/java/mysql-connector-java.jar
->
-> \# 配置mysql数据库的信息, mysql账号、密码、地址
->
-> db\_root\_user=root
->
-> db\_root\_password=******
->
-> db\_host=master
->
-> \# 安装时会使用上面的root用户创建如下数据库
->
-> db\_name=ranger
->
-> db\_user=ranger
->
-> db\_password=******
->
-> \#登录ranger-admin的用户名密码:admin/******
->
-> rangerAdmin\_password=******
->
-> rangerTagsync\_password=******
->
-> rangerUsersync\_password=******
->
-> keyadmin\_password=******
->
-> \# 不使用solr做审计存储,置为空即可
->
-> audit\_store=
->
-> \# \* audit\_solr\_url Elasticsearch Host(s). E.g. ***.***.***.***
->
-> audit\_elasticsearch\_urls=
->
-> audit\_elasticsearch\_port=
->
-> audit\_elasticsearch\_protocol=
->
-> audit\_elasticsearch\_user=
->
-> audit\_elasticsearch\_password=
->
-> audit\_elasticsearch\_index=
->
-> audit\_elasticsearch\_bootstrap\_enabled=true
->
-> \#master或本机ip
->
-> policymgr\_external\_url=http://master:6080
->
-> policymgr\_http\_enabled=true
->
-> policymgr\_https\_keystore\_file=
->
-> policymgr\_https\_keystore\_keyalias=rangeradmin
->
-> policymgr\_https\_keystore\_password=******
->
-> unix\_user=ranger
->
-> unix\_user\_pwd=******
->
-> unix\_group=ranger
-
-(1) 使用 root 用户执行如下脚本,安装 RangerAdmin:
-
-> ./setup.sh
-
-(1) 启动 Ranger Admin
-
-> \# stop关闭 restar重启
->
-> ews/ranger-admin-services.sh start
->
-> \#\#或 ews/ranger-admin start
->
-> 注:服务器关机后,需重启 ranger admin 服务。
-
-(1) web 登录验证是否安装
-
-> 访问网址:http://xxxx:6080/login.jsp
-
-2. 用户名:admin
-
-> ②密码:******
->
-> 如下成功登录,表示安装成功。
-
-
-
-修订记录
-========
-
- 发布日期 修订记录
- ------------ ----------------
- 2022-08-30 第一次正式发布
-
-相关视频:[openLooKeng内核认证与鉴权流程](https://space.bilibili.com/627629884/channel/collectiondetail?sid=1061144)
diff --git a/app/zh/information/blog/20230129-02/img/image3.png b/app/zh/information/blog/20230129-02/img/image3.png
deleted file mode 100644
index aaf4dad334d39961352c94d1750d4e5844de8339..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-02/img/image3.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-02/img/image4.png b/app/zh/information/blog/20230129-02/img/image4.png
deleted file mode 100644
index b935f5a639e781d1d7513137f4c1f924d1439961..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-02/img/image4.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-02/img/image5.png b/app/zh/information/blog/20230129-02/img/image5.png
deleted file mode 100644
index 7ad63ece7a54f52322ae617137f1cee100868de0..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-02/img/image5.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-02/img/image6.png b/app/zh/information/blog/20230129-02/img/image6.png
deleted file mode 100644
index d0946aa8f176bacb54c0c500ff6fc7b0d818f98a..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-02/img/image6.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-02/index.md b/app/zh/information/blog/20230129-02/index.md
deleted file mode 100644
index 78da27a2bea37e9d7f894854244a5802eea98fba..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/20230129-02/index.md
+++ /dev/null
@@ -1,197 +0,0 @@
----
-title: 'openLooKeng整合LDAP+Kerberos+Ranger'
-date: '2022-12-25'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', 'Ranger', 'Kerberos', 'LDAP', '认证鉴权', '安全认证']
-archives: '2022-12'
-author: 'tushengxia'
-description: 'openLooKeng整合LDAP+Kerberos+Ranger'
----
-
-整合流程
-========
-
-openLooKeng整合Ranger
----------------------
-
-参考文档:《OpenLooKeng在用户鉴权上的流程梳理》
-
-openLooKeng整合Kerberos+LDAP
-----------------------------
-
-参考文档:《openLooKeng 在认证方面的流程梳理》
-
-配置认证用户映射
-----------------
-
-认证用户映射定义了从认证系统中的用户到openLooKeng用户的映射规则。认证用户映射对于具有例如alice@example或CN=Alice
-Smith, OU=Finance, O=Acme,
-C=US等复杂用户名的Kerberos或LDAP认证特别重要。
-
-可以使用简单的正则表达式模式匹配配置用户映射,也可以在单独的配置文件中配置更复杂的映射规则。
-
-### 模式映射规则
-
-模式映射规则将认证用户映射到正则表达式中的第一个匹配组。如果正则表达式与认证用户不匹配,则拒绝认证。
-
-每个身份认证系统都具有单独的用户映射模式的属性,以在启用多个身份认证系统时允许不同的映射:
-
- ------------------------------ -------------------------------------------------------------
- **认证方式** **属性**
- Username and Password (LDAP) http-server.authentication.password.user-mapping.pattern
- Kerberos http-server.authentication.krb5.user-mapping.pattern
- Certificate http-server.authentication.certificate.user-mapping.pattern
- Json Web Token http-server.authentication.jwt.user-mapping.pattern
- ------------------------------ -------------------------------------------------------------
-
-### 文件映射规则
-
-文件映射规则允许认证用户配置更复杂的映射。这些映射规则是从配置属性中定义的JSON文件加载的,基于从上到下第一个匹配的规则进行映射,如果没有规则匹配,则认证被拒绝。每个规则由以下字段组成:
-
- -------------- ------------ -------------- ------------------------------
- **字段名称** **默认值** **是否必填** **描述**
- pattern (none) 是 用于认证用户匹配的正则表达式
- user \$1 否 替换模式匹配的字符串
- allow true 否 布尔值,指示是否允许认证
- -------------- ------------ -------------- ------------------------------
-
-每个身份认证系统都有一个用于用户映射文件的单独属性,以在启用多个身份认证系统时允许不同的映射:
-
- ------------------------------ ----------------------------------------------------------
- **认证方式** **属性**
- Username and Password (LDAP) http-server.authentication.password.user-mapping.file
- Kerberos http-server.authentication.krb5.user-mapping.file
- Certificate http-server.authentication.certificate.user-mapping.file
- Json Web Token http-server.authentication.jwt.user-mapping.file
- ------------------------------ ----------------------------------------------------------
-
-### 配置方式
-
-1. 创建映射文件
-
- 映射文件放在openLooKeng的data目录下
-
- vim mapping.json
-
-> {
->
-> "rules": \[
->
-> {
->
-> "pattern": ".\*O=(.+), OU=.\*"
->
-> },
->
-> {
->
-> "pattern": "(.+)"
->
-> }
->
-> \]
->
-> }
-
-注意:pattern的规则和keystore.jks有关
-
-> keytool -genkeypair -validity 3650 -alias openLooKeng -keyalg RSA
-> -dname "CN=\*.example.com,O=olk,OU=olk,L=olk,ST=olk,C=olk" -ext
-> "SAN=IP:***.***.***.***,IP:***.***.***.***,DNS:localhost,DNS:worker1.example.com,DNS:coordinator.example.com"
-> -keystore keystore.jks
-
-如以以上示例所示:将keystore映射为olk用户(.\*O=(.+), OU=.\*)
-
-1. 给olk用户增加impersonateuser权限
-
- 
-
-2. coordinator节点config.properties文件配置
-
-> http-server.authentication.certificate.user-mapping.file=mapping.json
-
-3. 重启openLooKeng
-
-> auncher restart
-
-### 验证
-
-#### Web UI
-
-https://coordinator.example.com:8443
-
-
-
-使用LDAP用户登录
-
-#### 命令行LDAP认证
-
-> ./bin/hetu-cli \\
->
-> --server https://coordinator.example.com:8443 \\
->
-> --keystore-path keystore.jks \\
->
-> --keystore-password ****** \\
->
-> --truststore-path keystore.jks \\
->
-> --truststore-password ****** \\
->
-> --user olk \\
->
-> --password
-
-#### 命令行Kerberos认证
-
-> ./bin/hetu-cli \\
->
-> --server https://coordinator.example.com:8443 \\
->
-> --krb5-config-path /etc/krb5.conf \\
->
-> --krb5-principal openLooKeng@EXAMPLE.COM \\
->
-> --krb5-keytab-path lk.keytab \\
->
-> --krb5-remote-service-name openLooKeng \\
->
-> --keystore-path keystore.jks \\
->
-> --keystore-password ******
-
-常见问题
-========
-
-Access Denied: User CN=\*.example.com, O=olk, OU=olk, L=olk, ST=olk, C=olk cannot impersonate user olk
-------------------------------------------------------------------------------------------------------
-
-[[]{#_Toc671199467 .anchor}]{#_Toc401016727
-.anchor}
-
-出现的原因:用户映射文件配置不正确,无法将keystore映射到Ranger的用户
-
-解决方法:参考上面的映射文件配置
-
-Access Denied: User olk cannot impersonate user bigdata
--------------------------------------------------------
-
-
-
-问题描述:olk用户不能模拟bigdata用户,这里的olk用户是通过用户映射得到的用户,出现这个错误是因为olk用户没有impersonateuser权限
-
-解决方法:给olk用户增加impersonateuser权限
-
-
-
-修订记录
-========
-
- ------------ ----------------
- 发布日期 修订记录
- 2022-08-27 第一次正式发布
- ------------ ----------------
-
-
-相关视频:[openLooKeng内核认证与鉴权流程](https://space.bilibili.com/627629884/channel/collectiondetail?sid=1061144)
\ No newline at end of file
diff --git a/app/zh/information/blog/20230129-03/index.md b/app/zh/information/blog/20230129-03/index.md
deleted file mode 100644
index dfb7722324a3319b181f0080ec85f612c3cb74d8..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/20230129-03/index.md
+++ /dev/null
@@ -1,107 +0,0 @@
----
-title: 'openLooKeng内核认证和鉴权流程需求分析说明书'
-date: '2022-12-25'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', 'nginx', 'Kerberos', 'LDAP', '认证鉴权', '安全认证']
-archives: '2022-12'
-author: 'tushengxia'
-description: 'openLooKeng内核认证和鉴权流程需求分析说明书'
----
-
-引言
-====
-
-项目名称
---------
-
-优化openLooKeng内核认证和鉴权流程
-
-项目背景
---------
-
-项目目标
---------
-
-openLooKeng作为一个开源的大数据组件,可以方便地集成到各类大数据平台中。访问安全性和用户鉴权在大数据平台解决方案中尤为重要,目前在安全性问题上,openLooKeng组件已支持配置协调节点/CLI接口的Kerberos认证或者对接LDAP认证能力;在用户鉴权方面,openLooKeng已具备对接第三方安全框架的能力(Apache Ranger)或者openLooKeng内置的系统访问控制。基于当前社区已有的能力,我们希望可以进一步的优化openLooKeng内核认证和鉴权流程,让社区可以更加方便和简单的使用。
-
-需求分析说明
-============
-
-总体需求分析
-------------
-
- 编号 建设内容 说明
- ------ ------------------------------------------------- -------------------------
- 1 openLooKeng在用户鉴权上的流程梳理 对接Ranger
- 2 openLooKeng 在认证方面的流程梳理 对接Kerberos和LDAP
- 3 openLooKeng 安全集群对接Nginx的流程梳理 对接安全Nginx
- 4 支持openLooKeng对接LDAP后,不强依赖Kerberos认证 `增加CERTIFICATE参数即可`
- 5 修复Web UI对接Kerberos后,sql执行出错的问题 修复bug
-
-openLooKeng在用户鉴权上的流程梳理
----------------------------------
-
- 编号 建设内容 说明
- ------ ----------------------------------- ---------------------------------------
- 1.1 openLooKeng对接Ranger原理 《openLooKeng对接Ranger详细设计文档》
- 1.2 openLooKeng对接Ranger部署流程梳理 《openLooKeng对接Ranger流程梳理》
- 1.3 openLooKeng权限策略配置流程梳理 《openLooKeng对接Ranger流程梳理》
- 1.4 openLooKeng权限使用说明 《openLooKeng对接Ranger流程梳理》
- 1.5 openLooKeng鉴权场景使用案例 《openLooKeng对接Ranger流程梳理》
- 1.6 录制操作视频 输出视频
- 1.7 openLooKeng鉴权问题总结 《openLooKeng对接Ranger流程梳理》
-
-openLooKeng在认证方面的流程梳理
--------------------------------
-
- 编号 建设内容 说明
- ------ ----------------------------------------------- ----------------------------------------------
- 2.1 openLooKeng对接OpenLDAP和Kerberos原理 《openLooKeng对接LDAP+Kerberos详细设计文档》
- 2.2 openLooKeng对接OpenLDAP和Kerberos部署流程梳理 《openLooKeng对接LDAP+Kerberos流程梳理》
- 2.3 openLooKeng用户认证使用说明 《openLooKeng对接LDAP+Kerberos流程梳理》
- 2.4 openLooKeng对接OpenLDAP和Kerberos后整合Ranger 《openLooKeng整合LDAP+Kerberos+Ranger》
- 2.5 录制操作视频 输出视频
- 2.6 openLooKeng认证问题总结 总结FAQ
-
-openLooKeng 安全集群对接Nginx的流程梳理
----------------------------------------
-
- 编号 建设内容 说明
- ------ ------------------------------------------------------ --------------------------------------------------
- 3.1 openLooKeng对接Nginx原理 《openLooKeng安全集群对接安全Nginx详细设计文档》
- 3.2 openLooKeng对接Nginx部署流程梳理 《openLooKeng安全集群对接安全Nginx流程梳理》
- 3.3 openLooKeng对接Nginx后整合OpenLDAP,Kerberos,Ranger 《openLooKeng安全集群对接安全Nginx流程梳理》
- 3.4 录制操作视频 输出视频
- 3.5 openLooKeng对接Nginx问题总结 总结FAQ
-
-支持openLooKeng对接LDAP后,不强依赖Kerberos认证
------------------------------------------------
-
- -------------------------------------------------------------------------------------------------------
- 编号 建设内容 说明
- ------ --------------------------------------- --------------------------------------------------------
- 4.1 openLooKeng对接LDAP,不强依赖Kerberos 增加CERTIFICATE参数即可
-
- 在《openLooKeng对接LDAP+Kerberos流程梳理》文档中已说明
- -------------------------------------------------------------------------------------------------------
-
-修复Web UI对接Kerberos后,sql执行出错的问题
--------------------------------------------
-
- --------------------------------------------------------------------------------------------------------------
- 编号 建设内容 说明
- ------ --------------------------------------- ---------------------------------------------------------------
- 5.1 修复Web UI对接Kerberos后,sql执行出错 在创建keystore时将localhost 增加到DNS中
-
- 在《修复Web UI对接Kerberos后,sql执行出错的问题》文档中以说明
- --------------------------------------------------------------------------------------------------------------
-
-修订记录
-========
-
- 发布日期 修订记录
- ------------ ----------------
- 2022-08-30 第一次正式发布
-
-相关视频:[openLooKeng内核认证与鉴权流程](https://space.bilibili.com/627629884/channel/collectiondetail?sid=1061144)
\ No newline at end of file
diff --git a/app/zh/information/blog/20230129-04/img/image10.png b/app/zh/information/blog/20230129-04/img/image10.png
deleted file mode 100644
index 39c1c2cb31f0f07e7d95494d342cd39058432c2f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image10.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image11.png b/app/zh/information/blog/20230129-04/img/image11.png
deleted file mode 100644
index 46ce81803acfe8f69454302ff351e9985756f505..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image11.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image12.png b/app/zh/information/blog/20230129-04/img/image12.png
deleted file mode 100644
index 295a86952958edf917d87b43a61d262336ca9427..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image12.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image13.png b/app/zh/information/blog/20230129-04/img/image13.png
deleted file mode 100644
index a403d318471197d225244cb686f9c46e778f8d86..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image13.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image14.png b/app/zh/information/blog/20230129-04/img/image14.png
deleted file mode 100644
index 81ff017e9c1dea0013d4784c9df4ba496f80355d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image14.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image15.png b/app/zh/information/blog/20230129-04/img/image15.png
deleted file mode 100644
index 71552b2af1097c5477971064a36499019e894a15..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image15.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image16.png b/app/zh/information/blog/20230129-04/img/image16.png
deleted file mode 100644
index eece9c2bddf996cd7d4d4e35005668ad85ccc9ea..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image16.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image17.png b/app/zh/information/blog/20230129-04/img/image17.png
deleted file mode 100644
index 68d1847d51ae082146347a77f46872880b6a9889..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image17.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image18.png b/app/zh/information/blog/20230129-04/img/image18.png
deleted file mode 100644
index 661cdac51deb98118240579a1828c14dd759078d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image18.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image19.png b/app/zh/information/blog/20230129-04/img/image19.png
deleted file mode 100644
index 521ef0a735b2447d02cc236c4c6d7d28c86fbcc7..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image19.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image20.png b/app/zh/information/blog/20230129-04/img/image20.png
deleted file mode 100644
index df053169c7e99561518fa9a3c0e88f614d3c4e32..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image20.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image21.png b/app/zh/information/blog/20230129-04/img/image21.png
deleted file mode 100644
index c74e7082fedf167f2886182960ab2e686bccac0f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image21.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image22.png b/app/zh/information/blog/20230129-04/img/image22.png
deleted file mode 100644
index cd46b720751eb25b15ea4451e8f0bbe12ec168fd..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image22.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image23.png b/app/zh/information/blog/20230129-04/img/image23.png
deleted file mode 100644
index 0a78dc7ba0d7ec4a7f8268317e26d1e7b7f0823d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image23.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image24.png b/app/zh/information/blog/20230129-04/img/image24.png
deleted file mode 100644
index 85d65bb6fd52747277a2e04c0556fdea0075005a..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image24.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image25.png b/app/zh/information/blog/20230129-04/img/image25.png
deleted file mode 100644
index eb9d72c683aa4b9ba9efa52bced0ff6bae3c0962..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image25.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image26.png b/app/zh/information/blog/20230129-04/img/image26.png
deleted file mode 100644
index 9c2633b1246506ef6bbf75bff2c88a501ea23f52..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image26.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image27.png b/app/zh/information/blog/20230129-04/img/image27.png
deleted file mode 100644
index 28d9aca010d07099445591c4c1a959df46d043aa..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image27.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image28.png b/app/zh/information/blog/20230129-04/img/image28.png
deleted file mode 100644
index 9c92a631399b52ac319e81cc8e6b5b706d0f07a5..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image28.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image29.png b/app/zh/information/blog/20230129-04/img/image29.png
deleted file mode 100644
index 742a3d2dce64ae6d4d6a5f5c99f9ab35caf8455b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image29.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image3.png b/app/zh/information/blog/20230129-04/img/image3.png
deleted file mode 100644
index 341e3dff5f3c2ea24343a97bc3a82d9c6abed798..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image3.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image30.png b/app/zh/information/blog/20230129-04/img/image30.png
deleted file mode 100644
index 0849f820496c9abda9730ff3cbb06c3d911c1358..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image30.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image31.png b/app/zh/information/blog/20230129-04/img/image31.png
deleted file mode 100644
index db2dcfb198cacafac80d6f315ff3744c240e8f6f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image31.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image32.png b/app/zh/information/blog/20230129-04/img/image32.png
deleted file mode 100644
index 16d77c20fa9113144c1cab025faae0b33aab5e08..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image32.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image33.png b/app/zh/information/blog/20230129-04/img/image33.png
deleted file mode 100644
index 657fa346d0f2a0dc88fb3809382d1631e23a14f7..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image33.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image34.png b/app/zh/information/blog/20230129-04/img/image34.png
deleted file mode 100644
index a8225f264993b8e68b7e8653def02d90dc5d0257..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image34.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image35.png b/app/zh/information/blog/20230129-04/img/image35.png
deleted file mode 100644
index 54ad146d44862d57ba9504f8f78cc5928370efd4..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image35.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image36.png b/app/zh/information/blog/20230129-04/img/image36.png
deleted file mode 100644
index 5b273804ed5f9c8fd7ceea199c6fd3468cf54331..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image36.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image37.png b/app/zh/information/blog/20230129-04/img/image37.png
deleted file mode 100644
index 8bfd6731b8cd2028b3dc3aad56621c9360177ee8..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image37.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image38.png b/app/zh/information/blog/20230129-04/img/image38.png
deleted file mode 100644
index 2249916e83bccfd221374b2834e3eceb52c42303..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image38.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image39.png b/app/zh/information/blog/20230129-04/img/image39.png
deleted file mode 100644
index 291672f31e2b2dcb3df1258838f5a153158ddcf8..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image39.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image4.png b/app/zh/information/blog/20230129-04/img/image4.png
deleted file mode 100644
index e0c96f17ac1f044e152678f0082dbc0c0c1b1b7b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image4.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image40.png b/app/zh/information/blog/20230129-04/img/image40.png
deleted file mode 100644
index 38fe8886b253e9b4cce0c453bfa0e5f283712267..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image40.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image41.png b/app/zh/information/blog/20230129-04/img/image41.png
deleted file mode 100644
index 94f5ed1a73b40232adf5e75ecd44196afa42cd6d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image41.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image42.png b/app/zh/information/blog/20230129-04/img/image42.png
deleted file mode 100644
index e5f852c1228852f207e9b76b531bdacb8e5296a1..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image42.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image43.png b/app/zh/information/blog/20230129-04/img/image43.png
deleted file mode 100644
index 889f16a6a359978c6e2492cf1c373464e1751068..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image43.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image44.png b/app/zh/information/blog/20230129-04/img/image44.png
deleted file mode 100644
index 7fbfa253bf1ecdbcce593bfaacbfdcb8cbf61319..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image44.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image45.png b/app/zh/information/blog/20230129-04/img/image45.png
deleted file mode 100644
index 50475dd8019503ddedab405124685036d70d0b5d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image45.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image46.png b/app/zh/information/blog/20230129-04/img/image46.png
deleted file mode 100644
index ae304f4912d2332480c5787a8ea3111a112e926c..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image46.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image47.png b/app/zh/information/blog/20230129-04/img/image47.png
deleted file mode 100644
index 56ef3939af77ebf2e2728fd22ced0380d75c27a2..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image47.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image48.png b/app/zh/information/blog/20230129-04/img/image48.png
deleted file mode 100644
index ee9b28f27613d3afeb75d900b9a94eaed79ac74d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image48.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image49.png b/app/zh/information/blog/20230129-04/img/image49.png
deleted file mode 100644
index bde289a6880c877f5722d06d156ef6e0ee00905d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image49.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image5.png b/app/zh/information/blog/20230129-04/img/image5.png
deleted file mode 100644
index 4b5d24fc212a8fdbef212d10d18ed02b12a960b3..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image5.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image50.png b/app/zh/information/blog/20230129-04/img/image50.png
deleted file mode 100644
index 5fb54b5d2fdb24740152e5fff3b4c5846ecb4fd3..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image50.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image51.png b/app/zh/information/blog/20230129-04/img/image51.png
deleted file mode 100644
index 0dd59ca5f256789772eeb2b82bb06926b4a59db0..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image51.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image52.png b/app/zh/information/blog/20230129-04/img/image52.png
deleted file mode 100644
index 2d4bfefc15fffa4bc54ee8f829d888860c3dccb4..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image52.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image53.png b/app/zh/information/blog/20230129-04/img/image53.png
deleted file mode 100644
index 0b95100bdf6911630819d4f5834ddd69784f30dd..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image53.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image54.png b/app/zh/information/blog/20230129-04/img/image54.png
deleted file mode 100644
index 752010eb056065cf3b9961b2be529b2020576563..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image54.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image55.png b/app/zh/information/blog/20230129-04/img/image55.png
deleted file mode 100644
index 32933b9d13b1983de7c25585aa6b06a181f66d71..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image55.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image56.png b/app/zh/information/blog/20230129-04/img/image56.png
deleted file mode 100644
index e6ce34553f4390a6e3fb7b7f0fa070ee54dd876c..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image56.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image57.png b/app/zh/information/blog/20230129-04/img/image57.png
deleted file mode 100644
index 6a58df2e3c677f5da552c6e98542b949ff1ac1d2..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image57.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image58.png b/app/zh/information/blog/20230129-04/img/image58.png
deleted file mode 100644
index f3b3b79c8d4bee32b5b2fde8fb33f85462bdf6b9..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image58.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image59.png b/app/zh/information/blog/20230129-04/img/image59.png
deleted file mode 100644
index ca974b4a4b9cf66e25340e9857a8fff851b75368..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image59.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image6.png b/app/zh/information/blog/20230129-04/img/image6.png
deleted file mode 100644
index 229b86405b804628b20740b4f4ae7fc5e19883a2..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image6.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image60.png b/app/zh/information/blog/20230129-04/img/image60.png
deleted file mode 100644
index b5db44a2a3bf1cda7ae0e8cf703189974c5da1e0..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image60.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image61.png b/app/zh/information/blog/20230129-04/img/image61.png
deleted file mode 100644
index bbc1f576b5a7b6cafd9dba4c9beb666ecdd3b307..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image61.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image62.png b/app/zh/information/blog/20230129-04/img/image62.png
deleted file mode 100644
index c4249c5513df1e26e38a79c65d99eca75456b35d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image62.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image63.png b/app/zh/information/blog/20230129-04/img/image63.png
deleted file mode 100644
index d7f2abd719b7be3cb023700e7c42473b65c359cd..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image63.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image64.png b/app/zh/information/blog/20230129-04/img/image64.png
deleted file mode 100644
index fdf53cf79596c228df5ab651411a2b9a00660a33..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image64.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image7.png b/app/zh/information/blog/20230129-04/img/image7.png
deleted file mode 100644
index 22cb9221678504faf217665c0d8bca50aae078fc..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image7.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image8.png b/app/zh/information/blog/20230129-04/img/image8.png
deleted file mode 100644
index c09844a04d8a3c38526a4227d55539fd3421480d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image8.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/img/image9.png b/app/zh/information/blog/20230129-04/img/image9.png
deleted file mode 100644
index c426133b22c3dfa42ee222af2004d10d8bebc451..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-04/img/image9.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-04/index.md b/app/zh/information/blog/20230129-04/index.md
deleted file mode 100644
index 3f53afe0a91cc327ada3077a10aea68778415fc9..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/20230129-04/index.md
+++ /dev/null
@@ -1,1020 +0,0 @@
----
-title: 'openLooKeng对接Ranger详细设计文档'
-date: '2022-12-25'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', 'Ranger', '认证鉴权', '安全认证']
-archives: '2022-12'
-author: 'tushengxia'
-description: 'openLooKeng对接Ranger详细设计文档'
----
-
-Ranger架构
-===========
-
-Ranger总体架构
---------------
-
-
-
-1. ### Ranger UserSync
-
- 定期从LDAP/Unix/File中加载用户,上报给RangerAdmin。
-
-
-
-### Ranger Admin
-
-1. 以RESTFUL形式提供策略的增删改查接口,同时内置一个Web管理页面。
-
-2. 与Ranger Plugin进行交互,将权限同步至Ranger Plugin所在节点
-
-
-
-1. ### Plugin
-
- 嵌入到各系统执行流程中,定期从RangerAdmin拉取策略,根据策略执行访问决策树,并且记录访问审计
-
-
-
-系统插件
---------
-
-系统插件主要负责三件事:
-
-1. 定期从RangerAdmin拉取策略,默认30秒
-
-2. 根据策略执行访问决策树
-
-3. 实时记录访问审计
-
- 以上执行逻辑是通用的,可由所有系统插件引用,因此剩下的问题是如何把这些逻辑嵌入到各个系统的访问决策流程中去。实现可扩展接口:多数的系统在实现时都有考虑功能扩展性的问题,一般会为核心的模块暴露出可扩展的接口,访问控制模块也不例外。Ranger通过实现访问控制接口,将自己的逻辑嵌入各个系统。
-
-### 系统插件拉取策略
-
-主动到RangerAdmin拉取策略,而非RangerAdmin把策略下发给各个插件,策略有变化,拉取新的策略更新内存中鉴权引擎,同时保存一份备份文件在本地,RangerAdmin挂掉后,组件也重启时,可以使用本地的备份继续鉴权。删除RangerAdmin上面的service,会使插件鉴权不可用。
-
-### 系统插件性能考虑
-
-1. 关闭策略中的audit审计日志
-
-2. 插件鉴权引擎定时对策略排序,优先匹配高命中的策略
-
-openLookeng Plugin类图
-======================
-
-RangerSystemAccessControlFactory
---------------------------------
-
-
-
-RangerSystemAccessControl
--------------------------
-
-
-
-RangerConfig
-------------
-
-
-
-OpenLooKengRangerPlugin
------------------------
-
-
-
-OpenLooKengAccessType
----------------------
-
-
-
-RangerOpenLooKengAccessRequest
-------------------------------
-
-
-
-RangerOpenLooKengResource
--------------------------
-
-
-
-RangerSystemAccessControl
--------------------------
-
-
-
-AccessControlManager
----------------------------------------------------------------------------------------
-
-openLooKeng 鉴权时序图
-======================
-
-加载系统访问权限控制组件
-------------------------
-
-
-
-
-
-检查主体是否被允许为指定的用户
-------------------------------
-
-
-
-检查identity是否允许模拟指定的用户
-----------------------------------
-
-
-
-取出identity可见的Catalog列表
------------------------------
-
-
-
-检查identity是否允许show catalogs
----------------------------------
-
-
-
-检查identity是否允许访问catalogs
---------------------------------
-
-
-
-检查identity是否允许更新指定的Table
------------------------------------
-
-
-
-检查identity是否允许在Catalog中创建Schema
------------------------------------------
-
-
-
-检查identity是否允许删除Catalog中的Schema
------------------------------------------
-
-
-
-检查identity是否允许重命名Schema
---------------------------------
-
-
-
-检查identity是否允许在Catalog中执行show schemas
------------------------------------------------
-
-
-
-取出identity可见的Schema列表
-----------------------------
-
-
-
-检查identity是否允许创建Table
------------------------------
-
-
-
-检查identity是否允许删除Table
------------------------------
-
-
-
-检查identity是否允许重命名Table
--------------------------------
-
-
-
-检查identity是否允许注释Table
------------------------------
-
-
-
-检查identity是否允许查看Table的元数据
--------------------------------------
-
-
-
-取出identity可见的Table列表
----------------------------
-
-
-
-检查identity是否允许查看Table的列信息
--------------------------------------
-
-
-
-取出identity可见的column列表
-----------------------------
-
-
-
-检查identity是否允许增加列
---------------------------
-
-
-
-检查identity是否允许删除列
---------------------------
-
-
-
-检查identity是否允许重命名列
-----------------------------
-
-
-
-检查identity是否允许插入数据到表中
-----------------------------------
-
-
-
-检查identity是否允许删除数据
-----------------------------
-
-
-
-检查identity是否允许创建Index
------------------------------
-
-
-
-检查identity是否允许删除Index
------------------------------
-
-
-
-检查identity是否允许重命名Index
--------------------------------
-
-
-
-检查identity是否允许更新Index
------------------------------
-
-
-
-检查identity是否允许查看Index
------------------------------
-
-
-
-检查identity是否允许创建View
-----------------------------
-
-
-
-检查identity是否允许删除View
-----------------------------
-
-
-
-检查identity是否允许在创建View时选择columns
--------------------------------------------
-
-
-
-检查identity是否允许将表授权给其它用户
---------------------------------------
-
-
-
-检查identity是否允许回收权限
-----------------------------
-
-
-
-检查identity是否允许设置系统属性
---------------------------------
-
-
-
-检查identity是否允许为Catalog设置属性
--------------------------------------
-
-
-
-检查identity是否允许查询Columns
--------------------------------
-
-
-
-检查identity是否允许创建角色
-----------------------------
-
-
-
-检查identity是否允许删除角色
-----------------------------
-
-
-
-检查identity是否允许授权给角色
-------------------------------
-
-
-
-检查identity是否允许回收角色权限
---------------------------------
-
-
-
-检查identity是否允许查看角色列表
---------------------------------
-
-
-
-检查identity是否允许查看当前Catalog上角色列表
----------------------------------------------
-
-
-
-检查identity是否允许查看当前角色的权限
---------------------------------------
-
-
-
-获取表中的列过滤器
-------------------
-
-
-
-获取与给定表、列和标识相关联的列掩码
-------------------------------------
-
-
-
-openLooKeng Plugin源码解析
-===========================
-
-openLooKengRangerPlugin
------------------------
-
-openLooKengRangerPlugin实现Plugin接口,RangerSystemAccessControlFactory获取系统访问控制工厂
-
-public class openLooKengRangerPlugin\
-implements Plugin\
-{\
-@Override\
-public Iterable<SystemAccessControlFactory>
-getSystemAccessControlFactories()\
-{\
-ArrayList<SystemAccessControlFactory> list = new
-ArrayList<>();\
-SystemAccessControlFactory factory = new
-RangerSystemAccessControlFactory();\
-list.add(factory);\
-return list;\
-}\
-}
-
-SystemAccessControlFactory
---------------------------
-
-SystemAccessControlFactory:系统访问控制工厂,
-用于创建SystemAccessControl,是openLooKeng提供的顶层工程类,需要在插件中实现
-
-public interface SystemAccessControlFactory\
-{\
-String getName();\
-\
-SystemAccessControl create(Map<String, String> config);\
-}
-
-RangerSystemAccessControlFactory
---------------------------------
-
-RangerSystemAccessControlFactory实现了SystemAccessControlFactory,同于创建RangerSystemAccessControl
-
-RangerSystemAccessControlFactory的NAME属性和access-control.properties配置文件中access-control.name=ranger匹配
-
-public class RangerSystemAccessControlFactory\
-implements SystemAccessControlFactory {\
-private static final String *NAME* = "ranger";\
-\
-@Override\
-public String getName() {\
-return *NAME*;\
-}
-
-### SystemAccessControlFactory\#create
-
-SystemAccessControlFactory的create方法用于创建RangerSystemAccessControl,采用反射创建
-
-@Override\
-public SystemAccessControl create(Map<String, String> config) {\
-*requireNonNull*(config, "config is null");\
-\
-try {\
-Bootstrap app = new Bootstrap(\
-binder -> {\
-*configBinder*(binder).bindConfig(RangerConfig.class);\
-binder.bind(RangerSystemAccessControl.class).in(Scopes.*SINGLETON*);\
-}\
-);\
-\
-Injector injector = app\
-.strictConfig()\
-.doNotInitializeLogging()\
-.setRequiredConfigurationProperties(config)\
-.initialize();\
-\
-return injector.getInstance(RangerSystemAccessControl.class);\
-} catch (Exception e) {\
-*throwIfUnchecked*(e);\
-throw new RuntimeException(e);\
-}\
-}
-
-SystemAccessControl
--------------------
-
-SystemAccessControlFactory的 create方法返回
-SystemAccessControl,里面定义了openLooKeng权限的 plugin必须实现的方法
-
-
-
-RangerSystemAccessControl
--------------------------
-
-RangerSystemAccessControl实现了SystemAccessControl接口,并对SystemAccessControl
-的方法做了实现
-
-public class RangerSystemAccessControl\
-implements SystemAccessControl {\
-private static final String *RANGER\_PLUGIN\_TYPE* = "openLooKeng";\
-private static final String
-*RANGER\_PRESTO\_AUTHORIZER\_IMPL\_CLASSNAME* =
-"org.apache.ranger.authorization.openLooKeng.authorizer.RangerSystemAccessControl";\
-\
-final private RangerPluginClassLoader rangerPluginClassLoader;\
-final private SystemAccessControl systemAccessControlImpl;
-
-[]{#_Toc1565453014
-.anchor}注意:RangerSystemAccessControl是shim模块下的类
-
-
-
-### RangerSystemAccessControl\#RangerSystemAccessControl
-
-@Inject\
-public RangerSystemAccessControl(RangerConfig config) {\
-try {\
-rangerPluginClassLoader =
-RangerPluginClassLoader.*getInstance*(*RANGER\_PLUGIN\_TYPE*,
-this.getClass());\
-\
-//获取另一个RangerSystemAccessControl类对象,具体鉴权的方法在另一个RangerSystemAccessControl中实现\
-@SuppressWarnings("unchecked")\
-Class<SystemAccessControl> cls =
-(Class<SystemAccessControl>)
-Class.*forName*(*RANGER\_PRESTO\_AUTHORIZER\_IMPL\_CLASSNAME*, true,
-rangerPluginClassLoader);\
-\
-activatePluginClassLoader();\
-\
-Map<String, String> configMap = new HashMap<>();\
-\
-if (config.getKeytab() != null && config.getPrincipal() != null) {\
-//用于针对Ranger进行身份验证的Keytab\
-configMap.put("ranger.keytab", config.getKeytab());\
-//使用keytab对Ranger进行身份验证的主体\
-configMap.put("ranger.principal", config.getPrincipal());\
-}\
-\
-//使用Hadoop用户组信息而不是openLooKeng组\
-configMap.put("ranger.use\_ugi",
-Boolean.*toString*(config.isUseUgi()));\
-\
-if (config.getHadoopConfigPath() != null) {\
-//hadoop配置的路径。在类路径中默认为openLooKeng-ranger-site.xml\
-configMap.put("ranger.hadoop\_config", config.getHadoopConfigPath());\
-}\
-\
-//通过反射创建RangerSystemAccessControl对象,将配置文件中的参数传递给RangerSystemAccessControl的构造函数\
-systemAccessControlImpl =
-cls.getDeclaredConstructor(Map.class).newInstance(configMap);\
-} catch (Exception e) {\
-throw new RuntimeException(e);\
-} finally {\
-deactivatePluginClassLoader();\
-}\
-}
-
-RangerSystemAccessControl
--------------------------
-
-RangerSystemAccessControl实现了SystemAccessControl接口,并对SystemAccessControl
-的方法做了实现
-
-public class RangerSystemAccessControl\
-implements SystemAccessControl {\
-private static Logger *LOG* =
-LoggerFactory.*getLogger*(RangerSystemAccessControl.class);\
-\
-final public static String *RANGER\_CONFIG\_KEYTAB* = "ranger.keytab";\
-final public static String *RANGER\_CONFIG\_PRINCIPAL* =
-"ranger.principal";\
-final public static String *RANGER\_CONFIG\_USE\_UGI* =
-"ranger.use\_ugi";\
-final public static String *RANGER\_CONFIG\_HADOOP\_CONFIG* =
-"ranger.hadoop\_config";\
-final public static String *RANGER\_openLooKeng\_DEFAULT\_HADOOP\_CONF*
-= "openLooKeng-ranger-site.xml";\
-final public static String *RANGER\_openLooKeng\_SERVICETYPE* =
-"openLooKeng";\
-final public static String *RANGER\_openLooKeng\_APPID* =
-"openLooKeng";\
-\
-final private RangerBasePlugin rangerPlugin;\
-\
-private boolean useUgi = false;
-
-### RangerSystemAccessControl\#RangerSystemAccessControl
-
-1. 在RangerSystemAccessControl的构造函数中会根据是否配置了ranger.keytab和ranger.principal来进行用户登录认证
-
-2. 创建并初始化RangerBasePlugin
-
-public RangerSystemAccessControl(Map<String, String> config) {\
-super();\
-\
-Configuration hadoopConf = new Configuration();\
-if (config.get(*RANGER\_CONFIG\_HADOOP\_CONFIG*) != null) {\
-URL url =
-hadoopConf.getResource(config.get(*RANGER\_CONFIG\_HADOOP\_CONFIG*));\
-if (url == null) {\
-*LOG*.warn("Hadoop config " +
-config.get(*RANGER\_CONFIG\_HADOOP\_CONFIG*) + " not found");\
-} else {\
-hadoopConf.addResource(url);\
-}\
-} else {\
-URL url =
-hadoopConf.getResource(*RANGER\_openLooKeng\_DEFAULT\_HADOOP\_CONF*);\
-if (*LOG*.isDebugEnabled()) {\
-*LOG*.debug("Trying to load Hadoop config from " + url + " (can be
-null)");\
-}\
-if (url != null) {\
-hadoopConf.addResource(url);\
-}\
-}\
-UserGroupInformation.*setConfiguration*(hadoopConf);\
-\
-if (config.get(*RANGER\_CONFIG\_KEYTAB*) != null &&
-config.get(*RANGER\_CONFIG\_PRINCIPAL*) != null) {\
-String keytab = config.get(*RANGER\_CONFIG\_KEYTAB*);\
-String principal = config.get(*RANGER\_CONFIG\_PRINCIPAL*);\
-\
-*LOG*.info("Performing kerberos login with principal " + principal + "
-and keytab " + keytab);\
-\
-try {\
-//使用keytab进行用户登录\
-UserGroupInformation.*loginUserFromKeytab*(principal, keytab);\
-} catch (IOException ioe) {\
-*LOG*.error("Kerberos login failed", ioe);\
-throw new RuntimeException(ioe);\
-}\
-}\
-\
-if (config.getOrDefault(*RANGER\_CONFIG\_USE\_UGI*,
-"false").equalsIgnoreCase("true")) {\
-useUgi = true;\
-}\
-\
-//创建并初始化RangerBasePlugin\
-rangerPlugin = new RangerBasePlugin(*RANGER\_openLooKeng\_SERVICETYPE*,
-*RANGER\_openLooKeng\_APPID*);\
-rangerPlugin.init();\
-rangerPlugin.setResultProcessor(new RangerDefaultAuditHandler());\
-}
-
-RangerBasePlugin
-----------------
-
-RangerBasePlugin 的构造方法,如 serviceType = “openLooKeng”, appId =
-“openLooKeng”
-
-public RangerBasePlugin(String serviceType, String appId) {\
-this(new RangerPluginConfig(serviceType, (String)null, appId,
-(String)null, (String)null, (RangerPolicyEngineOptions)null));\
-}
-
-RangerPluginConfig
-------------------
-
-serviceType = “openLooKeng”, appId = “openLooKeng”
-
-public RangerPluginConfig(String serviceType, String serviceName, String
-appId, String clusterName, String clusterType, RangerPolicyEngineOptions
-policyEngineOptions) {\
-this.addResourcesForServiceType(serviceType);\
-this.addResourcesForServiceName(this.serviceType, this.serviceName);\
-if (policyEngineOptions == null) {\
-policyEngineOptions = new RangerPolicyEngineOptions();\
-policyEngineOptions.configureForPlugin(this, this.propertyPrefix);\
-}\
-this.policyEngineOptions = policyEngineOptions;\
-*LOG*.info(policyEngineOptions);\
-}
-
-### RangerPluginConfig\#addResourcesForServiceType
-
-serviceType = “openLooKeng”
-
-private void addResourcesForServiceType(String serviceType) {\
-String auditCfg = "ranger-" + serviceType + "-audit.xml";\
-String securityCfg = "ranger-" + serviceType + "-security.xml";\
-String sslCfg = "ranger-" + serviceType + "-policymgr-ssl.xml";\
-if (!this.addResourceIfReadable(auditCfg)) {\
-this.addAuditResource(serviceType);\
-}\
-if (!this.addResourceIfReadable(securityCfg)) {\
-this.addSecurityResource(serviceType);\
-}\
-if (!this.addResourceIfReadable(sslCfg)) {\
-this.addSslConfigResource(serviceType);\
-}\
-}
-
-### RangerPluginConfig\#addResourcesForServiceName
-
-serviceType = “openLooKeng”, serviceName = “openLooKeng”,
-
-private void addResourcesForServiceName(String serviceType, String
-serviceName) {\
-if (StringUtils.*isNotBlank*(serviceType) &&
-StringUtils.*isNotBlank*(serviceName)) {\
-String serviceAuditCfg = "ranger-" + serviceType + "-" + serviceName +
-"-audit.xml";\
-String serviceSecurityCfg = "ranger-" + serviceType + "-" + serviceName
-+ "-security.xml";\
-String serviceSslCfg = "ranger-" + serviceType + "-" + serviceName +
-"-policymgr-ssl.xml";\
-this.addResourceIfReadable(serviceAuditCfg);\
-this.addResourceIfReadable(serviceSecurityCfg);\
-this.addResourceIfReadable(serviceSslCfg);\
-}\
-}
-
-### RangerBasePlugin构造函数
-
-public RangerBasePlugin(RangerPluginConfig pluginConfig) {\
-this.logHistoryList = new Hashtable();\
-this.logInterval = 30000;\
-this.accessTrigger = new DownloadTrigger();\
-this.pluginConfig = pluginConfig;\
-this.pluginContext = new RangerPluginContext(pluginConfig);\
-Set<String> superUsers =
-this.toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() +
-".super.users"));\
-Set<String> superGroups =
-this.toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() +
-".super.groups"));\
-Set<String> auditExcludeUsers =
-this.toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() +
-".audit.exclude.users"));\
-Set<String> auditExcludeGroups =
-this.toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() +
-".audit.exclude.groups"));\
-Set<String> auditExcludeRoles =
-this.toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() +
-".audit.exclude.roles"));\
-this.setSuperUsersAndGroups(superUsers, superGroups);\
-this.setAuditExcludedUsersGroupsRoles(auditExcludeUsers,
-auditExcludeGroups, auditExcludeRoles);\
-RangerScriptExecutionContext.*init*(pluginConfig);\
-this.chainedPlugins = this.initChainedPlugins();\
-}
-
-RangerBasePlugin\#init
-----------------------
-
-RangerBasePlugin 的 init 里构建PolicyRefresher,PolicyRefresher
-是一个线程,
-
-public void init() {\
-this.cleanup();\
-AuditProviderFactory providerFactory =
-AuditProviderFactory.*getInstance*();\
-if (!providerFactory.isInitDone()) {\
-if (this.pluginConfig.getProperties() != null) {\
-providerFactory.init(this.pluginConfig.getProperties(),
-this.getAppId());\
-} else {\
-*LOG*.error("Audit subsystem is not initialized correctly. Please check
-audit configuration. ");\
-*LOG*.error("No authorization audits will be generated. ");\
-}\
-}\
-\
-this.refresher = new PolicyRefresher(this);\
-*LOG*.info("Created PolicyRefresher Thread(" + this.refresher.getName()
-+ ")");\
-this.refresher.setDaemon(true);\
-this.refresher.startRefresher();\
-Iterator var2 = this.chainedPlugins.iterator();\
-\
-while(var2.hasNext()) {\
-RangerChainedPlugin chainedPlugin = (RangerChainedPlugin)var2.next();\
-chainedPlugin.init();\
-}\
-\
-}
-
-RangerBasePlugin\#grantAccess
------------------------------
-
-public void grantAccess(GrantRevokeRequest request,
-RangerAccessResultProcessor resultProcessor) throws Exception {\
-if (*LOG*.isDebugEnabled()) {\
-*LOG*.debug("==> RangerBasePlugin.grantAccess(" + request + ")");\
-}\
-\
-boolean isSuccess = false;\
-\
-try {\
-RangerPolicyEngine policyEngine = this.policyEngine;\
-if (policyEngine != null) {\
-request.setZoneName(policyEngine.getUniquelyMatchedZoneName(request));\
-}\
-\
-this.getAdminClient().grantAccess(request);\
-isSuccess = true;\
-} finally {\
-this.auditGrantRevoke(request, "grant", isSuccess, resultProcessor);\
-}\
-\
-if (*LOG*.isDebugEnabled()) {\
-*LOG*.debug("<== RangerBasePlugin.grantAccess(" + request + ")");\
-}\
-\
-}
-
-PolicyRefresher 的构造方法
---------------------------
-
-public PolicyRefresher(RangerBasePlugin plugIn) {\
-if (*LOG*.isDebugEnabled()) {\
-*LOG*.debug("==> PolicyRefresher(serviceName=" +
-plugIn.getServiceName() + ").PolicyRefresher()");\
-}\
-\
-RangerPluginConfig pluginConfig = plugIn.getConfig();\
-String propertyPrefix = pluginConfig.getPropertyPrefix();\
-this.plugIn = plugIn;\
-this.serviceType = plugIn.getServiceType();\
-this.serviceName = plugIn.getServiceName();\
-this.cacheDir = pluginConfig.get(propertyPrefix + ".policy.cache.dir");\
-String appId = StringUtils.*isEmpty*(plugIn.getAppId()) ?
-this.serviceType : plugIn.getAppId();\
-String cacheFilename = String.*format*("%s\_%s.json", appId,
-this.serviceName);\
-cacheFilename = cacheFilename.replace(File.*separatorChar*, '\_');\
-cacheFilename = cacheFilename.replace(File.*pathSeparatorChar*, '\_');\
-this.cacheFileName = cacheFilename;\
-Gson gson = null;\
-\
-try {\
-gson = (new
-GsonBuilder()).setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create();\
-} catch (Throwable var8) {\
-*LOG*.fatal("PolicyRefresher(): failed to create GsonBuilder object",
-var8);\
-}\
-\
-this.gson = gson;\
-this.disableCacheIfServiceNotFound =
-pluginConfig.getBoolean(propertyPrefix +
-".disable.cache.if.servicenotfound", true);\
-this.rangerAdmin = RangerBasePlugin.*createAdminClient*(pluginConfig);\
-this.rolesProvider = new RangerRolesProvider(this.getServiceType(),
-appId, this.getServiceName(), this.rangerAdmin, this.cacheDir,
-pluginConfig);\
-this.pollingIntervalMs = pluginConfig.getLong(propertyPrefix +
-".policy.pollIntervalMs", 30000L);\
-this.setName("PolicyRefresher(serviceName=" + this.serviceName + ")-" +
-this.getId());\
-if (*LOG*.isDebugEnabled()) {\
-*LOG*.debug("<== PolicyRefresher(serviceName=" + this.serviceName +
-").PolicyRefresher()");\
-}\
-\
-}
-
-RangerPluginContext\#createAdminClient
---------------------------------------
-
-public static RangerAdminClient createAdminClient(RangerPluginConfig
-pluginConfig) {\
-if (*LOG*.isDebugEnabled()) {\
-*LOG*.debug("==> RangerBasePlugin.createAdminClient(" +
-pluginConfig.getServiceName() + ", " + pluginConfig.getAppId() + ", " +
-pluginConfig.getPropertyPrefix() + ")");\
-}\
-\
-RangerAdminClient ret = null;\
-String propertyName = pluginConfig.getPropertyPrefix() +
-".policy.source.impl";\
-String policySourceImpl = pluginConfig.get(propertyName);\
-if (StringUtils.*isEmpty*(policySourceImpl)) {\
-if (*LOG*.isDebugEnabled()) {\
-*LOG*.debug(String.*format*("Value for property\[%s\] was null or empty.
-Unexpected! Will use policy source of type\[%s\]", propertyName,
-RangerAdminRESTClient.class.getName()));\
-}\
-} else {\
-if (*LOG*.isDebugEnabled()) {\
-*LOG*.debug(String.*format*("Value for property\[%s\] was \[%s\].",
-propertyName, policySourceImpl));\
-}\
-\
-try {\
-Class<RangerAdminClient> adminClass =
-Class.*forName*(policySourceImpl);\
-ret = (RangerAdminClient)adminClass.newInstance();\
-} catch (Exception var5) {\
-*LOG*.error("failed to instantiate policy source of type '" +
-policySourceImpl + "'. Will use policy source of type '" +
-RangerAdminRESTClient.class.getName() + "'", var5);\
-}\
-}\
-\
-if (ret == null) {\
-ret = new RangerAdminRESTClient();\
-}\
-\
-((RangerAdminClient)ret).init(pluginConfig.getServiceName(),
-pluginConfig.getAppId(), pluginConfig.getPropertyPrefix(),
-pluginConfig);\
-if (*LOG*.isDebugEnabled()) {\
-*LOG*.debug("<== RangerBasePlugin.createAdminClient(" +
-pluginConfig.getServiceName() + ", " + pluginConfig.getAppId() + ", " +
-pluginConfig.getPropertyPrefix() + "): policySourceImpl=" +
-policySourceImpl + ", client=" + ret);\
-}\
-\
-return (RangerAdminClient)ret;\
-}
-
-RangerAdminRESTClient\#init
----------------------------
-
-在此方法里,创建了和 Ranger Server 的 HTTP 连接。
-
-private void init(String url, String sslConfigFileName, int
-restClientConnTimeOutMs, int restClientReadTimeOutMs, Configuration
-config) {\
-if (*LOG*.isDebugEnabled()) {\
-*LOG*.debug("==> RangerAdminRESTClient.init(" + url + ", " +
-sslConfigFileName + ")");\
-}\
-\
-this.restClient = new RangerRESTClient(url, sslConfigFileName, config);\
-this.restClient.setRestClientConnTimeOutMs(restClientConnTimeOutMs);\
-this.restClient.setRestClientReadTimeOutMs(restClientReadTimeOutMs);\
-if (*LOG*.isDebugEnabled()) {\
-*LOG*.debug("<== RangerAdminRESTClient.init(" + url + ", " +
-sslConfigFileName + ")");\
-}\
-}
-
-PolicyRefresher\#run
---------------------
-
-先从 policyDownloadQueue 去出一个元素,执行一次 loadRoles() 和
-loadPolicy()
-
-public void run() {\
-if (*LOG*.isDebugEnabled()) {\
-*LOG*.debug("==> PolicyRefresher(serviceName=" + this.serviceName +
-").run()");\
-}\
-\
-while(true) {\
-DownloadTrigger trigger = null;\
-try {\
-trigger = (DownloadTrigger)this.policyDownloadQueue.take();\
-this.loadRoles();\
-this.loadPolicy();\
-continue;\
-} catch (InterruptedException var6) {\
-*LOG*.debug("PolicyRefresher(serviceName=" + this.serviceName +
-").run(): interrupted! Exiting thread", var6);\
-} finally {\
-if (trigger != null) {\
-trigger.signalCompletion();\
-}\
-}\
-if (*LOG*.isDebugEnabled()) {\
-*LOG*.debug("<== PolicyRefresher(serviceName=" + this.serviceName +
-").run()");\
-}\
-return;\
-}\
-}
-
-PolicyRefresher\#startRefresher
--------------------------------
-
-在此方法里,另启动一个 Timer, 定期(默认 30 秒)向 policyDownloadQueue
-里放一个 Object。也代表 PolicyRefresher 线程定期执行一次 loadRoles() 和
-loadPolicy()。
-
-public void startRefresher() {\
-this.loadRoles();\
-this.loadPolicy();\
-super.start();\
-this.policyDownloadTimer = new Timer("policyDownloadTimer", true);\
-\
-try {\
-this.policyDownloadTimer.schedule(new
-DownloaderTask(this.policyDownloadQueue), this.pollingIntervalMs,
-this.pollingIntervalMs);\
-if (*LOG*.isDebugEnabled()) {\
-*LOG*.debug("Scheduled policyDownloadRefresher to download policies
-every " + this.pollingIntervalMs + " milliseconds");\
-}\
-} catch (IllegalStateException var2) {\
-*LOG*.error("Error scheduling policyDownloadTimer:", var2);\
-*LOG*.error("\*\*\* Policies will NOT be downloaded every " +
-this.pollingIntervalMs + " milliseconds \*\*\*");\
-this.policyDownloadTimer = null;\
-}\
-\
-}
-
-PolicyRefresher\#loadRoles
---------------------------
-
-private void loadRoles() {\
-this.rolesProvider.loadUserGroupRoles(this.plugIn);\
-}
-
-PolicyRefresher\#loadUserGroupRoles
------------------------------------
-
-public void loadUserGroupRoles(RangerBasePlugin plugIn) {\
-RangerRoles roles = this.loadUserGroupRolesFromAdmin();
-
-plugIn.setRoles(roles);\
-}
-
-RangerRolesProvider\#loadUserGroupRolesFromAdmin
-------------------------------------------------
-
-private RangerRoles loadUserGroupRolesFromAdmin() throws
-RangerServiceNotFoundException {\
-roles = this.rangerAdmin.getRolesIfUpdated(this.lastKnownRoleVersion,
-this.lastActivationTimeInMillis);\
-return roles;\
-}
-
-RangerRolesProvider\#loadPolicy
--------------------------------
-
-从Ranger Admin加载策略,如果策略加载失败,加载本地缓存。
-
-private void loadPolicy() {\
-try {\
-ServicePolicies svcPolicies = this.loadPolicyfromPolicyAdmin();\
-if (svcPolicies == null && !this.policiesSetInPlugin) {\
-svcPolicies = this.loadFromCache();\
-}\
-if (svcPolicies != null) {\
-this.plugIn.setPolicies(svcPolicies);\
-this.policiesSetInPlugin = true;\
-this.setLastActivationTimeInMillis(System.*currentTimeMillis*());\
-this.lastKnownVersion = svcPolicies.getPolicyVersion() != null ?
-svcPolicies.getPolicyVersion() : -1L;\
-} else if (!this.policiesSetInPlugin && !this.serviceDefSetInPlugin) {\
-this.plugIn.setPolicies((ServicePolicies)null);\
-this.serviceDefSetInPlugin = true;\
-}\
-} catch (RangerServiceNotFoundException var7) {\
-if (this.disableCacheIfServiceNotFound) {\
-this.disableCache();\
-this.plugIn.setPolicies((ServicePolicies)null);\
-this.setLastActivationTimeInMillis(System.*currentTimeMillis*());\
-this.lastKnownVersion = -1L;\
-this.serviceDefSetInPlugin = true;\
-}\
-} catch (Exception var8) {\
-*LOG*.error("Encountered unexpected exception, ignoring..", var8);\
-}\
-}
-
-修订记录 {#修订记录 .154}
-========
-
- ------------ ----------------
- 发布日期 修订记录
- 2022-09-27 第一次正式发布
- ------------ ----------------
-
-相关视频:[openLooKeng内核认证与鉴权流程](https://space.bilibili.com/627629884/channel/collectiondetail?sid=1061144)
-
diff --git a/app/zh/information/blog/20230129-05/img/image10.png b/app/zh/information/blog/20230129-05/img/image10.png
deleted file mode 100644
index 39c1c2cb31f0f07e7d95494d342cd39058432c2f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image10.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image11.png b/app/zh/information/blog/20230129-05/img/image11.png
deleted file mode 100644
index 46ce81803acfe8f69454302ff351e9985756f505..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image11.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image12.png b/app/zh/information/blog/20230129-05/img/image12.png
deleted file mode 100644
index 295a86952958edf917d87b43a61d262336ca9427..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image12.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image13.png b/app/zh/information/blog/20230129-05/img/image13.png
deleted file mode 100644
index a403d318471197d225244cb686f9c46e778f8d86..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image13.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image14.png b/app/zh/information/blog/20230129-05/img/image14.png
deleted file mode 100644
index 81ff017e9c1dea0013d4784c9df4ba496f80355d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image14.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image15.png b/app/zh/information/blog/20230129-05/img/image15.png
deleted file mode 100644
index 71552b2af1097c5477971064a36499019e894a15..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image15.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image16.png b/app/zh/information/blog/20230129-05/img/image16.png
deleted file mode 100644
index eece9c2bddf996cd7d4d4e35005668ad85ccc9ea..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image16.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image17.png b/app/zh/information/blog/20230129-05/img/image17.png
deleted file mode 100644
index 68d1847d51ae082146347a77f46872880b6a9889..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image17.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image18.png b/app/zh/information/blog/20230129-05/img/image18.png
deleted file mode 100644
index 661cdac51deb98118240579a1828c14dd759078d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image18.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image19.png b/app/zh/information/blog/20230129-05/img/image19.png
deleted file mode 100644
index 521ef0a735b2447d02cc236c4c6d7d28c86fbcc7..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image19.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image20.png b/app/zh/information/blog/20230129-05/img/image20.png
deleted file mode 100644
index df053169c7e99561518fa9a3c0e88f614d3c4e32..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image20.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image21.png b/app/zh/information/blog/20230129-05/img/image21.png
deleted file mode 100644
index c74e7082fedf167f2886182960ab2e686bccac0f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image21.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image22.png b/app/zh/information/blog/20230129-05/img/image22.png
deleted file mode 100644
index cd46b720751eb25b15ea4451e8f0bbe12ec168fd..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image22.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image23.png b/app/zh/information/blog/20230129-05/img/image23.png
deleted file mode 100644
index 0a78dc7ba0d7ec4a7f8268317e26d1e7b7f0823d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image23.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image24.png b/app/zh/information/blog/20230129-05/img/image24.png
deleted file mode 100644
index 85d65bb6fd52747277a2e04c0556fdea0075005a..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image24.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image25.png b/app/zh/information/blog/20230129-05/img/image25.png
deleted file mode 100644
index eb9d72c683aa4b9ba9efa52bced0ff6bae3c0962..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image25.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image26.png b/app/zh/information/blog/20230129-05/img/image26.png
deleted file mode 100644
index 9c2633b1246506ef6bbf75bff2c88a501ea23f52..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image26.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image27.png b/app/zh/information/blog/20230129-05/img/image27.png
deleted file mode 100644
index 28d9aca010d07099445591c4c1a959df46d043aa..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image27.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image28.png b/app/zh/information/blog/20230129-05/img/image28.png
deleted file mode 100644
index 9c92a631399b52ac319e81cc8e6b5b706d0f07a5..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image28.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image29.png b/app/zh/information/blog/20230129-05/img/image29.png
deleted file mode 100644
index 742a3d2dce64ae6d4d6a5f5c99f9ab35caf8455b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image29.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image3.png b/app/zh/information/blog/20230129-05/img/image3.png
deleted file mode 100644
index 341e3dff5f3c2ea24343a97bc3a82d9c6abed798..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image3.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image30.png b/app/zh/information/blog/20230129-05/img/image30.png
deleted file mode 100644
index 0849f820496c9abda9730ff3cbb06c3d911c1358..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image30.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image31.png b/app/zh/information/blog/20230129-05/img/image31.png
deleted file mode 100644
index db2dcfb198cacafac80d6f315ff3744c240e8f6f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image31.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image32.png b/app/zh/information/blog/20230129-05/img/image32.png
deleted file mode 100644
index c69bf6b067ba4f94fba51c1515598f05a9bc7360..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image32.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image33.png b/app/zh/information/blog/20230129-05/img/image33.png
deleted file mode 100644
index c02292c6cab91bbfca9f63b025f77870f893a5a9..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image33.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image34.png b/app/zh/information/blog/20230129-05/img/image34.png
deleted file mode 100644
index cf0e99aa51adfa04fe496e862c4e5db57ed24de1..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image34.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image35.png b/app/zh/information/blog/20230129-05/img/image35.png
deleted file mode 100644
index 4514adc7e7c3e54759a037fde03e9620deafebeb..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image35.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image36.png b/app/zh/information/blog/20230129-05/img/image36.png
deleted file mode 100644
index 1fd54345f843e4c4485a765214eb8edfe2ab6cb8..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image36.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image37.png b/app/zh/information/blog/20230129-05/img/image37.png
deleted file mode 100644
index dcea94ae8be790ae624a7904b1ab8eea2dabf30f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image37.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image38.png b/app/zh/information/blog/20230129-05/img/image38.png
deleted file mode 100644
index c9c5a9ab85d4ec45e9af17fdef9b4a03e084ca90..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image38.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image39.png b/app/zh/information/blog/20230129-05/img/image39.png
deleted file mode 100644
index cd030d4dd74e4775568c69f0bbc26f2c3dcfc931..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image39.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image4.png b/app/zh/information/blog/20230129-05/img/image4.png
deleted file mode 100644
index e0c96f17ac1f044e152678f0082dbc0c0c1b1b7b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image4.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image40.png b/app/zh/information/blog/20230129-05/img/image40.png
deleted file mode 100644
index 80e211ba3abd7f45a9cf794aa05c0dce3f720d7d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image40.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image41.png b/app/zh/information/blog/20230129-05/img/image41.png
deleted file mode 100644
index 3e251a4faf2e8219009ba8905504b4a001c61b7d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image41.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image42.png b/app/zh/information/blog/20230129-05/img/image42.png
deleted file mode 100644
index 30788102b2d6667c08795e6a33749657a7672f52..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image42.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image43.png b/app/zh/information/blog/20230129-05/img/image43.png
deleted file mode 100644
index 966676a27ce1cfd0ca899fcc8eefaabb29dfc14b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image43.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image44.png b/app/zh/information/blog/20230129-05/img/image44.png
deleted file mode 100644
index 06b3d56ab84ccb862eb16ae556af0d3f4a17688f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image44.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image45.png b/app/zh/information/blog/20230129-05/img/image45.png
deleted file mode 100644
index 3dc6b58c4ecc0ad38ced1e0111c6a65c6f125c64..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image45.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image46.png b/app/zh/information/blog/20230129-05/img/image46.png
deleted file mode 100644
index 14b7c9432cb35c038e14cdac6b1fe4bc17e515ec..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image46.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image47.png b/app/zh/information/blog/20230129-05/img/image47.png
deleted file mode 100644
index ff9863d33e0d4ee0a0acafb8108933d68b334338..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image47.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image48.png b/app/zh/information/blog/20230129-05/img/image48.png
deleted file mode 100644
index da33bfdbdd838191b5e5b14aff9adf9d2db680e5..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image48.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image49.png b/app/zh/information/blog/20230129-05/img/image49.png
deleted file mode 100644
index 8eeaa08b3019eb00d783aa7537f71e6c84c5abda..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image49.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image5.png b/app/zh/information/blog/20230129-05/img/image5.png
deleted file mode 100644
index 4b5d24fc212a8fdbef212d10d18ed02b12a960b3..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image5.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image50.png b/app/zh/information/blog/20230129-05/img/image50.png
deleted file mode 100644
index ab6890afb44e7598dc3e431af95113cfb5cfee6b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image50.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image51.png b/app/zh/information/blog/20230129-05/img/image51.png
deleted file mode 100644
index cc017600e3848b78d0741ef6027f97f2b18f0080..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image51.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image52.png b/app/zh/information/blog/20230129-05/img/image52.png
deleted file mode 100644
index 9fd7cc78d90030ffab67446097b1dbb324ba4c40..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image52.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image53.png b/app/zh/information/blog/20230129-05/img/image53.png
deleted file mode 100644
index 19a8d042f22cd646f91bd4ac2b15ad21f1bdf8c1..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image53.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image54.png b/app/zh/information/blog/20230129-05/img/image54.png
deleted file mode 100644
index 48932148cee0a6767caf634a380c3bf953395e61..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image54.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image55.png b/app/zh/information/blog/20230129-05/img/image55.png
deleted file mode 100644
index b62f677029bfe64b07146ec1bd2aca3f6c7431d0..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image55.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image56.png b/app/zh/information/blog/20230129-05/img/image56.png
deleted file mode 100644
index 9b90336c17b5bd55ef8f54aca90d67c9659d9756..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image56.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image57.png b/app/zh/information/blog/20230129-05/img/image57.png
deleted file mode 100644
index 1df40037ff02088f8e7c40afc036976e192f25c1..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image57.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image58.png b/app/zh/information/blog/20230129-05/img/image58.png
deleted file mode 100644
index a3715cc665078baca578567c23d32f77ea16ece4..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image58.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image59.png b/app/zh/information/blog/20230129-05/img/image59.png
deleted file mode 100644
index 5f12eaff967f5a69caf470db587ef3a286b46a1a..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image59.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image6.png b/app/zh/information/blog/20230129-05/img/image6.png
deleted file mode 100644
index 229b86405b804628b20740b4f4ae7fc5e19883a2..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image6.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image60.png b/app/zh/information/blog/20230129-05/img/image60.png
deleted file mode 100644
index 773a65355ed97bd5270d1fcc1c80599b646f9211..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image60.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image61.png b/app/zh/information/blog/20230129-05/img/image61.png
deleted file mode 100644
index 4691f3708b60ae4c2c498d381982d01d67190320..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image61.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image62.png b/app/zh/information/blog/20230129-05/img/image62.png
deleted file mode 100644
index 137b906299c545c67198927ca827af6a77c387f9..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image62.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image63.png b/app/zh/information/blog/20230129-05/img/image63.png
deleted file mode 100644
index 2a4a4850d6a1aeef149978db6179382a33a06e98..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image63.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image64.png b/app/zh/information/blog/20230129-05/img/image64.png
deleted file mode 100644
index 632765110a860c18e7f1868309162d123a3c6443..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image64.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image65.png b/app/zh/information/blog/20230129-05/img/image65.png
deleted file mode 100644
index 40fa0c8d9a00ab820660194b3ad22a61a5b68d1b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image65.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image66.png b/app/zh/information/blog/20230129-05/img/image66.png
deleted file mode 100644
index 8d938505884c0129fe7b834266a701baa43bc187..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image66.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image67.png b/app/zh/information/blog/20230129-05/img/image67.png
deleted file mode 100644
index da2faf47a70492b02f0f7434685d9eb78875a217..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image67.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image68.png b/app/zh/information/blog/20230129-05/img/image68.png
deleted file mode 100644
index 348a468f97f25cdea1c80e5d6fabe8bd854bde69..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image68.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image69.png b/app/zh/information/blog/20230129-05/img/image69.png
deleted file mode 100644
index 372451002c28790bceb2b7044939586dd655cb02..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image69.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image7.png b/app/zh/information/blog/20230129-05/img/image7.png
deleted file mode 100644
index 22cb9221678504faf217665c0d8bca50aae078fc..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image7.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image70.png b/app/zh/information/blog/20230129-05/img/image70.png
deleted file mode 100644
index 7b69d5737a8d0a3fdc49a86f170ad46fe6a5a3e1..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image70.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image71.png b/app/zh/information/blog/20230129-05/img/image71.png
deleted file mode 100644
index c63a37f482efbf56322b87142766ca08738dc743..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image71.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image72.png b/app/zh/information/blog/20230129-05/img/image72.png
deleted file mode 100644
index b2cbc9eff9a368d3c17549fc09c217d03b983b2a..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image72.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image73.png b/app/zh/information/blog/20230129-05/img/image73.png
deleted file mode 100644
index 8a22410467025b2c139556d8b84f5f38047331ae..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image73.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image74.png b/app/zh/information/blog/20230129-05/img/image74.png
deleted file mode 100644
index 43c4461c8b5ae56ddb593732b2397a8008c9826e..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image74.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image75.png b/app/zh/information/blog/20230129-05/img/image75.png
deleted file mode 100644
index a1bda317b27fad95e0b8794fea8615624a9e2017..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image75.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image76.png b/app/zh/information/blog/20230129-05/img/image76.png
deleted file mode 100644
index ddc1e3489b4d5320a843ea1bf65e40b7be6e4442..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image76.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image77.png b/app/zh/information/blog/20230129-05/img/image77.png
deleted file mode 100644
index dabe608c5cbdfe3be05120ec8e3a205ec03c8576..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image77.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image78.png b/app/zh/information/blog/20230129-05/img/image78.png
deleted file mode 100644
index d6817e984e33a2655cb49efc0fe4949e3d8ea399..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image78.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image79.png b/app/zh/information/blog/20230129-05/img/image79.png
deleted file mode 100644
index 4597fb61ea51e5af1f08df08f05fe4468bdf5229..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image79.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image8.png b/app/zh/information/blog/20230129-05/img/image8.png
deleted file mode 100644
index c09844a04d8a3c38526a4227d55539fd3421480d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image8.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image80.png b/app/zh/information/blog/20230129-05/img/image80.png
deleted file mode 100644
index 4db71f85ca147f0af1811aa3637d5a69d7c0ee6a..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image80.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image81.png b/app/zh/information/blog/20230129-05/img/image81.png
deleted file mode 100644
index d14fb4af9cfd766451f31fdae98cdb8404840cad..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image81.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image82.png b/app/zh/information/blog/20230129-05/img/image82.png
deleted file mode 100644
index cb147e9682637c8f8ac94d0bb8b663cc473f0550..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image82.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image83.png b/app/zh/information/blog/20230129-05/img/image83.png
deleted file mode 100644
index b8ec5414ffc01050e7afe11832efab7168fc74d4..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image83.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image84.png b/app/zh/information/blog/20230129-05/img/image84.png
deleted file mode 100644
index 57591a7de123d4a08fb16c0ecd928e07abaa1a7d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image84.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image85.png b/app/zh/information/blog/20230129-05/img/image85.png
deleted file mode 100644
index b654dc00e4870ad1ac05930815f5c77f530eb57e..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image85.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image86.png b/app/zh/information/blog/20230129-05/img/image86.png
deleted file mode 100644
index 6ad860f03f4a8bc084d179b4a663875562c3e02c..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image86.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image87.png b/app/zh/information/blog/20230129-05/img/image87.png
deleted file mode 100644
index d05d83ff11cec2e6c2635e951f9a93adb7b77dc6..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image87.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image88.png b/app/zh/information/blog/20230129-05/img/image88.png
deleted file mode 100644
index 9b1469e8835458095c38be9d4d5ed093c9c1704b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image88.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image89.png b/app/zh/information/blog/20230129-05/img/image89.png
deleted file mode 100644
index 035f285d63fd7c95703a3e36f2abcc8e8eecba14..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image89.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/img/image9.png b/app/zh/information/blog/20230129-05/img/image9.png
deleted file mode 100644
index c426133b22c3dfa42ee222af2004d10d8bebc451..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-05/img/image9.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-05/index.md b/app/zh/information/blog/20230129-05/index.md
deleted file mode 100644
index 5b4d466cdbe7ef414a8609d74257d202a230c7a7..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/20230129-05/index.md
+++ /dev/null
@@ -1,901 +0,0 @@
----
-title: 'openLooKeng对接Ranger流程梳理'
-date: '2022-12-25'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', 'Ranger', '认证鉴权', '安全认证']
-archives: '2022-12'
-author: 'tushengxia'
-description: 'openLooKeng对接Ranger流程梳理'
----
-
-# 介绍
-
-1. 简要介绍
-
-openLooKeng对接Ranger配置访问控制的流程
-
-1. 建议的版本
-
-建议使用openLooKeng的版本为1.7.0
-
-建议使用Ranger的版本为2.1.0
-
-# 环境要求
-
-1. 操作系统要求
-
-操作系统要求如表2-2所示。
-
-1. []{#\_table5438964 .anchor}操作系统要求
-
----
-
-项目 版本 下载地址
-CentOS 7.6 https://www.centos.org/download/
-
----
-
-1. 环境要求
-
-openLooKeng
-
----
-
-节点 域名 ip
-coordinator(协调节点) coordinator1.example.com **_._**.**_._**
-worker(工作节点) worker1.example.com **_._**.**_._**
-worker(工作节点) worker1.example.com **_._**.**_._**
-
----
-
-Ranger
-
----
-
-节点 域名 ip
-RangerAdmin coordinator1.example.com **_._**.**_._**
-
----
-
-# openLooKeng对接Ranger
-
-> openLooKeng对接Ranger可以不依赖LDAP和Kerberos如果需要整合LDAP和Kerberos请参考《openLooKeng整合LDAP+Kerberos+Ranger》
-
-## 安装 Ranger Admin 插件
-
-### 下载Ranger Admin插件
-
-> 官方下载地址https://download.openlookeng.io/
->
-> 
->
-> 
->
-> 本文以1.7.0为例
->
-> 下载文件ranger-2.1.0-admin-openlookeng-1.7.0-plugin.tar.gz
-
-### 解压Ranger Admin插件
-
-解压到指定目录
-
-> tar -zxvf ranger-2.1.0-admin-openlookeng-1.7.0-plugin.tar.gz -C
-> /usr/local/soft/
-
-### Ranger 服务类型定义的注册
-
-> 使用 Ranger Admin提供的REST
-> API向Ranger注册服务类型定义。注册后,Ranger Admin 将提供UI
-> 以创建服务实例(在以前的版本中称为存储库)和服务类型策略。Ranger
-> 插件使用服务类型定义和策略来确定请求是否有访问权限以进行授权。如下示例所示,可以使用
-> curl 命令调用 REST API 接口注册服务:
->
-> Ranger的服务查询 REST API 示例:
->
-> 需要修改ranger的用户名和密码和ranger的主机名
->
-> curl -u admin:**\*\*** -X POST -H "Accept: application/json" -H
-> "Content-Type: application/json" -d
-> @service-defs/ranger-servicedef-openlookeng.json
-> http://master:6080/service/plugins/definitions
-
-注意:在Ranger Admin插件解压目录下执行
-
-### 复制openlookeng目录到Ranger Admin安装目录下的ranger-plugins目录:
-
-> scp -r openlookeng/ []{#OLE_LINK1
-> .anchor}/usr/local/soft/ranger-2.1.0-admin/ews/webapp/WEB-INF/classes/ranger-plugins/
-
-注意:在Ranger Admin插件解压目录下执行
-
-
-
-这时打开 Ranger Admin 可以看到多了“openLooKeng”服务:
-
-
-
-## 安装openLooKeng插件
-
-### 下载openLooKeng插件
-
-> 官方下载地址https://download.openlookeng.io/
->
-> 
->
-> 本文以1.7.0为例
->
-> 下载文件ranger-2.1.0-openlookeng-1.7.0-plugin.tar.gz
-
-### 解压openLooKeng插件
-
-tar -zxvf ranger-2.1.0-openlookeng-1.7.0-plugin.tar.gz -C
-/usr/local/soft/
-
-### 修改 install.properties文件
-
-> \# ranger admin 地址
->
-> POLICY_MGR_URL=http://master:6080
->
-> \# 这个随便取,回头在ranger admin中新建jdbc连接的时候需要填写这个
->
-> REPOSITORY_NAME=[]{#OLE_LINK3 .anchor}openlookengdev
->
-> \# openLooKeng 安装目录
->
-> COMPONENT_INSTALL_DIR_NAME=/usr/local/openlookeng/hetu
->
-> XAAUDIT.SOLR.ENABLE=false
->
-> \# 添加该条配置,不然后面执行会报错
->
-> XAAUDIT.SUMMARY.ENABLE=false
-
-备注:在openLooKeng插件插件解压路径下修改配置
-
-### 创建用户
-
-> groupadd hadoop
->
-> useradd -g hadoop openlookeng
-
-### 执行enable-openlookeng-plugin.sh
-
-> sh enable-openlookeng-plugin.sh
->
-> 注意:此时需要有openlookeng用户,若没有,自行创建。
->
-> 
-
-### 重启服务
-
-> 重启 Ranger Admin 服务:ews/ranger-admin-services.sh restart
->
-> 重启 openLooKeng 服务:bin/launcher restart
-
-### 验证
-
-打开 Admin Ranger:http://master:6080 可以看到OPENLOOKENG
-
-
-
-openLooKeng 的 Ranger 插件目前支持
-systemproperty、catalog、sessionproperty、schema、table 和 column
-等资源的权限管理。
-
-## 添加 openLooKeng Service
-
-1、打开 Admin
-Ranger:http://**_._**.**_._**:6080/,找到「OPENLOOKENG」点击「+」添加
-Service。
-
-
-
-> 2、输入 Service
-> Name、Username、Password、jdbc.driverClassName、jdbc.url
-> 这些信息:(1)Username填写:
-
-1. 未对接 openLDAP,用户任意;
-
-2. 对接了 openLDAP,填写 ldap 用户。
-
-> (2)Password填写:
-
-1. 对接 openLDAP 需填写;
-
-2. 未对接 openLDAP 不用填写。
-
-> (3)jdbc 填写:
-
-1. 非安全openLooKeng集群
-
- 非安全openLooKeng集群是指没有开启ldap和https
-
-jdbc:lk://master:8080/hive
-
-
-
-1. 安全openLooKeng集群:
-
-jdbc:lk://cdh01:8443?SSL=true&SSLKeyStorePath=/opt/openlookeng/hetu-server-1.7.0/keystore.jks&SSLKeyStorePassword=**\*\***
-
-
-
-注意:
-
-1. 需要指定Password,密码为ldap中的密码
-
-2. 第一次创建service不需要测试,直接点击Save,当openLooKeng
- plugin策略同步完成之后才能测试成功,当测试成功之后在使用ranger进行权限配置时才能获取到openLooKeng的元数据
-
-## 验证是否添加成功
-
-> 重启 openLooKeng,再打开 Ranger Admin Web
-> 页面,点击「Adult」->「Plugins」查看,出现如下表明 openLooKeng
-> 插件安装成功。
-
-
-
-点击「OPENLOOKENG」下的 openlookengdev,可以看到默认生成了 jdbc
-连接信息中 olk 用户的权限,该默认权限配置,
-
-注:没有进行配置的用户,没有任何权限,比如 olk2、test
-用户没配置,则没有任何权限。
-
-
-
-## 验证权限控制
-
-1、HTTP:
-
-(1)访问地址:http://openLooKeng-IP:8080
-
-
-
-(2)分别使用olk用户和其它用户登录,进入测试,执行sql。
-
-(1) olk用户:
-
-
-
-(1) 其它用户
-
- 执行sql报错
-
- 
-
- 没有权限
-
- 
-
-# openLooKeng对接安全Ranger(Ranger开启了Kerberos)
-
-## 创建principal
-
-> kadmin.local -q "addprinc -randkey openlookeng@EXAMPLE.COM"
->
-> kadmin.local -q "addprinc -randkey
-> openlookeng/coordinator.example.com@EXAMPLE.COM"
-
-## 生成keytab
-
-> kadmin -p admin/admin -q "ktadd -k ./lk.keytab
-> openlookeng@EXAMPLE.COM"
->
-> kadmin -p admin/admin -q "ktadd -k ./lk.keytab
-> openlookeng/coordinator.example.com@EXAMPLE.COM"
-
-## 安装Kerberos Client
-
-> yum install krb5-workstation krb5-libs krb5-auth-dialog
-
-## access-control.properties文件修改
-
-文件路径hetu-server-1.8.0/etc
-
-增加配置
-
-ranger.principal=openlookeng\
-ranger.keytab=/usr/local/soft/hetu-server-1.7.0/lk.keytab
-
-## 修改openLooKeng Service
-
-增加policy.download.auth.users
-
-
-
-注意:此处的value使用4.3中ranger.principal
-
-## 修改ranger-openlookeng-security.xml
-
-将ranger.plugin.openlookeng.policy.rest.url中的ip改成主机名
-
-
-
-## 重启openLooKeng
-
-launcher restart
-
-# 策略配置
-
-## 策略配置流程
-
-(1) 通过点击Add New Policy增加新的权限策略
-
-
-
-(1) Policy Details (基本信息)
-
- 
-
-(2) Allow Conditions(增加允许访问条件)
-
- 
-
-(3) Deny Conditions (增加拒绝访问条件)
-
- 
-
-(4) 点击add完成策略增加
-
-# 权限使用说明
-
-## Systemproperty
-
-### 权限列表
-
-Alert:设置系统会话属性
-
-### 增加权限
-
-(1) 点击编辑
-
- 
-
-(2) 增加一个用户
-
- 
-
-(3) 点击save
-
- 用户列表中会多一个用户
-
- 
-
-## Catalog
-
-### 权限列表
-
-use:如果 catalog 没授予 use,所有 catalog 下操作都无权限。
-
-select:只有授予 select 权限,show catalogs 才能显示对应的 catalog。
-
-create:新建 schema。
-
-show:显示当前 catalog 或者指定 catalog 下的所有授权 schema。
-
-### 增加权限
-
-(1) 点击编辑
-
-
-
-(1) 选择角色,用户组,或者用户增加catalog权限
-
- 本文以test用户为例,增加catalog所有权限
-
-
-
-(1) 使用test用户登录openLooKeng
-
- 
-
-(2) 执行show catalogs
-
- 
-
-## Sessionproperty
-
-### 权限列表
-
-alter:设置 catalog 会话属性
-
-### 增加权限
-
-(1) 点击编辑
-
- 
-
-(2) 设置权限
-
- 
-
-(3) 点击save
-
-## Impersonateuser
-
-### 权限列表
-
-impersonate : Kerberos或LDAP认证用户模拟openLooKeng查询用户。
-
-### 增加权限
-
-(1) 点击编辑
-
-
-
-(1) 设置权限
-
- 
-
-(2) 点击save
-
-## schema
-
-### 权限列表
-
-1. drop:删除已有 schema。
-
-2. alter:修改已有 schema 的定义。
-
-3. select:只有授予 select 权限,show schemas 才能显示对应的 schema。
-
-4. create:使用给定列新建表;通过 select 查询语句来新建视图。
-
-5. show:显示当前 schema 或者指定 schema 下的所有授权表。
-
-### 增加权限
-
-(1) 点击编辑
-
-
-
-(1) 修改权限
-
- 
-
-(2) 点击save保存
-
-(3) 使用test用户登录openLooKeng
-
-(4) 执行create database mysql.test ;
-
- 
-
-(5) 执行drop database mysql.test ;
-
- 
-
-## Table
-
-### 权限列表
-
-1. drop:删除已有表、视图、列。
-
-2. alter:修改已有表的定义;设置表声明;更新给定条件下的行数据。
-
-3. insert:插入行数据。
-
-4. delete:删除行数据。
-
-5. grant:给特定授权人授予特定权限。
-
-6. revoke:取消特定授权人的特定权限。
-
-7. select:只有授予 select 权限,show tables 才能显示对应的表。
-
-8. show:显示指定表下的所有授权列的类型和其他属性。
-
-### 增加权限
-
-(1) 点击编辑
-
-
-
-(1) 修改权限
-
- 
-
- 
-
-## column
-
-### 权限列表
-
-1. select:检索数据;只有授予 select 权限,show columns
- 才能显示对应的列。
-
-### 增加权限
-
-(1) 点击编辑
-
-
-
-(1) 修改权限
-
- 
-
- 
-
-
-
-(1) 使用test用户登录openLooKeng
-
-(2) 执行select \* from system.information_schema.tables;
-
- 
-
-## 列掩码
-
-列掩码只适用于select操作,指定用户只能获取掩码后的数据。
-openLooKeng目前支持8种掩码策略。
-
-### 掩码类型列表
-
----
-
-**掩码类型** **描述**
-Redact 小写字母转化为x,大写字母转化为X,数字转化为0。
-Partial mask: show last 4 显示最后四个字符;其他转化为’X’。
-Partial mask: show first 4 显示前四个字符;其他转化为’X’。
-Hash 使用sha256得到的哈希值。
-Nullify 替换为NULL。
-Unmasked (retain original value) 不进行掩码。
-Date: show only year Date类型:只显示年份。
-Custom 自定义。 例子: cast(concat({col}, “test”) as {type})
-
----
-
-### 使用示例
-
-(1) 增加策略
-
-
-
-(1) 配置策略
-
- 
-
-(2) 点击add
-
-(3) 使用test用户登录openLooKeng
-
-(4) 执行select \* from mysql.bigdata.students查询数据,发送name列被hash
-
- 
-
-## 行过滤
-
-行过滤只适用于select操作,指定用户只能获取过滤后的数据。通过将自定义过滤表达式作为SQL中的where条件实现数据过滤
-
-### 使用示例
-
-(1) 增加策略
-
-
-
-(1) 配置策略
-
- 过滤条件为age=22
-
-
-
-(1) 点击add
-
-(2) 使用test用户登录openLooKeng
-
-(3) 执行select \* from
-mysql.bigdata.students;查询数据,发现只有age=22的数据
-
- 
-
-# 场景使用案例
-
-## Mysql Catalog权限控制
-
-### 配置Mysql Catalog
-
-要配置MySQL连接器,在etc/catalog中创建一个目录属性文件,例如mysql.properties,将MySQL连接器挂载为mysql目录。使用以下内容创建文件,并根据设置替换连接属性:
-
-(1) vim mysql.properties
-
- 增加配置
-
-> connector.name=mysql
->
-> connection-url=jdbc:mysql://example.net:3306
->
-> connection-user=root
->
-> connection-password=**\*\***
-
-(1) 重启openLooKeng
-
-(2) 使用olk登录openLooKeng查看catalog,olk用户已在ranger中配置了所有权限
-
- 
-
-### Catalog级别权限控制
-
-(1) 点击增加策略
-
- 
-
-(2) 配置策略
-
- 
-
-
-
-(1) 使用test1用户登录openLooKeng
-
-(2) 执行查询查询可以mysql这个catalog中所有的表
-
- 
-
-### Schema级别权限控制
-
-(1) 增加策略
-
-
-
-(1) 增加完成
-
-
-
-(1) 使用test1用户登录openLooKeng
-
-(2) 执行select \* from mysql.bigdata.students;
-
-(3) 执行报错,需要catalog权限
-
- 
-
-(4) 增加catalog权限
-
- 
-
-(5) 执行select \* from mysql.bigdata.students;
-
-(6) 执行成功
-
- 
-
-### table级别权限控制
-
-(1) 配置表级别权限策略
-
- 
-
-(2) 表级别权限需要catalog权限,把test1用户增加到此策略中
-
- 
-
-
-
-(1) 使用test1用户登录openLooKeng
-
-(2) 查询students表可以查询
-
- 
-
-(3) 查询当前库下其它表,查询报错,没有权限
-
- 
-
-### column级别权限控制
-
-(1) 增加策略
-
- 
-
-(2) 查看策略
-
- 
-
-
-
-(1) 表级别权限需要catalog权限,把test1用户增加到此策略中
-
- 
-
-
-
-(1) 使用test1用户登录openLooKeng
-
-(2) 查询students表的id列,可以查询
-
- 
-
-(3) 查询查询students表的其它列,查询报错,没有权限
-
- 
-
-## Hive Catalog权限控制
-
-### 配置Hive Catalog
-
-要配置Hive连接器,在etc/catalog中创建一个目录属性文件,例如hive.properties,将MySQL连接器挂载为hive目录。使用以下内容创建文件,并根据设置替换连接属性:
-
-(1) vim hive.properties
-
- 增加配置
-
-> connector.name=hive-hadoop2
->
-> hive.metastore.uri=thrift://example.net:9083
-
-(1) 重启openLooKeng
-
-(2) 使用olk登录openLooKeng查看catalog,olk用户已在ranger中配置了所有权限
-
- 
-
-### Catalog级别权限控制
-
-(1) 配置策略
-
- 
-
-(2) 使用test1用户登录openLooKeng
-
-(3) 执行查询可以查询hive这个catalog中所有的表
-
- 
-
-### Schema级别权限控制
-
-(1) 增加策略
-
-
-
-(1) schema级别权限需要catalog权限,把test1用户增加到此策略中
-
- 
-
-(2) 使用test1用户登录openLooKeng
-
-(3) 执行select \* from hive.default.student;
-
- 
-
-### table级别权限控制
-
-(1) 配置策略
-
- 
-
-(2) 表级别权限需要catalog权限,把test1用户增加到此策略中
-
- 
-
-(3) 使用test1用户登录openLooKeng
-
-(4) 查询student表可以查询
-
- 
-
-(5) 查询当前库下其它表,查询报错,没有权限
-
- 
-
-### column级别权限控制
-
-(1) 配置策略
-
- 
-
-(2) 表级别权限需要catalog权限,把test1用户增加到此策略中
-
- 
-
-(3) 使用test1用户登录openLooKeng
-
-(4) 查询student表的id列,可以查询
-
- 
-
-(5) 查询查询student表的其它列,查询报错,没有权限
-
- 
-
-# 问题总结
-
-## 在ranger里面不显示openLooKeng插件
-
-1. 问题描述
-
-完成Ranger
-服务类型定义的注册后,登录http://ranger-IP:6080/login.jsp看不到openLooKeng插件
-
-
-
-1. 解决方法
-
-服务没有注册成功,可能是ranger的用户名密码不正确,或者主机名不正确。
-
-Ranger 服务类型定义的注册
-
-> curl -u admin:admin -X POST -H "Accept: application/json" -H
-> "Content-Type: application/json" -d
-> @service-defs/ranger-servicedef-openlookeng.json
-> http://master:6080/service/plugins/definitions
-
-(1) 主机名错误
-
-
-
-(1) 用户名密码错误
-
- 
-
- 正确结果
-
- 
-
- 
-
-## Access Denied: Cannot select from columns \[schema_name\] in table or view schemata
-
-
-
-1. 问题描述
-
-当在ranger中给某个用户增加某个schema的权限后,执行show schemas from
-jmx报错
-
-用户没有schemata的权限
-
-2. 解决方法
-
- 给这个用户增加schemata的权限
-
- 
-
- 注意:其它权限问题解决方法类似
-
-## Error getting policies; Received NULL response!!. secureMode=true, user=\*\*\*@EXAMPLE.COM (auth:KERBEROS), serviceName=\*\*\*
-
-openLooKeng中的日志
-
-
-
-Ranger中的日志
-
-
-
-1. 问题描述:下载策略失败,用户没有权限下载策略
-
-2. 解决方法
-
- access-control.properties中的ranger.principal和ranger中的policy.download.auth.users要保持一致
-
-## No valid credentials provided (Mechanism level: No valid credentials provided (Mechanism level: Server not found in Kerberos database (7) - LOOKING_UP_SERVER))
-
-1. 问题描述:没有提供有效凭证,
-
-主要原因是DNS正向和反向解析不一致
-
-1. 解决方法:
-
- 修改ranger-openlookeng-security.xml文件,将ranger.plugin.openlookeng.policy.rest.url中的ip改成主机名
-
- 
-
-# 修订记录 {#修订记录 .154}
-
----
-
-发布日期 修订记录
-2022-07-27 第一次正式发布
-2022-08-30 修改插件获取方式为官方下载
-2022-08-31 增加策略配置
-2022-09-01 增加权限使用说明
-2022-09-02 问题总结
-2022-09-02 增加mysql catalog权限使用案例
-2022-09-02 增加hive catalog权限使用案例
-
----
-
-相关视频:[openLooKeng内核认证与鉴权流程](https://space.bilibili.com/627629884/channel/collectiondetail?sid=1061144)
diff --git a/app/zh/information/blog/20230129-06/img/image10.png b/app/zh/information/blog/20230129-06/img/image10.png
deleted file mode 100644
index 36ce3a789c44f828d369ad543ae84559a2460b1b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image10.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image11.png b/app/zh/information/blog/20230129-06/img/image11.png
deleted file mode 100644
index 7ab263ba5df081c6831b3a3ce831f3dee9f767ff..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image11.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image12.png b/app/zh/information/blog/20230129-06/img/image12.png
deleted file mode 100644
index 75117585659977bde051fb8e1ca5dfa359d28e4f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image12.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image13.png b/app/zh/information/blog/20230129-06/img/image13.png
deleted file mode 100644
index 6099a63f96db5b52b515f7c1110a2b21ac3b4d35..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image13.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image14.png b/app/zh/information/blog/20230129-06/img/image14.png
deleted file mode 100644
index 8252ed628e7f9f2e4a2cd2351a87b7f1b7d41aef..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image14.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image15.png b/app/zh/information/blog/20230129-06/img/image15.png
deleted file mode 100644
index 5f9d7bd829c0d52ee0eb41fe1cc71bd4457c1f48..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image15.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image16.png b/app/zh/information/blog/20230129-06/img/image16.png
deleted file mode 100644
index 872e579b32f9e01130dad3cf51e7cf424d0ed489..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image16.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image17.png b/app/zh/information/blog/20230129-06/img/image17.png
deleted file mode 100644
index b217949a8811205428b614c4db60685a51795377..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image17.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image18.png b/app/zh/information/blog/20230129-06/img/image18.png
deleted file mode 100644
index df120081e9d80857bc0d0de81e416135351900fd..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image18.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image19.png b/app/zh/information/blog/20230129-06/img/image19.png
deleted file mode 100644
index 44be7b93482752b7b80d7053c6ff63f81085776e..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image19.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image20.png b/app/zh/information/blog/20230129-06/img/image20.png
deleted file mode 100644
index 621d49648123fad3d080f8e7a3f40e0b0c03c3a9..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image20.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image21.png b/app/zh/information/blog/20230129-06/img/image21.png
deleted file mode 100644
index 500e7bb9e388dd28afd4263c34e5d6f0286dd46f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image21.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image22.png b/app/zh/information/blog/20230129-06/img/image22.png
deleted file mode 100644
index de4c2ed05d2e13f50832eeab1531b181b8e6d555..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image22.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image23.png b/app/zh/information/blog/20230129-06/img/image23.png
deleted file mode 100644
index 9f34155594c026430156b76bd75c6489c51ffac0..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image23.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image24.png b/app/zh/information/blog/20230129-06/img/image24.png
deleted file mode 100644
index 562d4f56c535c320682c1efe0a3607431255499f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image24.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image25.png b/app/zh/information/blog/20230129-06/img/image25.png
deleted file mode 100644
index 3b4950964035bd578a246ef53e78c092fbdd6938..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image25.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image26.png b/app/zh/information/blog/20230129-06/img/image26.png
deleted file mode 100644
index 0d40537f2559cbe114d6b972c730de80890049dc..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image26.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image27.gif b/app/zh/information/blog/20230129-06/img/image27.gif
deleted file mode 100644
index 6d21a6ba0b608ca83b82acfb093bfef15cc2c0f3..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image27.gif and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image28.png b/app/zh/information/blog/20230129-06/img/image28.png
deleted file mode 100644
index 0b38c738be12f1d2690977bd25a756d8ea6c9713..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image28.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image29.png b/app/zh/information/blog/20230129-06/img/image29.png
deleted file mode 100644
index 1da354f484f0bb8e0a913a0801ade11592ed03d8..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image29.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image3.png b/app/zh/information/blog/20230129-06/img/image3.png
deleted file mode 100644
index 331417d57ebf5a5fae21a7764f55576880408f00..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image3.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image30.png b/app/zh/information/blog/20230129-06/img/image30.png
deleted file mode 100644
index ae9ca2ff619674b267ef4967ee52f865d65acb34..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image30.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image31.png b/app/zh/information/blog/20230129-06/img/image31.png
deleted file mode 100644
index 52d51fe32d4e52e48eff3853a31b768c09d2b1c2..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image31.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image32.png b/app/zh/information/blog/20230129-06/img/image32.png
deleted file mode 100644
index 3abfa00368d4e3ef427b9bdd67dc916432fbca29..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image32.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image33.png b/app/zh/information/blog/20230129-06/img/image33.png
deleted file mode 100644
index afd4f927d40f8b001f91ae4e5a418c2ee0a584c8..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image33.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image34.png b/app/zh/information/blog/20230129-06/img/image34.png
deleted file mode 100644
index 5bf2f63a8d0998eac49b55518d5ec571c98ea2f1..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image34.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image35.png b/app/zh/information/blog/20230129-06/img/image35.png
deleted file mode 100644
index 1d55edceb13c1b5d17e99a9940a1734d65a83a8a..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image35.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image36.png b/app/zh/information/blog/20230129-06/img/image36.png
deleted file mode 100644
index 1cc6cb3637daf86f8cf2b38aa5ea1db2905660b6..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image36.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image37.png b/app/zh/information/blog/20230129-06/img/image37.png
deleted file mode 100644
index b5f72f5bcdad40e5b294410b800d45e0c1608333..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image37.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image38.png b/app/zh/information/blog/20230129-06/img/image38.png
deleted file mode 100644
index 8ca21077bc4e0d025702985ef91148d65a549b96..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image38.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image39.png b/app/zh/information/blog/20230129-06/img/image39.png
deleted file mode 100644
index c70f4a9c2b2de7bb4d7b67aaa02c3f071d2110ab..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image39.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image4.png b/app/zh/information/blog/20230129-06/img/image4.png
deleted file mode 100644
index a3270e39f24c54fceee132a8de73e63bffeb58a7..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image4.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image40.png b/app/zh/information/blog/20230129-06/img/image40.png
deleted file mode 100644
index 93ec1e905cda7150ccadf609a52916169f569d89..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image40.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image41.png b/app/zh/information/blog/20230129-06/img/image41.png
deleted file mode 100644
index ecfcd59580352b50ab0c43bbad6bb323b5902fa0..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image41.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image42.png b/app/zh/information/blog/20230129-06/img/image42.png
deleted file mode 100644
index e70ccc34819e7c274f3845fdbb15aafbdf3c800e..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image42.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image43.png b/app/zh/information/blog/20230129-06/img/image43.png
deleted file mode 100644
index 3af9d5964ee5e0ff844828e84500bb196a07234d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image43.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image44.png b/app/zh/information/blog/20230129-06/img/image44.png
deleted file mode 100644
index 9c7b18d136d006c8ff6d757a0b7f51cd19c86d12..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image44.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image45.png b/app/zh/information/blog/20230129-06/img/image45.png
deleted file mode 100644
index a78eb5bde345c6f350c359c8cbc5770ec7740302..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image45.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image46.png b/app/zh/information/blog/20230129-06/img/image46.png
deleted file mode 100644
index 2c466acab62a827b92a682ae3c314c9e1b95b580..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image46.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image47.png b/app/zh/information/blog/20230129-06/img/image47.png
deleted file mode 100644
index 81185939714de447b659a8dd645c6e507bb21c13..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image47.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image48.png b/app/zh/information/blog/20230129-06/img/image48.png
deleted file mode 100644
index 45f9e25aaf0f4f12f373223e1022778f8d6cd8dc..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image48.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image49.png b/app/zh/information/blog/20230129-06/img/image49.png
deleted file mode 100644
index 49fa394aef6e7c6e3ccfd5311af4e891a255801f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image49.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image5.png b/app/zh/information/blog/20230129-06/img/image5.png
deleted file mode 100644
index ca776aa60c6c076fb46a0d559da078fffc709420..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image5.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image50.png b/app/zh/information/blog/20230129-06/img/image50.png
deleted file mode 100644
index 8e101a2226eeaacc6104a46e375e37b1ade70d01..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image50.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image51.png b/app/zh/information/blog/20230129-06/img/image51.png
deleted file mode 100644
index 878d5c6165c694bbe45b4f4e872d90538131d5e3..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image51.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image52.png b/app/zh/information/blog/20230129-06/img/image52.png
deleted file mode 100644
index 01312256a79ec56a7aa60fd4c7f9196f58c3152a..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image52.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image53.png b/app/zh/information/blog/20230129-06/img/image53.png
deleted file mode 100644
index 1b7c3acb9bd25be94a9e52ae7a8eaa9832a4bf97..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image53.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image54.png b/app/zh/information/blog/20230129-06/img/image54.png
deleted file mode 100644
index 122a1a9c37bb9653454b0f991b74be12cbf3218b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image54.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image55.png b/app/zh/information/blog/20230129-06/img/image55.png
deleted file mode 100644
index cf667ceb64876e1ded2822661aa272c4ce28ce62..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image55.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image56.png b/app/zh/information/blog/20230129-06/img/image56.png
deleted file mode 100644
index b634c9d35e21e947a113e622be38ab6f756c606e..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image56.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image6.png b/app/zh/information/blog/20230129-06/img/image6.png
deleted file mode 100644
index 93d8d028cb8e8939f22179ebf1894e255cd4dbdc..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image6.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image7.png b/app/zh/information/blog/20230129-06/img/image7.png
deleted file mode 100644
index 0bde5e7c8af50ade0f1d19c23de70a1335b24627..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image7.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image8.png b/app/zh/information/blog/20230129-06/img/image8.png
deleted file mode 100644
index dcb9bc9c263ac6823b538c0317f04f261a731c8a..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image8.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/img/image9.png b/app/zh/information/blog/20230129-06/img/image9.png
deleted file mode 100644
index 8c543cd5fdc74eb2397ae89213c9577da839b509..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-06/img/image9.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-06/index.md b/app/zh/information/blog/20230129-06/index.md
deleted file mode 100644
index bbe827a5c2c550e7bbe6cdf343d7febeda76c7c4..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/20230129-06/index.md
+++ /dev/null
@@ -1,409 +0,0 @@
----
-title: 'openLooKeng对接LDAP+Kerberos详细设计文档'
-date: '2022-12-25'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', 'LDAP', 'Kerberos', '认证鉴权', '安全认证']
-archives: '2022-12'
-author: 'tushengxia'
-description: 'oopenLooKeng对接LDAP+Kerberos详细设计文档'
----
-
-Kerberos介绍
-=============
-
-Kerberos是一种一种网络身份验证协议,只包括验证环节,不负责授权。它旨在通过使用密钥加密技术为客户端/服务器应用程序提供强身份验证。
-
-用户只需输入一次身份验证信息,就可凭借此验证获得的票据授予票据(ticket-granting
-ticket)访问多个接入Kerberos的服务,即SSO(Single Sign On,单点登录)。
-
-在Kerberos认证中,最主要的问题是如何证明“你是你”的问题,如当一个Client去访问Server服务器上的某服务时,Server如何判断Client是否有权限来访问自己主机上的服务,同时保证在这个过程中的通讯内容即使被拦截或篡改也不影响通讯的安全性,这正是Kerberos解决的问题。
-
-### 认证过程
-
-Kerberos的认证过程可分为三个阶段:
-
-(1)初始验证:客户端向KDC中的AS发送用户信息,以请求TGT;
-
-(2)获取服务票据:客户端拿着之前获得的TGT向KDC中的TGS请求访问某个服务的票据;
-
-(3)服务验证:客户端拿到票据(Ticket)后再到该服务的提供端验证身份,然后使用建立的加密通道与服务通信。
-
-### Kerberos认证时序图
-
-
-
-### 架构与工作原理
-
-
-
-(1)AS\_REQ
-是在初始化一个用户(kinit)的时候发出的用户认证请求,这个请求是发给KDC中的Authentication
-Server (AS);
-
-(2)AS\_REP 是 AS回复给client的信息,其中包括TGT (用TGS secret
-key加密过的) and the session key (用发请求的用户的secret key加密过的);
-
-(3)TGS\_REQ 是client为了一个service ticket向Ticket Granting Server
-(TGS)的信息. 其中包括上一条信息中获得的TGT (用TGS secret key加密过的)
-,一个客户端产生的一个authenticator(用session key加密过的);
-
-(4)TGS\_REP 是TGS回复给TGS\_REQ的信息. 其中包括service
-ticket(用appservice的secret key加密过),和 一个TGS产生的service
-sessinon key(之前AS产生的一条session key给加密过的);
-
-(5)AP\_REQ 是一条客户端发给appserver的访问服务的请求,其中包括service
-ticket和一个authenticator(使用TGS产生的service session key 加密过的);
-
-(6)AP\_REP
-是appserver回复给client的一条信息,证明这个appserver确实是客户端希望访问的server。不过这条信息也不一定总是要回复的。比如当客户端和appserver需要相互认证的时候,客户端向appserver发出请求,这个时候就不需要回复信息。
-
-LDAP介绍
-=========
-
-LDAP,即轻量级目录访问协议,是一种成熟、灵活且受良好支持的与目录服务器交互的基于标准的机制。它通常用于身份验证和存储有关用户、组和应用程序的信息,但
-LDAP 目录服务器是一种相当通用的数据存储,可用于各种应用程序。
-
-LDAP的基本模型
---------------
-
-1.1目录树概念 {#目录树概念 .ListParagraph}
--------------
-
-目录树:在一个目录服务系统中,整个目录信息集可以表示为一个目录信息树,树中的每个节点是一个条目。
-
-条目:每个条目就是一条记录,每个条目有自己的唯一可区别的名称(DN)。
-
-对象类:与某个实体类型对应的一组属性,对象类是可以继承的,这样父类的必须属性也会被继承下来。
-
-属性:描述条目的某个方面的信息,一个属性由一个属性类型和一个或多个属性值组成,属性有必须属性和非必须属性。
-
-1.2 DC、UID、OU、CN、SN、DN、RDN {#dcuidoucnsndnrdn .ListParagraph}
---------------------------------
-
-
-
-1.3 基本模型 {#基本模型 .ListParagraph}
-------------
-
-### 1.3.1 信息模型 {#信息模型 .ListParagraph}
-
-规定了 LDAP 目录信息的表示方式以及数据的存储结构。
-
-
-
-### 1.3.2 功能模型 {#功能模型 .ListParagraph}
-
-规定了可以对 LDAP 目录进行的操作(如查询、更新、认证等)。
-
-
-
-### 1.3.3 命名模型 {#命名模型 .ListParagraph}
-
-规定了 LDAP 目录中数据如何进行组织和区分。
-
-
-
-### 1.3.4 安全模型 {#安全模型 .ListParagraph}
-
-规定了保护 LDAP 目录中的数据的安全措施。
-
-
-
-1.4 LDAP的简单使用 {#ldap的简单使用 .ListParagraph}
-------------------
-
-统一身份认证主要是改变原有的认证策略,使需要认证的软件都通过LDAP进行认证,在统一身份认证之后,用户的所有信息都存储在AD
-Server中。终端用户在需要使用公司内部服务的时候,都需要通过AD服务器的认证。
-
-
-
-操作步骤:
-
-1\. 连接到LDAP服务器;
-
-2\. 绑定到LDAP服务器;
-
-3\. 在LDAP服务器上执行所需的任何操作;
-
-4\. 释放LDAP服务器的连接;
-
-认证原理
---------
-
-openLooKeng引擎层面的认证指的是当用户通过客户端连接openLooKeng引擎时的认证,目前openLooKeng支持基于LDAP的用户名密码方式的安全认证,整个认证过程原理图如下:
-
-
-
-当系统启动时,会由PasswordAuthenticatorManager模块调addPasswordAuthenticatorFactory加载PasswordAuthenticatorFactory
-实现类,当plugin插件加载完成时,PasswordAuthenticatorManager会调用loadPasswordAuthenticator
-方法完成具体 PasswordAuthenticator
-实例的创建。Ldap是继承了Password认证方式实现的,其中LdapAuthenticatorFactory类继承了PasswordAuthenticatorFactory类,内部认证原理与PasswordAuthenticator认证相同。
-
-1. openLooKeng对接LDAP&Kerberos类图
- =================================
-
- 1. PrestoServer
- ------------
-
-
-
-Bootstrap
----------
-
-
-
-ConfigurationFactory
---------------------
-
-
-
-Elements
---------
-
-
-
-AbstractConfigurationAwareModule
---------------------------------
-
-
-
-ServerSecurityModule
---------------------
-
-
-
-PluginManager
--------------
-
-
-
-KerberosAuthenticator
----------------------
-
-
-
-PasswordAuthenticator
----------------------
-
-
-
-openLooKeng对接LDAP&Kerberos时序图
-===================================
-
-4.1 Bootstrap {#bootstrap-1 .ListParagraph}
--------------
-
-Bootstrap类中的initialize方法负责对所有的模块进行初始化。
-
-
-
-4.2 ConfigurationFactory {#configurationfactory-1 .ListParagraph}
-------------------------
-
-
-
-Elements
---------
-
-
-
-AbstractConfigurationAwareModule
---------------------------------
-
-
-
-ServerSecurityModule
---------------------
-
-
-
-PluginManager
--------------
-
-
-
-源码解析
-=========
-
-5.1 FilterChain {#filterchain .ListParagraph}
----------------
-
-Servlet(Server Applet)是 Java Servlet 的简称,是使用 Java
-语言编写的运行在服务器端的程序。具有独立于平台和协议的特性,主要功能在于交互式地浏览和生成数据,生成动态Web内容。Servlet
-主要用于处理客户端传来的 HTTP 请求,并返回一个响应,它能够处理的请求有
-doGet() 和 doPost() 等。Servlet 应用程序的体系结构如下图所示:
-
-
-
-openLooKeng提供的WEB服务即是基于Servlet实现的,在基于Servlet的Web应用中,可以部署Filter过滤器,Servlet
-过滤器是可用于 Servlet 编程的 Java 类,有以下目的:
-
-> • 在客户端的请求访问后端资源之前,拦截这些请求。
->
-> • 在服务器的响应发送回客户端之前,处理这些响应。
->
-> 若有多个Filter都拦截同一目标资源,则它们就组成了一个Filter链(也称过滤器链),Filter链的拦截过程如下图所示:
-
-
-
-通过FilterChain可以进行很多的过滤操作,用户认证即包含在其中。
-
-5.2 加载过程 {#加载过程 .ListParagraph}
-------------
-
-### 5.2.1 PrestoServer {#prestoserver-1 .ListParagraph}
-
-PrestoServer是整个程序的路口,所有模块及插件都在此配置并加载,以及启动服务。
-
-
-
-[[]{#加载过程-1 .anchor}]{#prestoserver-2
-.anchor}其中的ServerSecurityModule即负责用户认证。
-
-当所有模块都添加完成后则开始进入初始化及配置步骤。
-
-
-
-### 5.2.2 Bootstrap {#bootstrap-2 .ListParagraph}
-
-Bootstrap类中的initialize方法负责对所有的模块进行初始化
-
-
-
-[]{#bootstrap-3
-.anchor}在initialize方法中将所有的模块传入ConfigurationFactory的registerConfigurationClasses进行注册。
-
-### ConfigurationFactory
-
-在ConfigurationFactory中又将所有的模块传入Elements中的getElements方法中进行配置。
-
-
-
-### Elements
-
-在Elements类中会根据不同的环境进行不同的配置,这里以DEVELOPMENT开发环境为例,在getElements方法中,会将所有的模块传入install方法中。
-
-
-
-install方法同属于Elements类,其会对每个模块调用configure方法
-
-
-
-### AbstractConfigurationAwareModule
-
-ServerSecurityModule集成并实现了AbstractConfigurationAwareModule类,在AbstractConfigurationAwareModule类中实现了configure方法,在该方法中会调用setup对模块进行配置
-
-
-
-### ServerSecurityModule
-
-ServerSecurityModule对AbstractConfigurationAwareModule抽象类中的setup方法进行了具体实现
-
-
-
-Servlet的FilterChain即在此注册,以及跟验证相关的PaswordAuthenticatorManager也在此加载
-
-
-
-同样根据在config.properties文件中的配置:http-server.authentication.type,进而加载对应的验证器
-
-
-
-可以通过代码看到,除了支持Kerberos和Password认证之外还支持CERTIFICATE和JWT认证
-
-
-
-至此ServerSecurityModule模块加载完毕。
-
-当所有模块都完成初始化及配置后,在PrestoServer类中又会通过loadPlugins方法对所有的插件进行加载
-
-
-
-### PluginManager
-
-PluginManager类会负责所有插件的加载
-
-
-
-下图是程序中包含的所有插件,跟验证相关的presto-password-authenticators插件会在此被加载
-
-
-
-PASSWORD验证方式默认是基于LDAP的密码认证
-
-
-
-loadPlugin方法中最后会通过installPlugin方法对插件进行安装
-
-
-
-在installPlugin方法中会根据不同的插件进行不同的配置,对于presto-password-authenticators插件则会根据http-server.authentication.type的配置完成不同验证器的加载
-
-
-
-当插件都加载完毕后在PrestoServer中会对ServerSecurityModule类中setup方法注册的PaswordAuthenticatorManager调用loadPasswordAuthenticator加载验证器
-
-
-
-在PaswordAuthenticatorManager中会根据在password-authenticator.properties文件中配置的验证方式加载对应的密码验证器
-
-
-
-password-authenticator.properties配置:
-
-
-
-loadPasswordAuthenticator方法:
-
-
-
-至此密码验证器加载完毕,之后PrestoServer启动完成。
-
-验证过程
---------
-
-当请求来到时会先经过前面注册的AuthenticationFilter,该类实现了Filter的doFilter方法,在doFilter方法中,会基于配置的验证方式和它对应的验证器,然后通过调用authenticate方法对每个请求进行验证
-
-
-
-### Kerberos认证
-
-Kerberos认证主要在KerberosAuthenticator类中完成的,在类初始化的时候会根据配置的信息如krb5.conf、keytab和principal等信息初始化好loginContext。
-
-
-
-当有请求带来时,会从request中取出用户信息或是token进行认证,如下:
-
-
-
-具体验证通过下列authenticate方法实现:
-
-
-
-当成功认证后会返回KerberosPrincipal继续传递给下一个Filter。
-
-### Password (LDAP)认证
-
-除了上面说的支持Kerberos的认证外,还支持密码认证,默认支持的是基于LDAP的密码认证。密码的认证是主要在PasswordAuthenticator类中的authenticate完成的
-
-
-
-在PasswordAuthenticator的authenticate方法中又调用了createAuthenticatedPrincipal方法进行验证,只有在LdapAuthenticator中对该方法进行了具体实现
-
-
-
-最终会在LdapAuthenticator的authenticate方法里面会调用LDAP相关的API跟LDAP服务器进行交互完成认证
-
-
-
-修订记录 {#修订记录 .154}
-========
-
- ------------ ------------------
- 发布日期 修订记录
- 2022-09-27 第一次正式发布
- 2022-10-20 增加时序图、类图
- ------------ ------------------
-
-相关视频:[openLooKeng内核认证与鉴权流程](https://space.bilibili.com/627629884/channel/collectiondetail?sid=1061144)
diff --git a/app/zh/information/blog/20230129-07/img/image3.png b/app/zh/information/blog/20230129-07/img/image3.png
deleted file mode 100644
index 1ad277f1fc6cd4e968dc72a8c21e8d1910eb8c5c..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-07/img/image3.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-07/img/image4.png b/app/zh/information/blog/20230129-07/img/image4.png
deleted file mode 100644
index 770a5185edcfa8622ee1cb555f4d2e89a33a10b4..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-07/img/image4.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-07/img/image5.png b/app/zh/information/blog/20230129-07/img/image5.png
deleted file mode 100644
index 47d57ed33535f4b43ffa864184e5e8e1701e9883..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-07/img/image5.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-07/img/image6.png b/app/zh/information/blog/20230129-07/img/image6.png
deleted file mode 100644
index 501e4c47f0369969feb63bf862b9340b595b6d0b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-07/img/image6.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-07/img/image7.png b/app/zh/information/blog/20230129-07/img/image7.png
deleted file mode 100644
index b5149511ccc104b571b73de2bffd3095e4f0a062..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-07/img/image7.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-07/index.md b/app/zh/information/blog/20230129-07/index.md
deleted file mode 100644
index 0fe584b7a7f5439334aa1a63689e5b58434758bd..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/20230129-07/index.md
+++ /dev/null
@@ -1,424 +0,0 @@
----
-title: 'openLooKeng对接LDAP+Kerberos流程梳理'
-date: '2022-12-25'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', 'LDAP', 'Kerberos', '认证鉴权', '安全认证']
-archives: '2022-12'
-author: 'tushengxia'
-description: 'openLooKeng对接LDAP+Kerberos流程梳理'
----
-
-介绍
-=====
-
-1. 简要介绍
-
-openLooKeng对接LDAP和kerberos实现用户登录认证
-
-1. 建议的版本
-
-建议使用openLooKeng的版本为1.7.0
-
-环境要求
-=========
-
-1. 操作系统要求
-
-操作系统要求如表2-2所示。
-
-1. []{#_table5438964 .anchor}操作系统要求
-
- -------- ------ ----------------------------------
- 项目 版本 下载地址
- CentOS 7.6 https://www.centos.org/download/
-
- -------- ------ ----------------------------------
-
-1. 环境要求
-
-openLooKeng
-
- ------------------------- -------------------------- -----------------
- 节点 域名 ip
- coordinator(协调节点) coordinator1.example.com ***.***.***.***
- worker(工作节点) worker1.example.com ***.***.***.***
- worker(工作节点) worker2.example.com ***.***.***.***
- ------------------------- -------------------------- -----------------
-
-LDAP
-
- ------ -------------------------- -----------------
- 节点 域名 ip
- LDAP coordinator1.example.com ***.***.***.***
- ------ -------------------------- -----------------
-
-Kerberos
-
- ---------- -------------------------- -----------------
- 节点 域名 ip
- Kerberos coordinator1.example.com ***.***.***.***
- ---------- -------------------------- -----------------
-
-\
-openLooKeng对接LDAP\
-(不依赖Kerberos)
-====================
-
-LDAP认证
---------
-
-> 通过配置openLooKeng,可以为客户端(如cli\_ldap)或JDBC、ODBC驱动启用HTTPS访问的前端LDAP认证。目前只支持涉及用户名和密码的简单的LDAP认证机制。openLooKeng客户端将用户名和密码发送给协调节点,协调节点使用外部LDAP服务验证这些凭据。
->
-> 为了给openLooKeng启用LDAP身份验证,需要修改openLooKeng协调节点上的配置。Worker节点上的配置不需要更改,只为仅验证从客户端到协调节点的通信。但是,如果您希望使用SSL/TLS来确保openLooKeng节点之间的通信安全,需要配置/security/internal-
-> communication。
-
-安全LDAP协议
-------------
-
-> openLooKeng需要安全的LDAP
-> (LDAPS)协议,所以请确保在LDAP服务器上启用了TLS。
-
-部署流程
---------
-
-> 您需要将LDAP服务器的TLS证书导入到openLooKeng协调节点默认的Java信任库中,以保证TLS连接的安全。可以使用下面的示例中的keytool命令将证书ldap.crt导入到协调节点上的信任库中。
->
-> 除此之外,使用HTTPS访问openLooKeng协调节点。您可以通过在协调节点上创建server\_java\_keystore来实现。
-
-### 导入证书(crt文件)到jdk
-
-> keytool -import -keystore
-> /usr/local/soft/jdk1.8.0\_171/jre/lib/security/cacerts -trustcacerts
-> -alias ldapserver -file ldap.crt
-
-密码:******, 注意:密码不可修改
-
-注意:仅在coordinator 操作
-
-### 生成openLooKeng的keystore
-
-创建所有 coordinator 和 worker 节点公用的 keystore。
-
-> keytool -genkeypair -validity 3650 -alias openlookeng -keyalg RSA
-> -dname "CN=\*.example.com,O=olk,OU=olk,L=olk,ST=olk,C=olk" -ext
-> "SAN=IP:***.***.***.***,DNS:localhost,DNS:coordinator.example.com,DNS:worker1.example.com,DNS:worker2.example.com"
-> -keystore keystore.jks
-
-密码:******
-
-### 将worker节点和coordinator节点主机名增加到hosts中
-
-> \# vim /etc/hosts
->
-> ***.***.***.*** coordinator.example.com
->
-> ***.***.***.*** worker1.example.com
->
-> ***.***.***.*** worker2.example.com
-
-注意:所有节点都需要增加
-
-### 同步keystore.jks到所有worker节点
-
-> scp keystore.jks worker\_ip:/<keystore path>
-
-注意:所有节点必须使用同一份keystore
-
-### 修改openLooKeng配置文件
-
-1. coordinator节点config.properties文件配置
-
-> \# vim config.properties
->
-> \#修改配置
->
-> discovery.uri=https://coordinator.example.com:8443
->
-> http-server.authentication.type=PASSWORD,CERTIFICATE
->
-> http-server.https.enabled=true
->
-> http-server.https.port=8443
->
-> http-server.https.keystore.path=<keystore path>
->
-> http-server.https.keystore.key=<keystore password>
->
-> node.internal-address=coordinator.example.com
->
-> http-server.http.enabled=false
->
-> internal-communication.https.required=true
->
-> internal-communication.https.keystore.path=<keystore path>
->
-> internal-communication.https.keystore.key=<keystore password>
-
-注意:在http-server.authentication.type配置中增加CERTIFICATE即不依赖Kerberos认证
-
-1. coordinator节点jvm.config文件配置
-
-> \# 增加配置
->
-> -Djavax.net.ssl.trustStore=/usr/local/soft/jdk1.8.0\_171/jre/lib/security/cacerts
->
-> -Djavax.net.ssl.trustStorePassword=changeit
-
-1. coordinator节点 password-authenticator.properties文件配置
-
-> \#增加配置
->
-> password-authenticator.name=ldap
->
-> ldap.url=ldaps://***.***.***.***:636
->
-> ldap.user-bind-pattern=uid=\${USER},ou=People,dc=chen,dc=com
->
-> ldap.cache-ttl=1s
->
-> \#\# 其中,ldap.url 为 ldap server 的地址。
-
-1. worker节点config.properties文件配置
-
-> discovery.uri=https://coordinator.example.com:8443
->
-> http-server.https.enabled=true
->
-> http-server.https.port=8443
->
-> http-server.https.keystore.path=<keystore path>
->
-> http-server.https.keystore.key=<keystore password>
->
-> node.internal-address=worker1.example.com
->
-> http-server.http.enabled=false
->
-> internal-communication.https.required=true
->
-> internal-communication.https.keystore.path=<keystore path>
->
-> internal-communication.https.keystore.key=<keystore password>
-
-### 重启openLookeng和LDAP
-
-> systemctl restart slapd
->
-> launcher restart
-
-验证
-----
-
-### Web UI
-
-https://coordinator.example.com:8443
-
-
-
-使用LDAP账号登录
-
-### 命令行测试
-
-> hetu-cli \\
->
-> --server https://coordinator.example.com:8443 \\
->
-> --keystore-path keystore.jks \\
->
-> --keystore-password ****** \\
->
-> --truststore-path keystore.jks \\
->
-> --truststore-password ****** \\
->
-> --user olk \\
->
-> --password
-
-输入LDAP密码:******
-
-\
-openLooKeng对接Kerberos
-=======================
-
-部署流程
---------
-
-### 创建principal
-
-> kadmin.local -q "addprinc -randkey openlookeng@EXAMPLE.COM"
->
-> kadmin.local -q "addprinc -randkey
-> openlookeng/coordinator.example.com@EXAMPLE.COM"
->
-> kadmin.local -q "addprinc -randkey
-> openlookeng/worker1.example.com@EXAMPLE.COM"
-
-### 生成keytab
-
-> kadmin -p admin/admin -q "ktadd -k ./lk.keytab
-> openlookeng@EXAMPLE.COM"
->
-> kadmin -p admin/admin -q "ktadd -k ./lk.keytab
-> openlookeng/coordinator.example.com@EXAMPLE.COM"
->
-> kadmin -p admin/admin -q "ktadd -k ./lk.keytab
-> openlookeng/worker1.example.com@EXAMPLE.COM"
->
-> 注意:openLooKeng 主机名必须为小写,如
-> worker1.example.com。所有节点主机名的 principal 都要创建。
-
-### 同步lk.keytab文件到 openLooKeng 集群所有节点上
-
-> scp lk.keytab
-> coordinator.example.com:/usr/local/soft/hetu-server-1.7.0
->
-> scp lk.keytab worker1.example.com:/usr/local/soft/hetu-server-1.7.0
-
-### 同步/etc/krb5.conf到openLookeng所有节点
-
-> scp /etc/krb5.conf coordinator.example.com:/etc
->
-> scp /etc/krb5.conf worker1.example.com:/etc
-
-### 修改openLooKeng配置文件
-
-启用 kerberos 认证,并指定 keytab 和 krb5 文件
-
-1. coordinator节点config.properties文件配置
-
-> internal-communication.kerberos.enabled=true
->
-> http-server.authentication.type=PASSWORD,CERTIFICATE,KERBEROS
->
-> http.server.authentication.krb5.service-name=HTTP
->
-> http.server.authentication.krb5.keytab=:/usr/local/soft/hetu-server-1.7.0/lk.keytab
->
-> http.authentication.krb5.config=/etc/krb5.conf
-
-1. worker节点config.properties文件配置
-
-> internal-communication.kerberos.enabled=true
->
-> http-server.authentication.type=CERTIFICATE,KERBEROS
->
-> http.server.authentication.krb5.service-name=HTTP
->
-> http.server.authentication.krb5.keytab=:/usr/local/soft/hetu-server-1.7.0/lk.keytab
->
-> http.authentication.krb5.config=/etc/krb5.conf
-
-1. 所有节点jvm.config文件配置
-
-> -Dsun.security.krb5.debug=true
->
-> -Dlog.enable-console=true
->
-> -Djava.security.krb5.conf=/etc/krb5.conf
-
-### 安装Kerberos Client
-
-> yum install krb5-workstation krb5-libs krb5-auth-dialog
-
-### 重启openLooKeng
-
-> launcher restart
-
-验证
-----
-
-### Web UI
-
-https://coordinator.example.com:8443
-
-> 
-
-选择Kerberos,使用LDAP账号登录,
-
-注意:仅支持火狐浏览器
-
-### 命令行LDAP认证
-
-> ./bin/hetu-cli \\
->
-> --server https://coordinator.example.com:8443 \\
->
-> --keystore-path keystore.jks \\
->
-> --keystore-password ****** \\
->
-> --truststore-path keystore.jks \\
->
-> --truststore-password ****** \\
->
-> --user olk \\
->
-> --password
-
-### 命令行Kerberos认证
-
-> ./bin/hetu-cli \\
->
-> --server https://coordinator.example.com:8443 \\
->
-> --krb5-config-path /etc/krb5.conf \\
->
-> --krb5-principal openlookeng@EXAMPLE.COM \\
->
-> --krb5-keytab-path lk.keytab \\
->
-> --krb5-remote-service-name openlookeng\\
->
-> --keystore-path keystore.jks \\
->
-> --keystore-password ******
-
-常见问题
-========
-
-#### *javax.net.ssl.SSLPeerUnverifiedException: Hostname localhost not verified: certificate: sha256/e3NmHTPCihGr3Gt0hj2q8oq0MZk5bedid18F67tk1mg= DN: CN=\*.example.com, OU=Unknown, O=Unknown, L=Unknown, ST=Unknown, C=Unknown subjectAltNames: \[\]*
-
-
-
-问题描述:openLookeng整合LDAP后访问WebUI之行sql报错
-
-问题分析:主机名localhost 未增加到keystore中
-
-解决方法:在创建keystore时将localhost 增加到DNS中
-
-> keytool -genkeypair -validity 3650 -alias openlookeng -keyalg RSA
-> -dname "CN=\*.example.com,O=olk,OU=olk,L=olk,ST=olk,C=olk" -ext
-> "SAN=IP:***.***.***.***,DNS:localhost,DNS:worker1.example.com,DNS:coordinator.example.com"
-> -keystore keystore.jks
-
-#### *Service announcement failed after 11.41ms. Next request will happen within 1000.00ms*
-
-
-
-
-问题描述:openLookeng整合LDAP后日志出现重复错误日志,同时在命令行执行sql报错
-
-
-
-问题分析:openLookeng节点的主机名未增加到keystore中
-
-解决方法:在创建keystore时将openLookeng所有节点的主机名增加到DNS中
-
-> keytool -genkeypair -validity 3650 -alias openlookeng -keyalg RSA
-> -dname "CN=\*.example.com,O=olk,OU=olk,L=olk,ST=olk,C=olk" -ext
-> "SAN=IP:***.***.***.***,DNS:localhost,DNS:worker1.example.com,DNS:coordinator.example.com"
-> -keystore keystore.jks
-
-修订记录
-========
-
- ------------ ----------------
- 发布日期 修订记录
- 2022-08-27 第一次正式发布
- 2022-11-09 修改格式
- ------------ ----------------
-
-相关视频:[openLooKeng内核认证与鉴权流程](https://space.bilibili.com/627629884/channel/collectiondetail?sid=1061144)
-
diff --git a/app/zh/information/blog/20230129-08/img/image3.png b/app/zh/information/blog/20230129-08/img/image3.png
deleted file mode 100644
index 64f3251d5fd53139ddb0c54fb5ed3358180fc19c..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-08/img/image3.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-08/img/image4.png b/app/zh/information/blog/20230129-08/img/image4.png
deleted file mode 100644
index 2fbefe9775aaf176cbe27740f9d4b63a611fed1b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-08/img/image4.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-08/img/image5.png b/app/zh/information/blog/20230129-08/img/image5.png
deleted file mode 100644
index d853ce6e85b9e924ad173184fd2ccb62437a5e2f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-08/img/image5.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-08/index.md b/app/zh/information/blog/20230129-08/index.md
deleted file mode 100644
index 790ad4c5cfd489ac5f746587083209b3a9f22ed3..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/20230129-08/index.md
+++ /dev/null
@@ -1,176 +0,0 @@
----
-title: 'openLooKeng对接Nginx详细设计文档'
-date: '2022-12-25'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', 'nginx', 'SSL', '认证鉴权', '安全认证']
-archives: '2022-12'
-author: 'tushengxia'
-description: 'openLooKeng对接Nginx详细设计文档'
----
-
-Nginx SSL加密
-==============
-
-SSL证书介绍
----------------
-
-SSL 证书就是遵守
-SSL协议,由受信任的数字证书颁发机构CA,在验证服务器身份后颁发,具有服务器身份验证和数据传输加密功能。
-
-SSL证书通过在客户端浏览器和Web服务器之间建立一条SSL安全通道(Secure
-socket layer(SSL)安全协议是由Netscape
-Communication公司设计开发。该安全协议主要用来提供对用户和服务器的认证;对传送的数据进行加密和隐藏;确保数据在传送中不被改变,即数据的完整性,现已成为该领域中全球化的标准。由于SSL技术已建立到所有主要的浏览器和WEB服务器程序中,因此,仅需安装服务器证书就可以激活该功能了),即通过它可以激活SSL协议,实现数据信息在客户端和服务器之间的加密传输,可以防止数据信息的泄露。保证了双方传递信息的安全性,而且用户可以通过服务器证书验证他所访问的网站是否是真实可靠。数位签名又名数字标识、签章
-(即 Digital Certificate,Digital ID
-),提供了一种在网上进行身份验证的方法,是用来标志和证明网路通信双方身份的数字信息文件,概念类似日常生活中的司机驾照或身份证相似。
-数字签名主要用于发送安全电子邮件、访问安全站点、网上招标与投标、网上签约、网上订购、安全网上公文传送、网上办公、网上缴费、网上缴税以及网上购物等安全的网上电子交易活动。
-
-SSL认证原理
------------
-
-安全套接字层(SSL) 技术通过加密信息和提供鉴权,保护您的网站安全。一份 SSL
-证书包括一个公共密钥和一个私用密钥。公共密钥用于加密信息,私用密钥用于解译加密的信息。浏览器指向一个安全域时,SSL
-同步确认服务器和客户端,并创建一种加密方式和一个唯一的会话密钥。它们可以启动一个保证消息的隐私性和完整性的安全会话。
-
-SSL的工作原理中包含如下三个协议:
-
-1. 握手协议(Handshake protocol)
-
-2. 记录协议(Record protocol)
-
-3. 警报协议(Alert protocol)
-
-### 握手协议
-
-握手协议是客户机和服务器用SSL连接通信时使用的第一个子协议,握手协议包括客户机与服务器之间的一系列消息。SSL中最复杂的协议就是握手协议。该协议允许服务器和客户机相互验证,协商加密和MAC算法以及保密密钥,用来保护在SSL记录中发送的数据。握手协议是在应用程序的数据传输之前使用的。
-
-### 记录协议
-
-记录协议在客户机和服务器握手成功后使用,即客户机和服务器鉴别对方和确定安全信息交换使用的算法后,进入SSL记录协议,记录协议向SSL连接提供两个服务:
-
-1. 保密性:使用握手协议定义的秘密密钥实现
-
-2. 完整性:握手协议定义了MAC,用于保证消息完整性
-
-### 警报协议
-
-客户机和服务器发现错误时,向对方发送一个警报消息。如果是致命错误,则算法立即关闭SSL连接,双方还会先删除相关的会话号,秘密和密钥。每个警报消息共2个字节,第1个字节表示错误类型,如果是警报,则值为1,如果是致命错误,则值为2;第2个字节制定实际错误类型。
-
-SSL证书功能
------------
-
-服务器部署了 SSL
-证书后可以确保用户在浏览器上输入的机密信息和从服务器上查询的机密信息从用户电脑到服务器之间的传输链路上是高强度加密传输的,是不可能被非法篡改和窃取的。同时向网站访问者证明了服务器的真实身份,此真实身份是通过第三方权威机构验证的。也就是说有两大作用:数据加密和身份认证。
-
-1)确认网站真实性(网站身份认证):用户需要登录正确的网站进行在线购物或其它交易活动,但由于互联网的广泛性和开放性,使得互联网上存在着许多假冒、钓鱼网站,用户如何来判断网站的真实性,如何信任自己正在访问的网站,可信网站将帮你确认网站的身份。当用户需要确认网站身份的时候,只需要点击浏览器地址栏里面的锁头标志即可。
-
-2)保证信息传输的机密性:用户在登录网站在线购物或进行各种交易时,需要多次向服务器端传送信息,而这些信息很多是用户的隐私和机密信息,直接涉及经济利益或私密,如何来确保这些信息的安全呢?可信网站将帮您建立一条安全的信息传输加密通道。
-
-在SSL会话产生时,服务器会传送它的证书,用户端浏览器会自动的分析服务器证书,并根据不同版本的浏览器,从而产生40位或128位的会话密钥,用于对交易的信息进行加密。所有的过程都会自动完成,对用户是透明的,因而,服务器证书可分为两种:最低40位和最低128位(这里指的是SSL会话时生成加密密钥的长度,密钥越长越不容易破解)证书。
-
-最低40位的服务器证书在建立会话时,根据浏览器版本不同,可产生40位或128位的SSL会话密钥用来建立用户浏览器与服务器之间的安全通道。而最低128位的服务器证书不受浏览器版本的限制可以产生128位以上的会话密钥,实现高级别的加密强度,无论是IE或Netscape浏览器,即使使用强行攻击的办法破译密码,也需要10年。
-
-HTTPS加密、解密过程
--------------------
-
-HTTPS有两部分组成:HTTP + SSL /
-TLS,也就是在HTTP上又加了一层处理加密信息的模块。服务端和客户端的信息传输都会通过TLS进行加密,所以传输的数据都是加密后的数据。
-
-
-
-SSL证书分类和应用环境
-----------------------
-
-目前市面上的SSL证书都是通过第三方SSL证书机构颁发的,常见可靠的第三方
-SSL证书颁发机构有DigiCert、GeoTrust、GlobalSign、Comodo等。
-
-根据不同使用环境,SSL证书可分为如下几种:
-
-1. 增强型SSL证书(EV SSL,即为Extended Validation
- SSL),适合银行金融类电子商务网站(网上购物)使用,证书里显示单位名称,显示绿色地址栏;有EV超安SSL和EV超安SSL
- Pro两个型号。
-
-2. 机构验证型SSL证书(OV SSL, 即为Organization Validation
- SSL),适合电子商务、电子政务网站、企事业单位管理系统、电子邮件系统使用,证书里显示单位名称;
- 有OV超真 SSL和OV 超真SSL Pro两个型号。
-
-3. 个人验证型SSL证书(IV SSL,即为IdentityValidation
- SSL),适用于个人专业网站使用,显示个人姓名;
-
-4. 域名验证型SSL证书(DV SSL 即为Domain Validation
- SSL),不显示任何信息,是最基础级的SSL
- 证书。通常是验证域名下某个指定文件的内容,或者验证与域名相关的某条
- TXT 记录;
-
-配置HTTPS需要的证书文件
------------------------
-
-Nginx配置HTTPS并不复杂,主要有两个步骤:
-
-1. 签发第三方可信任的SSL证书
-
-2. Nginx下配置 HTTPS
-
-要获取第三方CA机构的证书文件,需要证书申请者提供三个证书文件:
-
-1. 私钥文件(以.key结尾)
-
-2. 证书签署请求文件(以.crs结尾)
-
-3. 证书文件(以.crt结尾)
-
-其中:私钥文件由证书申请者生成,它是证书申请者的私钥文件,证书签署请求文件(crs文件),此文件里面包含申请者的DN(Distinguished
-Name,标识名)和公钥信息,此文件由证书申请者生成。将crs文件提交给第三方证书颁发机构,然后第三方CA机构会返回给申请者一个CRT文件。
-
-openLooKeng HA及反向代理
-------------------------
-
-要尽可能的显示HA的优势,建议客户端(例如openLooKeng CLI,
-JDBC等)不直接连接特定的协调节点,而是通过反向代理连接多个协调节点,例如使用负载平衡器或者Kubernetes服务,这样即使某个协调节点无法正常工作,客户端也可以继续使用其它的协调节点。
-
-### 反向代理要求
-
-通过反向代理连接时,要求客户端在执行语句查询期间连接到同一协调器,确保在语句查询运行时客户端和协调器之间保持恒定心跳。这可以通过配置Nginx反向代理实现,例如Nginx的ip\_hash。
-
-### ip\_hash
-
-ip\_hash是根据用户请求过来的ip,然后映射成hash值,然后分配到一个特定的服务器里面;
-
-使用ip\_hash这种负载均衡以后,可以保证用户的每一次会话都只会发送到同一台特定的Tomcat里面,它的session不会跨到其他的tomcat里面去的;
-
-
-
-首先通过将ip地址映射成一个hash值,然后将hash值对coordinator的数量3取模,得到coordinator的索引0、1、2;
-
-比如:5%3=2,则把这个请求发送到coordinator2服务器,以此类推;
-
-这样一来,只要用户的IP不发生改变,当前用户的会话就能够一直保持;
-
-
-
-nginx的ip\_hash算法是取ip地址的前三段数字进行hash映射,如果只有最后一段不一样,也会发送到同一个coordinator里面
-
-### 在nginx里面使用ip\_hash
-
-```java
-upstream backend {
- ip\_hash;
- server coordinator.example.com:8443;
- server coordinator1.example.com:8445;
-}
-```
-
-直接添加`ip\_hash`关键字即可,后续同一ip的访问将只会请求同一个服务器。
-
-### 注意事项
-
-一旦使用了ip\_hash,当我们需要移除一台服务器的时候,不能直接删除这个配置项,而是需要在这台服务器配置后面加上关键字down,表示不可用;因为如果直接移除配置项,会导致hash算法发生更改,后续所有的请求都会发生混乱;
-
-修订记录
-========
-
- 发布日期 修订记录
- ------------ ----------------
- 2022-09-27 第一次正式发布
-
-相关视频:[openLooKeng内核认证与鉴权流程](https://space.bilibili.com/627629884/channel/collectiondetail?sid=1061144)
diff --git a/app/zh/information/blog/20230129-09/img/image10.png b/app/zh/information/blog/20230129-09/img/image10.png
deleted file mode 100644
index 770a5185edcfa8622ee1cb555f4d2e89a33a10b4..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-09/img/image10.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-09/img/image11.png b/app/zh/information/blog/20230129-09/img/image11.png
deleted file mode 100644
index 26bc5dfa305f913e7cc9c9da3008baeafbb33fa6..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-09/img/image11.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-09/img/image12.png b/app/zh/information/blog/20230129-09/img/image12.png
deleted file mode 100644
index 1f89f472840494d84210dac82fdfb91fbb9c1cde..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-09/img/image12.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-09/img/image3.png b/app/zh/information/blog/20230129-09/img/image3.png
deleted file mode 100644
index 341e3dff5f3c2ea24343a97bc3a82d9c6abed798..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-09/img/image3.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-09/img/image4.png b/app/zh/information/blog/20230129-09/img/image4.png
deleted file mode 100644
index e0c96f17ac1f044e152678f0082dbc0c0c1b1b7b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-09/img/image4.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-09/img/image5.png b/app/zh/information/blog/20230129-09/img/image5.png
deleted file mode 100644
index 4b5d24fc212a8fdbef212d10d18ed02b12a960b3..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-09/img/image5.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-09/img/image6.png b/app/zh/information/blog/20230129-09/img/image6.png
deleted file mode 100644
index 229b86405b804628b20740b4f4ae7fc5e19883a2..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-09/img/image6.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-09/img/image7.png b/app/zh/information/blog/20230129-09/img/image7.png
deleted file mode 100644
index 22cb9221678504faf217665c0d8bca50aae078fc..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-09/img/image7.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-09/img/image8.png b/app/zh/information/blog/20230129-09/img/image8.png
deleted file mode 100644
index c09844a04d8a3c38526a4227d55539fd3421480d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-09/img/image8.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-09/img/image9.png b/app/zh/information/blog/20230129-09/img/image9.png
deleted file mode 100644
index aaf4dad334d39961352c94d1750d4e5844de8339..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-09/img/image9.png and /dev/null differ
diff --git a/app/zh/information/blog/20230129-09/index.md b/app/zh/information/blog/20230129-09/index.md
deleted file mode 100644
index 51a62ad2121dd15fb28e1185cc875f9f46527d99..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/20230129-09/index.md
+++ /dev/null
@@ -1,693 +0,0 @@
----
-title: 'openLooKeng安全集群对接安全Nginx流程梳理'
-date: '2022-12-25'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', 'nginx', 'SSL', '认证鉴权', '安全认证']
-archives: '2022-12'
-author: 'tushengxia'
-description: 'openLooKeng安全集群对接安全Nginx流程梳理'
----
-
-# 部署多个安全coordinator节点
-
-注意:多个coordinator需要使用同一个Kerberos,LDAP和Ranger
-
-## 安装 Ranger Admin 插件
-
-注意:Ranger Admin 插件只需要在ranger所在的节点配置即可
-
-### 下载Ranger Admin插件
-
-> 官方下载地址https://download.openlookeng.io/
->
-> 
->
-> 
->
-> 本文以1.7.0为例
->
-> 下载文件ranger-2.1.0-admin-openlookeng-1.7.0-plugin.tar.gz
-
-### 解压Ranger Admin插件
-
-解压到指定目录
-
-> tar -zxvf ranger-2.1.0-admin-openlookeng-1.7.0-plugin.tar.gz -C
-> /usr/local/soft/
-
-### Ranger 服务类型定义的注册
-
-> 使用 Ranger Admin提供的REST
-> API向Ranger注册服务类型定义。注册后,Ranger Admin 将提供UI
-> 以创建服务实例(在以前的版本中称为存储库)和服务类型策略。Ranger
-> 插件使用服务类型定义和策略来确定请求是否有访问权限以进行授权。如下示例所示,可以使用
-> curl 命令调用 REST API 接口注册服务:
->
-> Ranger的服务查询 REST API 示例:
->
-> 需要修改ranger的用户名和密码和ranger的主机名
->
-> curl -u admin:admin123 -X POST -H "Accept: application/json" -H
-> "Content-Type: application/json" -d
-> @service-defs/ranger-servicedef-openlookeng.json
-> http://master:6080/service/plugins/definitions
-
-注意:在Ranger Admin插件解压目录下执行
-
-### 复制openlookeng目录到Ranger Admin安装目录下的ranger-plugins目录:
-
-> scp -r openlookeng/ []{#OLE_LINK1
-> .anchor}/usr/local/soft/ranger-2.1.0-admin/ews/webapp/WEB-INF/classes/ranger-plugins/
-
-注意:在Ranger Admin插件解压目录下执行
-
-
-
-这时打开 Ranger Admin 可以看到多了“openLooKeng”服务:
-
-
-
-## openLooKeng对接ranger
-
-注意:每一个coordinator节点都需要配置
-
-### 下载openLooKeng插件
-
-> 官方下载地址https://download.openlookeng.io/
->
-> 
->
-> 本文以1.7.0为例
->
-> 下载文件ranger-2.1.0-openlookeng-1.7.0-plugin.tar.gz
-
-### 解压openLooKeng插件
-
-tar -zxvf ranger-2.1.0-openlookeng-1.7.0-plugin.tar.gz -C
-/usr/local/soft/
-
-### 修改 install.properties文件
-
-> \# ranger admin 地址
->
-> POLICY_MGR_URL=http://master:6080
->
-> \# 这个随便取,回头在ranger admin中新建jdbc连接的时候需要填写这个
->
-> REPOSITORY_NAME=[]{#OLE_LINK3 .anchor}openlookengdev
->
-> \# openLooKeng 安装目录
->
-> COMPONENT_INSTALL_DIR_NAME=/usr/local/openlookeng/hetu
->
-> XAAUDIT.SOLR.ENABLE=false
->
-> \# 添加该条配置,不然后面执行会报错
->
-> XAAUDIT.SUMMARY.ENABLE=false
-
-备注:在openLooKeng插件插件解压路径下修改配置
-
-### 创建用户
-
-> groupadd hadoop
->
-> useradd -g hadoop openlookeng
-
-### 执行enable-openlookeng-plugin.sh
-
-> sh enable-openlookeng-plugin.sh
->
-> 注意:此时需要有openlookeng用户,若没有,自行创建。
->
-> 
-
-### 重启服务
-
-> 重启 openLooKeng 服务:bin/launcher restart
-
-## openLooKeng对接LDAP+Kerberos
-
-注意:所有coordinator 都要配置
-
-### 导入证书(crt文件)到jdk
-
-注意:crt文件从LDAP节点获取
-
-> keytool -import -keystore
-> /usr/local/soft/jdk1.8.0_171/jre/lib/security/cacerts -trustcacerts
-> -alias ldapserver -file ldap.crt
-
-密码:**\*\***, 注意:密码不可修改
-
-注意:仅在coordinator 操作
-
-### 生成openLooKeng的keystore
-
-注意:keystore在某一个coordinator 中生成即可,将生成的文件同步到所有节点
-
-创建所有 coordinator 和 worker 节点公用的 keystore。
-
-> keytool -genkeypair -validity 3650 -alias openlookeng -keyalg RSA
-> -dname "CN=\*.example.com,O=olk,OU=olk,L=olk,ST=olk,C=olk" -ext
-> "SAN=IP:**_._**.**_._**,DNS:localhost,DNS:coordinator1.example.com,DNS:coordinator2.example.com,DNS:worker1.example.com,DNS:worker2.example.com"
-> -keystore keystore.jks
-
-密码:**\*\***
-
-### 同步keystore.jks到所有节点
-
-注意:所有节点必须使用同一份keystore
-
-> scp keystore.jks ip:/<keystore path>
-
-### 安装Kerberos Client
-
-注意:所有节点都需要安装
-
-> yum install krb5-workstation krb5-libs krb5-auth-dialog
-
-### 创建principal
-
-注意:principal在某一个coordinator
-中生成即可,将生成的文件同步到所有节点
-
-> kadmin.local -q "addprinc -randkey openlookeng@EXAMPLE.COM"
->
-> kadmin.local -q "addprinc -randkey
-> openlookeng/coordinator1.example.com@EXAMPLE.COM"
->
-> kadmin.local -q "addprinc -randkey
-> openlookeng/coordinator2.example.com@EXAMPLE.COM"
->
-> kadmin.local -q "addprinc -randkey
-> openlookeng/worker1.example.com@EXAMPLE.COM"
->
-> kadmin.local -q "addprinc -randkey
-> openlookeng/worker2.example.com@EXAMPLE.COM"
-
-### 生成keytab
-
-> kadmin -p admin/admin -q "ktadd -k ./lk.keytab
-> openlookeng@EXAMPLE.COM"
->
-> kadmin -p admin/admin -q "ktadd -k ./lk.keytab
-> openlookeng/coordinator1.example.com@EXAMPLE.COM"
->
-> kadmin -p admin/admin -q "ktadd -k ./lk.keytab
-> openlookeng/coordinator2.example.com@EXAMPLE.COM"
->
-> kadmin -p admin/admin -q "ktadd -k ./lk.keytab
-> openlookeng/worker1.example.com@EXAMPLE.COM"
->
-> kadmin -p admin/admin -q "ktadd -k ./lk.keytab
-> openlookeng/worker2.example.com@EXAMPLE.COM"
->
-> 注意:openLooKeng 主机名必须为小写,如
-> worker1.example.com。所有节点主机名的 principal 都要创建。
-
-### 同步lk.keytab文件到 openLooKeng 集群所有节点上
-
-> scp lk.keytab
-> coordinator1.example.com:/usr/local/soft/hetu-server-1.7.0
->
-> scp lk.keytab
-> coordinator2.example.com:/usr/local/soft/hetu-server-1.7.0
->
-> scp lk.keytab worker1.example.com:/usr/local/soft/hetu-server-1.7.0
->
-> scp lk.keytab worker2.example.com:/usr/local/soft/hetu-server-1.7.0
-
-### 同步/etc/krb5.conf到openLookeng所有节点
-
-> scp /etc/krb5.conf coordinator1.example.com:/etc
->
-> scp /etc/krb5.conf coordinator2.example.com:/etc
->
-> scp /etc/krb5.conf worker1.example.com:/etc
->
-> scp /etc/krb5.conf worker2.example.com:/etc
-
-### 修改openLooKeng配置文件
-
-1. coordinator节点config.properties文件配置
-
- 所有coordinator都需要修改
-
-> \# vim config.properties
->
-> \#修改配置
->
-> discovery.uri=https://coordinator.example.com:8443
->
-> http-server.authentication.type=PASSWORD,CERTIFICATE,KERBEROS
->
-> http-server.https.enabled=true
->
-> http-server.https.port=8443
->
-> http-server.https.keystore.path=<keystore path>
->
-> http-server.https.keystore.key=<keystore password>
->
-> internal-communication.kerberos.enabled=true
->
-> http.server.authentication.krb5.service-name=HTTP
->
-> http.server.authentication.krb5.keytab=:/usr/local/soft/hetu-server-1.7.0/lk.keytab
->
-> http.authentication.krb5.config=/etc/krb5.conf
->
-> node.internal-address=coordinator.example.com
->
-> http-server.http.enabled=false
->
-> internal-communication.https.required=true
->
-> internal-communication.https.keystore.path=<keystore path>
->
-> internal-communication.https.keystore.key=<keystore password>
-
-1. coordinator节点jvm.config文件配置
-
-> \# 增加配置
->
-> -Djavax.net.ssl.trustStore=/usr/local/soft/jdk1.8.0_171/jre/lib/security/cacerts
->
-> -Djavax.net.ssl.trustStorePassword=changeit
->
-> -Dsun.security.krb5.debug=true
->
-> -Dlog.enable-console=true
->
-> -Djava.security.krb5.conf=/etc/krb5.conf
-
-1. coordinator节点 password-authenticator.properties文件配置
-
-> \#增加配置
->
-> password-authenticator.name=ldap
->
-> ldap.url=ldaps://**_._**.**_._**:636
->
-> ldap.user-bind-pattern=uid=\${USER},ou=People,dc=chen,dc=com
->
-> ldap.cache-ttl=1s
->
-> \#\# 其中,ldap.url 为 ldap server 的地址。
-
-1. worker节点config.properties文件配置
-
-> discovery.uri=https://coordinator.example.com:8443
->
-> http-server.https.enabled=true
->
-> http-server.https.port=8443
->
-> http-server.https.keystore.path=<keystore path>
->
-> http-server.https.keystore.key=<keystore password>
->
-> internal-communication.kerberos.enabled=true
->
-> http-server.authentication.type=CERTIFICATE,KERBEROS
->
-> http.server.authentication.krb5.service-name=HTTP
->
-> http.server.authentication.krb5.keytab=:/usr/local/soft/hetu-server-1.7.0/lk.keytab
->
-> http.authentication.krb5.config=/etc/krb5.conf
->
-> node.internal-address=worker1.example.com
->
-> http-server.http.enabled=false
->
-> internal-communication.https.required=true
->
-> internal-communication.https.keystore.path=<keystore path>
->
-> internal-communication.https.keystore.key=<keystore password>
-
-1. []{#\_Toc1373921670 .anchor}worker节点jvm.properties文件配置
-
-> \# 增加配置
->
-> -Dsun.security.krb5.debug=true
->
-> -Dlog.enable-console=true
->
-> -Djava.security.krb5.conf=/etc/krb5.conf
-
-## 配置认证用户映射
-
-注意:所有coordinator节点都需要配置
-
-1. 创建映射文件
-
- 映射文件放在openLooKeng的data目录下
-
- vim mapping.json
-
-> {
->
-> "rules": \[
->
-> {
->
-> "pattern": ".\*O=(.+), OU=.\*"
->
-> },
->
-> {
->
-> "pattern": "(.+)"
->
-> }
->
-> \]
->
-> }
-
-注意:pattern的规则和keystore.jks中的dname有关
-
-"pattern": ".\*O=(.+), OU=.\*" 配置到olk用户名
-
-keytool -genkeypair -validity 3650 -alias openlookeng -keyalg RSA -dname
-"CN=\*.example.com,O=olk,OU=olk,L=olk,ST=olk,C=olk" -ext
-"SAN=IP:**_._**.**_._**,DNS:localhost,DNS:coordinator1.example.com,DNS:coordinator2.example.com,DNS:worker1.example.com,DNS:worker2.example.com"
--keystore keystore.jks
-
-如以以上示例所示:将keystore映射为olk用户(.\*O=(.+), OU=.\*)
-
-1. 给olk用户增加impersonateuser权限
-
- 
-
-2. coordinator节点config.properties文件配置
-
- 注意:所有coordinator节点都需要修改
-
-> http-server.authentication.certificate.user-mapping.file=mapping.json
-
-## 重启openLookeng和LDAP
-
-> systemctl restart slapd
->
-> launcher restart
-
-## 验证
-
-测试所有coordinator节点可用性
-
-### Web UI
-
-https://coordinator.example.com:8443
-
-> 
-
-选择Kerberos,使用LDAP账号登录,
-
-注意:仅支持火狐浏览器
-
-### 命令行LDAP认证
-
-> ./bin/hetu-cli \\
->
-> --server https://coordinator.example.com:8443 \\
->
-> --keystore-path keystore.jks \\
->
-> --keystore-password **\*\*** \\
->
-> --truststore-path keystore.jks \\
->
-> --truststore-password **\*\*** \\
->
-> --user olk \\
->
-> --password
-
-### 命令行Kerberos认证
-
-> ./bin/hetu-cli \\
->
-> --server https://coordinator.example.com:8443 \\
->
-> --krb5-config-path /etc/krb5.conf \\
->
-> --krb5-principal openlookeng@EXAMPLE.COM \\
->
-> --krb5-keytab-path lk.keytab \\
->
-> --krb5-remote-service-name openlookeng\\
->
-> --keystore-path keystore.jks \\
->
-> --keystore-password **\*\***
-
-# 部署具有高可用(HA)的openLooKeng
-
-## 配置协调节点和工作节点属性
-
-### 在所有协调节点上的etc/config.properties中添加如下配置。
-
-> hetu.multiple-coordinator.enabled=true
->
-> hetu.embedded-state-store.enabled=true
-
-### 在所有工作节点上的etc/config.properties中添加如下配置。
-
-> hetu.multiple-coordinator.enabled=true
-
-上述属性说明如下:
-
-hetu.multiple-coordinator.enabled: 启用多个协调节点。
-
-hetu.embedded-state-store.enabled: 协调节点启用嵌入式状态存储。
-
-说明:建议在所有协调节点(或至少2个)上启用嵌入式状态存储,以确保节点/网络关闭时,服务可以保持高可用性。
-
-## 配置状态存储区
-
-注意:需要依赖hadoop集群
-
-### 配置协调节点
-
-(1) 创建文件etc\\state-store.properties并添加以下属性。
-
-> state-store.type=hazelcast
->
-> state-store.name=query
->
-> state-store.cluster=cluster1
->
-> hazelcast.discovery.mode=tcp-ip
->
-> hazelcast.discovery.port=5701
->
-> hazelcast.discovery.tcp-ip.seeds=**_._**.**_._**:5701,**_._**.**_._**:5701
->
-> hazelcast.discovery.tcp-ip.profile=hdfs-config-default
-
-(1) 创建文件etc\\filesystem\\hdfs-config-default.properties
-并添加以下属性。
-
-> fs.client.type=hdfs
->
-> hdfs.config.resources=/opt/hdfs/core-site.xml,/opt/hdfs/hdfs-site.xml
->
-> hdfs.authentication.type=NONE
->
-> fs.hdfs.impl.disable.cache=true
-
-### 配置工作节点
-
-(1) 创建文件etc\\state-store.properties并添加以下属性。
-
-> state-store.type=hazelcast
->
-> state-store.name=query
->
-> state-store.cluster=cluster1
->
-> hazelcast.discovery.mode=tcp-ip
->
-> hazelcast.discovery.tcp-ip.seeds=**_._**.**_._**:5701,**_._**.**_._**:5701
->
-> hazelcast.discovery.tcp-ip.profile=hdfs-config-default
-
-(1) 创建文件etc\\filesystem\\hdfs-config-default.properties并添加以下属性。
-
-> fs.client.type=hdfs
->
-> \#Assume core-site.xml and hdfs-site.xml are located in /opt/hdfs
-> directory
->
-> hdfs.config.resources=/opt/hdfs/core-site.xml,/opt/hdfs/hdfs-site.xml
->
-> hdfs.authentication.type=NONE
->
-> fs.hdfs.impl.disable.cache=true
-
-## 重启openLooKeng
-
-> launcher restart
-
-## 测试
-
-分别访问多个coordinator节点测试是否可用
-
-
-
-# Nginx搭建
-
-## 安装gcc-c++编译器
-
-> yum install gcc-c++
->
-> yum install -y openssl openssl-devel
-
-## 安装pcre包
-
-> yum install -y pcre pcre-devel
-
-## 安装zlib包
-
-> yum install -y zlib zlib-devel
-
-## 下载Nginx安装包
-
-> wget https://nginx.org/download/nginx-1.19.9.tar.gz
-
-## 解压并进入nginx目录
-
-> tar -zxvf nginx-1.19.9.tar.gz
->
-> cd nginx-1.19.9
-
-## 编译安装
-
-> ./configure --prefix=/usr/local/soft/nginx
-> --with-http_stub_status_module --with-http_ssl_module
->
-> make
->
-> make install
-
-## nginx常用指令
-
-> /usr/local/soft/nginx/sbin/nginx -s reload \# 重新载入配置文件
->
-> /usr/local/soft/nginx/sbin/nginx -s reopen \# 重启 Nginx
->
-> /usr/local/soft/nginx/sbin/nginx -s stop \# 停止 Nginx
-
-# Nginx开启Https
-
-## 安装OpenSSL
-
-> yum install -y openssl openssl-devel
-
-## 检查Nginx是否支持https_ssl_module
-
-> nginx -V
-
-观察是否带有“–with-http_ssl_module”
-
-没有需要重新编译安装
-
-## 生成没有加密的私钥
-
-> openssl genrsa > nginx.key 2048
-
-## 根据私钥生成公钥
-
-> openssl req -new -x509 -key nginx.key > nginx.pem
-
-## 生成加密私钥
-
-> openssl genrsa -des3 -out server.pass.key 2048
-
-参数: genrsa:生成RSA私钥;-des3:des3算法;-out
-server.pass.key:生成的私钥文件名;2048:私钥长度
-
-## 去除私钥中的密码
-
-> openssl rsa -in server.pass.key -out server.key
-
-## 生成CSR(证书签名请求文件)
-
-> openssl req -new -key server.key -out server.csr
-
-## 生成自签名的SSL证书
-
-> openssl x509 -req -days 1825 -in server.csr -signkey server.key -out
-> server.crt
-
-# Nginx对接openLooKeng
-
-## 修改Nginx配置文件
-
-vim /usr/local/soft/nginx/conf/nginx.conf
-
-> http {
->
-> upstream backend {
->
-> ip_hash;
->
-> server coordinator.example.com:8443;
->
-> server coordinator1.example.com:8445;
->
-> }
->
-> server {
->
-> listen 4430 ssl;
->
-> server_name **_._**.**_._**;
->
-> ssl_certificate ssl/nginx.pem;
->
-> ssl_certificate_key ssl/nginx.key;
->
-> ssl_protocols SSLv3 TLSv1;
->
-> location / {
->
-> proxy_pass https://backend;
->
-> proxy_set_header Host **_._**.**_._**:4430;
->
-> }
->
-> }
->
-> }
-
-说明:将nginx.pem和nginx.key文件放在nginx_home/conf/ssl目录下
-
-## 重启nginx
-
-> nginx -s reopen
->
-> nginx -s reload
-
-## 测试
-
-访问Nginx
-
-https://**_._**.**_._**:4430
-
-
-
-# 修订记录
-
----
-
-发布日期 修订记录
-2022-09-20 第一次正式发布
-
----
diff --git a/app/zh/information/blog/20230129-10/img/image1.jpg b/app/zh/information/blog/20230129-10/img/image1.jpg
deleted file mode 100644
index b8f8988a51af891b380d3b3da21acb34bee6bdf2..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-10/img/image1.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20230129-10/img/image2.jpg b/app/zh/information/blog/20230129-10/img/image2.jpg
deleted file mode 100644
index 2f822d22331b421da24597dcefccad27a04851de..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-10/img/image2.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20230129-10/img/image3.jpg b/app/zh/information/blog/20230129-10/img/image3.jpg
deleted file mode 100644
index 1fb50dc395dc4e3d8645a4c84d693d8ee399537f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-10/img/image3.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20230129-10/img/image4.jpg b/app/zh/information/blog/20230129-10/img/image4.jpg
deleted file mode 100644
index bbe7e72a4eae02c35eb1dd0e72cef27cf14ec0a6..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-10/img/image4.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20230129-10/img/image5.jpg b/app/zh/information/blog/20230129-10/img/image5.jpg
deleted file mode 100644
index ebff64c850a24d738200458027dc2507318bdcdb..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-10/img/image5.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20230129-10/img/image6.jpg b/app/zh/information/blog/20230129-10/img/image6.jpg
deleted file mode 100644
index c37cfc3a32b0724f070f5f15d639ffc73d80d8bd..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-10/img/image6.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20230129-10/img/image7.jpg b/app/zh/information/blog/20230129-10/img/image7.jpg
deleted file mode 100644
index 82aed6237ed0ca357f6fbf4f29b23e37b6049ce9..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-10/img/image7.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20230129-10/img/image8.jpg b/app/zh/information/blog/20230129-10/img/image8.jpg
deleted file mode 100644
index f0a502ca4a8314f148bec6ddd4a83fb258d421fc..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/20230129-10/img/image8.jpg and /dev/null differ
diff --git a/app/zh/information/blog/20230129-10/index.md b/app/zh/information/blog/20230129-10/index.md
deleted file mode 100644
index a6980fd2b9aecd30dac06465e39550adcaf853df..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/20230129-10/index.md
+++ /dev/null
@@ -1,557 +0,0 @@
----
-title: 'OpenLDAP安装文档'
-date: '2022-12-25'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', 'nginx', 'Kerberos', 'LDAP', '认证鉴权', '安全认证']
-archives: '2022-12'
-author: 'tushengxia'
-description: 'OpenLDAP安装文档'
----
-
-OpenLDAP安装步骤
-============
-
-关闭系统防火墙
---------
-
-```java
-systemctl stop firewalld
-
-systemctl disable firewalld
-```
-
-关闭SELinux
---------
-
-```java
-setenforce 0
-sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
-```
-
-安装OpenLDAP
---------
-
-```java
-yum -y install openldap openldap-clients openldap-servers migrationtools openldap-devel
-```
-
-修改OpenLDAP的slapd.ldif配置文件
-------------
-
-```java
-进入ldap服务目录
-cd /usr/share/openldap-servers
-## 复制slapd.ldif文件到root根目录
-cp slapd.ldif /root/
-## 进入root根目录
-cd /root/
-## 编辑复制后的文件
-vim slapd.ldif
-## 追加以下内容:
-############ slapd.ldif 内容如下##################
-# See slapd-config(5) for details on configuration options.
-# This file should NOT be world readable.
-#
-
-dn: cn=config
-objectClass: olcGlobal
-cn: config
-olcArgsFile: /var/run/openldap/slapd.args
-olcPidFile: /var/run/openldap/slapd.pid
-#
-# TLS settings TLS可以注释掉
-#
-#olcTLSCACertificatePath: /etc/openldap/certs
-#olcTLSCertificateFile: "OpenLDAP Server"
-#olcTLSCertificateKeyFile: /etc/openldap/certs/password
-#
-
-
-#
-# Schema settings (ldap的schema设置,可以默认,不用修改)
-#
-
-dn: cn=schema,cn=config
-objectClass: olcSchemaConfig
-cn: schema
-## cn: schema 后面必须接一个 空行
-
-#include: file:///etc/openldap/schema/core.ldif
-
-## 注意顺序.(高能预警:有坑)注意上面必须得有个空行,否则报错,随时空行就对了.
-include: file:///etc/openldap/schema/corba.ldif
-include: file:///etc/openldap/schema/core.ldif
-include: file:///etc/openldap/schema/cosine.ldif
-include: file:///etc/openldap/schema/duaconf.ldif
-include: file:///etc/openldap/schema/dyngroup.ldif
-include: file:///etc/openldap/schema/inetorgperson.ldif
-include: file:///etc/openldap/schema/java.ldif
-include: file:///etc/openldap/schema/misc.ldif
-include: file:///etc/openldap/schema/nis.ldif
-include: file:///etc/openldap/schema/openldap.ldif
-include: file:///etc/openldap/schema/ppolicy.ldif
-include: file:///etc/openldap/schema/collective.ldif
-#
-# Frontend settings(使用默认信息,不用改)
-#
-
-dn: olcDatabase=frontend,cn=config
-objectClass: olcDatabaseConfig
-objectClass: olcFrontendConfig
-olcDatabase: frontend
-#
-#
-
-#
-# Configuration database(ldap数据库db信息,使用默认不用改)
-#
-
-dn: olcDatabase=config,cn=config
-objectClass: olcDatabaseConfig
-olcDatabase: config
-olcAccess: to * by dn.base="gidNumber=0+uidNumber=0,cn=peercred,cn=external,c
- n=auth" manage by * none
-
-#
-# Server status monitoring
-# 自定义域 cn=Manager,dc=chen,dc=com
-## (chen位置,可以修改,但是下面都需要统一哦)
-#
-
-dn: olcDatabase=monitor,cn=config
-objectClass: olcDatabaseConfig
-olcDatabase: monitor
-olcAccess: to * by dn.base="gidNumber=0+uidNumber=0,cn=peercred,cn=external,c
- n=auth" read by dn.base="cn=Manager,dc=chen,dc=com" read by * none
-
-#
-# Backend database definitions
-#
-
-dn: olcDatabase=hdb,cn=config
-objectClass: olcDatabaseConfig
-objectClass: olcHdbConfig
-olcDatabase: hdb
-olcSuffix: dc=chen,dc=com
-olcRootDN: cn=Manager,dc=chen,dc=com
-olcRootPW: {SSHA}/JVZxfQtyJw+c7Ajt4lwYDDozZJ+yoct
-olcDbDirectory: /var/lib/ldap
-olcDbIndex: objectClass eq,pres
-olcDbIndex: ou,cn,mail,surname,givenname eq,pres,sub
-olcDbIndex: uidNumber,gidNumber,loginShell eq,pres
-olcDbIndex: uid,memberUid eq,pres,sub
-
-olcDbIndex: nisMapName,nisMapEntry eq,pres,sub
-######################################## end ############################
-```
-
-注意:
-几处改动的地方如下:
-1. TLS settings TLS可以注释掉
-2. #include: file:///etc/openldap/schema/core.ldif注释或删除该行,复制粘贴该行下面的xxx.ldif
-3. 三处dc=dc-domain,改为自己的,也可以参考dc=chen(dc类似于文件系统的目录)
-4. 最后增加一下配置:
-(1) olcRootPW: {SSHA}SXPP7UseIlnqTeOrrb6PrbCFIWSYvZeM
-(2) olcDbIndex: uidNumber,gidNumber,loginShell eq,pres
-(3) olcDbIndex: uid,memberUid eq,pres,sub
-(4) olcDbIndex: nisMapName,nisMapEntry eq,pres,sub
-其中olcRootPW后面的value值可以通过slappasswd命令生成:`slappasswd -s ******`
-
-
-为openLdap重新生成配置
----------------------------------
-
-1. 删除老配置
-
-```java
-rm -rf /etc/openldap/slapd.d/*
-```
-
-2. 使用/root/slapd.ldif新的配置文件重新生成配置
-
-```java
-slapadd -F /etc/openldap/slapd.d -n 0 -l /root/slapd.ldif
-```
-注意:如果提示报错, 检查slapd.ldif的内容是否都正确
-
-3. 测试配置返回“config file testing succeeded”则表示配置文件正确
-
-```java
-slaptest -u -F /etc/openldap/slapd.d
-```
-
-4. 将新生成的配置,属主转交给ldap用户
-
-```java
-chown -R ldap. /etc/openldap/slapd.d/
-```
-
-
-安装openLDAP的数据库文件
--------------------------------
-
-1. 将`/usr/share/openldap-servers/`目录下的`DB_CONFIG.example`文件拷贝至`/var/lib/ldap`目录下并重命名为`DB_CONFIG`
-
-```java
-cp /usr/share/openldap-servers/DB_CONFIG.example /var/lib/ldap/DB_CONFIG
-```
-
-
-2. 修改数据库文件属主给ldap用户
-
-```java
-chown -R ldap. /var/lib/ldap/
-```
-
-启动ldap主服务
-----------------------
-
-```java
-systemctl enable slapd ##开机启动
-systemctl start slapd ##启动
-systemctl status slapd ##查看状态
-systemctl stop slapd ##停止
-```
-
-生成ldap所需的结构
-----------------------
-
-1. 手工创建并编辑需要导入ldap的根及管理账号信息
-
-```java
-vim root.ldif
-## 手工编辑信息如下
-###########################
-## ldap将生成的’根‘信息
-dn: dc=chen,dc=com
-dc: chen
-objectClass: top
-objectClass: domain
-
-## ldap将生成的’管理员账号‘信息
-dn: cn=Manager,dc=chen,dc=com
-objectClass: organizationalRole
-cn: Manager
-```
-
-
-
-2. 导入 “根” 与 “管理员账号” 信息
-
-```java
-ldapadd -D "cn=Manager,dc=chen,dc=com" -W -x -f root.ldif
-```
-
-注意
-密码是:slapd.ldif中配置的密码(******)
-
-3. 查看是否导入成功
-
-```java
-ldapsearch -b "dc=chen,dc=com" -D "cn=Manager,dc=chen,dc=com" -W
-```
-
-4. 修改默认的根域信息
-
-```java
-## 进入目录
-cd /usr/share/migrationtools/
-## 编辑基础文件
-vim migrate_common.ph
-## 修改如下2个参数. 调整为生成的根信息
-$DEFAULT_MAIL_DOMAIN = "chen.com";
-$DEFAULT_BASE = "dc=chen,dc=com";
-```
-
-5. 创建openLdap所需的系统基础文件(base.ldif文件)
-
-```java
-## 创建基础文件
-vim base.ldif
-###########保留如下内容,其它可看着删除#############
-## 如下为系统基本用户信息(linux系统用户信息)
-dn: ou=People,dc=chen,dc=com
-ou: People
-objectClass: top
-objectClass: organizationalUnit
-
-dn: ou=Group,dc=chen,dc=com
-ou: Group
-objectClass: top
-objectClass: organizationalUnit
-########################
-```
-
-
-
-6. 导出系统用户组信息(group.ldif文件)
-
-```java
-## 创建组文件
-vim group.ldif
-## 增加配置
-dn: cn=root,ou=Group,dc=chen,dc=com
-objectClass: posixGroup
-objectClass: top
-cn: root
-userPassword: {crypt}x
-gidNumber: 0
-
-dn: cn=chen,ou=Group,dc=chen,dc=com
-objectClass: posixGroup
-objectClass: top
-cn: chen
-userPassword: {crypt}x
-gidNumber: 1101
-```
-
-
-
-7. 创建系统用户信息(user.ldif文件)
-
-```java
-## 创建用户文件
-vim user.ldif
-##增加配置
-dn: uid=root,ou=People,dc=chen,dc=com
-uid: root
-cn: root
-objectClass: account
-objectClass: posixAccount
-objectClass: top
-objectClass: shadowAccount
-userPassword: {crypt}$6$uwFhiB6t4noSw097$.3rE8EYGfBclUE2hmHQ6vwO6Yv96eHARxPXvZGxaggQw1JTRU/c9Y7vifdomBJBsz1DUrtKmpkUtB.XygufF10
-shadowMin: 0
-shadowMax: 99999
-shadowWarning: 7
-loginShell: /bin/bash
-uidNumber: 0
-gidNumber: 0
-homeDirectory: /root
-gecos: root
-
-dn: uid=olk,ou=People,dc=chen,dc=com
-uid: olk
-cn: olk
-objectClass: account
-objectClass: posixAccount
-objectClass: top
-objectClass: shadowAccount
-userPassword: {SSHA}0/b2fgqHSJUc/SWTUPOyByW08PocyVeS
-shadowLastChange: 18353
-shadowMin: 0
-shadowMax: 99999
-shadowWarning: 7
-loginShell: /bin/bash
-uidNumber: 1101
-gidNumber: 1101
-homeDirectory: /home/olk
-gecos: olk
-```
-
-其中userPassword后面的value值可以通过slappasswd命令生成:`slappasswd -s ******`
-
-8. 将基础文件、用户、组导入OpenLDAP
-
-```java
-##会提示输入密码(密码是slapd.ldif里面的配置)
-ldapadd -D "cn=Manager,dc=chen,dc=com" -W -x -f base.ldif
-ldapadd -D "cn=Manager,dc=chen,dc=com" -W -x -f group.ldif
-ldapadd -D "cn=Manager,dc=chen,dc=com" -W -x -f user.ldif
-```
-
-生成OpenLDAP证书
------------------------------------
-
-1. 复制openssl配置文件到/tmp下
-
-```java
-cp /etc/pki/tls/openssl.cnf /tmp/openssl.cnf
-```
-2. 编辑/tmp/openssl.cnf
-
-```java
-## 此文件的格式类似ini的配置文件格式,找到[req]段落,加上下面的配置:
-req_extensions = v3_req
-## 这段配置表示在生成 CSR 文件时读取名叫 v3_req 的段落的配置信息,因此我们再在此配置文件中加入一段名为 v3_req 的配置:
-subjectAltName = @alt_names
-## 这段配置中最重要的是在最后导入名为 alt_names 的配置段,因此我们还需要添加一个名为 [ alt_names ] 的配置段:
-[ alt_names ]
-IP = ***.***.***.***
-## 注:ip 为OpenLDAP 服务器 ip
-```
-
-
-
-3. 生成OpenLDAP证书(crt文件)
-
-(1) 进入目录
-
-```
-cd /etc/openldap/certs/
-```
-
-(2) 生成私钥key
-
-```
-openssl genrsa -out ldap.key 1024
-```
-
-(3) 基于openssl文件和key文件,生成csr签名文件
-
-```
-openssl req -new -key ldap.key -out ldap.csr -config /tmp/openssl.cnf
-```
-
-
-
-
-(4) 查看证书请求文件的内容:
-
-```
-openssl req -text -noout -in ldap.csr
-```
-
-
-
-(5) 生成签名
-
-```
-openssl x509 -req -days 3650 -in ldap.csr -signkey ldap.key -out ldap.crt -extensions v3_req -extfile /tmp/openssl.cnf
-```
-
-配置ldap开启ssl
------------------------------------
-
-1. 创建certs.ldif
-
-```
-# 文件内容
-# 说明: (⾼能预警:有坑) 该配置为tls相关配置,配置⽂件必须为此格式,此顺序,此⽅式导⼊,
-dn: cn=config
-changetype: modify
-replace: olcTLSCACertificatePath
-olcTLSCACertificatePath: /etc/openldap/certs
--
-replace: olcTLSCertificateFile
-olcTLSCertificateFile: /etc/openldap/certs/ldap.crt
--
-replace: olcTLSCertificateKeyFile
-olcTLSCertificateKeyFile: /etc/openldap/certs/ldap.key
--
-replace: olcTLSVerifyClient
-olcTLSVerifyClient: never
-```
-
-2. 执行命令导入配置到 /etc/openldap/slapd.d/cn=config.ldif 文件中:
-
-```
-ldapmodify -Y EXTERNAL -H ldapi:/// -f certs.ldif
-```
-
-3. 编辑/etc/openldap/ldap.conf
-
-```
-vim /etc/openldap/ldap.conf
-# 增加配置
-TLS_REQCERT allow
-TLS_CERT /etc/openldap/certs/ldap.crt
-TLS_KEY /etc/openldap/certs/ldap.key
-# ldap节点ip
-URI ldaps://***.***.***.***
-# ldap base名称
-BASE dc=chen,dc=com
-```
-
-
-
-4. 配置同时启动ldap和ldaps
-
-```
-vim /etc/sysconfig/ldap (没有就⼿动创建)
-SLAPD_LDAP=yes
-SLAPD_LDAPI=yes
-SLAPD_LDAPS=yes
-vim /etc/sysconfig/slapd
-# 配置slapd启⽤监听ldaps
-vim /etc/sysconfig/slapd
-# 修改配置
-SLAPD_URLS="ldapi:/// ldap:/// ldaps:///"
-```
-
-5. 重启OpenLDAP
-
-```
-service slapd restart
-```
-
-PhpLDAPAdmin安装
-========
-
-yum安装
------------------------------------
-
-```
-yum -y install httpd php php-ldap php-gd php-mbstring php-pear php-bcmath php-xml
-yum -y install epel-release
-yum --enablerepo=epel -y install phpldapadmin
-```
-
-修改配置文件
------------------------------------
-
-1. /etc/httpd/conf.d/phpldapadmin.conf
-
-```
-# 修改配置
-
-
-1.统一的资源目录管理
-
-支持统一的元数据管理,包括元数据的模型设计、模型审核、模型发布、模型变更,以及模型版本管理、关系管理等;
-
-2.全面灵活的数据采集
-
-采用调度中心+决策中心相结合的分布式数据采集处理架构,支持多样化的采集策略和多种抽取、汇总方式;支持丰富的数据处理组件,可视化任务编排、配置、规则定义及发布;针对结构化资源和非结构化资源,以数据映射、数据脱敏、数据过滤的工具化手段进行数据处理;
-
-3.集中式数据质量管控
-
-支持数据质量核查规则配置与管理,在数据采集清洗过程中完成数据质量核查与告警;支持对数据质量告警的监控和数据质量问题的可视化呈现;提供知识库管理和查询,支持数据血统分析和影响分析;
-
-4.模型版本管理、关系管理
-
-支持以数据服务的形式封装数据,提供统一的数据开发共享能力;数据服务封装和开发,支持 WebService 协议、FTP 协议和数据库等各类接口;支持服务发布、订阅及审核管理,同时支持对服务调用情况进行监控,以确保平台采集管理的数据正常实现交换共享。
-
-
-
-在解决以上传统大数据大数据平台问题之外,北明数据资产管理平台 V4.0 版本采用华为开源 openLooKeng 引擎,构建轻量级流、批、交互式全场景数据查询交互分析,支持从不同数据中心或不同数据源获取数据,并对外进行共享开放,减少数据迁移、接口开发的工作量。同时支持一次性从不同数据中心、不同数据库获取数据并输出到目标端,从而减少数据的搬运成本,提高交换的效率,弥补跨 DC、跨库数据的数据联合分析与即时查询空白。
-
-
-
-### openLooKeng 在北明数据资产管理平台 V4.0 的价值:
-
-1.跨数据源查询;
-
-2.SQL 算子下推数据;
-
-3.跨数据中心:选择最优的跨数据中心执行计划,拆分子查询到不同 DC 的多个计算引擎执行;
-
-4.标准 SQL 语法:SQL2003;
-
-5.数据即时查询:数据采样、小数据交互式查询等。
-
-
diff --git a/app/zh/information/blog/blogguidance/index.md b/app/zh/information/blog/blogguidance/index.md
deleted file mode 100644
index f5f5d04bd4aad9b9cae622fbec2db952ef9d4611..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/blogguidance/index.md
+++ /dev/null
@@ -1,115 +0,0 @@
----
-title: '如何发表博客'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng']
-author: 'openLooKeng'
-date: '2021-07-20'
-description: '如何发表博客'
----
-
-## 准备
-
-1. 参考 标签嵌入图片, 但你的图片资源需要放入当前目录下(即 your-gitee-id 目录下),输入图片名称作为 src 值:
-
-```
-
-```
-
-1. Commit 你的博客
-
-```
-git add
-
-拷贝生成的 keystore.jks 到集群所有节点上
-
-### 1.2 所有节点编辑/etc/hosts 文件,在 hostname 后面增加 example.com 部分(因为上面创建的 keystore 是\*.example.com),如下
-
-```
-ip1 host1 host1.example.com
-ip2 host2 host2.example.com
-ip3 host3 host3.example.com
-```
-
-### 1.3 配置 openLooKeng 的 config.properties
-
-所有节点配置:
-
-```
-node.internal-address=hostname.example.com //hostname为各节点主机名
-http-server.http.enabled=false
-http-server.https.enabled=true
-http-server.https.port=9090 //设置https端口号
-http-server.https.keystore.path=/opt/hetu/keystore.jks //keystore存放路径
-http-server.https.keystore.key=****** //生成keystore时输入的密码
-internal-communication.https.required=true
-internal-communication.https.keystore.path=/opt/hetu/keystore.jks //keystore存放路径
-internal-communication.https.keystore.key=****** //生成keystore时输入的密码
-```
-
-**注:**文件路径和密码根据实际修改
-
-### 1.4 验证
-
-重启 openLooKeng 服务,使用 hetu-cli 连接:
-
-```
-java -jar hetu-cli-1.4.0-SNAPSHOT-executable.jar --server https://host1.example.com:9090 --keystore-path /opt/hetu/keystore.jks --keystore-password ******
-```
-
-### 2 开启 openLooKeng 的 Kerberos 认证
-
-### 2.1 安装 Kerberos Server
-
-#### 2.1.1 安装
-
-```
-yum install -y krb5-server krb5-lib krb5-workstation
-```
-
-#### 2.1.2 修改配置文件
-
-**配置/etc/krb5.conf**
-
-```
-# Configuration snippets may be placed in this directory as well
-
-includedir /etc/krb5.conf.d/
-[logging]
- default = FILE:/var/log/krb5libs.log
- kdc = FILE:/var/log/krb5kdc.log
- admin_server = FILE:/var/log/kadmind.log
-
-[libdefaults]
- dns_lookup_realm = false
- ticket_lifetime = 24h
- renew_lifetime = 7d
- forwardable = true
- rdns = false
- pkinit_anchors = FILE:/etc/pki/tls/certs/ca-bundle.crt
- default_realm = EXAMPLE.COM
- default_ccache_name = /tmp/krb5cc_%{uid}
-
-[realms]
- EXAMPLE.COM = {
- kdc = host4 //kerberos服务器主机名
- admin_server = host4
- }
-
-[domain_realm]
- .example.com = EXAMPLE.COM
- example.com = EXAMPLE.COM
-```
-
-**注:**
-
-1. 修改 default_realm 与[realms]中命名相同;
-2. kdc、admin_server 配置为本地的 hostname;
-3. 配置 default_ccache_name。
-
-拷贝 krb5.conf 文件到 openLooKeng 集群所有节点上,config.properties 认证配置会用到。
-
-**配置** **/var/kerberos/krb5kdc/kdc.conf**
-
-```
-[kdcdefaults]
- kdc_ports = 88
- kdc_tcp_ports = 88
-
-[realms]
- EXAMPLE.COM = {
- \#master_key_type = aes256-cts
- acl_file = /var/kerberos/krb5kdc/kadm5.acl
- dict_file = /usr/share/dict/words
- admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab
- supported_enctypes = aes256-cts:normal aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal camellia256-cts:normal camellia128-cts:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal
- }
-```
-
-**注:**kdc.conf 中 realm 名与 krb5.conf 相同
-
-#### 2.1.3 创建数据库
-
-```
-/usr/sbin/kdb5_util create -r EXAMPLE.COM -s
-```
-
-其中需要设置管理员密码,创建完成会在/var/kerberos/krb5kdc/下面生成 principal 文件,若重建数据库则需先删除/var/kerberos/krb5kdc 下面 principal 相关文件
-
-#### 2.1.4 创建管理员账号
-
-编辑/var/kerberos/krb5kdc/kadm5.acl,增加
-
-```
-*/admin@EXAMPLE.COM *
-```
-
-**注:**代表全部权限
-
-#### 2.1.5 重启 kerberos 服务
-
-```
-systemctl restart krb5kdc
-systemctl restart kadmin
-```
-
-### **2.2** **Kerberos client 安装(client 和 server 共部署时,不需要安装)**
-
-#### 2.2.1 安装
-
-```
-yum install krb5-workstation krb5-libs krb5-auth-dialog
-```
-
-#### 2.2.2 配置
-
-复制 kerberos 的 server 的/etc/krb5.conf 到 client 的/etc/krb5.conf
-
-#### 2.2.3 验证客户端可以访问 KDC
-
-```
-kinit admin/admin
-kadmin -p 'admin/admin' -w '******' -s 'ip4' -q 'list_principals'
-```
-
-**注:**-w 为访问密码,-s 为 kerberos KDC IP
-
-### 2.3 openLooKeng 对接 Kerberos
-
-#### 2.3.1 生成 openLooKeng 的 keytab
-
-在装有 Kerberos 客户端的机器上执行下面语句(**生成服务名为 HTTP,webUI 访问会用到**)
-
-创建 principal:
-
-```
-kadmin -p admin/admin -q "addprinc -randkey HTTP@EXAMPLE.COM"
-kadmin -p admin/admin -q "addprinc -randkey HTTP/host1@EXAMPLE.COM"
-kadmin -p admin/admin -q "addprinc -randkey HTTP/host2@EXAMPLE.COM"
-kadmin -p admin/admin -q "addprinc -randkey HTTP/host3@EXAMPLE.COM"
-```
-
-生成 keytab:
-
-```
-kadmin -p admin/admin -q "ktadd -k /opt/openlookeng/lks/lk.keytab HTTP@EXAMPLE.COM"
-kadmin -p admin/admin -q "ktadd -k /opt/openlookeng/lks/lk.keytab HTTP/host1@EXAMPLE.COM"
-kadmin -p admin/admin -q "ktadd -k /opt/openlookeng/lks/lk.keytab HTTP/host2@EXAMPLE.COM"
-kadmin -p admin/admin -q "ktadd -k /opt/openlookeng/lks/lk.keytab HTTP/host3@EXAMPLE.COM"
-```
-
-**注:**openLooKeng 主机名必须为小写,如 host1。所有节点主机名的 principal 都要创建
-
-拷贝 lk.keytab 文件到 openLooKeng 集群所有节点上,config.properties 认证配置会用到。
-
-#### 2.3.2 配置 openLooKeng 的 config.properties
-
-所有节点增加:
-
-```
-http-server.authentication.type=KERBEROS
-http.server.authentication.krb5.service-name= HTTP
-http.server.authentication.krb5.keytab=/opt/openlookeng/lks/lk.keytab
-http.authentication.krb5.config=/opt/openlookeng/lks/krb5.conf
-internal-communication.kerberos.enabled=true
-```
-
-#### 2.3.4 配置 openLooKeng 的 jvm.config
-
-所有节点增加:
-
-```
--Dsun.security.krb5.debug=true
--Dlog.enable-console=true
--Djava.security.krb5.conf=/opt/openlookeng/lks/krb5.conf
-```
-
-#### 2.3.5 验证:
-
-重启 openLooKeng 服务,使用 hetu-cli 连接:
-
-```
-java -Dsun.security.krb5.debug=true -jar hetu-cli-1.4.0-SNAPSHOT-executable.jar \
---server https://host1.example.com:9090 \
---keystore-path /opt/hetu/keystore.jks \
---keystore-password ****** \
---krb5-config-path /opt/openlookeng/lks/krb5.conf \
---krb5-principal HTTP \
---krb5-keytab-path /opt/openlookeng/lks/lk.keytab \
---krb5-remote-service-name HTTP \
---debug
-```
-
-**注:**路径、密码、principal 根据实际情况修改
-
-### 3 openLooKeng(开启 kerberos)浏览器访问 webUI
-
-openLooKeng 开启 kerberos 认证,采用的是 ticket 的认证的方式,因此不管在浏览器输什么用户名、密码都无法认证通过。
-
-**解决方法:**
-
-**配置浏览器使用 ticker 认证方式,不用传统的用户名、密码认证方式。**
-
-**当前对于 ticket 认证方式支持比较好的是 firefox 浏览器,以下以 firefox 浏览器配置为例说明如何配置 ticket 认证,访问开启 kerberos 认证的 openLooKeng 服务。**
-
-### 3.1 安装 windows Kerberos 客户端
-
-Kerberos windows 客户端叫 KFW,下载地址:http://web.mit.edu/kerberos/dist/index.html,默认安装即可。默认情况会安装到C:\Program Files\MIT\Kerberos 路径,同时在 C:\ProgramData\MIT\Kerberos5 路径下生成 kerberos 的配置文件 krb5.ini。
-
-### 3.2 配置 kerberos 客户端
-
-配置环境变量:
-
-KRB5_CONFIG ------kerberos 配置文件
-
-KRB5CCNAME ------kerberos 认证缓存文件路径
-
-
-
-**注:**重启系统,使配置生效
-
-### 3.3 配置浏览器
-
-在浏览器输入 about:config,搜索 network.negotiate,配置如下:
-
-- network.negotiate-auth.delegation-uris、network.negotiate-auth.trusted-uris 为 coordinator 的 hostname+域名,比如 host1.example.com
-
-- network.negotiate-auth.gsslib 为 KFW 安装路径下 gssapid 的路径(根据浏览器是 32 位、64 位选择对应文件)
-
- 比如:C:\Program Files\MIT\Kerberos\bin\gssapi64.dll
-
-- network.negotiate-auth.using-native-gsslib 修改为 false
-
-- network.negotiate-auth.allow-non-fqdn 修改为 true
-
-搜索 network.auth.use-sspi ,设置为 false
-
-### 3.4 配置 KFW
-
-可以从 KDC 服务器上拷贝/etc/krb5.conf 中的内容,然后写入 krb5.ini 文件中。
-
-**注:**需要删除或者注释掉 default_ccache_name 的设置
-
-```
-[logging]
-default = FILE:/var/log/krb5libs.log
-kdc = FILE:/var/log/krb5kdc.log
-admin_server = FILE:/var/log/kadmind.log
-[libdefaults]
-dns_lookup_realm = false
-dns_lookup_kdc = false
-ticket_lifetime = 24h
-\#renew_lifetime = 7d
-forwardable = true
-default_realm = EXAMPLE.COM
-
-[realms]
-EXAMPLE.COM = {
-kdc = host4
-admin_server = host4
-}
-
-[domain_realm]
-.example.com = EXAMPLE.COM
-example.com = EXAMPLE.COM
-```
-
-### 3.5 完成认证
-
-配置完成后,使用创建的用户认证(kerberos 客户端或命令行)
-
-使用 Kerberos 客户端:
-
-
-
-使用命令行:
-
-
-
-如果是机机用户,使用 keytab 认证,需要拷贝 keytab 文件到 windows 机器上,再使用命令认证
-
-
-
-**注:**
-
-1. 在 C:\Windows\System32\drivers\etc\hosts 中需增加 KDC、coordinator 节点 hostname 到 IP 的映射关系;
-2. 使用 KFW 的 kinit,不能使用 java 的 kinit
-
-### 3.6 登录 openLooKeng WEB UI
-
-https://host1.example.com:9090
-
-
-
-
-
-**注:**
-
-1. 配置 openLooKeng 的 config.properties 中 http.server.authentication.krb5.service-name 必须为 HTTP;
diff --git a/app/zh/information/blog/configguide-02/2021-09-24-config-11.png b/app/zh/information/blog/configguide-02/2021-09-24-config-11.png
deleted file mode 100644
index 22b9f7ac6abca03657cf6266d853e9857c7e5b63..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-02/2021-09-24-config-11.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-02/2021-09-24-config-12.png b/app/zh/information/blog/configguide-02/2021-09-24-config-12.png
deleted file mode 100644
index 24da7572eb041f1b13d59fc992634fbe04894d29..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-02/2021-09-24-config-12.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-02/index.md b/app/zh/information/blog/configguide-02/index.md
deleted file mode 100644
index c12bee75d3ddad4598b824a46f32bec0b953116a..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/configguide-02/index.md
+++ /dev/null
@@ -1,175 +0,0 @@
----
-title: 'openLooKeng AA安全配置指导(二)----对接nginx'
-date: '2021-09-24'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', 'Kerberos', 'nginx']
-archives: '2021-09'
-author: 'senny456'
-description: '指导如何配置openLooKeng AA开启Kerberos后,对接nginx代理执行sql'
----
-
-**环境信息**
-
-| Role | IP | Hostname | Hostname+domain name |
-| ------------------ | --- | -------- | -------------------- |
-| coordinator&worker | ip1 | host1 | host1.example.com |
-| coordinator&worker | ip2 | host2 | host2.example.com |
-| worker | ip3 | host3 | host3.example.com |
-| Kerberos/nginx | ip4 | host4 | host4.example.com |
-
-openLooKeng AA 开启 Kerberos,参见[openLooKeng AA 安全配置指导(一)----对接 Kerberos](https://openlookeng.io/zh/information/blog/configguide-01/)
-
-### 1 openLooKeng(开启 kerberos)对接 Nginx
-
-### 1.1 Nginx 安装
-
-1. 开源 nginx 的版本选型
-
- 下载地址: http://openresty.org/cn/download.html
-
- 选用 openresty-***.***.***.*** 版本,需从 nginx 官网下载
-
-2. 开源 nginx 的依赖模块组件
-
- nginx_upstream_check_module
-
- 下载地址:https://github.com/yaoweibin/nginx_upstream_check_module
-
- ngx_http_consistent_hash
-
- 下载地址:https://github.com/replay/ngx_http_consistent_hash
-
-3. 安装步骤:
-
- - 依赖安装:
-
- ```
- yum -y install gcc zlib zlib-devel pcre-devel openssl openssl-devel
- ```
-
- - 安装 openresty
-
- ```
- ./configure --prefix=/usr/local/openresty/ --with-http_stub_status_module --with-luajit --without-http_redis2_module --with-http_iconv_module --with-http_ssl_module --with-stream --add-module=/opt/nginx_upstream_check_module --add-module=/opt/ngx_http_consistent_hash
- ```
-
- gmake
-
- gmake install
-
- - 添加环境变量
-
- ```
- vi /etc/profile
- 添加: PATH=/usr/local/openresty/nginx/sbin:$PATH
- source /etc/profile
- ```
-
-4. 启动、停止、重启的命令
-
- 启动: nginx -c /usr/local/openresty/nginx/conf/nginx.conf
-
- 重启: nginx -s reload
-
- 停止: nginx -s stop
-
-### 1.2 Nginx.conf 配置
-
-[nginx.conf](nginx.conf)
-
-注意修改以下内容
-
-
-
-host1.example.com:9090 为 openlookeng coordinator 节点的域名和端口号
-
-client.cer 是从 openLooKeng 的 keystore 中提取的证书
-
-lk.key 是从 openLooKeng 的 keystore 中提取的私钥
-
-**注意:证书和私钥,nginx 用户需要用权限读取**
-
-
-
-### 1.3 证书和私钥提取方式(参考)
-
-提取私钥
-
-1. 先转为 pfs 格式(使用 openLooKeng 中生成的 keystore.jks)
-
- ```
- keytool -v -importkeystore -srckeystore keystore.jks -srcstoretype jks -srcstorepass ****** -destkeystore lk.pfx -deststoretype pkcs12 -deststorepass ****** -destkeypass ******
- ```
-
-2. 显示到界面
-
- ```
- openssl pkcs12 -in lk.pfx -nocerts -nodes
- openssl pkcs12 -in lk.pfx -nocerts -nodes -out lk.key (保存到key)
- ```
-
-3. 导出证书
-
- ```
- keytool -export -alias openLooKeng -keystore keystore.jks -rfc -file client.cer
- ```
-
-### 1.4 添加 nginx 的 keytab
-
-创建 principal:
-
-```
-kadmin -p admin/admin -q "addprinc -randkey HTTP/host4@EXAMPLE.COM"
-```
-
-添加 keytab:
-
-```
-kadmin -p admin/admin -q "ktadd -k /opt/openlookeng/lks/lk.keytab HTTP/host4@EXAMPLE.COM"
-```
-
-**拷贝并替换 openLooKeng 集群所有节点上的 keytab 文件**
-
-### 1.5 配置 openLooKeng 的 config.properties
-
-coordinator 节点上增加如下配置:
-
-```
-failure-detector.http-client.authentication.krb5.service-principal-pattern=${SERVICE}@host4
-query-info.http-client.authentication.krb5.service-principal-pattern=${SERVICE}@host4
-workerInfo.http-client.authentication.krb5.service-principal-pattern=${SERVICE}@host4
-memoryManager.http-client.authentication.krb5.service-principal-pattern=${SERVICE}@host4
-scheduler.http-client.authentication.krb5.service-principal-pattern=${SERVICE}@host4
-node-manager.http-client.authentication.krb5.service-principal-pattern=${SERVICE}@host4
-exchange.http-client.authentication.krb5.service-principal-pattern=${SERVICE}@host4
-event.http-client.authentication.krb5.service-principal-pattern=${SERVICE}@host4
-discovery.http-client.authentication.krb5.service-principal-pattern=${SERVICE}@host4
-http.server.authentication.krb5.principal-hostname=host4
-```
-
-worker 节点上增加如下配置:
-
-```
-node-manager.http-client.authentication.krb5.service-principal-pattern=${SERVICE}@host4
-exchange.http-client.authentication.krb5.service-principal-pattern=${SERVICE}@host4
-event.http-client.authentication.krb5.service-principal-pattern=${SERVICE}@host4
-discovery.http-client.authentication.krb5.service-principal-pattern=${SERVICE}@host4
-http.server.authentication.krb5.principal-hostname=host4
-```
-
-**其中:host4 为 nginx 节点的主机名**
-
-### 1.6 验证
-
-```
-java -jar hetu-cli-1.4.0-SNAPSHOT-executable.jar \
---server https://host4.example.com:443 \
---keystore-path /opt/hetu/keystore.jks \
---keystore-password ****** \
---krb5-config-path /opt/openlookeng/lks/krb5.conf \
---krb5-principal HTTP \
---krb5-keytab-path /opt/openlookeng/lks/lk.keytab \
---krb5-remote-service-name HTTP \
---debug
-```
diff --git a/app/zh/information/blog/configguide-02/nginx.conf b/app/zh/information/blog/configguide-02/nginx.conf
deleted file mode 100644
index 1a0d5f733585ce4dbfc20e0a11da496e83aec1e1..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/configguide-02/nginx.conf
+++ /dev/null
@@ -1,104 +0,0 @@
-user nginx; # 指定nginx的运行用户为nginx
-worker_processes 8;
-worker_cpu_affinity 00000001 00000010 00000100 00001000 00010000 00100000 01000000 10000000;
-
-error_log logs/error.log info;
-
-#pid logs/nginx.pid;
-
-worker_rlimit_nofile 65535; ## nginx进程可以打开的最大文件句柄数的上限
-events {
- use epoll;
- worker_connections 65240;
-}
-
-http {
- include mime.types;
- default_type application/octet-stream;
- underscores_in_headers on;
- server_tokens off;
- ssi off;
-
- log_format main '$remote_addr - $remote_user [$time_local] "$request" '
- '$status $body_bytes_sent "$http_referer" '
- '"$http_user_agent" "$http_x_forwarded_for"';
-
- access_log logs/access.log main;
-
- autoindex off;
- port_in_redirect off;
-
- keepalive_timeout 125;
- client_header_timeout 125;
- client_body_timeout 125;
-
- fastcgi_buffer_size 1024k;
- fastcgi_buffers 8 1024k;
- fastcgi_busy_buffers_size 1024k;
-
- proxy_buffer_size 1024k;
- proxy_buffers 32 128k;
- proxy_busy_buffers_size 1024k;
-
- proxy_temp_file_write_size 2048k;
-
- client_header_buffer_size 128k;
- large_client_header_buffers 32 64k;
-
-
- upstream gotoaa {
- ip_hash;
- server host1.example.com:9090 weight=1; ##更改为coordinator的IP和端口
- server host2.example.com:9090 weight=1; ##更改为coordinator的IP和端口, 多个可新增
- check interval=3000 rise=2 fall=5 timeout=1000 type=http;
-
- }
-
-
- server {
- listen 443 ssl default_server;
- server_name host4.example.com;
- add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
-
- ## 证书和秘钥从被代理的openLooKeng集群中的keystore文件中提取
- ssl_certificate /usr/local/openresty/nginx/ssls/148/client.cer; # 证书,路径必须是nginx用户可读取的路径
- ssl_certificate_key /usr/local/openresty/nginx/ssls/148/lk.key; #秘钥,路径必须是nginx用户可读取的路径
- ssl_session_timeout 5m; # 超时时间5分钟,避免攻击者建立大量无效链接或慢速攻击
- ssl_session_cache shared:SSL:10m; #使用10M共享内存,ssl会话缓存,可以使客户端在一定时间内复用这个ssl会话,减少资源占用;
- ssl_protocols TLSv1.2 TLSv1.3;
- ssl_ciphers "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256";
- ##ssl_ciphers ALL:!aNULL;
- ssl_prefer_server_ciphers on; # 协商时优先使用服务器指定的加密算法
-
- client_header_buffer_size 64k;
- large_client_header_buffers 8 64k;
- client_body_buffer_size 64k;
- client_max_body_size 1m;
-
- add_header X-XSS-Protection "1; mode=block"; #提供xss防护功能
- add_header X-Frame-Options DENY; # 提供点击劫持防护功能
- add_header X-Content-Type-Options nosniff; # 提供禁用浏览器的类型猜测功能
- add_header Strict-Transport-Security " max-age=31536000; includeSubDomains "; #启用HSTS功能,强制走HTTPS
- add_header Content-Security-Policy "default-src 'self'"; #提供对加载资源的安全控制功能
- add_header Cache-control "no-cache, no-store, must-revalidate"; #以下三个是页面缓存控制响应头,涉及敏感数据的页面必须设置
- add_header Pragma no-cache;
- add_header Expires 0;
-
- location / {
- proxy_hide_header X-Powered-By;
- proxy_pass https://gotoaa;
- proxy_set_header Host $host:$server_port;
- proxy_set_header X-Real-IP $remote_addr;
- }
-
-
-
-
- #error_page 404 /404.html;
- error_page 500 502 503 504 /50x.html;
- # location = /50x.html {
- # root html;
- # }
- }
-
-}
diff --git a/app/zh/information/blog/configguide-03/2021-09-24-config-14.png b/app/zh/information/blog/configguide-03/2021-09-24-config-14.png
deleted file mode 100644
index 22dd4af1182d13c8cd0f52c26d5f44a483752130..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-03/2021-09-24-config-14.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-03/2021-09-24-config-15.png b/app/zh/information/blog/configguide-03/2021-09-24-config-15.png
deleted file mode 100644
index baed68cdcba1bbe8b908431aede2aa504a52c839..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-03/2021-09-24-config-15.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-03/2021-09-24-config-16.png b/app/zh/information/blog/configguide-03/2021-09-24-config-16.png
deleted file mode 100644
index e184d45ed620c1e9a0ce840f767e43395f37f43b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-03/2021-09-24-config-16.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-03/2021-09-24-config-17.png b/app/zh/information/blog/configguide-03/2021-09-24-config-17.png
deleted file mode 100644
index 51b6c8862f6cd54acf5a825e77de3a051a9e0c68..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-03/2021-09-24-config-17.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-03/2021-09-24-config-18.png b/app/zh/information/blog/configguide-03/2021-09-24-config-18.png
deleted file mode 100644
index bd9cbba8a834bc02b8fc6c4fd37676a932265c0d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-03/2021-09-24-config-18.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-03/2021-09-24-config-19.png b/app/zh/information/blog/configguide-03/2021-09-24-config-19.png
deleted file mode 100644
index 52b3fc7094b1007aa02430a38ca6ec221e839cad..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-03/2021-09-24-config-19.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-03/2021-09-24-config-20.png b/app/zh/information/blog/configguide-03/2021-09-24-config-20.png
deleted file mode 100644
index 06684b312b8db4803858282d1440820222bda859..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-03/2021-09-24-config-20.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-03/2021-09-24-config-21.png b/app/zh/information/blog/configguide-03/2021-09-24-config-21.png
deleted file mode 100644
index 9b80bf70f35b9f1de5c02849eefd30a97a18d6fb..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-03/2021-09-24-config-21.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-03/2021-09-24-config-22.png b/app/zh/information/blog/configguide-03/2021-09-24-config-22.png
deleted file mode 100644
index 9057e1a58e3fe7125a98933c8f18f921bf968e33..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-03/2021-09-24-config-22.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-03/2021-09-24-config-23.png b/app/zh/information/blog/configguide-03/2021-09-24-config-23.png
deleted file mode 100644
index ef098e4e7320480d59259ce511adc0f34d28f5d3..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-03/2021-09-24-config-23.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-03/2021-09-24-config-24.png b/app/zh/information/blog/configguide-03/2021-09-24-config-24.png
deleted file mode 100644
index 04733640e7654b1045e94cfe806bd78c41610dd0..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-03/2021-09-24-config-24.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-03/2021-09-24-config-25.png b/app/zh/information/blog/configguide-03/2021-09-24-config-25.png
deleted file mode 100644
index 240d62900b8a92cc67353773ec4872c9eea4fb05..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-03/2021-09-24-config-25.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-03/2021-09-24-config-27.png b/app/zh/information/blog/configguide-03/2021-09-24-config-27.png
deleted file mode 100644
index b670d546613d430bf1e856558da5f12fb62d9143..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-03/2021-09-24-config-27.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-03/2021-09-24-config-28.png b/app/zh/information/blog/configguide-03/2021-09-24-config-28.png
deleted file mode 100644
index 7fdb346a08d62cb3ab9a646051bd40de6665b7ef..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-03/2021-09-24-config-28.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-03/2021-09-24-config-29.png b/app/zh/information/blog/configguide-03/2021-09-24-config-29.png
deleted file mode 100644
index 9194e9b768dbdb0da978b6a2cd9152763f1d9654..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-03/2021-09-24-config-29.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-03/index.md b/app/zh/information/blog/configguide-03/index.md
deleted file mode 100644
index 11e37d3018e896ac0a5e3c0f7beac753535e5edb..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/configguide-03/index.md
+++ /dev/null
@@ -1,395 +0,0 @@
----
-title: 'openLooKeng AA安全配置指导(三)----对接OpenLDAP'
-date: '2021-09-24'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', 'SSL', 'OpenLDAP']
-archives: '2021-09'
-author: 'senny456'
-description: '指导在开启openLooKeng SSL后,如果对接OpenLDAP做用户认证'
----
-
-**环境信息**
-
-| Role | IP | Hostname | Hostname+domain name |
-| ------------------ | --- | -------- | -------------------- |
-| coordinator&worker | ip1 | host1 | host1.example.com |
-| coordinator&worker | ip2 | host2 | host2.example.com |
-| worker | ip3 | host3 | host3.example.com |
-| Kerberos/OpenLDAP | ip4 | host4 | host4.example.com |
-
-假设 openLooKeng 集群已配置好 kerberos 认证(当前对接 OpenLDAP 必须开启 kerberos 认证,参见连接:[openLooKeng AA 安全配置指导(一)----对接 Kerberos](https://openlookeng.io/zh/information/blog/configguide-01/))
-
-### 1 OpenLDAP 安装
-
-### 1.1 安装 OpenLDAP
-
-```
-yum -y install openldap openldap-servers openldap-clients openldap-devel compat-openldap
-```
-
-### 1.2 初始化 OpenLDAP 配置
-
-```
-cp /usr/share/openldap-servers/DB_CONFIG.example /var/lib/ldap/DB_CONFIG
-
-cp /usr/share/openldap-servers/slapd.ldif /root/.
-```
-
-### 1.3 修改配置文件 slapd.ldif
-
-```
-###############################slapd.ldif 内容如下#####################3######
-
-#
-# See slapd-config(5) for details on configuration options.
-# This file should NOT be world readable.
-#
-dn: cn=config
-objectClass: olcGlobal
-cn: config
-olcArgsFile: /var/run/openldap/slapd.args
-olcPidFile: /var/run/openldap/slapd.pid
-# TLS settings TLS #
-olcTLSCACertificatePath: /etc/openldap/certs
-olcTLSCertificateFile: /etc/openldap/certs/ldap.crt
-olcTLSCertificateKeyFile: /etc/openldap/certs/ldap.key
-olcTLSVerifyClient: never
-# Schema settings
-dn: cn=schema,cn=config
-objectClass: olcSchemaConfig
-cn: schema
-#include: file:///etc/openldap/schema/core.ldif注意顺序.(高能预警:有坑)注意上面必须得有个空行,否则报错,随时空行就对了.
-include: file:///etc/openldap/schema/corba.ldif
-include: file:///etc/openldap/schema/core.ldif
-include: file:///etc/openldap/schema/cosine.ldif
-include: file:///etc/openldap/schema/duaconf.ldif
-include: file:///etc/openldap/schema/dyngroup.ldif
-include: file:///etc/openldap/schema/inetorgperson.ldif
-include: file:///etc/openldap/schema/java.ldif
-include: file:///etc/openldap/schema/misc.ldif
-include: file:///etc/openldap/schema/nis.ldif
-include: file:///etc/openldap/schema/openldap.ldif
-include: file:///etc/openldap/schema/ppolicy.ldif
-include: file:///etc/openldap/schema/collective.ldif
-#
-# Frontend settings
-dn: olcDatabase=frontend,cn=config
-objectClass: olcDatabaseConfig
-objectClass: olcFrontendConfig
-olcDatabase: frontend
-#
-# Configuration database
-dn: olcDatabase=config,cn=config
-objectClass: olcDatabaseConfig
-olcDatabase: config
-olcAccess: to * by dn.base="gidNumber=0+uidNumber=0,cn=peercred,cn=external,c
- n=auth" manage by * none
-#
-# Server status monitoring
-# 自定义域 cn=Manager,dc=example,dc=com (example位置,随便填,但是下面统一)
-#
-dn: olcDatabase=monitor,cn=config
-objectClass: olcDatabaseConfig
-olcDatabase: monitor
-olcAccess: to * by dn.base="gidNumber=0+uidNumber=0,cn=peercred,cn=external,c
- n=auth" read by dn.base="cn=Manager,dc=example,dc=com" read by * none
-
-#
-# Backend database definitions
-dn: olcDatabase=hdb,cn=config
-objectClass: olcDatabaseConfig
-objectClass: olcHdbConfig
-olcDatabase: hdb
-olcSuffix: dc=example,dc=com
-olcRootDN: cn=Manager,dc=example,dc=com
-olcRootPW: ****
-olcDbDirectory: /var/lib/ldap
-olcDbIndex: objectClass eq,pres
-olcDbIndex: ou,cn,mail,surname,givenname eq,pres,sub
-olcDbIndex: uidNumber,gidNumber,loginShell eq,pres
-olcDbIndex: uid,memberUid eq,pres,sub
-olcDbIndex: nisMapName,nisMapEntry eq,pres,sub
-########################################end###################################
-```
-
-\#附:生成密码(可以不管这里。上面的 olcRootPW 密文可以由这个生成并替换)
-\#slappasswd -s ****
-\##{SSHA}R09wEQwdZ2PzL/9fbRGhfEzW6FR17Ioi
-
-并修改对应的字段
-
-
-
-### 1.4 重新生成配置文件信息
-
-rm -rf /etc/openldap/slapd.d/\* ##删除之前配置
-
-slapadd -F /etc/openldap/slapd.d -n 0 -l /root/slapd.ldif ## 生成新配置
-
-slaptest -u -F /etc/openldap/slapd.d ##测试配置
-
-授权:chown -R ldap. /etc/openldap/slapd.d/
-
- chown -R ldap. /var/lib/ldap/
-
-重启服务:/etc/init.d/slapd restart(或者 systemctl restart slapd)
-
-### 1.5 导入管理员账号信息
-
-生成 root.ldif
-
-```
-dn: dc=example,dc=com
-dc: example
-objectClass: top
-objectClass: domain
-
-dn: cn=Manager,dc=example,dc=com
-objectClass: organizationalRole
-cn: Manager
-```
-
-```
-ldapadd -D "cn=Manager,dc=example,dc=com" -W -x -f root.ldif
-```
-
-\## 导入密码是:slapd.ldif 中配置的 ****
-
-```
-ldapsearch -h test -b "dc=example,dc=com" -D "cn=Manager,dc=example,dc=com" -W
-```
-
-\##查看是否导入成功(test 是 hostname,可用 ip 代替)
-
-### 1.6 界面端
-
-1. 使用同一局域网 windows 下的 LdapAdmin
-
- 1)点击 start 中的 connect,并选择 new connection。
-
-
-
- 2)点击 fetch DNs 获取基础 DN
-
-
-
- 3)去掉匿名连接的 √,输入用户名密码,此处密码为前面配置的 ******
-
-
-
- 点击 Test connection 测试是否连接成功
-
-
-
-2. 创建用户:
-
- 1)在根目录右键 new 中选择 orgnaizational unit…
-
-
-
- 2)Name 中填 People 并在 People 下创建用户
-
-
-
-
-
-
-
- 3)设置用户密码
-
-
-
-
-
-### 2 openlookeng 对接 OpenLDAP
-
-### 2.1 修改 openssl 配置文件
-
-1. 复制 openssl 配置文件到 /tmp 下:
-
- ```
- cp /etc/pki/tls/openssl.cnf /tmp/openssl.cnf
- ```
-
-2. 编辑/tmp/openssl.cnf
-
- 此文件的格式是类似 ini 的配置文件格式,找到 **[ req ]** 段落,加上下面的配置:
-
- ```
- req_extensions = v3_req
- ```
-
-
-
- 这段配置表示在生成 CSR 文件时读取名叫 v3_req 的段落的配置信息,因此我们再在此配置文件中加入一段名为 v3_req 的配置:
-
- ```
- [ v3_req ]
- # Extensions to add to a certificate request
- basicConstraints = CA:FALSE
- keyUsage = nonRepudiation, digitalSignature, keyEncipherment
- subjectAltName = @alt_names
- ```
-
-
-
- 这段配置中最重要的是在最后导入名为 alt_names 的配置段,因此我们还需要添加一个名为 [ alt_names ] 的配置段:
-
- [ alt_names ]
-
- IP = ip4
-
- 注:ip 为 OpenLDAP 服务器 ip
-
-### 2.2 生成 openldap 证书
-
-```
-cd /etc/openldap/certs/
-openssl genrsa -out ldap.key 1024
-openssl req -new -key ldap.key -out ldap.csr -config /tmp/openssl.cnf
-```
-
-查看证书请求文件的内容:
-
-```
-openssl req -text -noout -in ldap.csr
-```
-
-生成签名
-
-```
-openssl x509 -req -days 3650 -in ldap.csr -signkey ldap.key -out ldap.crt -extensions v3_req -extfile /tmp/openssl.cnf
-```
-
-### 2.3 配置 ldap 开启 ssl
-
-#### 2.3.1 编辑/root/slapd.ldif
-
-```
-olcTLSCACertificatePath: /etc/openldap/certs
-olcTLSCertificateFile: /etc/openldap/certs/ldap.crt
-olcTLSCertificateKeyFile: /etc/openldap/certs/ldap.key
-olcTLSVerifyClient: never
-```
-
-**注:配置的证书路径应与[2.2](#_生成openldap证书)步骤中生成的一致**
-
-#### 2.3.2 编辑/etc/openldap/ldap.conf
-
-```
-TLS_REQCERT allow
-TLS_CERT /etc/openldap/certs/ldap.crt
-TLS_KEY /etc/openldap/certs/ldap.key
-URI ldaps://ip4 //需修改为OpenLDAP服务器ip
-BASE dc=example,dc=com
-```
-
-#### 2.3.3 重新生成配置并重启服务
-
-service slapd stop(或者 systemctl stop slapd)
-
-rm -rf /etc/openldap/slapd.d/\* ##删除之前配置
-
-slapadd -F /etc/openldap/slapd.d -n 0 -l /root/slapd.ldif ## 生成新配置
-
-slaptest -u -F /etc/openldap/slapd.d ##测试配置
-
-授权:chown -R ldap. /etc/openldap/slapd.d/
-
- chown -R ldap. /var/lib/ldap/
-
-#### 2.3.4 配置同时启动 ldap 和 ldaps
-
-编辑/etc/sysconfig/ldap (没有就手动创建)
-
-SLAPD_LDAP=yes
-
-SLAPD_LDAPI=no
-
-SLAPD_LDAPS=yes
-
-#### 2.3.5 启动并查看是否已经成功
-
-service slapd restart(或者 systemctl start slapd)
-
-netstat -tnlp |grep 389
-
-netstat -tnlp |grep 636
-
-**注:若 636 未启动**
-
-**vi /etc/sysconfig/slapd**
-
-**SLAPD_URLS="ldapi:/// ldap:/// ldaps:///" //配置 slapd 启用监听 ldaps**
-
-### 2.4 导入证书
-
-在 openldap 和 openLooKeng 服务器上使用 keytool 导入证书 (密码:****)
-
-(拷贝 openldap 的 ldap.crt 证书到 openLooKeng 服务器)
-
-```
-keytool -import -trustcacerts -file ldap.crt -alias ldapserver -keystore /opt/jdk1.8.0_191/jre/lib/security/cacerts
-```
-
-### 2.5 配置 openlookeng
-
-**新增配置**
-
-1. 配置 config.properties,增加
-
- ```
- http-server.authentication.type=PASSWORD,KERBEROS
- ```
-
-2. 配置 jvm.config,增加
-
- ```
- -Djavax.net.ssl.trustStore=/opt/jdk1.8.0_191/jre/lib/security/cacerts
- -Djavax.net.ssl.trustStorePassword=changeit
- ```
-
-3. 增加 password-authenticator.properties 配置文件
-
- ```
- password-authenticator.name=ldap
- ldap.url=ldaps://ip4:636
- ldap.user-bind-pattern=cn=${USER},ou=People,dc=example,dc=com
- ldap.cache-ttl=1s
- ```
-
- **注:**
-
- **ldap.url:ldap 服务器的 ip 和端口**
-
- **ldap.user-bind-pattern=cn:根据为 ldap 上用户配置**
-
-### 2.6 重启 openlookeng 服务
-
-```
-./bin/launcher restart
-```
-
-### 2.7 验证
-
-**webUI:**
-
-登录 https://ip1:9090/ui/login.html
-
-IP:openLooKeng coordinator 节点的 IP 或者域名
-
-
-
-**CLI:**
-
-```
-java -jar hetu-cli-1.4.0-SNAPSHOT-executable.jar \
---server https://ip1.example.com:9090 \
---keystore-path /opt/hetu/keystore.jks \
---keystore-password ****** \
---user lk \
---password
-```
-
-
diff --git a/app/zh/information/blog/configguide-04/2021-09-24-config-30.png b/app/zh/information/blog/configguide-04/2021-09-24-config-30.png
deleted file mode 100644
index 0f635453dc81402a260554777c3ecc73d9ee94de..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-04/2021-09-24-config-30.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-04/2021-09-24-config-31.png b/app/zh/information/blog/configguide-04/2021-09-24-config-31.png
deleted file mode 100644
index 35c3519c5499ebef080ef748b8a0bad414723fb5..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-04/2021-09-24-config-31.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-04/2021-09-24-config-32.png b/app/zh/information/blog/configguide-04/2021-09-24-config-32.png
deleted file mode 100644
index 31472bf578dbc077b4c997336da3e06cd2cec00a..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-04/2021-09-24-config-32.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-04/2021-09-24-config-33.png b/app/zh/information/blog/configguide-04/2021-09-24-config-33.png
deleted file mode 100644
index 81ce0dba0b9950baafd5914ff2fd2964b779344a..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-04/2021-09-24-config-33.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-04/2021-09-24-config-35.png b/app/zh/information/blog/configguide-04/2021-09-24-config-35.png
deleted file mode 100644
index 1b14a7331f45960ea164aee2e47d7fb825dafead..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-04/2021-09-24-config-35.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-04/2021-09-24-config-36.png b/app/zh/information/blog/configguide-04/2021-09-24-config-36.png
deleted file mode 100644
index 415c20cb44ed0bba19805a331d80d464ba6f2362..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-04/2021-09-24-config-36.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-04/2021-09-24-config-37.png b/app/zh/information/blog/configguide-04/2021-09-24-config-37.png
deleted file mode 100644
index b2c3354d324530156b048f5fea37308edc8b6f09..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-04/2021-09-24-config-37.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-04/2021-09-24-config-38.png b/app/zh/information/blog/configguide-04/2021-09-24-config-38.png
deleted file mode 100644
index 9cd9787a3e700f663afd43fa7b591e550ae28967..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-04/2021-09-24-config-38.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-04/2021-09-24-config-39.png b/app/zh/information/blog/configguide-04/2021-09-24-config-39.png
deleted file mode 100644
index 96fbc5059c5985ef6923b9eff954086224637acb..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-04/2021-09-24-config-39.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-04/2021-09-24-config-40.png b/app/zh/information/blog/configguide-04/2021-09-24-config-40.png
deleted file mode 100644
index 9af0b091a4e883ac07f7681992abeb849923dac8..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-04/2021-09-24-config-40.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-04/2021-09-24-config-41.png b/app/zh/information/blog/configguide-04/2021-09-24-config-41.png
deleted file mode 100644
index e3ff4964060c1f12590e2888860a5a8dc24c70cf..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-04/2021-09-24-config-41.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-04/2021-09-24-config-42.png b/app/zh/information/blog/configguide-04/2021-09-24-config-42.png
deleted file mode 100644
index 8998d8ca691a0f2392bffb06a3f7bba3144309da..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-04/2021-09-24-config-42.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-04/2021-09-24-config-43.png b/app/zh/information/blog/configguide-04/2021-09-24-config-43.png
deleted file mode 100644
index bf157f238349b0c7968a3fd1958985eafdff15f6..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/configguide-04/2021-09-24-config-43.png and /dev/null differ
diff --git a/app/zh/information/blog/configguide-04/index.md b/app/zh/information/blog/configguide-04/index.md
deleted file mode 100644
index 6fd5c05856b2c1cc08ac9f295d5ed86178491265..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/configguide-04/index.md
+++ /dev/null
@@ -1,234 +0,0 @@
----
-title: 'openLooKeng AA安全配置指导(四)----对接ranger'
-date: '2021-09-24'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', 'Ranger', 'OpenLDAP']
-archives: '2021-09'
-author: 'senny456'
-description: '指导配置openLooKeng对接ranger做权限控制'
----
-
-### 1 Ranger 编译和部署
-
-代码路径:https://github.com/apache/ranger/tree/release-ranger-2.1.0, release-ranger-2.1.0 分支。
-
-在 Ranger 的 2.1.0 版本上开发 openLookeng 插件,所以需要编译和部署 ranger-2.1.0。
-
-### 1.1 代码编译
-
-```
-git clone https://github.com/apache/ranger.git --branch release-ranger-2.1.0
-\# 进入代码根目录
-cd ranger
-mvn clean compile package install -DskipTests
-ls target/
-\# target 目录下为所有压缩包
-```
-
-### 1.2 安装 MySQL
-
-安装可参考网上教程:https://www.cnblogs.com/lzhdonald/p/12511998.html
-
-本地安装完成后,连接数据库:mysql -u root -p,新增 openlookeng 用户并赋予权限
-
-```
-create database ranger;
-CREATE USER 'openlookeng'@'%' IDENTIFIED BY 'openlookeng123';
-GRANT ALL ON ranger.* TO 'openlookeng'@'%';
-FLUSH PRIVILEGES;
-```
-
-### 1.3 安装 Ranger Admin
-
-安装可参考官方教程:https://cwiki.apache.org/confluence/display/RANGER/Ranger+Installation+Guide
-
-```
-# 进入Ranger代码编译生成的target目录下
-tar -zxf ranger-2.1.0-admin.tar.gz
-cd ranger-2.1.0-admin/
-vi install.properties
-```
-
-install.properties 修改如下部分
-
-```
-#mysql 数据库信息
-db_root_user=root
-db_root_password=******
-db_host=xxx.xxx.xxx.xxx
-
-# DB UserId used for the Ranger schema
-# 提前在mysql中创建数据库和用户
-db_name=ranger
-db_user=openlookeng
-db_password=******
-
-# 禁用审计功能
-#audit_store=solr
-```
-
-将 mysql 的驱动包放置到/usr/share/java/mysql-connector-java.jar。
-
-```
-CREATE USER 'openlookeng'@'%' IDENTIFIED BY '******';
-GRANT ALL ON ranger.* TO 'openlookeng'@'%';
-FLUSH PRIVILEGES;
-```
-
-执行./setup.sh
-
-
-
-启动 Ranger Admin 服务: service ranger-admin start
-
-
-
-### 1.4 检查是否成功
-
-访问 Ranger 控制台:http://ranger-IP:6080,默认账号密码:admin/****,出现登录界面且登录成功,说明 Ranger Admin 安装成功
-
-
-
-### 2 安装 Ranger openLooKeng Plugin
-
-代码路径:https://gitee.com/chen-peikun/openlookeng-ranger-plugin, master 分支。
-
-按照下面操作指导进行编译和部署即可(基于 Ranger 的 2.1.0 版本)。
-
-### 2.1 代码编译
-
-```
-git clone https://gitee.com/chen-peikun/openlookeng-ranger-plugin.git
-# 进入代码根目录
-cd openlookeng-ranger-plugin
-mvn clean compile package install
-ls target/
-# target 目录下为所有压缩包
-```
-
-### 2.2 在 Ranger Admin 中安装 Ranger openLooKeng 的服务端插件
-
-```
-# 代码openlookeng-ranger-plugin根目录
-# 进入Ranger代码编译生成的target目录下
-cd target
-tar -zxf ranger-2.1.0-admin-openlookeng-plugin.tar.gz
-cd ranger-2.1.0-admin-openlookeng-plugin/
-
-# 将openlookeng目录拷贝到Ranger Admin的ranger-plugins目录下
-# 示例:Ranger Admin路径为/home/ranger-2.1.0-admin
-cp -r openlookeng /home/ranger-2.1.0-admin/ews/webapp/WEB-INF/classes/ranger-plugins/
-
-# 使用curl命令,将service-defs目录下的ranger-servicedef-openlookeng.json服务定义文件注册到Ranger Admin服务中
-# "password"是Ranger Admin的admin账户密码
-# "ranger-admin-host:port"使用对应的Ranger Admin配置的host和port
-curl -u admin:password -X POST -H "Accept: application/json" -H "Content-Type: application/json" -d @service-defs/ranger-servicedef-openlookeng.json http://ranger-admin-host:port/service/plugins/definitions
-
-# 【重启Ranger Admin服务】
-service ranger-admin restart
-```
-
-### 2.3 在 Ranger 控制台新增 openlookengdev 服务
-
-访问 Ranger 控制台:http://ranger-IP:6080,默认账号密码:admin/****,新增 openLooKeng 服务:
-
-
-
-Username 填写:**lk**, openLooKeng 控制台(http://openLooKeng-IP:8080)默认使用**lk 用户**进行数据查询和展示,使用**lk 用户**创建服务,ranger 默认会给创建服务用户全部权限。
-
-Passworld:对接 openLDAP 需填写;未对接 openLDAP 不用填写。
-
-jdbc.url 填写:
-
- 非安全 openLooKeng 集群:jdbc:lk://openLooKeng-IP:http-port/catalog
-
- 安全 openLooKeng 集群:jdbc:lk://openLooKeng-IP:https-port/hive?SSL=true
-
- 如果配置 nginx:jdbc:lk://nginx-IP:https-port/hive?SSL=true
-
-
-
-测试连接成功,可以使用。
-
-### 2.4 在 openLooKeng 中安装 Ranger openLooKeng 的客户端插件
-
-```
-# 代码openlookeng-ranger-plugin根目录
-# 进入Ranger代码编译生成的target目录下
-cd target
-tar -zxf ranger-2.1.0-openlookeng-plugin.tar.gz
-cd ranger-2.1.0-openlookeng-plugin/
-vi install.properties
-```
-
-install.properties 修改如下部分
-
-```
-# Location of Policy Manager URL
-# Example: POLICY_MGR_URL=http://policymanager.xasecure.net:6080
-POLICY_MGR_URL=ttp:///ranger-IP:6080
-
-# This is the repository name created within policy manager
-# Example: REPOSITORY_NAME=openlookengdev
-# Ranger 控制台创建的Service名
-REPOSITORY_NAME=openlookengdev
-
-#Presto component installed directory
-#COMPONENT_INSTALL_DIR_NAME=../openlookeng
-COMPONENT_INSTALL_DIR_NAME=/root/hetu-server-1.0.0-SNAPSHOT
-
-#为了简单,此处不开启审计功能
-XAAUDIT.SOLR.ENABLE=false
-
-#虽然文档中没有提及,不设置的话,enable-presto-plugin.sh脚本执行出错
-XAAUDIT.SUMMARY.ENABLE=false
-```
-
-以 root 用户执行脚本: ./enable-presto-plugin.sh
-
-**如果是 AA 环境,coordinator 节点上都需要执行上面脚本**
-
-检查 openLooKeng 配置文件目录是否生成 access-control.properties
-
-
-
-检查 openLooKeng 的 plugin 目录下是否生成 ranger 目录,其中 jar 包连接到 ranger-2.1.0-openlookeng-plugin/lib
-
-
-
-重启 openLookeng\*\*
-
-进入 openLookeng 安装目录下的 bin 子目录,新建 lk 用户并授权,使用 lk 用户执行:./launcher restart
-
-### 2.5 添加权限控制,验证是否成功
-
-访问 Ranger 控制台:http://ranger-ip:6080,点击 openlookengdev 服务,进行权限控制:
-
-
-
-添加资源权限控制:
-
-
-
-show schemas/tables/columns 等显示元数据信息操作,对应的 catalog/schema/table/column 需要授权 select 权限,还必须给对应 Catalog/Schame(information_schema)/Table(schemata,tables,columns)/Column(\*)授予 select 权限)
-
-**示例如下:**
-
-【配置前】test 账号没有配置权限
-
-
-
-1)在 Ranger 上授权 test 访问 Catalog(hive2)的 use 权限,**注意:管理表权限必须先设置 catalog 的 use 权限**
-
-
-
-2)在 Ranger 上授权 test 访问 Catalog(hive2)/Schame(default)/Table(user)/Column(id,info_age)数据的权限
-
-【配置权限】给 cc_call_center_sk,cc_call_center_id 两个列访问权限
-
-
-
-【配置后】
-
-
diff --git a/app/zh/information/blog/gravitys169/1.jpg b/app/zh/information/blog/gravitys169/1.jpg
deleted file mode 100644
index b504c61a006578676f21a24f9f5cf9e9c00f81f5..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/gravitys169/1.jpg and /dev/null differ
diff --git a/app/zh/information/blog/gravitys169/2.jpg b/app/zh/information/blog/gravitys169/2.jpg
deleted file mode 100644
index 648d378ae6c583789bbc110dd75bac7159990a6d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/gravitys169/2.jpg and /dev/null differ
diff --git a/app/zh/information/blog/gravitys169/index.md b/app/zh/information/blog/gravitys169/index.md
deleted file mode 100644
index e37b961143e3b71c696fe8a56fb6f3f6933c47a9..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/gravitys169/index.md
+++ /dev/null
@@ -1,129 +0,0 @@
----
-title: '数据虚拟化引擎openLooKeng介绍'
-date: '2020-06-30'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', '大数据', '数据虚拟化']
-archives: '2020-06'
-author: 'gravitys169'
-description: '介绍数据虚拟化openLooKeng是什么,解决什么问题,关键特性以及应用场景'
----
-
-## 大数据分析的现状及问题
-
-21 世纪是信息爆炸的世纪,随着 IT 技术的飞速发展,越来越多的应用源源不断的产生数以亿计的数据。在过去的近一个世纪里,科学家与工程师发明了各种各样的数据管理系统来存储与管理各种各样的数据:关系型数据库、NoSQL 数据库,文档数据库、Key-value 数据库,对象存储系统等等。形态多样的数据管理系统为企业组织在管理数据上带来便利的同时,随之而来的是管理与充分利用这些数据系统存储的数据的难题。无论是关系型数据库中的 PostgreSQL 或者 MySQL,抑或是 Hadoop 体系下的 Hive 或者 HBase,这些目前业界通用的数据管理系统都有自成体系的一套 SQL 方言。数据分析师想要分析某一种数据管理系统的数据,就得熟练掌握某一种 SQL 方言;为了对不同数据源进行联合查询,那么就得在应用程序逻辑中使用不同的客户端去连接不同的数据源,**整个分析过程架构复杂,编程入口多,系统集成困难**,这对于涉及海量数据的数据分析师而言这样的分析过程十分痛苦。
-
-
-
-为了解决多数据源形成的数据孤岛的联合查询问题,业界正在广泛使用数据仓库这一解决方案。数据仓库在过去的数年里快速发展,它通过抽取(Extract)、转换(Transform)、加载(Load)各种各样数据源中的数据,经过 ETL 这一整套流程,将加工后的数据集中保存在专题数据仓库中,供数据分析师或用户使用。但随着数据规模的进一步增长,不得不指出的是,业界已经逐渐认识到将数据搬运到数据仓库的过程是昂贵的,除了数据仓库的硬件或软件的成本,维护与更新整个 ETL 逻辑系统的人力成本也逐渐成为数据仓库的重要开销之一。数据仓库 ETL 流程同时也是笨重且耗时的,为了获取到想要的数据,数据分析师或用户不得不妥协于数据仓库 T+1 的数据分析模式,想要快速进行业务分析探索对于数据分析师来说一直是一个待解的难题。
-
-**人们为了解决各种各样的数据管理系统的数据孤岛问题,针对不同的业务应用又发明了专题数据仓库,但随着业务应用的增多,日益增多的专题数据仓库又变成了数据孤岛。所以英勇的“屠龙勇士”随着时间的流逝都不可避免的会变成“恶龙”吗?是否有一种系统架构简洁、编程入口统一、系统集成度好的解决方案呢?也许今天,我们是时候回到最初的起点,来从头看看大数据数据分析的另一种范式了。**
-
-## 数据虚拟化引擎 openLooKeng:我们不搬运数据,我们是数据的”连接器“
-
-所以当我们回头来看数据仓库碰到的各种各样的问题的时候,聪明的您很容易发现,数据仓库这个”屠龙勇士“之所以逐渐变成“恶龙”是因为它在**不停的搬运数据**,搬运数据正是导致数据仓库的建立与分析过程繁重、费时、昂贵的“元凶”。既然搬运数据导致了这些问题,那么让我们回到大数据分析的出发点,考虑下“林中的另一条路”,而这条路正是 openLooKeng 正在走的**变数据搬运为数据连接**的路。
-
-简明扼要的讲,openLooKeng 数据虚拟化引擎分析数据的方式是通过各种各样的数据源 Connector 连接到各个数据源系统,用户在发起查询时,通过各个 Connector 实时的去获取数据并进行高性能的计算,从而在秒级或分钟级内得到分析结果。这与以往的数据仓库通过 T+1 的 ETL 数据搬运过程处理好数据再给用户使用的方式有很大差异。
-
-与以往数据分析师需要学习各种各样的 SQL 方言不同的是,现在数据分析师只需要熟练掌握 ANSI SQL2003 语法。而各种各样的数据管理系统在 SQL 标准上的差异则由 openLooKeng 作为中间层进行了屏蔽,用户不用再学习各种 SQL 方言,这些繁杂的 SQL 方言转换的工作都将由 openLooKeng 来完成。通过将用户从各种各样的 SQL 方言中“解放”出来,用户可以专注于构建高价值的业务应用查询分析逻辑,这些分析逻辑形成的无形资产往往才是企业商业智能的核心,openLooKeng 正是出于帮助用户快速构建高价值的业务分析逻辑这一目的来构建自己的整个技术架构的。由于无需搬运数据,用户的分析查询灵感可以快速的使用 openLooKeng 进行验证,从而达到比以往 T+1 的数据仓库分析处理过程更快的分析效果。
-
-
-
-让我们站得更高一点来看,既然 openLooKeng 可以通过 Connector 连接到关系型数据库、NoSQL 数据库等数据管理系统,那么可不可以将 openLooKeng 自身也作为一个 Connector 呢?答案是肯定的。当我们将 openLooKeng 自身也作为一个数据源提供给另一个 openLooKeng 集群时,可以得到这样的好处:之前由于跨地域或者跨 DC 的网络带宽或者时延限制,导致的多个数据中心之间的数据要实现实时联邦查询基本上是不可用的,而现在 openLooKeng 集群 1 将本地数据进行计算后将结果再传递给 openLooKeng 集群 2 进行进一步分析,避免了大量原始数据的传输,从而规避了跨域跨 DC 查询的网络问题。
-
-openLooKeng 的统一 SQL 入口,丰富的南向数据源生态,一定程度上解决了以往跨源查询架构复杂、编程入口太多、系统集成度差的问题,实现了数据从“搬运”到“连接”的模式转换,方便了用户快速实现海量数据的价值变现。
-
-## openLooKeng 的关键特性
-
-也许在看了上面的介绍之后,您已经迫不及待的想知道 openLooKeng 能在哪些场景下使用了,从而来解决目前业务应用的痛点问题。但在继续介绍 openLooKeng 适用的业务场景之前,让我们先来看看 openLooKeng 的一些关键特性,以便于您更深入的理解 openLooKeng 为什么适合这些业务场景,甚至您也可以基于 openLooKeng 的这些能力进一步探索更多的业务场景。
-
-#### 专为海量数据设计的内存计算框架
-
-openLooKeng 从一诞生便是针对 TB 甚至 PB 级海量数据的查询分析任务而设计的,其对于 Hadoop 文件系统具有天然的亲和性,其 SQL on Hadoop 的分布式处理架构,采用了存储与计算分离的设计理念,可方便的实现计算或存储节点的水平扩展。同时 openLooKeng 内核采用基于内存的计算框架,所有数据的处理都在内存中以并行的流水线式作业完成,可提供秒级到分钟级的查询时延响应。
-
-#### ANSI SQL2003 语法的支持
-
-openLooKeng 支持 ANSI SQL2003 语法,用户使用 openLooKeng 语法进行查询时,无论底层数据源是 RDBMS 还是 NoSQL 或者其他数据管理系统,借助 openLooKeng 的 Connector 框架,数据可以依然存放在原始的数据源中,从而实现数据“0 搬迁”的查询。
-
-通过 openLooKeng 的统一 SQL 入口,可实现对底层各种数据源 SQL 方言的屏蔽,用户无需再关心底层数据源的 SQL 方言便可获取到该数据源的数据,方便了用户消费数据。
-
-#### 多种多样的数据源 Connector
-
-正如数据管理系统的多种多样一样,openLooKeng 针对这些数据管理系统开发了多种多样的数据源 Connector,包括 RDBMS(Oracle Connector、HANA Connector 等),NoSQL(Hive Connector、HBase Connector 等),全文检索数据库(ElasticSearch Connector 等)。openLooKeng 可以通过这些多样的 Connector 方便的获取到数据源数据,从而进一步进行基于内存的高性能联合计算。
-
-#### 跨域跨 DC 的 DataCenter Connector
-
-openLooKeng 不仅提供跨多种数据源联合查询的能力,还将跨源查询的能力进一步延伸,开发了跨域跨 DC 查询的 DataCenter Connector。通过这个新 Connector 可以连接到远端另外的 openLooKeng 集群,从而提供在不同数据中心间协同计算的能力。 其中的关键技术如下:
-
-并行数据访问:worker 可以并发访问数据源以提高访问效率, 客户端也可以并发从服务端获取数据以加快数据获取速度。
-
-数据压缩:在数据传输期间进行序列化之前,先使用 GZIP 压缩算法对数据进行压缩,以减少通过网络传输的数据量。
-
-跨 DC 动态过滤:过滤数据以减少从远端提取的数据量,从而确保网络稳定性并提高查询效率。
-
-#### 高性能的查询优化技术
-
-openLooKeng 在内存计算框架的基础上,还利用许多查询优化技术来满足高性能的交互式查询的需要。
-
-- **索引**
-
- openLooKeng 提供基于 Bitmap Index、Bloom Filter 以及 Min-max Index 等索引。通过在现有数据上创建索引,并且把索引结果存储在数据源外部,在查询计划编排时便利用索引信息过滤掉不匹配的文件,减少需要读取的数据规模,从而加速查询过程。
-
-- Cache
-
- openLooKeng 提供丰富多样的 Cache,包括元数据 cache、执行计划 cache、ORC 行数据 cache 等。通过这些多样的 cache,可加速用户多次对同一 SQL 或者同一类型 SQL 的查询时延响应。
-
-- 动态过滤
-
- 所谓的动态过滤是指是在运行时(run time)将 join 一侧表的过滤信息的结果应用到另一侧表的过滤器的优化方法,openLooKeng 不仅提供了多种数据源的动态过滤优化特性,还将这一优化特性应用到了 DataCenter Connector,从而加速不同场景关联查询的性能。
-
-- 算子下推
-
- openLooKeng 通过 Connector 框架连接到 RDBMS 等数据源时,由于 RDBMS 具有较强的计算能力,一般情况下将算子下推到数据源进行计算可以获取到更好的性能。openLooKeng 目前支持多种数据源的算子下推,包括 Oracle、HANA 等,特别地,针对 DC Connector 也实现了算子下推,从而实现了更快的查询时延响应。
-
-#### 高可用特性
-
-- HA AA 双活
-
- openLooKeng 引入了高可用的 AA 特性,支持 coordinator AA 双活机制,能够保持多个 coordinator 之间的负载均衡,同时也保证了 openLooKeng 在高并发下的可用性。
-
-- Auto-scaling
-
- openLooKeng 的弹性伸缩特性支持将正在执行任务的服务节点平稳退服,同时也能将处于不活跃状态的节点拉起并接受新的任务。openLooKeng 通过提供“已隔离”与“隔离中”等状态接口供外部资源管理者(如 Yarn、Kubernetes 等)调用,从而实现对 coordinator 和 worker 节点的弹性扩缩容。
-
-## openLooKeng 的常见应用场景
-
-通过上述对 openLooKeng 关键特性的介绍,想必您的脑海中已经浮现出了不少 openLooKeng 的应用场景,下面让我们一起来看看它在现实业务的应用场景吧。
-
-#### 高性能的交互式查询场景
-
-openLooKeng 基于内存的计算框架,充分利用内存并行处理、索引、Cache、分布式的流水线作业等技术手段来快速的进行查询分析,可以处理 TB 甚至 PB 级的海量数据。以往使用 Hive、Spark 甚至 Impala 来构建查询任务的交互式分析应用系统都可以使用 openLooKeng 查询引擎来进行换代升级,从而获取更快的查询性能。
-
-#### 跨源异构的查询场景
-
-正如前文所述,RDBMS、NoSQL 等数据管理系统在客户的各种应用系统中广泛使用;为了处理这些数据而建立起来的 Hive 或者 MPPDB 等专题数据仓库也越来越多。而这些数据库或者数据仓库往往彼此孤立形成独立的数据孤岛,数据分析师常常苦于:
-
-- 查询各种数据源需要使用不同的连接方式或者客户端,以及运行不同的 SQL 方言,这些不同导致额外的学习成本以及复杂的应用开发逻辑
-- 如果不将各种数据源的数据再次汇聚到一起,则无法对不同系统的数据进行联邦查询
-
-使用 openLooKeng 可实现 RDBMS、NoSQL 等数据库以及 Hive 或 MPPDB 等数据仓库的联合查询,借助 openLooKeng 的跨源异构查询能力,数据分析师可实现海量数据的分钟级甚至秒级查询分析。
-
-#### 跨域跨 DC 的查询场景
-
-对于省-市、总部-分部这样两级或者多级数据中心的场景,用户常常需要从省级(总部)数据中心查询市级(分部)数据中心的数据,这种跨域查询的主要瓶颈在于多个数据中心之间的网络问题(带宽不足、时延大、丢包等),从而导致查询时延长、性能不稳定等。
-
-openLooKeng 专为这种跨域查询设计了跨域跨 DC 的解决方案 DataCenter Connector,通过 openLooKeng 集群之间传输计算结果的方式,避免了大量原始数据的网络传输,规避了带宽不足、丢包等带来的网络问题,一定程度上解决了跨域跨 DC 查询的难题,在跨域跨 DC 的查询场景有较高的实用价值。
-
-#### 计算存储分离的场景
-
-openLooKeng 自身是不带存储引擎的,其数据源主要来自各种异构的数据管理系统,因而是一个典型的存储计算分离的系统,可以方便的进行计算、存储资源的独立水平扩展。openLooKeng 存储计算分离的技术架构可实现集群节点的动态扩展,实现不断业务的资源弹性伸缩,适合于需要计算存储分离的业务场景。
-
-#### 快速进行数据探索的场景
-
-如前文所述,客户为了查询多种数据源中的数据,通常的做法是通过 ETL 过程建立专门的数据仓库,但这样带来昂贵的人力成本、ETL 时间成本等问题。对于需要快速进行数据探索而不想构建专门的数据仓库的客户,将数据复制并加载到数据仓库的做法显得既费时又费力,而且还可能得不到用户想要的分析结果。
-
-openLooKeng 可通过标准语法定义出一个虚拟的数据集市,结合跨源异构的查询能力连接到各个数据源,从而在这个虚拟的数据集市语义层定义出用户需要探索的各种分析任务。使用 openLooKeng 的这种数据虚拟化能力,客户可快速的建立起基于各种数据源的探索分析服务,而无需构建复杂的、专门的数据仓库,从而节约人力与时间成本,对于想快速进行数据探索从而开发新业务的场景使用 openLooKeng 是最佳的选择之一。
-
-## 展望未来
-
-数据虚拟化引擎 openLooKeng 在探索跨域跨 DC 的交互式查询场景上有了一定的进展。展望未来,还有不少问题需要我们去验证和解决,比如除了交互式分析场景,如何解决 openLooKeng 在流处理和批处理上的问题?用户还需要什么样的数据源 Connector?衷心期待广大用户和开发者加入到 openLooKeng 开源社区中来,携手开发 openLooKeng 项目,解决更多的用户问题,让大数据更简单。
diff --git a/app/zh/information/blog/index.md b/app/zh/information/blog/index.md
deleted file mode 100644
index b23c9a85496a336d0f165933760149a4c530d12c..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/index.md
+++ /dev/null
@@ -1,5 +0,0 @@
-
-
-
-
-传统的数据集成技术正面临着严峻的挑战:应用各自为政,数仓互不相通,数据存放的格式和数据安全的管理都是烟囱式模型,无法跨业务集成和使用。在这种情况下,数据的可获得性往往通过 ETL 来实现,这就造成了业务系统集成工作量巨大、安全系数低等多种困境。数据孤岛现象突出,资源无法共享,利用率低。对于客户或者应用而言,亟需数据融合,整合资源,消除数据孤岛,从而满足业务发展需求。
-
-
-
-数据虚拟化技术应运而生,它可以屏蔽数据的格式、地域、系统等属性,最终为用户呈现出统一的视图。数据虚拟化能够统一管理全域数据,并通过按需读取所需数据的方式,实现数据的快速融合分析和可信管理。数据虚拟化不再是仅做数据的收集器,而是会走向数据连接,消灭数据烟囱。
-
-目前从业界来看,数据虚拟化技术在未来将拥有非常好的前景。Gartner 认为在 2019 年数据虚拟化将节省 40%的数据集成成本,并预测到 2020 年将有 50%的组织和企业采用数据虚拟化作为他们的数据集成技术,到 2022 年这个数据将变为 60%(Gartner Market Guide for Data Virtualization,16 November 2018)。
-
-
-
-根据 Gartner 的预测,在不远的未来,将会有越来越多的企业通过引入数据虚拟化技术来应对和解决他们的大数据业务需求;与此同时多个研究分析机构或媒体对于数据虚拟化市场也持看好姿态,均预测在未来几年数据虚拟化的市场将呈现非常可观的增长趋势。
-
-
-
-### 数据虚拟化的价值
-
-传统的大数据分析技术如 ETL,更多的是以“数据收集”为法则,来实现对数据的处理并提供给上层应用进行分析。但在当前大数据领域的新局势之下,客户或者应用需要耗费大量的 ETL 开销后才能使用数据,同时还要面对 ETL 本身非常复杂的开发过程。
-
-与传统 ETL 不同,数据虚拟化旨在实现从“收集”到“连接”的革命,通过连接数据实现数据全域统一管理。数据虚拟化具有为用户提供实时获取全面数据信息的能力,包括数据接入、数据治理、数据分析、数据检索、数据追溯、数据云化等,这些能力使得当下很多行业的复杂数据服务场景都能够被有效地应对并执行。
-
-
-
-人们对于海量数据的运用预示着新一波生产率增长和消费者盈余浪潮的到来,因此在提供诸多数据服务能力的同时,数据虚拟化最终将为用户带来一系列的价值:
-
-1. 增加业务用户的生产能力并提升效率;
-
-1. 减少开发资源,降低开发成本;
-
-1. 实时响应,更快的数据访问过程;
-
-
diff --git a/app/zh/information/blog/omniRuntime/index.md b/app/zh/information/blog/omniRuntime/index.md
deleted file mode 100644
index a5b91b62c919721bff8e4930120af9aa8d0c6a0f..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/omniRuntime/index.md
+++ /dev/null
@@ -1,149 +0,0 @@
----
-title: 'OmniRuntime:为下一代分析提供动力'
-date: '2021-08-06'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', '大数据', 'OmniRuntime']
-archives: '2021-08'
-author: 'gravitys169'
-description: '介绍OmniRuntime是什么,组成模块和有益效果。'
----
-
-在数据规模和多样性爆炸式增长的过程中,现代分析引擎在为企业提供低延迟和低成本的实时分析方面发挥着至关重要的作用。此外,当今的应用程序希望这些分析系统提供高并发查询和高吞吐量,同时支持对结构化和非结构化数据的查询。
-
-为了满足这些要求,现代分析引擎和数据库通常依赖于将查询计划编译为本机代码。手动编写算子的方式,如 filter,join 和 group by,通常包含处理不同参数和数据类型所需的控制流。本机代码生成的目的是通过消除不必要的代码并将其专门用于底层平台,从而最大限度地减少指令数量。
-
-这些技术被广泛用于许多分析引擎,如 Spark、Flink,以优化类似的功能。然而,当前的解决方案在功能开发的易用性和异构硬件的支持方面往往存在局限性。我们设计了通用数据分析加速器**OmniRuntime**,以解决这一限制,以进一步改进当下流行的分析引擎,如 openLooKeng、Spark 和 Hive。
-
-为了提供这些功能,OmniRuntime 提供了五个核心组件,如下图所示:
-
-
-
- 图 1. OmniRuntime 核心组件示意图
-
-- **OmniJit**:一个透明、易于使用的即时编译框架,不需要对任何低级代码生成框架(如 LLVM 或 Janino)的了解。
-
-- **OmniVec**:一种列式内存格式,提供高性能内存访问、全内存生命周期管理和内置 SIMD 操作。它还支持所有常用的数据类型,如 int、double、varchar 和 decimal 等。
-
-- **OmniOperator**:native 算子库,充分利用当今异构计算环境中可用硬件的计算能力。当与 OmniJit 结合使用时,OmniOperators 能够动态地适应工作负载、参数和数据配置文件,以实现最佳性能。
-
-- **OmniCache**:一种关系型缓存,性能优于传统的面向块的缓存,命中率极高。
-
-- **OmniData**:快速数据访问和协作层,提供数据层和计算层之间的双向通信和数据传输。
-
-在下一节中,我们将提供每个组件的说明。
-
-### OmniJIT
-
-OmniJit 组件是 Omniruntime 的核心,对分析工作负载的整体性能提高做出了重大贡献。
-
-OmniJit 旨在为普通开发人员提供即时编译支持。它提供了一个易于使用的高级语言框架,如 C/C++。OmniJit 自动识别运算符中最关键的性能代码部分并优化它们。生成的运算符是专门使用查询、运行时和硬件特定的计算机代码的。OmniJit 依赖于运行时信息来基于查询上下文应用最佳优化。这可以是数据集基数、列大小、数据类型、可用硬件(SIMD、加速器等)以及许多其他信息,以产生最佳运算符优化。
-
-通过使用 OmniJit,分析引擎开发人员将不再需要与 LLVM 或 Janino 等系统提供的复杂低级 API 交互,以提取最佳性能。
-
-
-
- 图 2. OmniJit 分支裁剪和循环展开优化
-
-### OmniOperator
-
-OmniOperator 表示处理特定查询数据的计算逻辑代码。SQL 查询可以由许多不同的算子组成,OmniJit 负责动态优化它们。OmniJit 优化了 C++算子,以生成具有最小执行指令计数的可执行文件。
-
-算子接口的结构与火山模型相似,因为它遵循类似的接口生命周期:实例化、AddInput、GetOuput 和 Close。所有 OmniOperators 都提供相同的标准接口,实现对分析引擎不透明。
-
-通过利用标准接口,我们可以轻松地将这些运算符暴露给上层计算引擎。此外,它还帮助我们提供一致的开发生命周期体验,同时允许跨各种分析平台的可移植性。开发人员可以让 OmniJit 使用自动化方法优化运算符。或者,他可以指导优化策略,如参数固定、循环开发和代码中特定核心方法的矢量化自适应执行。
-
-与本机平台运算符相比,生成的 OmniOperators 执行的指令计数更少,资源消耗更低,开发开销也更低。下图显示了 Java Analytics 平台中的整体 OmniOperator 生命周期。由于大多数分析引擎都是用 Java 编码的,我们还提供了一个 JNI 接口来促进集成。
-
-
-
- 图 3. OmniOperator 引擎与 Native Operator 的交互
-
-### OmniVector
-
-OmniVector 是 OmniRuntime 的另一个组件。OmniVector 定义了标准的列式内存格式。OmniVector 设计为可移植的、独立于列的内存数据格式。OmniVector 支持丰富的数据类型系统,旨在满足各种分析数据系统的需求。这种内存数据格式为数据密集型应用程序提供高性能和高可扩展性。
-
-OmniVector 的核心代码是在 C++中实现的,带有高级语言绑定,以实现交叉兼容性。它提供了一个异步接口,允许读取和写入操作由各种组件并行进行。这允许我们在写入或持久化操作被继续时公开 OmniVector 的内容。
-
-下图显示了 OmniVec 的整体架构。
-
-
-
- 图 4. OmniVec Binging 和 Native 架构
-
-基于作用域的全生命周期管理的 OmniVector 不仅消除了内存泄漏的可能性,而且提供了高性能的内存访问。每个 OmniVector 在其生命周期中都会经历几个步骤。每个步骤都由 OmniVctor 操作触发:
-
-1. 作用域创建:为了进行有效的内存管理并避免页面故障开销,向量在特定的执行作用域内分配。这允许高效的内存池,从而最大限度地减少内存管理开销。
-
-2. OmniVector 分配:通过使用上面创建的范围,开发人员现在可以分配新的向量。
-
-3. OmniVector 修改:API 支持 Set 和 Put 操作。前者在特定对应的索引位置操作单个值。后者是批处理 put 方法,在该方法中,数组插入到指定的开始位置。
-
-4. OmniVector 读取:与修改一样,我们支持单值检索或批处理操作。
-
-5. OmniVector 释放:一旦不使用向量,就可以将其释放到作用域内存池
-
-6. 作用域释放:释放所有向量后,我们可以释放作用域和关联的内存池。
-
-由于此设计,OmniVector 支持以下功能:
-
-- 零拷贝操作
-- 支持数据生命周期管理和内存泄漏检测。
-- 支持复杂的数据结构,如 MAP、列表和结构。
-- SIMD 指令优化和硬件加速接口
-- 自动溢出到存储
-- 高性能内存分配和池化。
-
-### OmniJIT、OmniVctor 和 OmniOperator 性能数据
-
-我们在 openLooKeng、Spark 和 Hive 等流行的大数据系统中集成了 OmniRuntime 和 OmniJit。然后,我们使用 TPC-H 在 openLooKeng 上进行了基准测试,实验结果表明集成了 Omniruntime 框架的性能明显优于原始分析引擎。
-
-
-
- 图 5. OmniJit 优化的算子效果
-
-### OmniCache
-
-OmniCache 是 OmniRuntime 中的关系型缓存。OmniCache 不仅缓存数据,还维护数据与缓存中的数据之间的关系。OmniCache 构造虚拟数据集,缓存从物理集数据或其他虚拟数据集派生的关系数据,并使用 SQL SELECT 语句定义用于缓存的关系数据。
-
-这种方法比传统的基于文件块的缓存系统具有优势。传统缓存系统中整个文件块必须缓存或丢弃,这可能会导致许多数据换入和换出操作,导致缓存命中率非常低。
-
-为了维护关系信息,OmniCache 将其状态和结构信息提供给查询优化器,以最大限度地提高命中率。公开的缓存元数据有助于分析引擎优化查询计划,并首先访问缓存中的数据,而不是通过慢速数据存储。此外,缓存还能够存储中间查询结果。它允许通过使查询计划程序直接访问先前计算的数据,而不是执行如图 6 所示的完整查询计划来加速操作。
-
-
-
- 图 6. OmniCache 操作原理图
-
-Omnicache 具备以下功能:
-
-- 缓存管理:通过使用物化视图命令生成和管理物化视图。
-- SQL 重写:SQL 重写使用关系代数和成本模型执行,以有效利用缓存数据。
-- 缓存存储:全局内存池的关系型缓存,基于堆外内存实现,可进行高效的数据存储和访问。
-
-### OmniData
-
-OmniData 是 OmniRuntime 的快速数据访问和协作层,旨在减少数据存储层和计算层之间的数据传输,这在现代存算分离的数据中心中非常有效。
-
-OmniData 不会直接从存储端加载文件,然后处理数据,而是将特定操作卸载到存储端进行近数据处理。目标是减少所需的网络通信量和整体计算量。
-
-OmniData 通过将查询执行划分为几个与数据分布匹配的阶段来实现这一目标。然后,代表子处理操作的每个阶段都被发送到存储节点附近或存储节点上执行。亲和性调度用于避免存储节点计算能力过载,并保持较高的整体吞吐量。阶段性操作允许本地数据加载和处理。
-
-除了其近数据处理功能之外,OmniData 还提供以下功能:
-
-- 数据加密和压缩以利用本地的异构计算设备,硬件加速专用的加密和压缩引擎可以大幅提高数据处理吞吐量。
-
-- 动态数据过滤:我们利用动态代码生成来支持智能网卡操作,以便最大限度地提高处理吞吐量,同时减少使用的资源。
-
-- 网络堆栈卸载:进一步利用 TOE 和零拷贝加速,用于传输本地计算的结果。
-
-
-
- 图 7. OmniData 操作示意图
-
-### 总结
-
-OmniRuntime 使用 OmniJIT、OminVector、OmniOperator、OmniCache 和 OmniData 的组合,为下一代分析平台提供通用的数据处理基础。OmniRuntime 通过为不同分析引擎提供具有上下文优化功能的通用引擎,显著减轻了创建自定义优化的负担。
-
-Omniruntime 框架支持异构硬件环境,包括各种处理器(x86、ARM)、加速器(GPU、FPGA、..)。
-
-OmniRuntime 强大的下一代数据处理能力使数据分析引擎能够满足当前和未来的高并发、高吞吐量、结构化和非结构化查询的业务需求。
diff --git a/app/zh/information/blog/omniRuntime/omniruntime1.png b/app/zh/information/blog/omniRuntime/omniruntime1.png
deleted file mode 100644
index 082ca2836901d77b92c75314d2b7c5c6c06890d1..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/omniRuntime/omniruntime1.png and /dev/null differ
diff --git a/app/zh/information/blog/omniRuntime/omniruntime2.png b/app/zh/information/blog/omniRuntime/omniruntime2.png
deleted file mode 100644
index 08770b5ec4c2485da294c678107ffb435ec3137b..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/omniRuntime/omniruntime2.png and /dev/null differ
diff --git a/app/zh/information/blog/omniRuntime/omniruntime3.png b/app/zh/information/blog/omniRuntime/omniruntime3.png
deleted file mode 100644
index 1ec06c15aec125bfaac5944f28f4eae754be71ba..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/omniRuntime/omniruntime3.png and /dev/null differ
diff --git a/app/zh/information/blog/omniRuntime/omniruntime4.png b/app/zh/information/blog/omniRuntime/omniruntime4.png
deleted file mode 100644
index 0e8a69c18c52e8111c9239b654c4114d25a750cf..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/omniRuntime/omniruntime4.png and /dev/null differ
diff --git a/app/zh/information/blog/omniRuntime/omniruntime5.png b/app/zh/information/blog/omniRuntime/omniruntime5.png
deleted file mode 100644
index a7506480d2d1d3f112c840c3e384799787829e6a..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/omniRuntime/omniruntime5.png and /dev/null differ
diff --git a/app/zh/information/blog/omniRuntime/omniruntime6.png b/app/zh/information/blog/omniRuntime/omniruntime6.png
deleted file mode 100644
index 03c0741daa9db5a974d35ba05c7401c592fbda17..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/omniRuntime/omniruntime6.png and /dev/null differ
diff --git a/app/zh/information/blog/omniRuntime/omniruntime7.png b/app/zh/information/blog/omniRuntime/omniruntime7.png
deleted file mode 100644
index a9720f96605ce661fd0705219e7b630a79493081..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/omniRuntime/omniruntime7.png and /dev/null differ
diff --git a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-01.png b/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-01.png
deleted file mode 100644
index 3f5e29e30ccb720b6b7c3a218312153b1fd3e79a..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-01.png and /dev/null differ
diff --git a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-02.png b/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-02.png
deleted file mode 100644
index 5560b6711b7f54cee98dc7d2a47de5b87d1e2af4..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-02.png and /dev/null differ
diff --git a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-03.png b/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-03.png
deleted file mode 100644
index acf3b16a2592a8e06828853dd60f574cb6fee744..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-03.png and /dev/null differ
diff --git a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-04.png b/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-04.png
deleted file mode 100644
index e1fe4e4401a9fbcdb2d595de3f96b036281cfed3..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-04.png and /dev/null differ
diff --git a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-05.png b/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-05.png
deleted file mode 100644
index c637efec486a4a28f4b86c6e800eea743c570d93..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-05.png and /dev/null differ
diff --git a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-06.png b/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-06.png
deleted file mode 100644
index 92cc16fe536eda2a3599692e8a834320b888e690..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-06.png and /dev/null differ
diff --git a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-07.png b/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-07.png
deleted file mode 100644
index 32c882d69a08c12153a586c4aedad65c2802bbfd..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-07.png and /dev/null differ
diff --git a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-08.png b/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-08.png
deleted file mode 100644
index d454eb596debdba12090f59e37d940a76aee9a9f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-08.png and /dev/null differ
diff --git a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-09.png b/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-09.png
deleted file mode 100644
index 9bc728922bf3f80dd561738543bf3fc6c33a31df..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-09.png and /dev/null differ
diff --git a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-10.png b/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-10.png
deleted file mode 100644
index bc4838dc3c6d93e794a355ce711537604b1c2f0d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-10.png and /dev/null differ
diff --git a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-11.png b/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-11.png
deleted file mode 100644
index b56e868d214b9df2cbd553a6f36c078ce0d772fd..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-11.png and /dev/null differ
diff --git a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-12.png b/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-12.png
deleted file mode 100644
index cee707deb61c0d4d1417a100f7dcab1ed958d6d4..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-12.png and /dev/null differ
diff --git a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-13.png b/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-13.png
deleted file mode 100644
index 1a4896f9baae458dc491e3cbfcc8e6b56043cdd9..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-13.png and /dev/null differ
diff --git a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-14.png b/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-14.png
deleted file mode 100644
index c977544c82d3e6e07754097d5d15a1deabb102e1..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-14.png and /dev/null differ
diff --git a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-15.png b/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-15.png
deleted file mode 100644
index c3653bc8debd9cde5d19dff840c98b6298f245da..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-15.png and /dev/null differ
diff --git a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-16.png b/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-16.png
deleted file mode 100644
index ae747c1a18cc11e2228be99b220f507214b21faf..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-16.png and /dev/null differ
diff --git a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-17.png b/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-17.png
deleted file mode 100644
index bda7457e5c91ead7818460c4e9350bd49e9faddf..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/qcon2020/2020-12-06-date-QCon-17.png and /dev/null differ
diff --git a/app/zh/information/blog/qcon2020/index.md b/app/zh/information/blog/qcon2020/index.md
deleted file mode 100644
index 8a08b1c4a542889d1a3547b51c123550337873a9..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/qcon2020/index.md
+++ /dev/null
@@ -1,185 +0,0 @@
----
-title: 'openLooKeng架构解析与性能优化'
-date: '2021-01-27'
-module: 'blog'
-customLayout: 'true'
-tags: ['大数据', '数据虚拟化', 'openLooKeng']
-archives: '2020-12'
-author: 'Daniel Luo'
-description: 'openLooKeng架构解析和性能优化实践'
----
-
-### 1 大数据分析现状和背景
-
-#### 1.1 大数据分析现状
-
-
-
-从 2008 年 Hadoop 成为 Apache 顶级项目之后,大数据技术经历了一个繁荣的发展阶段,各种组件层出不穷,上图显示了,当前查询分析软件有 300+,这也导致了当前大数据平台就像堆积木一样,数据在各个组件之间流转,需要冗长的 ETL 过程,数据存在多个副本,开发者需要多种系统的编程语言,开发难度高,使用复杂;
-
-
-
-如上图所示,市场占有率最高也仅仅 11%,老牌数据库厂商 Oracle 仅仅占有 9%,而 others 占了 53%,即没有巨头,新进入者有巨大的机会;
-
-
-
-从上图可以看出大数据的市场服务占比在 40%左右,而数据的服务占比在个位数,这里的服务指使用大数据的服务成本,包括开发,运维和维护,这也间接说明了大数据使用复杂的问题。
-
-#### 1.2 大数据分析面临的挑战
-
-
-
-大数据当前面临的挑战归结起来有三点:
-
-1. 数据经过 ETL 之后,存在多个副本,多个数据烟囱,管理复杂;
-2. 引擎接口不统一,跨源关联分析的编程模型复杂,开发难度大;
-3. 跨 DC 的数据访问,数据需要经过多次的数据搬迁,比如 DC1 访问 DC2,数据先要从数据源->DC2 的前置机->共享交互平台->DC1 的前置机->DC1 的数据分析引擎。
-
-上述的挑战就要求分析引擎提供:
-(1)批/交互式融合分析;
-(2)跨源数据分析;
-(3)跨域协同分析。
-
-### 2 openLooKeng 架构
-
-openLooKeng 是一个统一高效的数据虚拟化融合分析引擎,我们的愿景是让大数据变简单,即要解决上述问题。
-北向接口方面,openLooKeng 提供 ODBC、JDBC 以及 REST 接口,以 ANSI 2003 SQL 为载体提供统一数据访问接口,BI 工具、AI 工具可以有效地通过所提供的接口与 openLooKeng 集成,简化系统设计。南向接口方面,通过数据源连接框架,Data Source Connector 提供多种数据源的访问能力,无论是大数据生态的 Hive 或者 Hbase,或是 OLTP 数据库 PostgreSQL 以及 MySQL,都可以方便的接入。此外,openLooKeng 提供跨数据中心 Data Center Connector,提供高性能跨域协同计算。
-
-
-
-openLooKeng 基础的交互式查询能力是基于 Presto 开源版本构筑的。但 openLooKeng 在技术场景、引擎内核技术、南北向应用生态等方面相对于 Presto 有较大差异。openLooKeng 是一个类 MPP 架构的分布式处理系统,包含协调器 Coordinator 以及 Worker 两种角色,通过实现 AA 高可用性,整体系统无单点故障问题。同时,openLooKeng 内部采用向量化列式处理,针对大数据场景列式处理性能更快,且可以充分利用 CPU 并行潜力。通过基于内存的流水线处理结构,openLooKeng 可以实现高性能并行处理。
-
-
-
-同时,openLooKeng 提供以下三个特性:
-(1)高可用 Coordinator AA,防止单节点失效,
-(2)DC Connector 提供跨域分析的能力,
-(3)VDM(虚拟数据集市)简化数据的开发流程。
-
-#### 2.1 高可用 Coordinator
-
-
-
-上图显示高可用 Coordinator 的架构,图中的 StateStore 是一个分布式缓存,当前使用的是 hazelcast,它有三作用:提供 lock 服务,用于选出提供 discovery 服务的 Coordinator
-
-DC connector 和 data source connector 的实现类似,只不过数据源是一个 DC。
-
-DC coonector 的数据流如上图所示:
-(1)获取分片,当前 DC 的分片是配置设置的一个值;
-(2)DC1 调度分片;
-(3)DC1 worker 收到分片处理请求之后,向 DC2 发送一个 Post 请求,实际上是一个 SQL 请求;
-(4)DC1 的 worker 一直调用 Http Get 从 DC2 获取数据,直到数据获取完成。
-
-通过 DC connector 打通异地数据中心数据访问,跨 DC 数据协同查询无需依赖数据中转平台,且通过算子下推与跨域动态过滤技术,可获得广域网部署,局域网的性能体验。
-
-#### 2.3 虚拟数据集市 VDM
-
-
-
-通过 VDM,可方便的对底层的数据源、数据表进行管理,通过建立轻量级的视图来实现对不同数据源的模式化访问,使得用户不需要每次查询都关心数据的分布以及访问方式,从而简化数据开发过程。
-
-### 3 性能优化实践
-
-#### 3.1 优化技巧全景
-
-
-
-优化的技巧主要包括:
-(1)在数据源侧,更适应 openLooKeng,针对 Hive 数据源分桶/分区、小文件合并、查询字段排序等策略可以让数据源更加适配 openLooKeng,提升整体性能。
-(2)引擎层,增强交互式查询能力,包括多种缓存加速
-
-当前,Presto 在分区列上已经可以进行分区过滤,但是非分区上不可以。为了应对上图中的 sql,openLooKeng 提供稀疏索引来进行分区裁剪。
-
-Bloom filter 索引,确定每个 split 是否包含要搜索的值,并只对可能包含该值的 split 进行读操作:
-(1)可以快速判断一个集合中有无某个值;
-(2)需要预先通过 create index 进行索引创建,openLooKeng 提供类似于 DB 的 create index 命令,可以创建 bloom filter、min/max 和 bitmap 索引;
-(3)通过在 coordinator 侧过滤,减少不必要的 split 生成与处理。
-
-
-
-上图是真实客户场景的测试结果,客户的场景分析如下:
-(1)单表数据量很大,只有天分区,测试数据包含 30 天;
-(2)谓词包含 OR 以及 AND;
-(3)单表点查询,聚合操作,无 join。客户的要求是 100 并发下,30 天的数据查询在 3 秒内返回。
-
-在没有创建索引前,单并发的查询性能在 8~10 秒左右。通过添加索引,可以看出:只 50 并发下都满足了需要,30 天 100 并发的查询还存在一定差距。后续的优化思路包括:索引 OR 支持,并且下推 OR 操作;聚合场景的 Aggregation Stage Cache 和 StarTree Index。
-
-#### 3.3 动态过滤
-
-
-
-TPC-DS 是 TPC 标准组织推出的一个广泛使用的行业标准决策支持基准,用于评估数据处理引擎的性能。在 TPC-DS 是一个典型的星型&雪花型的数据仓库组织方式,包含 7 张事实表和 17 张维度表,事实表数据量极大,而维度表相对较小,查询很少有谓词直接应用到事实表,事实表查询条件通过维度表相连接得到。
-
-问题:传统谓词下推等优化很难应用,因为 probe 侧的表无法做到有效过滤,几乎是全表进行扫描参与 join,导致 join 数据量巨大导致执行时间过长。
-
-针对这种场景,openLooKeng 采用动态过滤(Dynamic Filtering)技术。依靠 join 条件以及 build 侧表读出的数据,运行时生成动态过滤条件
-
-TPC-DS 的性能结果如上图所示,开启动态过滤,TPC-DS 测试用例的执行时间减少了 38.9%。
-
-#### 3.4 全局动态过滤
-
-
-
-为了提高跨 DC 的访问性能,openLooKeng 把动态过滤应用到跨 DC 的数据访问,即全局动态过滤特性。设计思路如下:
-
-DC2 coordinator 侧:
-
-(1)增加业务用户的生产能力并提升效率;
-(2)通过 session 判断当前 query 是否存在 cross-region-dynamic-filter,判断依据是 dc-1 的 DC Connector 在下推子语句到 dc-2 时,会监测是否有 dynamicFilter,若存在,则会在 HTTP Request 中设置属性标签,在 dc-2 的 coordinator 接收语句时,将 session 中的设置启用 cross-region-dynamic-filter;
-(3)coordinator 完成子语句的 Analyze 会生成 Plan,从 Plan 中提取子语句的列名到 Plan 中各个 PlanNode 的 outputSymbol 的映射关系,同时,在这个过程中,判断 TableScanNode 是否 DC table,若是,则标记下来,可能存在需要继续下推 filter 的可能;将 mapping 和 DC 标记记录存入到 hazelcast。
-
-DC2 worker 侧:
-
-(1)CrossRegionDynamicFilterOperator 从 hazelcast 根据 queryId 获取 filter,通过 OutputNode 中 columnNames 和 outputSymbols 的映射判断是否存在需要过滤的列,若存在,则对 Page 中的该列进行过滤;
-(2)对从 Connector 获取的 Page 数据进行过滤;涉及的 Operator 有三个,TableScanOperator、ScanFilterAndProjectOperator、TableScanWorkerProcessorOperator; 从 hazelcast 中根据 queryId 拉取 filter,mapping,DC 标记记录,然后根据规则对 Page 的对应列进行 filter,以及生成新的 bloomFilter 并存入 hazelcast;
-(3)DC Connector 从 hazelcast 获取是否需要下推到远端集群的 filter。
-
-性能测试如下:
-
-
-
-SQL-2 性能提升这么明显于以下几个原因:
-(1)通过动态过滤,DC2 右表的 TableScan 操作之后返回给上一阶段的数据明显降低了几个数量级,从而极大的减少了数据 Join 的时长;
-(2)减少了 DC2 到 DC1 的数据传输;
-(3)在 DC1 上进行 join 的数据量极大减少了。
-
-#### 3.5 TPC-DS 性能测试
-
-
-
-优化技巧总结:
-(1)动态过滤:q64,q78,q75,q40,q49
-(2)Semi join 转 inner join:q95,q14,q93,q58
-(3)window function + filter 转 Top(N+M):q67
-(4)使用 group by 消除 self join: q95
-(5)join reorder: q64
-
-### 4 总结
-
-1. 统一北向数据访问接口,丰富南向数据源,实现跨数据源数据免搬迁融合分析。
-
-2. DC connector 提供跨域分析能力,并且通过全局动态过滤,算子下推,压缩断点续传,Coordinator AA 等技术来提高 DC connector 的性能和稳定性。
-
-3. 通过元数据 cache,执行计划优化,索引,动态过滤,算子下推等特性,整体提高 openLooKeng 的性能。
diff --git a/app/zh/information/blog/ufolr/connector/2021-03-11-connector01.jpg b/app/zh/information/blog/ufolr/connector/2021-03-11-connector01.jpg
deleted file mode 100644
index 249acbebdf5feaa47118ba7f2307303d9ed59dab..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/ufolr/connector/2021-03-11-connector01.jpg and /dev/null differ
diff --git a/app/zh/information/blog/ufolr/connector/2021-03-11-connector02.jpg b/app/zh/information/blog/ufolr/connector/2021-03-11-connector02.jpg
deleted file mode 100644
index eaa879edec8c4b10babe795644e43a74f546e98e..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/ufolr/connector/2021-03-11-connector02.jpg and /dev/null differ
diff --git a/app/zh/information/blog/ufolr/connector/2021-03-11-connector03.jpg b/app/zh/information/blog/ufolr/connector/2021-03-11-connector03.jpg
deleted file mode 100644
index 082540f7045dbbcc982d31431a43ef989080bc31..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/ufolr/connector/2021-03-11-connector03.jpg and /dev/null differ
diff --git a/app/zh/information/blog/ufolr/connector/2021-03-11-connector04.jpg b/app/zh/information/blog/ufolr/connector/2021-03-11-connector04.jpg
deleted file mode 100644
index c336eb1b59b04a058dcbf115999ec8545a1d20a8..0000000000000000000000000000000000000000
Binary files a/app/zh/information/blog/ufolr/connector/2021-03-11-connector04.jpg and /dev/null differ
diff --git a/app/zh/information/blog/ufolr/connector/index.md b/app/zh/information/blog/ufolr/connector/index.md
deleted file mode 100644
index e81c4e75f70c5b50d76dda8127a2d79fd31f3784..0000000000000000000000000000000000000000
--- a/app/zh/information/blog/ufolr/connector/index.md
+++ /dev/null
@@ -1,264 +0,0 @@
----
-title: 'openLooKeng Connector 原理'
-date: '2021-03-11'
-module: 'blog'
-customLayout: 'true'
-tags: ['openLooKeng', '大数据']
-archives: '2020-06'
-author: 'ufolr'
-description: '朋友们,如果您已经对 openLooKeng 的整体方案有了解,知道 openLooKeng 有一个Connector 来连接数据源。如果您还想更深入了解 openLooKeng Connector 的实现原理,ufolr 的原创笔记应该能帮到您。'
----
-
-## 前言
-
-朋友们,如果您已经对 openLooKeng 的整体方案有了解,知道 openLooKeng 有一个 Connector 来连接数据源。如果您还想更深入了解 openLooKeng Connector 的实现原理,ufolr 的原创笔记应该能帮到您。
-
-## openLooKeng Connector 原理
-
-### 1 什么是 Connector?
-
-我们知道在整个 Presto 工程中所有的功能都是以插件的形式实现,而 Connector 则是一种负责 Presto 与数据源进行交互的插件,不同的数据库对应于不同的 Connector。
-
-#### 1.1 SPI (Service Provider Interface)
-
-SPI 是 JDK 内置的服务提供/发现机制,它通过在 ClassPath 路径下的 META-INF/services/目录中定义的文件,自动加载文件里所定义的类。该机制为很多框架的扩展提供了可能,如知名的 JDBC 就使用了该机制。
-
-SPI 的特点:
-
-- SPI 文件名为实现 Service 接口的全限定名
-
-- SPI 文件内容为实现该接口的具体文件
-
-- 使用 ServiceLoader.load(Class class) 动态加载 Service 接口的实现类
-
-- 如果 SPI 实现的类在外部 Jar 包中,则需要将该 jar 包放在当前程序的 classpath 下
-
-- Service 的实现类必须有无参构造方法
-
-#### 1.2 PrestoPlugin (插件)
-
-openLooKeng 中有一个 presto-spi 模块,该模块即定义了 openLooKeng 对外暴露的 SPI 接口,实现对应的接口即可实现连接器、类型、函数、系统访问控制等的功能。特别的,openLooKeng 的插件即实现了 Plugin SPI 的模块:
-
-
-
-### 3 Connector 开发
-
-#### 3.1 Maven 工程配置
-
-1. 在 openLooKeng 源码根目录创建与 Connector 名对应的目录,并在该目录中创建 pom.xml
-
-2. 将当前模块加入 io.hetu.core 组中,版本需要和当前 openLooKeng 工程版本一致:
-
-```
-
-
-#### 3.2 非 JDBC 数据源
-
-对于非 JDBC 数据源,我们需要根据数据源的情况,手动实现一个完整 Connector 的几个必须组件。
-
-
-
-其调用逻辑如下图:
-
-
-
- Metadata
-
-实现 io.prestosql.spi.connector.ConnectorMetadata 接口,提供了对数据源元数据的管理,例如列举 Schemas、Tables、Columns 信息,创建表、删除表,创建视图、删除视图等一系列 DDL 操作。
-
- SplitManager
-
-Split 详细描述了一个数据分片的具体内容,每种数据源的 Split 都不尽相同,数据源中定义的 Split 即为 Source Stage 中调度的 Split。Split 的分片合理性直接决定了 Hetu 读取数据源数据的效率。如果一个 Split 分片涵盖的数据量过大,数据不均匀,则会拖慢整个 SQL;若过小,则会造成大量的 CPU 资源都耗费在调度 Split 的操作上。
-
-Splits 通过 SplitManager 获取,最终是通过接口 io.prestosql.spi.connector.ConnectorSplitManager 的实现类,也就是各个 Connector 的 SplitManager 获取 Split,该接口包括了以下方法用于获取 Split:
-
-
-
-另外,传统的数据问题在于,没有人能够轻松地查询所有可用数据,这些单独存储的数据我们称为数据孤岛。每个数据孤岛都必须单独查询,然后必须手动合并查询结果。该流程不仅耗费大量成本、时间,而且效率低下。为了整合数据,通常企业会采用以下一项或多项数据整合策略:
-
-抽取、转换和加载(ETL):该流程从不同数据孤岛中复制数据,然后将这些数据转移至中央位置(例如,数据仓库)
-企业服务总线(ESB):可为应用程序建立通信系统,使其共享信息。
-数据虚拟化:该技术为数据孤岛内的数据创建实时整合视图,并且可以把数据提供给应用程序、分析人员和业务用户。
-
-大部分数据整合技术都是先复制数据,然后把数据副本转移到新的合并储备库。相反,数据虚拟化技术无需复制数据,即可提供整合数据的实时视图。
-
-我们采用了开源的 openLooKeng 作为我们的数据虚拟化引擎,它具有极简的数据分析体验,灵活、可扩展及高可靠的特性。
-
-### 02 管理数据沼泽
-
-今天对任何企业而言,管理大量数据都充满挑战。不仅是数据的数量,企业还必须管理多种数据类型⸺包括结构化数据、非结构化数据及半结构化数据,并且这些数据类型还来自多个数据源。
-
-
-
-这些不同的数据类型必须经常从数据源抽取出来,转换成不同的格式后,加载到消费应用程序上(这一过程被称为“ETL”),然后企业才能使用这些数据类型。ETL 流程往往是脚本化流程或手工流程,需要 IT 部门的协助,以计划分批处理的方式进行,这个过程缺少灵活性,还会带来更多的复杂性和延迟。
-
-### 03 资蛛侠数据服务介绍
-
-大部分数据整合解决方案都是将数据副本向将要合并数据源迁移,不过数据虚拟化技术却提供了一种完全不同的方法。数据虚拟技术没有移动数据,而是提供一个数据整合视图,让数据保留在原来的位置。企业不必支付数据的移动和存放费用,但却可以获得数据整合带来的优势。
-
-
-
-数据虚拟化技术不仅能够实现与传统数据整合技术相同的许多转换和质保功能⸺例如, ETL、数据复制、数据联邦、企业服务总线(ESB)等,而且能够借助现代技术,以较低成本,更加迅速、敏捷地提供实时数据整合。在许多情况下,数据虚拟化技术可以取代传统数据整合技术,并且减少对复制数据集市和数据仓库的需求。
-
-资蛛侠数据服务(以下统称为“产品”)利用数据虚拟化技术,将跨数据源关联查询、数据服务开发、服务审批与发布、上架 API 目录、消费与管理等数据服务全生命周期管理,打通异构数据源的关联和共享,以数据驱动业务的各个环节,快速赋能企业各种应用场景。
-
-
-
-产品提供的主要能力如下:
-
-1. 结构化与非结构化数据的语义整合
-
-产品利用数据虚拟化技术,能够将非结构化 Web 数据的语义对接到结构化数据的少数技术之一。统一的 SQL 语义,跨源跨结构的访问和整合企业内数据。
-
-
-
-2. 敏捷的数据服务开通
-
-产品提高了 API 的经济性。无论是原始数据源,还是衍生、整合或虚拟的数据源,都可通过统一的协议进行访问,并且能够在几分钟内即可实现受控访问。
-
-3. 消除不必要的数据移动
-
-有了我们数据服务产品,就无需再为报告取数目的进行数据复制,也不必再重写 ETL 脚本。产品基于企业现有的数据和应用架构进行操作,并且配置方式完全相同。
-
-
-
-4. 完整的数据沿袭和敏捷的业务规则
-
-在任何时间点,公司都可以了解和报告任何敏感数据集的完整数据沿袭,包括其原始来源,所有视图和修改。另外,系统运转的情况下设置数据屏蔽,以免相关数据被缺少必要凭证的用户查看。由于这些规则设置在平台中,因此可以在不同类型的系统之间快速有效地应用它们。
-
-5. 数据服务全生命周期管理
-
-数据服务的生命周期包括接入数据源、可见呈现、分析探索、分享、监控保护、消亡等状态。整个过程系统支持自动的监控管理及版本历史的记录保存,同时对全生命周期的数据服务状态流转过程支持流程审批管理。
-
-
-
-特别地,三方 API 注册到系统中,和系统创建的 API 一样,同样享受异常告警的待遇。
-
-### 04 结语
-
-资蛛侠数据服务 与传统的抽取、转换和加载(ETL)流程相比,产品落地后技术所需的开发人员要少得多。相对于每 4 名 ETL 开发人员工作量,您只需要一名数据虚拟化技术开发人员即可。不仅如此,它还能为用户带来:
-
-维护成本低于传统整合工具: 多次的物理复制、转移和存储数据成本高昂。数据虚拟化技术创建了虚拟数据层,从而消除了数据的复制需要和存储成本。
-
-加快了数据管理速度: 传统数据整合办法等上几个小时、甚至几天才能获得结果,而数据虚拟化技术却可以实时提供。
-
-超越了数据联邦: 数据虚拟化技术是数据联邦技术的超集。其中不仅包括高级性能优化能力,还包括自助服务搜索和发现能力。
-
-补充传统数据仓库技术: 数据虚拟化技术与传统的数据仓库工具并驾齐驱,互为补充。
-
-
-
-### openLooKeng 在览众资蛛侠数据服务平台的价值:
-
-1. 提供统一的 SQL 接口跨库访问多种数据源
-
-2. 提供近实时的高性能数据分析引擎
-
-3. 提供灵活、易扩展、高可靠的设计架构
diff --git a/app/zh/information/news/2019-11-19-openHetu-Announced/2019-11-19-openHetu-Announced-01.jpg b/app/zh/information/news/2019-11-19-openHetu-Announced/2019-11-19-openHetu-Announced-01.jpg
deleted file mode 100644
index 9a6ea2d0a500f7755aad30c2ae77f79c7a9b5b27..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2019-11-19-openHetu-Announced/2019-11-19-openHetu-Announced-01.jpg and /dev/null differ
diff --git a/app/zh/information/news/2019-11-19-openHetu-Announced/2019-11-19-openHetu-Announced-02.jpg b/app/zh/information/news/2019-11-19-openHetu-Announced/2019-11-19-openHetu-Announced-02.jpg
deleted file mode 100644
index 774f80bd8017f556d9fd538803edc0a7e4682da6..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2019-11-19-openHetu-Announced/2019-11-19-openHetu-Announced-02.jpg and /dev/null differ
diff --git a/app/zh/information/news/2019-11-19-openHetu-Announced/index.md b/app/zh/information/news/2019-11-19-openHetu-Announced/index.md
deleted file mode 100644
index 9829f1e3967dd8a986f04b8cb2ddbbc442830503..0000000000000000000000000000000000000000
--- a/app/zh/information/news/2019-11-19-openHetu-Announced/index.md
+++ /dev/null
@@ -1,39 +0,0 @@
----
-title: '华为面向鲲鹏计算产业,宣布启动数据基础设施战略并开源数据虚拟化引擎HetuEngine'
-date: '2019-11-19'
-module: 'news'
-customLayout: 'true'
-categories: ['Announcement']
-author: 'Huawei'
-description: '今日,华为在深圳举办2019全球数据基础设施论坛,面向鲲鹏计算产业,宣布全面启动数据基础设施战略,并开源数据虚拟化引擎HetuEngine(河图引擎),希望让伙伴像使用“数据库”一样使用“大数据”,让数据治理、使用更简单。'
----
-
-[中国,深圳,2019 年 11 月 19 日] 今日,华为在深圳举办 2019 全球数据基础设施论坛,面向鲲鹏计算产业,宣布全面启动数据基础设施战略,并开源数据虚拟化引擎 HetuEngine(河图引擎),希望让伙伴像使用“数据库”一样使用“大数据”,让数据治理、使用更简单。这是继今年 9 月基于“鲲鹏+昇腾”双引擎全面启航计算战略后,华为从数据角度对计算战略的再度阐述。华为数据基础设施战略围绕数据“采-存-算-管-用”的全生命周期,诠释了华为通过提供融合、智能、开放的数据基础设施,使能各行各业客户释放数据价值,让智能无所不及。
-
-智能时代,算力是新生产力,数据是新生产资料,而 5G、AI 和云成为新生产工具。自动驾驶、4K/8K 视频、AR/VR、物联网等应用的兴起,为企业带来海量的数据增长和分析处理需求,但需求与资源之间存在巨大落差。
-
-华为 Cloud & AI 产品与服务总裁侯金龙在题为《打造“融合、智能、开放”数据基础设施,携手迈入智能时代》的主题演讲中表示:“随着 5G、AI、云等技术的深度应用与融合,剧增的数据正在改变我们的生产和生活,但也带来找数难、取数难、用数难等挑战。华为面向鲲鹏计算产业,全面启动数据基础设施战略,提供融合、智能、开放的数据基础设施,对数据的采、存、算、管、用实施端到端的整合和优化,致力于让数据在全生命周期内好用,数据的每比特价值最大,每比特成本最优,让合作伙伴像使用‘数据库’一样使用‘大数据’。”
-
-
-
-#### 打造融合、智能、开放的数据基础设施
-
-基于“鲲鹏+昇腾”双引擎的强大算力,华为持续围绕数据构建计算、存储、智能化能力,加强研发投入和技术创新。华为数据基础设施包括数据存储、数据处理、数据管理系统、数据虚拟化引擎等,它包含以下三大特征:
-
-- **融合**:基于在存储、数据库、大数据等技术领域的突破,打破“存储内部系统墙”、“数据库与存储链路墙”、“大数据与存储配置墙”、“数据库与大数据协同墙”四堵墙。这四堵墙的打破,让数据融合更彻底,帮助客户实现 TCO 降低 30%以上、数据访问和处理性能提升 2 倍、分析效率提升 100%。
-
-- **智能**:基于 AI 芯片、存储和华为云的三层架构,通过云上云下结合,云上训练和云下推理,让系统越用越快、越用越省。其中,依托昇腾处理器的 AI 能力,自动学习和识别 IO 流,提升 Cache 预取命中率,系统整体性能提升 20%;依托鲲鹏处理器的多核算力,根据不同的数据类型,实时优化数据缩减算法,TCO 降低 25%;结合华为云自身大规模运维运营经验,当前可以实现提前 14 天预测硬盘故障,提前 60 天预测性能瓶颈,提前 365 天预测容量不足,其中 30%的故障可以自我修复。
-
-- **开放**:针对找数难、取数难、用数难的问题,推出数据虚拟化引擎 HetuEngine,屏蔽数据类型差异、地域差异、语法差异,让数据治理、使用简单。HetuEngine 拥有“一个入口、一个目录、一份数据、统一安全”四大核心能力,通过屏蔽数据基础设施的复杂度,让伙伴像使用“数据库”一样使用“大数据”,复用现有的生态、工具和技能,提升开发效率 2 到 10 倍。
-
-#### 开源数据虚拟化引擎 HetuEngine,实现更快、更好的业务系统对接
-
-为了更好的发展数据产业,让客户和合作伙伴更简单地对接应用与数据,华为在大会上宣布开源 HetuEngine。开源版本的河图引擎叫 openHetu,将于 2020 年 6 月上线。华为将开源内核,开发者可以基于开源代码进行定制,包括数据源扩展、SQL 执行策略等,实现应用快速对接,提升开发效率。
-
-
-
-华为始终践行“平台+生态”策略,通过硬件开放,软件开源,使能伙伴,共建开放、繁荣的鲲鹏计算产业生态,共同迈入智能时代。
-
-2019 年 11 月 18 日至 19 日,华为以“引领数据基础设施,携手迈入智能时代”为主题,在深圳召开 2019 全球数据基础设施论坛,展望智能时代发展趋势与机会、推进产业政策发展、展示最新数据基础设施产品与解决方案。欲了解更多大会信息,请点击:
-
-新闻来源:
-
-[中国,深圳,2020 年 7 月 1 日] 昨日,华为正式宣布开源数据虚拟化引擎 openLooKeng,开源社区官网(
-
-#### Talk is cheap. Show me the code.
-
-一场开源盛会少不了令人印象深刻的 Demo。
-日前在北京圆满落幕的 openEuler Summit 2020,就有一场近乎完美的 openEuler 生态多样性场景演示《openEuler&Friends》。数据虚拟化引擎 openLooKeng 有幸参与其中。
-
-
-
-这场 Live Demo,由 10 位来自不同领域的 openEuler 社区开发者同台协作完成,展示了 openEuler 在云计算、云原生及集群监控、储存、大数据、数据库、Web 等领域开源生态的丰富性。
-
-
-
-
-
-现场完成的数据大屏
-
-20 分钟内,框架部署全程零失误,应用运行表现简洁丝滑而流畅,现场千百名观众的掌声,是肯定,也是认可。
-
-
-
-在本次峰会上,openLooKeng 在 Demo Show 中担任什么角色,发挥了哪些可靠作用? 让我们听听来自北明软件有限公司的林舒凡老师的声音。林老师也是本次 Demo 现场 openLooKeng 项目的代表。
-
-
-
-
-
-从最初的“我”到现在的“我们”,openLooKeng 健康积极的成长态势离不开许多开源朋友们与合作伙伴们的贡献和支持。而这场 Demo 的成功,是对 openLooKeng 成长路上的肯定,技术上的认可。
-
-为用户提供极简的数据体验,持续不断地构建大数据技术竞争力,是 openLooKeng 前行的目标。我们希望,也真挚地欢迎未来能有越来越多的朋友们加入到 openLooKeng 这个大家庭,一起携手前行,共话大数据引擎技术,在享受技术盛宴的同时一起努力,Make Data Simplified!
diff --git a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-01.jpg b/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-01.jpg
deleted file mode 100644
index 1d39802e37ae7600c7eccdf104f4396f6cfb1d5c..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-01.jpg and /dev/null differ
diff --git a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-02.jpg b/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-02.jpg
deleted file mode 100644
index f0fa5bed55cc2a62d8777ee702f5c8b04180e3bc..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-02.jpg and /dev/null differ
diff --git a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-03.jpg b/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-03.jpg
deleted file mode 100644
index e7721f85a78b4d27a61ab4227ecfeba23ffcb660..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-03.jpg and /dev/null differ
diff --git a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-04.jpg b/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-04.jpg
deleted file mode 100644
index 4fd65a3ecb6b390b23afd445b1a82b064425661d..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-04.jpg and /dev/null differ
diff --git a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-05.jpg b/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-05.jpg
deleted file mode 100644
index 26b161d9cb3ab764d2ef5e1b09b2f5377e1e79d3..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-05.jpg and /dev/null differ
diff --git a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-06.jpg b/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-06.jpg
deleted file mode 100644
index d750a6d3e17617c9b7d46f8d24668c8d58fa7481..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-06.jpg and /dev/null differ
diff --git a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-07.jpg b/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-07.jpg
deleted file mode 100644
index 0aed611dad05f9bac3e61e2e388bd272cc5295a8..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-07.jpg and /dev/null differ
diff --git a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-08.jpg b/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-08.jpg
deleted file mode 100644
index 2f8e02095fce4196dfbb93ca6ed1fc752c42b425..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-08.jpg and /dev/null differ
diff --git a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-09.jpg b/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-09.jpg
deleted file mode 100644
index fcb428911b5b9ab9dfb95709d13b5700972b753a..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-09.jpg and /dev/null differ
diff --git a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-10.jpg b/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-10.jpg
deleted file mode 100644
index dfe2a84768a8e7c7183142b9c4286cf13d78b518..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2021-01-ccf-bdci-news/2021-02-01-news-10.jpg and /dev/null differ
diff --git a/app/zh/information/news/2021-01-ccf-bdci-news/index.md b/app/zh/information/news/2021-01-ccf-bdci-news/index.md
deleted file mode 100644
index 664418fe9d9b738b0fd13b7b602fc8f4c18d86a0..0000000000000000000000000000000000000000
--- a/app/zh/information/news/2021-01-ccf-bdci-news/index.md
+++ /dev/null
@@ -1,47 +0,0 @@
----
-title: '优秀!2020 CCF BDCI大赛结果出炉,openLooKeng赛题3支队伍登榜'
-date: '2021-01-24'
-module: 'news'
-customLayout: 'true'
-categories: ['Announcement']
-author: 'openLooKeng'
-description: '2020 CCF BDCI大赛日前落下了帷幕。openLooKeng非常荣幸参与到这场极具影响力的行业盛会中,并贡献了赛题「openLooKeng性能优化」。该赛题任务吸引了632支队伍,658人参加。历经3个多月的激烈角逐,单赛题奖项中,参与openLooKeng赛题的3支队伍脱颖而出,荣获本场竞赛的一、二等奖。'
----
-
-
-
-日前,2020 CCF 大数据与人工智能大赛(简称 2020 CCF BDCI)决赛在湖南长沙成功举办。作为大数据领域中的关键项目,openLooKeng 非常荣幸参与到这场极具影响力的行业盛会中,并贡献了赛题「openLooKeng 性能优化」。该赛题任务吸引了 632 支队伍,658 人参加。历经 3 个多月的激烈角逐,单赛题奖项中,参与 openLooKeng 赛题的 3 支队伍脱颖而出,荣获本场竞赛的一、二等奖。
-
-
-
-2021 年 1 月 23-24 日,由中国计算机学会主办,由 CCF 大数据专家委员会、CCF 自然语言处理专业委员会、CCF 数据库专业委员会、CCF 高性能计算专业委员会、CCF 人工智能与模式识别专业委员会、CCF 计算机安全专业委员会、CCF 计算机应用专业委员会七个专委以及教育部易班发展中心、长沙市科技局、长沙高新区管委会、湖南大学、数联众创共同承办的 2020 CCF 大数据与计算智能大赛决赛暨中国大数据技术大会在湖南长沙成功举办。
-
-
-
-创办于 2013 年的 CCF 大数据与计算智能大赛(CCF BDCI),如今已成为大数据与人工智能领域的极具影响力的赛事,是大数据综合赛事第一品牌。openLooKeng 非常荣幸为这场极具影响力的赛事贡献赛题——「openLooKeng 性能优化」。
-
-
-
-本次 openLooKeng 的赛题共吸引了 632 支队伍,658 人参加。历经 3 个月多的激烈角逐,参与 openLooKeng 赛题任务的 3 支队伍在单赛题奖项中脱颖而出,荣获本场竞赛一、二等奖。
-
-
-
-「中科大星辰队」荣获本场竞赛一等奖。该队队员均来自于中国科学技术大学先进数据系统实验室(ADSL),由李诚老师带队,两名大四本科生徐宇鸣、陈清源,三名大三本科生王章瀚、高楚晴、刘逸菲组成。赛事期间,在完成在校课业任务之外,他们出色地完成了本次赛题的任务,期间更进行了大量的测试分析,扎实的测试数据,最终摘取大赛硕果。
-
-荣获二等奖的是「进击的巨人」和「零叉零零」两支队伍。「进击的巨人」,由 3 名高校学生组成,分别是中国科学技术大学计算机学院的研一学生刘祥和信息科学技术学院研一学生邱明璞;东北大学计算机科学与技术专业研一学生秦杰杰。他们相识于同一所高中,对计算机领域有着浓厚的兴趣,十分享受比赛带来的乐趣。「零叉零零」由一名来自清华大学计算机系研二学生组成。喜欢算法的他在这场赛事中独挑大梁,并交出了不菲的成绩。
-
-
-
-
-
-
-
-
-
-
-
-恭喜这群朝气蓬勃的小可爱们获奖!
-
-愿你们在追梦路上勇往直前,不负韶华。
diff --git a/app/zh/information/news/2021-03-31/index.md b/app/zh/information/news/2021-03-31/index.md
deleted file mode 100644
index 16e560b22d491974c44401bbffe36a227d4776f2..0000000000000000000000000000000000000000
--- a/app/zh/information/news/2021-03-31/index.md
+++ /dev/null
@@ -1,76 +0,0 @@
----
-title: 'openLooKeng v1.2.0 正式发布'
-date: '2021-03-31'
-module: 'news'
-customLayout: 'true'
-categories: ['Announcement']
-author: 'openLooKeng'
-description: '暖春3月,openLooKeng迎来了新版本V1.2.0。新版本中增强了哪些性能?又新增了哪些技术?最全解析在这里'
----
-
-自开源以来,openLooKeng 社区得到越来越多朋友的支持。社区内,小伙伴们对 openLooKeng 的性能给予了肯定和赞赏,同时也给出了许多有价值的建议。暖春 3 月,在众人的期待下,openLooKeng 迎来了新版本 V1.2.0。openLooKeng V1.2.0 是在旧版本的基础上进行优化,并基于小伙伴们的体验和建议,新增一些技术,以提高引擎性能,争取为大家带来更丝滑流畅的体验。
-
- 于引擎内核来说,主要增强两个维度:融合分析场景和性能。
-
-### 查询容错增强,提高引擎执行的可靠性
-
-在批查询处理运行的过程中,当某个工作节点出现故障时,可在其他节点上恢复任务,比如对 Hive 数据源的 insert 和 create table as select 操作。针对长时间运行的批查询处理任务,相比上一个版本,稳定性和可靠性有了极大的提升。
-
-### 查询性能的优化:基于 StarTree 的查询预聚合能力增强
-
-StarTree 旨在优化低延迟、聚合查询语句。我们通过 StarTree 查询预聚合能力,为用户构建所需要的不同维度和不同聚合操作的 cube。在以后的查询过程中,如果遇到任何可匹配的聚合子查询,引擎将直接从 cube 中读取数据,避免在原始表上执行查询,从而提高查询性能。
-
-### 引入 CTE(公共表表达式)优化技术,减少内存使用
-
-在执行计划优化过程中引入 CTE(公共表表达式)技术。当一个复杂查询中,存在某个子查询(例如 with 语句)被多次使用,优化器会为重复的子查询自动生成一个 CTE 节点,该 CTE 节点的输出会被执行计划流程中的多个父节点消费。也就是说,重复的子查询只会执行一次,化繁为简, 同时也减少内存占用,引擎获得更好的性能。
-
-### 通用算子下推框架,让 Connectors 参与到执行计划优化中
-
-为了让算子下推过程更加简单灵活,我们采用新的通用算子下推框架,让每个 Connector 可以参与到执行计划优化中。引擎在进行执行计划优化时,能够自行应用 Connector 的优化规则,使得算子下推过程变得更加高效。
-
-### 增强 HIVE ORC 的数据维护性能
-
-该特性优化了数据写入和数据修改(更新和删除)的处理速度,并不影响『读性能』的流畅度;支持并发访问 Hive Metastore,提高元数据操作性能,从而进一步优化数据写入和修改的性能。
-
- 于引擎门户来说,主要集中在南向生态方面的优化。
-
-### HBase Connector 性能优化
-
-针对单表查询性能的提升,我们新增了分片算法。另外,该性能支持 HBase 访问 Snapshot 的模式,从而提升多并发查询性能。
-
-### 数据源 UDF(User-Defined Function)支持下推
-
-引擎支持外部函数(UDF)的注册,也支持将其下推到 JDBC 数据源。为了让大家拥有更好的体验,用户可以在不迁移自己数据源 UDF 的情况下,使用已有的 UDF, 提高 UDF 的复用度。
-
-以上便是 openLooKeng 新版本 V1.2.0 在性能优化上较为亮眼的地方。当然,作为大数据领域的关键项目,openLooKeng 十分看重引擎的易用性和安全性。针对这两点,新版本 V1.2.0 做出如下增强:
-
-### 易用性 | 增强 Admin Dashboard UI 用户体验
-
-优化定时全量加载导致的 UI 页面卡顿,
-
-增强查询历史和查询结果的分页显示,
-
-优化新增连接器的参数配置,
-
-UI 界面支持 Kerberos 和密码登录,支持查询历史按用户进行过滤。
-
-### 安全性 | 基于 Ranger 的细粒度权限管控:支持行过滤和列掩码
-
-增加行过滤和列掩码,增加认证用户模拟权限控制,提供更细化的权限控制粒度。
-
----
-
-看了这么多,是不是很想动手一试?感兴趣的朋友可以下载体验。
-
-新版本下载地址:
-
-
-标准认为,一个成熟的社区,需要同时具备四方面核心能力,即:社区治理、社区运营、社区开发以及基础设施保障社区健康长期运转,如下:
-
-
-
-openLooKeng 经过 OSCAR (云计算开源产业联盟) 专家重重评审、测试 22 道环节,成为首批获得《可信开源社区评估体系》认证的开源社区,并获得评估证书。
-
-
-
-感谢为社区的小伙伴们。
-
-如果您觉得开源很酷,欢迎加入我们社区,一起让大数据更简单。
-
-openLooKeng, Make Big Data Simplified
diff --git a/app/zh/information/news/2021-05-28/hackathon_001.png b/app/zh/information/news/2021-05-28/hackathon_001.png
deleted file mode 100644
index 0991de3388800e820d7726c51124213e4aefdd5a..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2021-05-28/hackathon_001.png and /dev/null differ
diff --git a/app/zh/information/news/2021-05-28/hackathon_002.png b/app/zh/information/news/2021-05-28/hackathon_002.png
deleted file mode 100644
index 9c30ae99033cc8a69d4042757e225c9e7a9f3d83..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2021-05-28/hackathon_002.png and /dev/null differ
diff --git a/app/zh/information/news/2021-05-28/hackathon_003.png b/app/zh/information/news/2021-05-28/hackathon_003.png
deleted file mode 100644
index 202fd5481dda5a71b7de2415055b8fa3ee609580..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2021-05-28/hackathon_003.png and /dev/null differ
diff --git a/app/zh/information/news/2021-05-28/hackathon_005.png b/app/zh/information/news/2021-05-28/hackathon_005.png
deleted file mode 100644
index 0fc5c71316e325df1f746d5ef10a3147fe53df2f..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2021-05-28/hackathon_005.png and /dev/null differ
diff --git a/app/zh/information/news/2021-05-28/hackathon_006.png b/app/zh/information/news/2021-05-28/hackathon_006.png
deleted file mode 100644
index d52667360bca6665aef8edb9df9be76100fe1b20..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2021-05-28/hackathon_006.png and /dev/null differ
diff --git a/app/zh/information/news/2021-05-28/hackathon_007.png b/app/zh/information/news/2021-05-28/hackathon_007.png
deleted file mode 100644
index 6e688544d16b8051ffbe2b621e7002148afcbe15..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2021-05-28/hackathon_007.png and /dev/null differ
diff --git a/app/zh/information/news/2021-05-28/hackathon_008.png b/app/zh/information/news/2021-05-28/hackathon_008.png
deleted file mode 100644
index 75b23388d6139a8e737cd07e778ccd8679d56325..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2021-05-28/hackathon_008.png and /dev/null differ
diff --git a/app/zh/information/news/2021-05-28/index.md b/app/zh/information/news/2021-05-28/index.md
deleted file mode 100644
index 9a875a13c3b118a700dc68c374fff7e2cb39e24c..0000000000000000000000000000000000000000
--- a/app/zh/information/news/2021-05-28/index.md
+++ /dev/null
@@ -1,83 +0,0 @@
----
-title: '第12届中国开源黑客松 | openLooKeng再现荣耀星光'
-date: '2021-05-30'
-module: 'news'
-customLayout: 'true'
-categories: ['Announcement']
-author: 'openLooKeng'
-description: '5月28日-29日,由中国开源云联盟指导,中国电子技术标准化研究院、华为、英特尔、腾讯联合主办的第12届中国开源黑客松在北京顺利举办。本次活动共20个项目参与,吸引来自不同高校、不同行业的 103 位开发者的参加。'
----
-
-金鼓喧阗,势如彍弩。限时 2 天的黑客马拉松,从 0 到 1 完成 2 个特性开发,500+代码行的合入,openLooKeng 7 人小队再现荣耀星光
-
-
-
-5 月 28 日-29 日,由中国开源云联盟指导,中国电子技术标准化研究院、华为、英特尔、腾讯联合主办的第 12 届中国开源黑客松在北京顺利举办。本次活动共 20 个项目参与,吸引来自不同高校、不同行业的 103 位开发者的参加。
-
-
-
-作为开源新成员,也是二次参与黑客松活动的数据虚拟化引擎 openLooKeng,短短两天,7 名成员从零开始搭建测试,完成了两个特性的开发:ClickHouse Connector 外部 UDF 注册和下推适配以及 Kylin Connector 的适配。团队代表成员之一,黎一泽荣获黑客松明星组员奖。
-
-
-
-**项目成果代码 URL**
-
-ClickHouse Connector
-
-
----
-
-### 第 12 届中国开源黑客松
-
-2021 年 5 月 28 日-29 日,第 12 届黑客松由中国电子技术标准化研究院,华为,英特尔,腾讯主办,中国开源云联盟,木兰开源社区承办。在 OIF 基金会、Linux 基金会、CNCF 基金会、Ceph 基金会、SODA 基金会、开放原子开源基金会、开源社、思否开发者社区、上海白玉兰开源开放研究院、OpenI 启智社区、绿色计算产业联盟等国内外开源组织的大力支持下开展。活动共有 20 个项目参加,吸引来自华为,腾讯,英特尔,蚂蚁集团,烽火,浪潮,联通数科,天翼云,电信集成,Linaro,北大杭研、中科院网络信息中心、灵雀云、杭州电子科技大学、58 同城等共 103 名开发者参加。
-
-
-
-中国开源黑客松活动,以代码开发为主要活动内容,致力于推动开源项目扩大开发者生态,促进开源项目上下游协同开发。自 2015 年以来,黑客松活动已经走过了一轮十二届。未来,中国开源黑客松将持续发力,愿更多的项目加入其中,更多的朋友参与进来,让开源生态更加繁荣。
-
-
-
-
diff --git a/app/zh/information/news/2021-06-30/index.md b/app/zh/information/news/2021-06-30/index.md
deleted file mode 100644
index 3d2322352ef09c0ddf6cab2b51330a510b466071..0000000000000000000000000000000000000000
--- a/app/zh/information/news/2021-06-30/index.md
+++ /dev/null
@@ -1,67 +0,0 @@
----
-title: 'openLooKeng v1.3.0 正式发布'
-date: '2021-06-30'
-module: 'news'
-customLayout: 'true'
-categories: ['Announcement']
-author: 'openLooKeng'
-description: '2021年6月底,openLooKeng将迎来新版本 v1.3.0。基于社区用户和开发者的体验和建议,openLooKeng 新版本 v1.3.0 在原基础上进行特性优化,并提供新功能,以提升引擎性能。'
----
-
-自 2020 年 6 月 30 日开源以来,openLooKeng 项目服务越来越多,包括华为云 DLI 服务、北明软件数据资产管理平台、览众科技资蛛侠等。为了持续为用户提供极简的数据分析体验,openLooKeng 保持 3 个月一个版本的迭代速率。2021 年 6 月底, openLooKeng 将迎来新版本 v1.3.0。基于社区用户和开发者的体验和建议,openLooKeng 新版本 v1.3.0 在原基础上进行特性优化,并提供新功能,以提升引擎性能。
-
-于引擎内核来说,openLooKeng v1.3.0 主要集中两个维度的增强:融合分析场景和性能提升。
-
-## 融合分析场景
-
-### Task recovery
-
-Task recovery 支持查询任务重试,即当查询任务或某个工作节点出现故障,可在其他节点上恢复查询任务。新版本 v1.3.0 在该基础上支持对 CTE(公共表表达式)和 spill-to-disk 特性的兼容,进一步增强引擎内核的批查询能力。
-
-## 性能提升
-
-### 1. Star Tree 索引
-
-Star Tree 索引提供一种预聚合技术,通过创建和管理用户所需要的不同的 Cubes(多维数据集)来优化冰山查询的延时。新版本 v1.3.0 增强包括支持对大型 Cubes 的创建(10 Billion 基数);支持在 Cubes 中创建过滤规则来过滤数据;支持对 Cubes 的增量插入。
-
-### 2. CTE(Common Table Expressions)
-
-对 CTE(具有 with 的语句)进行执行计划优化,使得 CTE 只被执行一次,并将其流式传输到查询中的其他引用,减少该查询所需的计算资源。新版本 v1.3.0 中,CTE 支持算子下推,进一步提高系统的总体并发。
-
-### 3. 新增优化器 CBO(Cost Based Optimization)
-
-针对已排序的表,openLooKeng 新版本 v1.3.0 新增排序聚合器。与原有的哈希聚合器相比,排序聚合器减少了对部分不必要的数据保存,节省了内存和处理时间,提高引擎查询性能。
-
-openLooKeng 注重引擎内核的增强,也重视南向生态的扩展,期待更多的小伙伴加入进来,共同发展。新版本 v1.3.0 提升原有连接器功能的同时,新增 Hudi、Greenplum、ClickHouse 等数据源连接器,为社区小伙伴们提供更好的数据体验。
-
-## 功能提升
-
-1.Oracle Connector 支持 Update 和 Delete 操作;
-
-2.JDBC Connector 支持多分片查询,通过提高并发来提升性能;
-
-3.Memory Connector 通过 hetuMetastore 支持数据和元数据的持久化,从而防止内存表信息丢失;提供新的数据布局以支持排序和索引,进而提高数据查询性能。
-
-## 新增数据源连接器
-
-1.Hudi Connector,支持对 Hudi 数据源的访问查询;
-
-2.Greenplum Connector,支持对 GreenPlum 数据源的基本读写操作,但不支持删除和更新;
-
-3.ClickHouse Connector,支持对 ClickHouse 数据源的基本读写操作;支持 SQL 语句下推到该数据源,支持外部函数的注册及函数下推能力。
-
-当然,为了让用户有更好的体验,提高引擎的可靠性,openLooKeng v1.3.0 还增强了资源隔离的特性:根据用户配置限制和资源使用情况进行节流调度或终止任务,从而确保资源的正确使用。
-
-### **openLooKeng 新版本 v1.3.0 下载地址**
-
-
-
-此外,OmniData 算子下推特性将计算侧的 Filter、Aggregation、Limit 算子下推到存储节点执行,实现近数据计算,利用多样算力缓解计算侧 CPU 的压力。
-
-OmniData 服务将算子处理结果通过网络传输到计算节点,从而减少无效数据在网络上的传输,有效提升大数据计算性能。
-
-
-
-**OmniData 适用范围:**
-
-1. 支持 openLooKeng v1.4.0 和 Spark 3.0.0,提供对应的引擎侧 OmniData 插件。
-
-2. 支持算子下推到 HDFS,支持 S3 的存储访问接口。
-
-3. 支持数据格式包括:TXT、ORC、Parquet。
-
-4. TaiShan 服务器,支持鲲鹏处理器的体系架构。
-
-OmniData 在大数据存算分离场景或大规模融合场景的适用性,极大符合 openLooKeng 的愿景:让大数据更简单。
-
-OmniData 特性的引入,将进一步提升 openLooKeng 引擎性能。
-
-OmniData Connector 更多详情请参考:
-
-
-
-
-2021 云原生产业大会:openLooKeng 获得“可信开源社区评估体系”认证证书
-
-当前,已有包括招商银行、光大银行、亚信科技、中科院、中国移动、中国联通等不同行业的 30+家合作伙伴加入 openLooKeng 社区,并在多个关键特性上积极贡献。openLooKeng 社区还与行业解决方案软件伙伴紧密合作,针对行业场景的特点,联合推出对应解决方案:
-
-- 与北明软件推出数据资产管理平台 v4.0,提供数据交换共享服务;
-- 与览众科技推出资蛛侠数据服务平台等。
-
-
-
-以上排序不分先后
-
-在吸引一批批领先企业与科研机构加入的同时,openLooKeng 有许许多多开源大数据爱好者的参与。我们感谢招行大数据技术专家吴酋珉、光大银行程小舰、鸿鹄元数总经理胡继云、学生开发者何正杰以及社区其他朋友们带来的分享与贡献,让社区迸发新的生命力。
-
-
-
-openLooKeng 社区用户间的协作联接图
-
-社区用户通过一次次的贡献与交流,互相联接,共同构筑 openLooKeng 社区美好蓝图
-
-当然,还有很多开源爱好者们也在幕后默默支持社区,他们不断在社区组织、社区论坛、交流群内穿梭,积极提问、热心解答、互相帮助,让社区愈加有活力。
-
-
-
-openLooKeng 社区贡献者(Gitee ID)
-
-- openLooKeng 下载量已突破 90000+;
-- 社区贡献者 300+;
-- 累计 Pull Request 提交达到 1900+;
-- 累计 Commit 数达到 1400+;
-- 社区交流群人数达到 600+。
-
-### 感谢所有与 openLooKeng 相伴的人,未来路上我们一起同行。
-
-### openLooKeng,Make Big Data Simplified.
diff --git a/app/zh/information/news/2021-11-29/banner1.jpg b/app/zh/information/news/2021-11-29/banner1.jpg
deleted file mode 100644
index 001dd685eced9c59263aa0ef885c2d0227a3fda6..0000000000000000000000000000000000000000
Binary files a/app/zh/information/news/2021-11-29/banner1.jpg and /dev/null differ
diff --git a/app/zh/information/news/2021-11-29/index.md b/app/zh/information/news/2021-11-29/index.md
deleted file mode 100644
index 865ea88f41dbeff67ad7e404b7f0dbaf7009b523..0000000000000000000000000000000000000000
--- a/app/zh/information/news/2021-11-29/index.md
+++ /dev/null
@@ -1,95 +0,0 @@
----
-title: '2021信创“大比武”鲲鹏赛道总决赛圆满落幕,openLooKeng两支队伍获奖'
-date: '2021-11-29'
-module: 'news'
-customLayout: 'true'
-categories: ['Announcement']
-author: 'openLooKeng'
-description: '信创“大比武”总决赛圆满落幕,全国十强队伍上线,参与openLooKeng赛题的两支队伍,北明大数据团队与鸿鹄团队分别荣获铜奖与优胜奖。恭喜!'
----
-
-11 月 29 日,2021 信创“大比武”鲲鹏基础软件开发赛道(以下简称“鲲鹏赛道”)总决赛圆满落幕。四个多月以来,选手们一路披荆斩棘,带来了一场又一场精彩绝伦的比赛。总决赛现场,经过评委老师们认真审核、综合评定,最终决出十支队伍荣获赛事大奖。
-
-其中,参与 openLooKeng 赛题的两支队伍,北明大数据团队与鸿鹄团队分别荣获铜奖与优胜奖。恭喜。
-
-
-
----
-
-## 铜奖
-
-### 北明大数据团队
-
-
-
-### 参赛宣言
-
-数据北明,罗庚引擎,融合分析,点数成金!
-
-### 作品名称
-
-查询日志持久化
-
-### 作品简介
-
-基于 openLooKeng 大数据技术,开发了数据服务跨源分析和日志持久化功能,实现了数据服务数据跨库融合分析,持久化日志,多维度查询增强,查询条件日志查询的性能提升,落地于政务、交通、金融行业等行业项目,应用在数据免搬迁,跨源数据融合分析场景。
-
-### 获奖感言
-
-感谢主办方、各位评委老师辛苦付出、感谢团队的小伙伴,感谢华为通过软件开源的方式,共享在基础软件领域的多年积累,使能伙伴发行 openEuler 和 openGauss 商业版,繁荣基础软件产业生态。参加本次鲲鹏赛道活动,不仅可以通过赛事掌握领域最新技术,还能在赛事过程中提升技术水平,夯实公司团队对信创基础软件能力掌握与重视。同时希望有更多数据开发者加入到 openlookeng 开源社区。目前,我们已经深入融合 openlookeng 形成商业解决方案,希望未来为鲲鹏产业发挥价值,提升鲲鹏生态的影响力。
-
----
-
-## 优胜奖
-
-### 鸿鹄团队
-
-
-
-### 参赛宣言
-
-鸿鹄展翅,鹏程万里!
-
-### 作品名称
-
-SQL 历史记录查询功能增强
-
-### 作品简介
-
-基于 openLooKeng 大数据技术,开发增强了 SQL 历史记录查询能力,对执行跨源分析 SQL 进行持久化处理,支持数据库和文件存储方式,便于历史 SQL 解析关系分析。同时,在 SQL 记录查询 UI 方面提升了交互性,丰富了筛选条件,优化了列表布局。目前,已将该成果落地于国内多家金融、政府、交通等行业数据中台项目,得到广泛好评。
-
-### 获奖感言
-
-感谢主办方给予我们展示团队的机会,通过本次比赛,我们感受到了开源的魅力,与其他优秀团队竞技的过程,使我们明白了自身的不足,促使我们去改正进步,比赛虽然已经结束了,但是我们拥抱开源和提升自我的脚步不会停止,最后希望能有越来越多的人能参与和支持国产开源项目。
-
----
-
-## 2021 信创“大比武”鲲鹏赛道总决赛获奖名单
-
-
-
-识别二维码观看总决赛回放
-
-
-
----
-
-## 关于 2021 信创“大比武”
-
-本届 2021 信创“大比武”鲲鹏赛道总决赛由信息技术应用创新工作委员会指导,华为主办,重庆鲲鹏创新中心、北京鲲鹏联合创新中心、openEuler 开源社区、openGauss 开源社区及 openLooKeng 开源社区共同承办。自开赛以来,来自各行业近百家企业以及高校的 300 余位开发者踊跃报名参赛,产生了数百个优秀作品。
-
-信创技术应用创新工作委员会副秘书长王莉和中国工程院院士、信创专家委主任委员廖湘科为鲲鹏赛道总决赛致辞,他们表示,鲲鹏赛道吸引了全国的操作系统、数据库及大数据从业者和爱好者,加入到基础软件的生态建设和完善,促进了技术创新和人才培育的结合,搭建了信创资源和行业应用的对接平台,推动了信息技术应用创新产业的良性发展。华为计算开源业务总经理堵俊平也在致辞中强调了在基础软件领域构建核心产业链的重要性,开源社区能够为计算产业持续发展奠定良好的基础。
-
-本次鲲鹏赛道搭建了一个开放交流、施展才能的舞台,华为作为主办方,通过命题设置、为参赛团队提供一对一的专家指导和关键技术深度指导,帮助参赛团队开发更具竞争力的优质项目,以赛促学,以赛促建,进一步深化了技术赋能商业、商业牵引技术,相互融合的良性循环。
-
-鲲鹏计算产业是基于鲲鹏处理器的基础软硬件设施、行业应用及服务,涵盖从底层硬件、基础软件到上层行业应用的全产业链条。作为鲲鹏计算产业的发起者和重要成员,华为秉持“硬件开放、软件开源、使能伙伴,发展人才”的策略,通过战略性、长周期的研发投入,和产业伙伴一起,持续推进全栈计算技术的创新发展。
-
-未来,华为还将持续投入开源技术创新、开源基础设施建设、开源软件能力建设,为汇聚产业伙伴的力量,吸引全球开发者共同构建开源社区,促进我国的开源生态和海外主流开源社区充分融合共享。
-
----
-
-欢迎关注 openLooKeng,加小助手微信(openLooKengoss)进入专属技术交流群。
-
-社区代码仓
-
- 近日, 中国科协召开的 2022“科创中国”年度会议上,中国工程院院士周济发布了 2021“科创中国”开源创新榜单。**数据虚拟化引擎 openLooKeng 入选了“科创中国”开源创新榜年度优秀开源产品**。此次上榜也代表着“科创中国”对 openLooKeng 在开源创新方面取得的成绩的认可。
-
-
-
-
-
-开源至今,openLooKeng 健康积极的成长态势离不开许多朋友们的支持和贡献。正因如此,openLooKeng 才得以入选此次优秀开源产品的榜单。这是对 openLooKeng 开源成果的肯定,也是对社区成长路上的认可。
-
-感谢一直以来陪伴 openLooKeng 的朋友们,未来让我们继续携手前行,共话大数据引擎技术;也欢迎更多的朋友们加入 openLooKeng 这个大家庭,在享受技术盛宴的同时一起努力,Make Big Data Simplified !
-
-2021“科创中国”榜单:
-
-
https://gitee.com/openlookeng/hetu-core/pulls/1451 |
-
-## 算子处理扩展:
-
-| 描述 | 链接 |
-| -------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- |
-| 增加通过用户自定义 worker 结点物理执行计划的生成,用户可以实现自己的算子 pipeline 代替原生实现,加速算子处理。 | PR: https://gitee.com/openlookeng/hetu-core/pulls/1436
https://gitee.com/openlookeng/hetu-core/pulls/1443 |
-
-## HIVE UDF 扩展:
-
-| 描述 | 链接 |
-| ----------------------------------------------------------------------------------------- | -------------------------------------------------- |
-| 增加 HIVE UDF 函数命名空间的适配,以支持执行基于 HIVE UDF 框架编写的 UDF(含 GenericUDF) | https://gitee.com/openlookeng/hetu-core/pulls/1455 |
-
-欢迎在 openLooKeng gitee 仓上提 Issue,分享您的体验感受与建议,您的声音或将成为 openLooKeng 引擎性能提升的关键。
-
-openLooKeng 开源社区官方网站:
-
-
-
-
-## Task Recovery
-
-
-
-
-
- 增加通过用户自定义 worker 节点物理执行计划的生成,用户可以实现自己的算子 pipeline 代替原生实现,加速算子处理。支持实现算子 Native 化计算。
-
-
-
-
-## openLooKeng WEB UI 增强
-
-
-
- 1
- 支持不开启快照特性时,实现失败任务重试
-
-
- 2
- 支持在 CLI 端调试模式下显示快照相关统计信息:包括正在捕获的快照数量、已经捕获的快照数量、恢复次数(包括查询重启)、捕获的所有快照大小和捕获快照所需时间、用于恢复的快照大小和恢复时间
-
-
- 3
- 资源管理:在任务消耗大量内存导致内存不足时自动暂停/恢复任务,用户也可以选择主动暂停和恢复任务(限于 CTAS 和 INSERT INTO SELECT 场景)
-
-
- 4
- 增加可配置的失败重试策略:① 超时重试策略 ② 最大次数重试策略
-
-
-
-5
- 新增 Gossip 失败检测机制:控制节点通过定期接收心跳来监控工作节点是否存活,Gossip 机制使得所有工作节点能够相互监测,确保所有节点所知的信息都是一致的,可以防止在集群中出现短暂网络故障时导致过早的查询失败
-
-
-
-
-## openLooKeng 集群管理平台
-
-
-
- 1
- openLooKeng 审计日志功能增强
-
-
- 2
- openLooKeng SQL 历史记录持久化
-
-
- 3
- SQL Editor 支持自动联想
-
-
- 4
- 支持常用 SQL 语句收藏
-
-
- 5
- 数据源加载异常时,前端可通过异常标记显示
-
-
-
-6
- UI 支持显示数据源信息
-
-
-
-
-openLooKeng 已开源两年,目前 openLooKeng 已有社区用户数达到 120K + ,源源不断的用户贡献让 openLooKeng 保持青春与活力,这份耕耘属于社区中每一位开发者,熠熠生辉的瞬间,是每一位开发者点亮的星火。
-
-从初识 openLooKeng 开始,到你在社区写下的第一行代码,再到你提出的每个 PR,都是独属于你与 openLooKeng 的独家记忆,在 openLooKeng 开源两周年的日子,“对于大数据虚拟化引擎 openLooKeng,你有哪些寄语送给它?”(送出生日祝福,请移步 issue 评论区:https://gitee.com/openlookeng-competition/events-issue/issues/I5EKF3?from=project-issue)
-
-openLooKeng 新版本 v1.7.0 下载地址:https://openlookeng.io/zh/download/
-
-openLooKeng 代码仓地址: https://gitee.com/openlookeng
-
-openLooKeng 社区官网: https://openlookeng.io/zh/
-
-openLooKeng 小助手微信号:openLooKengoss
-
-openLooKeng 两周年有奖征集祝福:https://gitee.com/openlookeng-competition/events-issue/issues/I5EKF3?from=project-issue
diff --git a/app/zh/information/news/2022-08-04/index.md b/app/zh/information/news/2022-08-04/index.md
deleted file mode 100644
index 1675179c845a6a11d4fe467595cabdd80862de4b..0000000000000000000000000000000000000000
--- a/app/zh/information/news/2022-08-04/index.md
+++ /dev/null
@@ -1,57 +0,0 @@
----
-title: '数据的价值仅仅是待挖掘的资产吗?'
-date: '2022-08-04'
-module: 'news'
-customLayout: 'true'
-categories: ['']
-author: 'Jenny Darmody'
-description: '公司收集的数据如果想最大化发挥价值,首先需要制定一个合适的数据管理策略。——Benoit Hudzia,华为'
----
-
-### 公司收集的数据如果想最大化发挥价值,首先需要制定一个合适的数据管理策略。——Benoit Hudzia,华为
-
-2006 年,英国数学家 Clive Humby 首次提出“数据是新石油”的说法。将近 20 年后,数据量指数级增长,如何利用这些数据本身就成了一门艺术。
-
-华为爱尔兰研究中心首席人工智能运营专家 Benoit Hudzia 认为,我们正逐步迈入“数据成为工业生态系统新沃土”的时代。
-
-“如今,互联网和智能手机丰富了数据类型和载体,也让数据变得更有价值。但仅仅将数据当做一种资产,简单挖掘而后弃之不用是非常不明智的。”Benoit 在接受 Silicon Republic 网站采访时这样表示。
-
-“数据应当被作为原材料,为客户提供价值并建立更好的关系。就像石油转化为能源需要发动机一样,数据也需要经过分析,获得知识、挖掘价值并创造商业利益。”
-
-“下一个阶段,我们要做的不仅是简单的提取和处理数据,还需要持续开发和增强数据价值,赋能长期的商业成功。”
-
-### “由此带来的快速增长的数据分析量需要更加有效的治理和管控。” ——Benoit Hudzia
-
-2008 年,Hudzia 在 SAP 的研发部门工作时,首次接触到大数据分析。
-
-“那是一段令人兴奋的日子,因为 SAP 刚刚发布了新的数据库平台 SAP HANA,能够帮我们即时了解业务运营,分析可用数据,并根据快速变化的业务条件做出快速反应。”
-
-在 2020 年入职华为之前,Hudzia 曾加入以色列一家初创企业从事私有云解决方案构建与运行,之后又在世界上最大的托管银行之一从事公有云运营工作。
-
-目前,Hudzia 在华为从事大数据解决方案新技术开发,主要负责开源大数据平台 openLooKeng。
-
-他解释说:“openLooKeng 是一个开放的引擎,可以对任何地方的数据进行现场分析,包括地理上的远程数据源。它通过 SQL 2003 接口提供数据全局视图,具有高可用性、弹性伸缩、内置缓存和索引支持。”
-
-Hudzia 拥有多年的数据分析经验,他认为数据管理正逐步成为公司数据战略的核心,简单的数据收集不足以彻底释放数据价值。
-
-他说:“为了利用数据来提供数据分析解决方案的最佳实践,组织必须建立一套连贯的管理策略和运营方案,来收集、组织、保护和存储数据,以便基于数据分析进行商业决策。”
-
-“数据在当今的金融、运输和电信系统中发挥着关键作用。如果这些系统要保持稳定和可信、公平和有效,那么快速增长的数据分析量就需要更加有效的治理和管控。”
-
-“通过在现实和虚拟世界中有效的数据管理,组织能够游刃有余地应对数据处理过程中的挑战,更有效地使用资源,节省时间和金钱,并最终取得商业成功。”
-
-### 纷至沓来的数据挑战
-
-Hudzia 说,要想正确发挥数据的作用,就必须把数据视为企业资产,基于中心和自助数据基础设施,让不同的业务域能自主运营自己的数据
-
-“然而,这种资产下放带来了一些挑战。管理多个数据产品及对应元数据比较困难,很可能会导致多个分散的数据管道,造成混乱。”他说。
-
-此外,法规在数据分析和管理中不可或缺,公司必须确保其管理数据的方式符合相关法规,如欧盟的《通用数据保护条例(GDPR)》。
-
-Hudzia 说:“在这方面,欧盟一直是先行者,颁布了诸多法规,如隐私盾、Schrems II、GDPR、人工智能法律框架等。”
-
-“此外,欧盟也在不断提醒我们,仅通过监管来确保数据主权是有风险的。监管之外,我们还应具备在本地部署基础设施和服务的能力。这就是为什么欧盟正在推动 Gaia-X 等大型项目,以确保建立一个合规的欧洲数据服务生态系统。”
-
-他补充说,其他国家也在研究数据法规,比如美国最近公开发布了《美国数据隐私和保护法》草案。
-
-“这一连串的政策和监管迫使大型企业重新考虑他们的数据和云战略。这些企业将需要提供超本地化服务的解决方案,同时满足监管规则和客户数据生态系统不断增长的需求。”
diff --git "a/app/zh/information/news/2022-09-22/img/\345\233\276\347\211\2072.png" "b/app/zh/information/news/2022-09-22/img/\345\233\276\347\211\2072.png"
deleted file mode 100644
index cab125e264bf0c3ca4bceb9f2a3f028b7d2d12ec..0000000000000000000000000000000000000000
Binary files "a/app/zh/information/news/2022-09-22/img/\345\233\276\347\211\2072.png" and /dev/null differ
diff --git "a/app/zh/information/news/2022-09-22/img/\345\233\276\347\211\2073.png" "b/app/zh/information/news/2022-09-22/img/\345\233\276\347\211\2073.png"
deleted file mode 100644
index 71efddaac5b8c7acd4b9a0d1f0e6759804ccb448..0000000000000000000000000000000000000000
Binary files "a/app/zh/information/news/2022-09-22/img/\345\233\276\347\211\2073.png" and /dev/null differ
diff --git "a/app/zh/information/news/2022-09-22/img/\345\273\226\347\231\273\345\256\217\345\256\230\347\275\221\347\205\247\347\211\207.jpg" "b/app/zh/information/news/2022-09-22/img/\345\273\226\347\231\273\345\256\217\345\256\230\347\275\221\347\205\247\347\211\207.jpg"
deleted file mode 100644
index c6b40d3047109f2834754db21fa44d64cda7b391..0000000000000000000000000000000000000000
Binary files "a/app/zh/information/news/2022-09-22/img/\345\273\226\347\231\273\345\256\217\345\256\230\347\275\221\347\205\247\347\211\207.jpg" and /dev/null differ
diff --git "a/app/zh/information/news/2022-09-22/img/\346\265\267\346\212\2451.png" "b/app/zh/information/news/2022-09-22/img/\346\265\267\346\212\2451.png"
deleted file mode 100644
index e98828f703158382dcd96cbffd0310a630c42fa1..0000000000000000000000000000000000000000
Binary files "a/app/zh/information/news/2022-09-22/img/\346\265\267\346\212\2451.png" and /dev/null differ
diff --git "a/app/zh/information/news/2022-09-22/img/\346\265\267\346\212\2452.png" "b/app/zh/information/news/2022-09-22/img/\346\265\267\346\212\2452.png"
deleted file mode 100644
index 906376c03e5cc6fd0aca13692590684867b35632..0000000000000000000000000000000000000000
Binary files "a/app/zh/information/news/2022-09-22/img/\346\265\267\346\212\2452.png" and /dev/null differ
diff --git "a/app/zh/information/news/2022-09-22/img/\347\250\213\344\270\200\350\210\260.jpg" "b/app/zh/information/news/2022-09-22/img/\347\250\213\344\270\200\350\210\260.jpg"
deleted file mode 100644
index cd887c7dd1eedc3fbb0f42549f7e37c60549f3b3..0000000000000000000000000000000000000000
Binary files "a/app/zh/information/news/2022-09-22/img/\347\250\213\344\270\200\350\210\260.jpg" and /dev/null differ
diff --git a/app/zh/information/news/2022-09-22/index.md b/app/zh/information/news/2022-09-22/index.md
deleted file mode 100644
index 2295b185ef123b1632312a97bd26b3b6afec9d22..0000000000000000000000000000000000000000
--- a/app/zh/information/news/2022-09-22/index.md
+++ /dev/null
@@ -1,63 +0,0 @@
----
-title: 'Apache Hadoop Meetup 2022 上海站'
-date: '2022-09-24'
-module: 'news'
-customLayout: 'true'
-categories: ['openLooKeng']
-author: 'openLooKeng'
-description: '公司收集的数据如果想最大化发挥价值,首先需要制定一个合适的数据管理策略。——Benoit Hudzia,华为'
----
-
-9 月 24 日@上海,Apache Hadoop Meetup 2022 将与大家见面。作为战略合作伙伴的 openLooKeng,此次将继续在大数据圈发力,推动“第四届中国 Apache Hadoop Meetup“成功举办!
-
-此次大会,旨在共同探讨大数据开源技术前沿动态的思考与经验,旨在促进工程师同行相互交流、学习、社交....
-
-## 一、openLooKeng 精彩议题预告
-
-**讲师:廖登宏 华为研发高级工程师**
-
-
-
- 1
- 提供图形化的指标监控及定制,及时获取系统的关键信息
-
-
- 2
- 提供服务属性的配置管理,满足实际业务的性能需求
-
-
- 3
- 提供集群、服务、实例的操作功能,满足一键启停等操作需求
-
-
- 4
- 提供 openLooKeng 在线部署功能,支持在线升级
-
-
- 5
- 提供节点伸缩服务
-
-
- 6
- 集成 openLooKeng 现有的 Web UI
-
-
-
-7
- openLooKeng Manager 工具支持用户登录/退出功能
-
-
-1.主论坛议题:
-
-**【从 openLooKeng 看大数据未来的趋势和发展】**
-
-简介:随着大数据技术的发展,数据虚拟化在满足多样性数据联合分析的需求的同时也面临诸多挑战,本议题带来在大数据上我们如何突破 CPU、IO 与 Network 等多种瓶颈的一些思考
-
-2.企业应用分论坛议题:
-
-**【openLooKeng 的批处理特性探索】**
-
-简介:面对大数据分析场景的日益复杂,如何通过一套架构一个引擎满足客户需求,从而简化客户的运营管理负担,是一个重要的优化方向,本议题将带来我们在 openLooKeng 上的一些交互式,批处理场景融合解决方案的思考。
-
-**讲师:程一舰 中国光大银行多源查询平台负责人**
-
-
-
-3.线上直播议题:**【openLooKeng 在光大银行多源查询场景的探索和实践】**
-
-简介:介绍银行在跨源查询场景的需求,openLooKeng 在跨源场景的应用以及在各数据源查询中的探索与优化。
-
-## 二、参与社区活动,领取心动伴手礼(先到先得!)
-
-
-
-
-
-## 三、活动报名入口
-
-
-
-## 四、社区活动兑奖指南
-
-**【怎么兑奖】**
-
-添加 openLooKeng 小助手
-
-发暗号“**大数据**”
-
-提前进入 openLooKeng 交流群
-
-惊喜奖品的等你拿!
-
-
diff --git a/app/zh/information/news/2022-09-30/index.md b/app/zh/information/news/2022-09-30/index.md
deleted file mode 100644
index 6d16b40658843a4af6b0f9521238dc79af79eddd..0000000000000000000000000000000000000000
--- a/app/zh/information/news/2022-09-30/index.md
+++ /dev/null
@@ -1,43 +0,0 @@
----
-title: 'openLooKeng新版本1.8.0正式发布'
-date: '2022-09-30'
-module: 'news'
-customLayout: 'true'
-categories: ['Announcement']
-author: 'openLooKeng'
-description: '神州奋起,国家繁荣,山河壮丽,岁月峥嵘,街上四处悬挂的红旗,高扬的国歌,无不在为这个欢喜的日子做铺垫,赶在这个举杯同庆之际,大数据虚拟化引擎openLooKeng也迎来了全新版本的升级,在社区开发者们三个月以来不断努力下,openLooKeng1.8.0版本正式发布。'
----
-
-神州奋起,国家繁荣,山河壮丽,岁月峥嵘,街上四处悬挂的红旗,高扬的国歌,无不在为这个欢喜的日子做铺垫,赶在这个举杯同庆之际,大数据虚拟化引擎 openLooKeng 也迎来了全新版本的升级,在社区开发者们三个月以来不断努力下,openLooKeng1.8.0 版本正式发布。
-
-下面就带大家快速浏览 openLooKeng1.8.0 版本的关键特性。
-
-
-### 任务级恢复
-
- 1.通过使用物化的exchange信息仅重试失败的任务来实现查询恢复,支持Select场景的任务重试和恢复
-
- 2.用于任务恢复的快照文件支持存放在HDFS上
-
- 3.在实现物化exchange数据时,支持直接序列化和压缩。
-### 查询资源管理
-
- 实现查询级别的资源监控,当任务超出资源组配置时,可支持任务的自动暂停/恢复、溢出可撤销内存、限制调度和终止查询
-### 连接器扩展
-
-singleData连接器为openGauss补充OLAP能力,增强数据分析竞争力。
-支持对接ShardingSphere和tidRange两种模式。
-
-请参考: https://gitee.com/openlookeng/hetu-core/tree/1.8.0/hetu-docs/zh
-
-
-以上则是openLooKeng新版本1.8.0的关键特性,欢迎小伙伴们下载体验~~
-
-openLooKeng 新版本 v1.8.0 下载地址:
-活动背景:“如何在开源社区崭露头角?”
-
-openLooKeng 是一款大数据虚拟化引擎,致力于让大数据更简单。为了鼓励更多的开发者参与到 openLooKeng 开源项目的共建中来,我们准备发起一个长期有效的激励活动!
-
-如果您是一个初学者,并期望这个项目为您打开开源世界的大门,那不放参与 openLooKeng 贡献,不仅可以积累亮眼的项目经历,而且还能领取很多社区心动奖品,贡献越多奖励越大,同时,贡献排名前十讲获得“openLooKeng 社区优秀开发者”荣誉证书,并公示到 openLooKeng 官网和各种媒体平台,openLooKeng 大数据开源社区期待你的加入!
-
-举办方:openLooKeng 社区
-
-## 一、参与开源贡献,你将获得什么?
-
-### 1. 开源社区贡献经历
-
-有趣的开源项目贡献之旅,积累亮眼的项目奖励,让你的工作履历增添一抹闪光点!
-
-### 2. openLooKeng 社区精美奖品
-
-
-
-
-
-
-
-## 三、openLooKeng 任务选择
-
-### 项目 1
-
-1. 项目标题:**Iceberg Connector 开发**
-2. 项目描述:`AApache Iceberg` 是一种开放的表格格式,专为巨大的 PB 级表格而设计,是目前市面上流行的三大开源数据湖方案之一。当前,openLooKeng 还没有支持 Iceberg,因此需要开发 `Iceberg Connector`。
-3. 项目难度:基础
-4. 社区导师:周思佩
-
-5. 项目产出要求:
- - a) Iceberg Connector 设计文档
- - b) Iceberg Connector 代码与相关测试用例
- - c) 性能测试结果
-6. 项目技术要求
- - a) SQL、Java 编程;
- - b) openLooKeng Connector 原理与设计
-7. 相关的开源软件仓库列表
- - a)