Merge pull request #296 from WeBankFinTech/dev-0.9.1

Dev 0.9.1

.gitignore

@@ -1,6 +1,7 @@
 *.iml
 .idea
 .DS_Store
+*.class
 
 assembly/target
 
@@ -20,4 +21,14 @@ plugins/azkaban/linkis-jobtype/target/
 plugins/linkis/linkis-appjoint-entrance/target/
 
 sendemail-appjoint/sendemail-core/target/
-visualis-appjoint/appjoint/target/
+visualis-appjoint/appjoint/target/
+
+dss-user-manager/target/
+logs
+### Example user template template
+### Example user template
+
+# IntelliJ project files
+
+out
+gen

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+  "editor.formatOnPaste": true,
+  "editor.formatOnType": true,
+  "editor.formatOnSave": true
+}

assembly/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <artifactId>dss</artifactId>
         <groupId>com.webank.wedatasphere.dss</groupId>
-        <version>0.9.0</version>
+        <version>0.9.1</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

conf/config.sh

@@ -77,4 +77,4 @@ AZKABAN_ADRESS_PORT=8081
 QUALITIS_ADRESS_IP=127.0.0.1
 QUALITIS_ADRESS_PORT=8090
 
-DSS_VERSION=0.9.0
+DSS_VERSION=0.9.1

datachecker-appjoint/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <artifactId>dss</artifactId>
         <groupId>com.webank.wedatasphere.dss</groupId>
-        <version>0.9.0</version>
+        <version>0.9.1</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

docs/en_US/ch1/DataSphereStudio_Compile_Manual.md

@@ -6,7 +6,7 @@
 
 ```xml
   <properties>
-          <dss.version>0.9.0</dss.version>
+          <dss.version>0.9.1</dss.version>
           <linkis.version>0.9.4</linkis.version>
           <scala.version>2.11.8</scala.version>
           <jdk.compile.version>1.8</jdk.compile.version>

docs/en_US/ch2/DSS_0.9.1_upgrade_notes.md

@@ -0,0 +1,85 @@
+# DSS 0.9.1 upgrade notes
+
+## Environmental description
+
+------
+
+1. The user who installs the node machine deployment of DSS must have the permission to create the directory in hdfs
+   1)If the hadoop cluster adopts the kerberos authentication mechanism, in order to prevent the ticket from expiring, you need to execute the knit -kt command on the client, for example:kinit -kt /etc/security/keytabs/hdfs.keytab  yarn/xxxxxxxx
+   2)If the hadoop cluster uses the simple authentication mechanism, use hdfs dfs chmod authorization, such as: hdfs dfs chmod 775 /user/hive/xx
+2. The user who deploys DSS on the DSS node has the permission to create hive database:
+   1)If hadoop cluster using simple authentication mechanism, you can try the following manner authorized:
+   hive>set system:user:name=dss;
+   hive> grant all to user dss;
+
+Currently, there is no automatic authorization in the script, and the user needs to execute the command manually.
+2)If the Hadoop cluster adopts kerberos authentication, the kinit command is automatically executed in our script to obtain the ticket, and there is no need to execute the command manually. The user only needs to configure the kerberos related parameters. For the specific configuration, see the kerberos configuration chapter.
+
+The newly created user should be configured in hive.users.in.admin.role in hive-site.xml.
+
+1. LDAP must be installed (user authentication only supports LDAP), and there must be ou=user and ou=group entries in ldap, such as:  ou=user,dc=baidu,dc=com和ou=group,dc=baidu,dc=com.
+   The ldap version supports 2.4.x, and the support of other versions is to be verified
+2. Install sshpass service, yum -y install sshpass
+
+## Upgrade installation instructions
+
+------
+
+The jar package involved in this change: Under the dss-server/lib directory:dss-application-0.9.1.jar,dss-server-0.9.1.jar,dss-user-manager-0.9.1.jar
+Front-end static files: web
+After replacing the above file with the latest one, then modify the following configuration file
+
+### Installation and configuration file instructions
+
+1. kerberos related
+
+Function description: If the Hadoop cluster uses the kerberos authentication mechanism, the newly created user will be granted kerberos permissions
+Configuration file path: dss-server/conf/linkis.properties
+
+```
+  	Parameters:
+         wds.linkis.kerberos.enable.switch --Whether the cluster adopts the kerberos authentication mechanism, 0-do not use kerberos 1-use kerberos. If the Hadoop cluster does not use the kerberos authentication mechanism, none of the following parameters need to be configured.
+         wds.linkis.kerberos.keytab.path  --The storage location of keytab on the DSS installation node can be arbitrarily specified, such as /etc/security/keytabs
+         wds.linkis.kerberos.kdc.node  --Deploy the KDC service node IP, such as 192.168.1.1
+         wds.linkis.kerberos.ssh.port  --Deploy the KDC service node SSH port number, generally 22
+         wds.linkis.kerberos.kdc.user.name --Deploy a linux user name on the KDC node, the user must have sudo permission (very important!!!) for ssh operation
+         wds.linkis.kerberos.kdc.user.password  --The login password of the kdc node user mentioned above, used for ssh operation
+         wds.linkis.kerberos.realm --Kerberos manages the domain name of the hadoop cluster, please consult the cluster operation and maintenance personnel
+         wds.linkis.kerberos.admin--A user granted the admin role on kerberos (such as hdfs, very important!!!, otherwise the authorization cannot be completed)
+```
+
+1. metastore related
+
+Function description:  create  hive databases for newly created user and grant the newly created user permission
+Parameter configuration file: dss-server/conf/linkis.properties
+
+```
+Parameters:
+      wds.linkis.metastore.hive.hdfs.base.path  --The path where hive warehouse data is stored on hdfs, such as /user/hive/warehouse
+      wds.dss.deploy.path --dss_linkis installation package path, such as /usr/local/dss_linkis
+```
+
+1. ldap related
+   Function description: Create a new Entry under ou=user and ou=group of ldap for user login authentication
+   Parameter configuration file path: tools/bin/ldap_user.py
+
+```
+LDAP_HOST -- Install the ldap service IP, such as 192.168.11.11
+LDAP_BASEDN --The upper dn of ou=user or ou=group, such as dc=example,dc=com
+LDAP_ADMIN -- The dn of the user logging in to the ldap service, such as cn=admin,dc=example,dc=cn
+LDAP_PASS --Password for logging in to the ldap service
+The first line in the ldap_user.py file #!/usr/local/tools/venv/bin/python, replace /user/local with the installation path of dss_linkis
+```
+
+## User manual
+
+------
+
+1. Access address [http://url](http://url/):port/#/userManger, you need to login as a super user (installation user)
+2. Server configuration:
+   If the hadoop cluster uses the simple authentication mechanism, the user needs to add the ip, login user name (with sudo permission), and password of each server in the yarn cluster.The underlying principle is that the server where dss is installed will ssh to each server in the yarn cluster, and then create a linux user.
+
+If the kerberos authentication mechanism adopted by the hadoop cluster, just add an ip (such as 127.0.0.1), username, and password. If not added, the interface will report an exception, and subsequent versions will fix this bug.
+
+1. WorkspaceRootPath, hdfsRootPath, resultRootPath, schedulerPath, DSS installation directory, Azkaban installation directory.The default value is consistent with the configuration in the installation configuration file config.sh. The directory can be either the hdfs directory, starting with hdfs:///, or the linux directory, starting with file:///.
+2. The bottom layer will create a hive database for the user, the database name: xx_default, and give the permission to add, delete, modify, and select.

docs/zh_CN/ch1/DSS编译文档.md

@@ -6,7 +6,7 @@
 
 ```xml
   <properties>
-          <dss.version>0.9.0</dss.version>
+          <dss.version>0.9.1</dss.version>
           <linkis.version>0.9.4</linkis.version>
           <scala.version>2.11.8</scala.version>
           <jdk.compile.version>1.8</jdk.compile.version>

docs/zh_CN/ch2/DSS_0.9.1_升级说明.md

@@ -0,0 +1,72 @@
+# DSS-0.9.1升级说明
+
+## 环境说明
+
+1. 安装DSS节点上部署DSS用户必须有hdfs创建目录的权限
+   1)如果hadoop集群采用kerberos认证机制，为防止票据过期，则需要在客户端执行 knit -kt命令,比如：kinit -kt /etc/security/keytabs/hdfs.keytab  yarn/xxxxxxxx
+   2)如果hadoop集群采用simple认证机制，则使用hdfs dfs chmod 授权，比如:hdfs dfs chmod 775 /user/hive/xx
+2. 安装DSS节点上部署DSS的用户具有创建hive database权限问题:
+   1)如果hadoop集群采用simple认证机制，可以尝试如下方式授权：
+   hive>set system:user:name=dss;
+   hive> grant all to user dss
+   目前并未在脚本中自动授权，需要用户手动执行命令。
+   2)如果hadoop集群采用kerberos认证，在我们的脚本中自动执行了kinit命令以获取票据，不需要手工执行命令，用户只需要要配置kerberos相关的参数，具体配置见kerberos配置章节。
+   新建的用户要在 hive-site.xml 中hive.users.in.admin.role配置。
+3. 必须安装有LDAP（用户认证只支持LDAP），ldap中必须有ou=user和ou=group条目，比如：ou=user,dc=baidu,dc=com和ou=group,dc=baidu,dc=com。ldap版本支持2.4.x,其他版本的支持情况待验证
+4. 安装sshpass服务，yum -y install sshpass
+
+## 版本升级安装说明
+
+本次改动涉及的的jar包:dss-server/lib目录下: dss-application-0.9.1.jar,dss-server-0.9.1.jar,dss-user-manager-0.9.1.jar
+前端静态文件：web
+将以上文件替换成最新的后，然后修改下面的配置文件
+
+### 安装及配置文件说明
+
+1. kerberos相关
+   功能说明：如果hadoop集群采用kerberos认证机制，则会给新建的用户授予kerberos权限
+   配置文件路径：dss-server/conf/linkis.properties
+
+```
+  	参数：
+         wds.linkis.kerberos.enable.switch --集群是否采用kerberos认证机制，0-不采用kerberos 1-采用kerberos。如果hadoop集群不采用kerberos认证机制，则下面的参数都不需要配置。
+         wds.linkis.kerberos.keytab.path  --keytab在DSS安装节点上的存放位置，可以任意指定，比如 /etc/security/keytabs  
+         wds.linkis.kerberos.kdc.node  --部署KDC服务节点IP,比如192.168.1.1
+         wds.linkis.kerberos.ssh.port  --部署KDC服务节点SSH端口号，一般都是22
+         wds.linkis.kerberos.kdc.user.name --部署KDC节点上的一个linux用户名,该用用户必须有sudo权限(重要，重要！！！)，用于ssh操作
+         wds.linkis.kerberos.kdc.user.password  --上面提到的kdc节点用户的登录密码，用于ssh操作
+         wds.linkis.kerberos.realm --kerberos管理hadoop集群的域名，请咨询集群运维人员。
+         wds.linkis.kerberos.admin--kerberos上的一个被授予admin角色的用户（如hdfs,非常重要！！！！，否则无法完成授权）
+```
+
+1. metastore相关
+   功能说明：给新建用户hive库，并授予新建用户权限
+   参数配置文件：dss-server/conf/linkis.properties
+
+```
+	参数：
+      wds.linkis.metastore.hive.hdfs.base.path  --hive仓库数据存储在在hdfs上的路径，比如 /user/hive/warehouse
+      wds.dss.deploy.path --dss_linkis安装包路径，比如 /usr/local/dss_linkis
+```
+
+3.ldap相关
+功能说明：在ldap的ou=user和ou=group下新建一个Entry，用于用户登录验证
+参数配置文件路径：安装包下的tools/bin/ldap_user.py
+
+```
+LDAP_HOST -- 安装ldap服务IP，比如192.168.11.11
+LDAP_BASEDN --ou=user或ou=group的上层dn，比如 dc=example,dc=com
+LDAP_ADMIN -- 登录ldap服务的用户dn 比如 'cn=admin,dc=example,dc=cn'
+LDAP_PASS --登录ldap服务的密码
+ldap_user.py文件中第一行 #!/usr/local/tools/venv/bin/python,将/user/local换成dss_linkis的安装路径
+```
+
+## 使用说明
+
+1. 访问地址 http://url:port/#/userManger,需要用超级用户(安装用户)登录
+2. 服务器配置：
+   如果hadoop集群采用的是simple认证机制，则用户需要添加yarn集群各服务器的ip、登录用户名（具有sudo权限）、密码。其底层的原理是安装dss的服务器会ssh到yarn集群的各服务器,然后创建linux用户。
+   如果hadoop集群采用的kerberos认证机制，则随便添加一个ip（比如127.0.0.1）,用户名,密码。如果不添加接口会报异常，后续版本会修复此bug
+3. workspaceRootPath,hdfsRootPath,resultRootPath,schedulerPath,DSS安装目录，Azkaban安装目录。
+   其默认值和安装配置文件config.sh里配置的保持一致。其目录既可以是hdfs目录,以hdfs:///开头，也可以是linux目录，以file:///开头
+4. 底层会给用户创建hive库,库名：xx_default,并赋予增删改查权限

dss-appjoint-auth/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <artifactId>dss</artifactId>
         <groupId>com.webank.wedatasphere.dss</groupId>
-        <version>0.9.0</version>
+        <version>0.9.1</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

dss-appjoint-core/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <artifactId>dss</artifactId>
         <groupId>com.webank.wedatasphere.dss</groupId>
-        <version>0.9.0</version>
+        <version>0.9.1</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

dss-appjoint-loader/pom.xml

@@ -22,12 +22,12 @@
     <parent>
         <artifactId>dss</artifactId>
         <groupId>com.webank.wedatasphere.dss</groupId>
-        <version>0.9.0</version>
+        <version>0.9.1</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>dss-appjoint-loader</artifactId>
-    <version>0.9.0</version>
+    <version>0.9.1</version>
 
     <dependencies>
         <dependency>

dss-application/pom.xml

@@ -23,7 +23,7 @@
     <parent>
         <artifactId>dss</artifactId>
         <groupId>com.webank.wedatasphere.dss</groupId>
-        <version>0.9.0</version>
+        <version>0.9.1</version>
     </parent>
     <artifactId>dss-application</artifactId>
 
@@ -47,6 +47,12 @@
             <artifactId>dss-appjoint-loader</artifactId>
             <version>${dss.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.htrace</groupId>
+            <artifactId>htrace-core</artifactId>
+            <version>3.2.0-incubating</version>
+            <scope>compile</scope>
+        </dependency>
 
     </dependencies>
 

dss-application/src/main/java/com/webank/wedatasphere/dss/application/conf/ApplicationConf.java

@@ -26,4 +26,27 @@
 public class ApplicationConf {
 
     public static final CommonVars<String> FAQ = CommonVars.apply("wds.linkis.application.dws.params","");
+
+    public static final String SUPER_USER_NAME = CommonVars.apply("wds.linkis.super.user.name","").getValue();
+    public static final String WORKSPACE_USER_ROOT_PATH = CommonVars.apply("wds.linkis.workspace.user.root.path","").getValue();
+    public static final String HDFS_USER_ROOT_PATH = CommonVars.apply("wds.linkis.hdfs.user.root.path","").getValue();
+    public static final String RESULT_SET_ROOT_PATH = CommonVars.apply("wds.linkis.result.set.root.path","").getValue();
+    public static final String WDS_SCHEDULER_PATH = CommonVars.apply("wds.linkis.scheduler.path","").getValue();
+    public static final String WDS_USER_PATH = CommonVars.apply("wds.linkis.user.path","hdfs:///user").getValue();
+    public static final String DSS_INSTALL_DIR = CommonVars.apply("wds.linkis.dss.install.dir","").getValue();
+    public static final String AZKABAN_INSTALL_DIR = CommonVars.apply("wds.linkis.azkaban.install.dir","").getValue();
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 }

dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/DSSUser.java

@@ -25,6 +25,14 @@ public class DSSUser {
     private String username;
     private String name;
     private Boolean isFirstLogin;
+    private boolean isSuperUser = false;
+
+    public boolean getIsSuperUser() {
+        return isSuperUser;
+    }
+    public void setIsSuperUser(boolean isSuperUser) {
+        this.isSuperUser = isSuperUser;
+    }
 
     public Long getId() {
         return id;

dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/WorkSpacePath.java

@@ -0,0 +1,59 @@
+package com.webank.wedatasphere.dss.application.entity;
+
+public class WorkSpacePath {
+    private String workspaceRootPath;
+    private String hdfsRootPath;
+    private String resultRootPath;
+    private String schedulerPath;
+    private String userPath;
+
+    public String getUserPath() {
+        return userPath;
+    }
+
+    public void setUserPath(String userPath) {
+        this.userPath = userPath;
+    }
+
+
+
+    public String getWorkspaceRootPath() {
+        return workspaceRootPath;
+    }
+
+    public void setWorkspaceRootPath(String workspaceRootPath) {
+        this.workspaceRootPath = workspaceRootPath;
+    }
+
+    public String getHdfsRootPath() {
+        return hdfsRootPath;
+    }
+
+    public void setHdfsRootPath(String hdfsRootPath) {
+        this.hdfsRootPath = hdfsRootPath;
+    }
+
+    public String getResultRootPath() {
+        return resultRootPath;
+    }
+
+    public void setResultRootPath(String resultRootPath) {
+        this.resultRootPath = resultRootPath;
+    }
+
+    public String getSchedulerPath() {
+        return schedulerPath;
+    }
+
+    public void setSchedulerPath(String schedulerPath) {
+        this.schedulerPath = schedulerPath;
+    }
+
+
+
+
+
+
+
+
+}