Skip to content

Commit 7123bcc

Browse files
wushengyeyouyawushengyeyouya
authored
Merge pull request #25 from WeDataSphere/dev-0.2.0-jobstate-interface
[Feature] JobState component in launcher module
2 parents 9f600c7 + 60306df commit 7123bcc

83 files changed

Lines changed: 1146 additions & 192 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎assembly/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
<parent>
2121
<artifactId>streamis</artifactId>
2222
<groupId>com.webank.wedatasphere.streamis</groupId>
23-
<version>0.1.0</version>
23+
<version>0.2.0</version>
2424
</parent>
2525
<modelVersion>4.0.0</modelVersion>
2626

‎db/streamis_ddl.sql‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ SET FOREIGN_KEY_CHECKS = 0;
88
DROP TABLE IF EXISTS `linkis_stream_configuration_config_key`;
99
CREATE TABLE `linkis_stream_configuration_config_key` (
1010
`id` bigint(20) NOT NULL,
11-
`key` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
11+
`key` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
1212
`name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
1313
`description` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
1414
`default_value` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
@@ -31,13 +31,13 @@ CREATE TABLE `linkis_stream_configuration_config_key` (
3131
-- ----------------------------
3232
DROP TABLE IF EXISTS `linkis_stream_configuration_config_value`;
3333
CREATE TABLE `linkis_stream_configuration_config_value` (
34-
`id` bigint(20) NOT NULL,
34+
`id` bigint(20) NOT NULL AUTO_INCREMENT,
3535
`configkey_id` bigint(20) NULL DEFAULT NULL,
36-
`config_value` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
36+
`config_value` varchar(500) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
3737
`type` int(10) NULL DEFAULT NULL,
3838
`job_id` bigint(20) NULL DEFAULT NULL,
39-
`job_name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
40-
`config_key` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
39+
`job_name` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
40+
`config_key` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
4141
PRIMARY KEY (`id`) USING BTREE,
4242
INDEX `key`(`config_key`) USING BTREE,
4343
INDEX `keyid`(`configkey_id`) USING BTREE
@@ -129,8 +129,8 @@ DROP TABLE IF EXISTS `linkis_stream_job`;
129129

130130
CREATE TABLE `linkis_stream_job` (
131131
`id` bigint(20) NOT NULL AUTO_INCREMENT,
132-
`project_name` varchar(50) DEFAULT NULL,
133-
`name` varchar(50) DEFAULT NULL,
132+
`project_name` varchar(100) DEFAULT NULL,
133+
`name` varchar(200) DEFAULT NULL,
134134
`status` tinyint(1) DEFAULT '0' COMMENT '1:已完成 ，2:等待重启 ，3:告警 ，4:慢任务 ，5:运行中 ，6:失败任务',
135135
`create_by` varchar(50) DEFAULT NULL,
136136
`create_time` datetime DEFAULT NULL,
@@ -182,7 +182,7 @@ DROP TABLE IF EXISTS `linkis_stream_project`;
182182
CREATE TABLE `linkis_stream_project` (
183183
`id` bigint(20) NOT NULL AUTO_INCREMENT,
184184
`workspace_id` bigint(20) DEFAULT NULL,
185-
`name` varchar(50) DEFAULT NULL,
185+
`name` varchar(100) DEFAULT NULL,
186186
`create_by` varchar(50) DEFAULT NULL,
187187
PRIMARY KEY (`id`) USING BTREE
188188
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT COMMENT='项目表';
@@ -211,14 +211,14 @@ DROP TABLE IF EXISTS `linkis_stream_task`;
211211
CREATE TABLE `linkis_stream_task` (
212212
`id` int(11) NOT NULL AUTO_INCREMENT,
213213
`job_version_id` bigint(20) NOT NULL,
214-
`job_id` varchar(50) DEFAULT NULL,
214+
`job_id` varchar(200) DEFAULT NULL,
215215
`version` varchar(50) DEFAULT NULL,
216216
`status` int(3) DEFAULT NULL,
217217
`start_time` datetime DEFAULT NULL,
218218
`last_update_time` datetime DEFAULT NULL,
219219
`err_desc` varchar(10240) DEFAULT NULL,
220220
`submit_user` varchar(50) DEFAULT NULL,
221-
`linkis_job_id` varchar(50) DEFAULT NULL,
221+
`linkis_job_id` varchar(200) DEFAULT NULL,
222222
`linkis_job_info` mediumtext,
223223
PRIMARY KEY (`id`) USING BTREE
224224
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT COMMENT='任务表';
@@ -239,4 +239,15 @@ CREATE TABLE `linkis_stream_alert_record` (
239239
PRIMARY KEY (`id`)
240240
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
241241

242+
DROP TABLE IF EXISTS `linkis_stream_project_privilege`;
243+
244+
CREATE TABLE `linkis_stream_project_privilege` (
245+
`id` bigint(20) NOT NULL AUTO_INCREMENT,
246+
`project_id` bigint(20) NOT NULL,
247+
`user_name` varchar(100) NOT NULL,
248+
`privilege` tinyint(1) DEFAULT '0' NOT NULL COMMENT '1:发布权限 ，2:编辑权限 ，3:查看权限 ，4:所有权限 ，5:发布编辑权限 ，6:发布查看权限 ，7:编辑查看权限 ',
249+
PRIMARY KEY (`id`) USING BTREE
250+
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='项目权限表';
251+
252+
242253
SET FOREIGN_KEY_CHECKS = 1;

‎pom.xml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222
<groupId>com.webank.wedatasphere.streamis</groupId>
2323
<artifactId>streamis</artifactId>
24-
<version>0.1.0</version>
24+
<version>0.2.0</version>
2525
<packaging>pom</packaging>
2626

2727
<name>Streamis Project Parent POM</name>
@@ -41,11 +41,13 @@
4141
<module>streamis-server</module>
4242
<module>streamis-project</module>
4343
<module>assembly</module>
44+
<module>streamis-appconn</module>
4445
</modules>
4546

4647
<properties>
47-
<linkis.version>1.0.3</linkis.version>
48-
<streamis.version>0.1.0</streamis.version>
48+
<linkis.version>1.1.1</linkis.version>
49+
<dss.version>1.1.0</dss.version>
50+
<streamis.version>0.2.0</streamis.version>
4951
<scala.version>2.11.12</scala.version>
5052
<jdk.compile.version>1.8</jdk.compile.version>
5153
<maven.version>3.3.3</maven.version>
@@ -67,7 +69,7 @@
6769
<jersey-bean-validation.version>2.21</jersey-bean-validation.version>
6870
<aspectj.version>1.9.5</aspectj.version>
6971
<xstream.version>1.4.19</xstream.version>
70-
<jobmanager.version>0.1.0</jobmanager.version>
72+
<jobmanager.version>0.2.0</jobmanager.version>
7173
<mysql.connector.version>5.1.47</mysql.connector.version>
7274
</properties>
7375

‎streamis-appconn/pom.xml‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<parent>
6+
<artifactId>streamis</artifactId>
7+
<groupId>com.webank.wedatasphere.streamis</groupId>
8+
<version>0.2.0</version>
9+
</parent>
10+
<modelVersion>4.0.0</modelVersion>
11+
12+
<artifactId>streamis-appconn</artifactId>
13+
14+
<properties>
15+
<maven.compiler.source>8</maven.compiler.source>
16+
<maven.compiler.target>8</maven.compiler.target>
17+
</properties>
18+
19+
<dependencies>
20+
<dependency>
21+
<groupId>com.webank.wedatasphere.dss</groupId>
22+
<artifactId>dss-appconn-core</artifactId>
23+
<version>${dss.version}</version>
24+
<exclusions>
25+
<exclusion>
26+
<artifactId>linkis-common</artifactId>
27+
<groupId>org.apache.linkis</groupId>
28+
</exclusion>
29+
<exclusion>
30+
<artifactId>json4s-jackson_2.11</artifactId>
31+
<groupId>org.json4s</groupId>
32+
</exclusion>
33+
</exclusions>
34+
</dependency>
35+
<dependency>
36+
<groupId>org.apache.linkis</groupId>
37+
<artifactId>linkis-common</artifactId>
38+
</dependency>
39+
<dependency>
40+
<groupId>org.apache.linkis</groupId>
41+
<artifactId>linkis-module</artifactId>
42+
</dependency>
43+
</dependencies>
44+
<build>
45+
<plugins>
46+
<plugin>
47+
<groupId>org.apache.maven.plugins</groupId>
48+
<artifactId>maven-deploy-plugin</artifactId>
49+
</plugin>
50+
<plugin>
51+
<groupId>net.alchim31.maven</groupId>
52+
<artifactId>scala-maven-plugin</artifactId>
53+
</plugin>
54+
<plugin>
55+
<groupId>org.apache.maven.plugins</groupId>
56+
<artifactId>maven-jar-plugin</artifactId>
57+
</plugin>
58+
<plugin>
59+
<groupId>org.apache.maven.plugins</groupId>
60+
<artifactId>maven-assembly-plugin</artifactId>
61+
<version>2.3</version>
62+
<inherited>false</inherited>
63+
<executions>
64+
<execution>
65+
<id>make-assembly</id>
66+
<phase>package</phase>
67+
<goals>
68+
<goal>single</goal>
69+
</goals>
70+
<configuration>
71+
<descriptors>
72+
<descriptor>src/main/assembly/distribution.xml</descriptor>
73+
</descriptors>
74+
</configuration>
75+
</execution>
76+
</executions>
77+
<configuration>
78+
<skipAssembly>false</skipAssembly>
79+
<finalName>out</finalName>
80+
<appendAssemblyId>false</appendAssemblyId>
81+
<attach>false</attach>
82+
<descriptors>
83+
<descriptor>src/main/assembly/distribution.xml</descriptor>
84+
</descriptors>
85+
</configuration>
86+
</plugin>
87+
</plugins>
88+
<resources>
89+
<resource>
90+
<directory>src/main/java</directory>
91+
<includes>
92+
<include>**/*.xml</include>
93+
</includes>
94+
</resource>
95+
<resource>
96+
<directory>src/main/resources</directory>
97+
<excludes>
98+
<exclude>**/application.yml</exclude>
99+
<exclude>**/bootstrap.yml</exclude>
100+
<exclude>**/log4j2.xml</exclude>
101+
</excludes>
102+
</resource>
103+
</resources>
104+
</build>
105+
</project>

‎streamis-appconn/src/main/assembly/distribution.xml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
<!--
2+
~ /*
3+
~ * Copyright 2021 WeBank
4+
~ *
5+
~ * Licensed under the Apache License, Version 2.0 (the "License");
6+
~ * you may not use this file except in compliance with the License.
7+
~ * You may obtain a copy of the License at
8+
~ *
9+
~ * http://www.apache.org/licenses/LICENSE-2.0
10+
~ *
11+
~ * Unless required by applicable law or agreed to in writing, software
12+
~ * distributed under the License is distributed on an "AS IS" BASIS,
13+
~ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
~ * See the License for the specific language governing permissions and
15+
~ * limitations under the License.
16+
~ */
17+
-->
18+
<assembly
19+
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3"
20+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
21+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
22+
<!-- Unique key -->
23+
<id>dss-streamis-appconn</id>
24+
<formats>
25+
<format>dir</format>
26+
</formats>
27+
<includeBaseDirectory>true</includeBaseDirectory>
28+
<baseDirectory>streamisAppConn</baseDirectory>
29+
30+
<dependencySets>
31+
<dependencySet>
32+
<outputDirectory>lib</outputDirectory>
33+
<useProjectArtifact>true</useProjectArtifact>
34+
<useTransitiveDependencies>true</useTransitiveDependencies>
35+
<unpack>false</unpack>
36+
<useStrictFiltering>true</useStrictFiltering>
37+
<useTransitiveFiltering>true</useTransitiveFiltering>
38+
</dependencySet>
39+
</dependencySets>
40+
41+
<fileSets>
42+
<!-- appconn.properties -->
43+
<fileSet>
44+
<directory>${basedir}/src/main/resources</directory>
45+
<includes>
46+
<include>appconn.properties</include>
47+
</includes>
48+
<fileMode>0777</fileMode>
49+
<outputDirectory>conf</outputDirectory>
50+
<lineEnding>unix</lineEnding>
51+
</fileSet>
52+
<!-- init.sql -->
53+
<fileSet>
54+
<directory>${basedir}/src/main/resources</directory>
55+
<includes>
56+
<include>init.sql</include>
57+
</includes>
58+
<fileMode>0777</fileMode>
59+
<outputDirectory>db</outputDirectory>
60+
</fileSet>
61+
</fileSets>
62+
</assembly>

‎streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/dss/appconn/StreamisAppConn.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package com.webank.wedatasphere.streamis.dss.appconn;
2+
3+
import com.webank.wedatasphere.dss.appconn.core.ext.ThirdlyAppConn;
4+
import com.webank.wedatasphere.dss.appconn.core.impl.AbstractOnlySSOAppConn;
5+
import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard;
6+
import com.webank.wedatasphere.dss.standard.app.structure.StructureIntegrationStandard;
7+
import com.webank.wedatasphere.streamis.dss.appconn.structure.StreamisStructureIntegrationStandard;
8+
import org.apache.linkis.common.conf.CommonVars;
9+
10+
public class StreamisAppConn extends AbstractOnlySSOAppConn implements ThirdlyAppConn {
11+
12+
public static final String STREAMIS_APPCONN_NAME = CommonVars.apply("wds.dss.appconn.streamis.name", "Streamis").getValue();
13+
14+
private StreamisStructureIntegrationStandard structureIntegrationStandard;
15+
16+
@Override
17+
public DevelopmentIntegrationStandard getOrCreateDevelopmentStandard() {
18+
return null;
19+
}
20+
21+
@Override
22+
public StructureIntegrationStandard getOrCreateStructureStandard() {
23+
return structureIntegrationStandard;
24+
}
25+
26+
@Override
27+
protected void initialize() {
28+
structureIntegrationStandard = new StreamisStructureIntegrationStandard();
29+
}
30+
}

‎streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/dss/appconn/structure/StreamisStructureIntegrationStandard.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package com.webank.wedatasphere.streamis.dss.appconn.structure;
2+
3+
import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureIntegrationStandard;
4+
import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectService;
5+
import com.webank.wedatasphere.streamis.dss.appconn.structure.project.StreamisProjectService;
6+
7+
/**
8+
* Structure integration standard
9+
*/
10+
public class StreamisStructureIntegrationStandard extends AbstractStructureIntegrationStandard {
11+
12+
/**
13+
* Singleton project service
14+
* @return
15+
*/
16+
@Override
17+
protected ProjectService createProjectService() {
18+
return new StreamisProjectService();
19+
}
20+
}

‎streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/dss/appconn/structure/project/StreamisProjectCreationOperation.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package com.webank.wedatasphere.streamis.dss.appconn.structure.project;
2+
3+
import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils;
4+
import com.webank.wedatasphere.dss.standard.app.sso.origin.request.action.DSSPostAction;
5+
import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureOperation;
6+
import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectCreationOperation;
7+
import com.webank.wedatasphere.dss.standard.app.structure.project.ref.DSSProjectContentRequestRef;
8+
import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectResponseRef;
9+
import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef;
10+
import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException;
11+
import com.webank.wedatasphere.streamis.dss.appconn.StreamisAppConn;
12+
import com.webank.wedatasphere.streamis.dss.appconn.utils.StreamisCommonUtil;
13+
import org.apache.linkis.server.conf.ServerConfiguration;
14+
15+
import java.util.List;
16+
17+
public class StreamisProjectCreationOperation extends AbstractStructureOperation<DSSProjectContentRequestRef.DSSProjectContentRequestRefImpl, ProjectResponseRef>
18+
implements ProjectCreationOperation<DSSProjectContentRequestRef.DSSProjectContentRequestRefImpl> {
19+
20+
private final static String PROJECT_CREATE_URL = "/api/rest_j/" + ServerConfiguration.BDP_SERVER_VERSION() + "/streamis/createProject";
21+
22+
@Override
23+
protected String getAppConnName() {
24+
return StreamisAppConn.STREAMIS_APPCONN_NAME;
25+
}
26+
27+
@Override
28+
public ProjectResponseRef createProject(DSSProjectContentRequestRef.DSSProjectContentRequestRefImpl dssProjectContentRequestRef) throws ExternalOperationFailedException {
29+
String url = getBaseUrl() + PROJECT_CREATE_URL;
30+
DSSPostAction streamisPostAction = new DSSPostAction();
31+
streamisPostAction.setUser(dssProjectContentRequestRef.getDSSProject().getCreateBy());
32+
streamisPostAction.addRequestPayload("projectName",dssProjectContentRequestRef.getDSSProject().getName());
33+
streamisPostAction.addRequestPayload("description", dssProjectContentRequestRef.getDSSProject().getDescription());
34+
streamisPostAction.addRequestPayload("workspaceName", dssProjectContentRequestRef.getDSSProject().getWorkspaceName());
35+
List<String> accessUsers = dssProjectContentRequestRef.getDSSProjectPrivilege().getAccessUsers();
36+
List<String> editUsers = dssProjectContentRequestRef.getDSSProjectPrivilege().getEditUsers();
37+
List<String> releaseUsers = dssProjectContentRequestRef.getDSSProjectPrivilege().getReleaseUsers();
38+
streamisPostAction.addRequestPayload("accessUsers",accessUsers);
39+
streamisPostAction.addRequestPayload("editUsers",editUsers);
40+
streamisPostAction.addRequestPayload("releaseUsers",releaseUsers);
41+
ResponseRef responseRef = StreamisCommonUtil.getExternalResponseRef(dssProjectContentRequestRef, ssoRequestOperation, url, streamisPostAction);
42+
@SuppressWarnings("unchecked")
43+
Long projectId = DSSCommonUtils.parseToLong(responseRef.getValue("projectId"));
44+
return ProjectResponseRef.newExternalBuilder()
45+
.setRefProjectId(projectId.longValue()).success();
46+
}
47+
48+
49+
}

0 commit comments

Comments
 (0)