From a00c06daf59a42b50589a3117bbf4004e74ce4ed Mon Sep 17 00:00:00 2001 From: liuchengguo 00679 Date: Thu, 22 Apr 2021 10:38:41 +0800 Subject: [PATCH 001/434] Add: datasource code first commit --- streamis-datasource/pom.xml | 13 + .../streamis-datasource-manager/pom.xml | 110 +++++++- .../src/main/assembly/distribution.xml | 222 ++++++++++++++++ .../dao/StreamisDatasourceAuthorityDao.java | 16 ++ .../dao/StreamisDatasourceFieldsDao.java | 16 ++ .../manager/dao/StreamisTableMetaDao.java | 15 ++ .../impl/StreamisDatasourceAuthorityDao.xml | 19 ++ .../dao/impl/StreamisDatasourceFieldsDao.xml | 23 ++ .../manager/dao/impl/StreamisTableMetaDao.xml | 27 ++ .../domain/StreamisDatasourceAuthority.java | 105 ++++++++ .../domain/StreamisDatasourceFields.java | 163 ++++++++++++ .../manager/domain/StreamisTableMeta.java | 224 ++++++++++++++++ .../restful/api/StreamisTableMetaApi.java | 244 ++++++++++++++++++ .../StreamisDatasourceAuthorityService.java | 16 ++ .../StreamisDatasourceFieldsService.java | 16 ++ .../service/StreamisTableMetaService.java | 18 ++ ...treamisDatasourceAuthorityServiceImpl.java | 21 ++ .../StreamisDatasourceFieldsServiceImpl.java | 21 ++ .../impl/StreamisTableMetaServiceImpl.java | 20 ++ .../streamis-datasource-server/pom.xml | 28 ++ .../src/bin/start-streamis-datasource.sh | 38 +++ .../src/bin/stop-streamis-datasource.sh | 47 ++++ .../src/conf/application.yml | 28 ++ .../src/conf/linkis.properties | 32 +++ .../src/conf/log4j.properties | 36 +++ .../src/conf/log4j2.xml | 38 +++ .../DataSourceServer2managerRestfulApi.java | 88 +++++++ .../api/DataSourceTransferRestfulApi.java | 59 +++++ .../streamis-datasource-transfer/pom.xml | 51 ++++ .../transfer/conf/TransferConfiguration.java | 24 ++ .../entity/StreamisDataSourceCode.java | 44 ++++ .../transfer/entity/StreamisField.java | 77 ++++++ .../transfer/entity/StreamisTableEntity.java | 45 ++++ .../datasource/transfer/entity/TableInfo.java | 71 +++++ .../transfer/service/DataSourceTransfer.java | 14 + .../transfer/service/LinkisDataSource.java | 29 +++ .../impl/AbstractLinkisDataSource.java | 23 ++ .../service/impl/DataSourceTransferImpl.java | 114 ++++++++ .../service/impl/KafkaLinkisDataSource.java | 26 ++ .../client/LinkisDataSourceClient.scala | 55 ++++ .../DataSourceTransferFlinksqlUtils.scala | 38 +++ 41 files changed, 2313 insertions(+), 1 deletion(-) create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/assembly/distribution.xml create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceAuthorityDao.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceFieldsDao.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisTableMetaDao.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceAuthorityDao.xml create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceFieldsDao.xml create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisTableMetaDao.xml create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceAuthority.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceFields.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisTableMeta.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceAuthorityService.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceFieldsService.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisTableMetaService.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceAuthorityServiceImpl.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceFieldsServiceImpl.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisTableMetaServiceImpl.java create mode 100644 streamis-datasource/streamis-datasource-server/src/bin/start-streamis-datasource.sh create mode 100644 streamis-datasource/streamis-datasource-server/src/bin/stop-streamis-datasource.sh create mode 100644 streamis-datasource/streamis-datasource-server/src/conf/application.yml create mode 100644 streamis-datasource/streamis-datasource-server/src/conf/linkis.properties create mode 100644 streamis-datasource/streamis-datasource-server/src/conf/log4j.properties create mode 100644 streamis-datasource/streamis-datasource-server/src/conf/log4j2.xml create mode 100644 streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java create mode 100644 streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java create mode 100644 streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/conf/TransferConfiguration.java create mode 100644 streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisDataSourceCode.java create mode 100644 streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisField.java create mode 100644 streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisTableEntity.java create mode 100644 streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/TableInfo.java create mode 100644 streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/DataSourceTransfer.java create mode 100644 streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/LinkisDataSource.java create mode 100644 streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/AbstractLinkisDataSource.java create mode 100644 streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java create mode 100644 streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/KafkaLinkisDataSource.java create mode 100644 streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala create mode 100644 streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/util/DataSourceTransferFlinksqlUtils.scala diff --git a/streamis-datasource/pom.xml b/streamis-datasource/pom.xml index 5e188370b..0df441d56 100644 --- a/streamis-datasource/pom.xml +++ b/streamis-datasource/pom.xml @@ -9,6 +9,10 @@ 4.0.0 + + 1.0.0-RC1 + + streamis-datasource pom @@ -17,5 +21,14 @@ streamis-datasource-server + + + com.webank.wedatasphere.linkis + linkis-common + ${linkis.version} + + + + \ No newline at end of file diff --git a/streamis-datasource/streamis-datasource-manager/pom.xml b/streamis-datasource/streamis-datasource-manager/pom.xml index 39fef68a7..5d3cbe7f0 100644 --- a/streamis-datasource/streamis-datasource-manager/pom.xml +++ b/streamis-datasource/streamis-datasource-manager/pom.xml @@ -8,8 +8,116 @@ 0.1.0-SNAPSHOT 4.0.0 - streamis-datasource-manager + + 1.0.0-RC1 + 2.7.2 + 2.2.6.RELEASE + 2.2.6.RELEASE + 2.3.7.RELEASE + 2.2.6.RELEASE + 25.1-jre + 2.8.5 + 2.11.3 + 2.11.12 + 1.8 + 2.15.2 + 2.11 + 4.1.44.Final + 3.5.3 + 2.16 + 2.23.1 + 9.4.20.v20190813 + 4.5.4 + 4.5.4 + 1.7.30 + 3.3.3 + 1.4.15 + 5.2.12.RELEASE + 5.3.6.RELEASE + 0.9.10 + 3.4.1 + UTF-8 + 3.19.0-GA + 3.2.2 + 2.6 + 2.4 + + + + com.webank.wedatasphere.linkis + linkis-rpc + ${linkis.version} + + + com.webank.wedatasphere.linkis + linkis-mybatis + ${linkis.version} + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + src/main/java + + **/*.xml + + + + src/main/resources + + **/*.properties + **/application.yml + **/bootstrap.yml + **/log4j2.xml + + + + \ No newline at end of file diff --git a/streamis-datasource/streamis-datasource-manager/src/main/assembly/distribution.xml b/streamis-datasource/streamis-datasource-manager/src/main/assembly/distribution.xml new file mode 100644 index 000000000..eafa84a85 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/assembly/distribution.xml @@ -0,0 +1,222 @@ + + + + linkis-metadata + + dir + + false + linkis-metadata + + + + + + lib + true + true + false + false + true + + + antlr:antlr:jar + aopalliance:aopalliance:jar + com.fasterxml.jackson.core:jackson-annotations:jar + com.fasterxml.jackson.core:jackson-core:jar + com.fasterxml.jackson.core:jackson-databind:jar + com.fasterxml.jackson.module:jackson-module-paranamer:jar + com.fasterxml.jackson.module:jackson-module-scala_2.11:jar + com.google.code.gson:gson:jar + com.google.guava:guava:jar + com.google.protobuf:protobuf-java:jar + com.netflix.archaius:archaius-core:jar + com.netflix.hystrix:hystrix-core:jar + com.netflix.netflix-commons:netflix-commons-util:jar + com.netflix.netflix-commons:netflix-statistics:jar + com.netflix.ribbon:ribbon:jar + com.netflix.ribbon:ribbon-core:jar + com.netflix.ribbon:ribbon-httpclient:jar + com.netflix.ribbon:ribbon-loadbalancer:jar + com.netflix.ribbon:ribbon-transport:jar + com.netflix.servo:servo-core:jar + com.ning:async-http-client:jar + com.sun.jersey.contribs:jersey-apache-client4:jar + com.sun.jersey:jersey-client:jar + com.sun.jersey:jersey-core:jar + com.sun.jersey:jersey-server:jar + com.sun.xml.bind:jaxb-impl:jar + com.webank.wedatasphere.linkis:linkis-common:jar + commons-beanutils:commons-beanutils:jar + commons-beanutils:commons-beanutils-core:jar + commons-cli:commons-cli:jar + commons-collections:commons-collections:jar + commons-configuration:commons-configuration:jar + commons-daemon:commons-daemon:jar + commons-dbcp:commons-dbcp:jar + commons-digester:commons-digester:jar + commons-io:commons-io:jar + commons-lang:commons-lang:jar + commons-net:commons-net:jar + commons-pool:commons-pool:jar + io.netty:netty:jar + io.netty:netty-all:jar + io.netty:netty-buffer:jar + io.netty:netty-codec:jar + io.netty:netty-codec-http:jar + io.netty:netty-common:jar + io.netty:netty-handler:jar + io.netty:netty-transport:jar + io.netty:netty-transport-native-epoll:jar + io.reactivex:rxjava:jar + io.reactivex:rxnetty:jar + io.reactivex:rxnetty-contexts:jar + io.reactivex:rxnetty-servo:jar + javax.activation:activation:jar + javax.annotation:javax.annotation-api:jar + javax.inject:javax.inject:jar + javax.servlet:javax.servlet-api:jar + javax.servlet.jsp:jsp-api:jar + javax.xml.bind:jaxb-api:jar + javax.xml.stream:stax-api:jar + net.databinder.dispatch:dispatch-core_2.11:jar + net.databinder.dispatch:dispatch-json4s-jackson_2.11:jar + org.antlr:antlr-runtime:jar + org.antlr:stringtemplate:jar + org.apache.commons:commons-compress:jar + org.apache.commons:commons-math3:jar + org.apache.curator:curator-framework:jar + org.apache.curator:curator-recipes:jar + org.apache.directory.api:api-asn1-api:jar + org.apache.directory.api:api-util:jar + org.apache.directory.server:apacheds-i18n:jar + org.apache.directory.server:apacheds-kerberos-codec:jar + org.apache.hadoop:hadoop-annotations:jar + org.apache.hadoop:hadoop-auth:jar + org.apache.hadoop:hadoop-common:jar + org.apache.hadoop:hadoop-hdfs:jar + org.apache.htrace:htrace-core:jar + org.apache.logging.log4j:log4j-api:jar + org.apache.zookeeper:zookeeper:jar + org.aspectj:aspectjweaver:jar + org.bouncycastle:bcpkix-jdk15on:jar + org.bouncycastle:bcprov-jdk15on:jar + org.codehaus.jettison:jettison:jar + org.eclipse.jetty:jetty-continuation:jar + org.eclipse.jetty:jetty-http:jar + org.eclipse.jetty:jetty-io:jar + org.eclipse.jetty:jetty-jndi:jar + org.eclipse.jetty:jetty-plus:jar + org.eclipse.jetty:jetty-security:jar + org.eclipse.jetty:jetty-server:jar + org.eclipse.jetty:jetty-servlet:jar + org.eclipse.jetty:jetty-util:jar + org.eclipse.jetty:jetty-webapp:jar + org.eclipse.jetty:jetty-xml:jar + org.fusesource.leveldbjni:leveldbjni-all:jar + org.hdrhistogram:HdrHistogram:jar + org.json4s:json4s-ast_2.11:jar + org.json4s:json4s-core_2.11:jar + org.json4s:json4s-jackson_2.11:jar + org.jsoup:jsoup:jar + org.mortbay.jetty:jetty:jar + org.mortbay.jetty:jetty-util:jar + org.scala-lang.modules:scala-parser-combinators_2.11:jar + org.scala-lang.modules:scala-xml_2.11:jar + org.scala-lang:scala-compiler:jar + org.scala-lang:scala-library:jar + org.scala-lang:scala-reflect:jar + org.scala-lang:scalap:jar + org.slf4j:jul-to-slf4j:jar + org.slf4j:slf4j-api:jar + org.springframework.boot:spring-boot:jar + org.springframework.boot:spring-boot-autoconfigure:jar + org.springframework.boot:spring-boot-starter:jar + org.springframework.boot:spring-boot-starter-aop:jar + org.springframework.cloud:spring-cloud-commons:jar + org.springframework.cloud:spring-cloud-context:jar + org.springframework.cloud:spring-cloud-netflix-archaius:jar + org.springframework.cloud:spring-cloud-netflix-ribbon:jar + org.springframework.cloud:spring-cloud-starter:jar + org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar + org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar + org.springframework.security:spring-security-crypto:jar + org.springframework.security:spring-security-rsa:jar + org.springframework:spring-aop:jar + org.springframework:spring-beans:jar + org.springframework:spring-context:jar + org.springframework:spring-core:jar + org.springframework:spring-expression:jar + + org.springframework:spring-web:jar + org.tukaani:xz:jar + org.yaml:snakeyaml:jar + xerces:xercesImpl:jar + xml-apis:xml-apis:jar + xmlenc:xmlenc:jar + + + + + + + ${basedir}/conf + + * + + 0777 + conf + unix + + + + + + diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceAuthorityDao.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceAuthorityDao.java new file mode 100644 index 000000000..2d088ed2e --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceAuthorityDao.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.streamis.datasource.manager.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceAuthority; + +/** + *

+ * 数据源授权表 Mapper 接口 + *

+ * + * @author c01013 + * @since 2021-04-02 + */ +public interface StreamisDatasourceAuthorityDao extends BaseMapper { + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceFieldsDao.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceFieldsDao.java new file mode 100644 index 000000000..656c459f2 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceFieldsDao.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.streamis.datasource.manager.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceFields; + +/** + *

+ * 数据源字段表 Mapper 接口 + *

+ * + * @author c01013 + * @since 2021-04-02 + */ +public interface StreamisDatasourceFieldsDao extends BaseMapper { + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisTableMetaDao.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisTableMetaDao.java new file mode 100644 index 000000000..cd0bc271b --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisTableMetaDao.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.streamis.datasource.manager.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisTableMeta; +/** + *

+ * StreamTableMeta表 Mapper 接口 + *

+ * + * @author c01013 + * @since 2021-04-02 + */ +public interface StreamisTableMetaDao extends BaseMapper { + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceAuthorityDao.xml b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceAuthorityDao.xml new file mode 100644 index 000000000..6eccc1b7b --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceAuthorityDao.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + id, streamis_table_meta_id, authority_scope, authority_id, grant_user + + + diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceFieldsDao.xml b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceFieldsDao.xml new file mode 100644 index 000000000..515fe23d0 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceFieldsDao.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + id, streamis_table_meta_id, field_name, field_type, field_is_primary, field_is_partition, verify_rule, field_alias, field_description + + + diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisTableMetaDao.xml b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisTableMetaDao.xml new file mode 100644 index 000000000..47906b301 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisTableMetaDao.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + id, name, tags, description, layer, scope, params, workspace_name, linkis_datasource_name, create_by, create_time, update_by, update_time + + + diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceAuthority.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceAuthority.java new file mode 100644 index 000000000..ef9c14b0b --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceAuthority.java @@ -0,0 +1,105 @@ +package com.webank.wedatasphere.streamis.datasource.manager.domain; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.activerecord.Model; +import java.io.Serializable; + +/** + *

+ * 数据源授权表 + *

+ * + * @author c01013 + * @since 2021-04-02 + */ +@TableName("streamis_datasource_authority") +public class StreamisDatasourceAuthority extends Model { + + private static final long serialVersionUID = 1L; + + /** + * 自增主键 + */ + @TableId(value = "id", type = IdType.AUTO) + private Long id; + /** + * streamis_table_meta 的id + */ + @TableField("streamis_table_meta_id") + private Long streamisTableMetaId; + /** + * 授权范围,1:workspace，2:project,3:task + */ + @TableField("authority_scope") + private String authorityScope; + /** + * 授权范围的id,如果是workspace级别授权，则是workspaceid，如果是project 则是projectId + */ + @TableField("authority_id") + private String authorityId; + /** + * 授权用户名 + */ + @TableField("grant_user") + private String grantUser; + + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getStreamisTableMetaId() { + return streamisTableMetaId; + } + + public void setStreamisTableMetaId(Long streamisTableMetaId) { + this.streamisTableMetaId = streamisTableMetaId; + } + + public String getAuthorityScope() { + return authorityScope; + } + + public void setAuthorityScope(String authorityScope) { + this.authorityScope = authorityScope; + } + + public String getAuthorityId() { + return authorityId; + } + + public void setAuthorityId(String authorityId) { + this.authorityId = authorityId; + } + + public String getGrantUser() { + return grantUser; + } + + public void setGrantUser(String grantUser) { + this.grantUser = grantUser; + } + + @Override + protected Serializable pkVal() { + return this.id; + } + + @Override + public String toString() { + return "StreamisDatasourceAuthority{" + + ", id=" + id + + ", streamisTableMetaId=" + streamisTableMetaId + + ", authorityScope=" + authorityScope + + ", authorityId=" + authorityId + + ", grantUser=" + grantUser + + "}"; + } +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceFields.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceFields.java new file mode 100644 index 000000000..a1f62094c --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceFields.java @@ -0,0 +1,163 @@ +package com.webank.wedatasphere.streamis.datasource.manager.domain; + + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.activerecord.Model; + +import java.io.Serializable; + +/** + *

+ * 数据源字段表 + *

+ * + * @author c01013 + * @since 2021-04-02 + */ +@TableName("streamis_datasource_fields") +public class StreamisDatasourceFields extends Model { + + private static final long serialVersionUID = 1L; + + /** + * 自增主键 + */ + @TableId(value = "id", type = IdType.AUTO) + private Long id; + /** + * streamis_table_meta 的id，外键 + */ + @TableField("streamis_table_meta_id") + private Long streamisTableMetaId; + /** + * 数据源字段名字 + */ + @TableField("field_name") + private String fieldName; + /** + * 字段类型，string、int等 + */ + @TableField("field_type") + private String fieldType; + /** + * 是否主键 + */ + @TableField("field_is_primary") + private Integer fieldIsPrimary; + /** + * 是否分区 + */ + @TableField("field_is_partition") + private Integer fieldIsPartition; + /** + * 校验规则 + */ + @TableField("verify_rule") + private String verifyRule; + /** + * 字段别名 + */ + @TableField("field_alias") + private String fieldAlias; + /** + * 描述 + */ + @TableField("field_description") + private String fieldDescription; + + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getStreamisTableMetaId() { + return streamisTableMetaId; + } + + public void setStreamisTableMetaId(Long streamisTableMetaId) { + this.streamisTableMetaId = streamisTableMetaId; + } + + public String getFieldName() { + return fieldName; + } + + public void setFieldName(String fieldName) { + this.fieldName = fieldName; + } + + public String getFieldType() { + return fieldType; + } + + public void setFieldType(String fieldType) { + this.fieldType = fieldType; + } + + public Integer getFieldIsPrimary() { + return fieldIsPrimary; + } + + public void setFieldIsPrimary(Integer fieldIsPrimary) { + this.fieldIsPrimary = fieldIsPrimary; + } + + public Integer getFieldIsPartition() { + return fieldIsPartition; + } + + public void setFieldIsPartition(Integer fieldIsPartition) { + this.fieldIsPartition = fieldIsPartition; + } + + public String getVerifyRule() { + return verifyRule; + } + + public void setVerifyRule(String verifyRule) { + this.verifyRule = verifyRule; + } + + public String getFieldAlias() { + return fieldAlias; + } + + public void setFieldAlias(String fieldAlias) { + this.fieldAlias = fieldAlias; + } + + public String getFieldDescription() { + return fieldDescription; + } + + public void setFieldDescription(String fieldDescription) { + this.fieldDescription = fieldDescription; + } + + @Override + protected Serializable pkVal() { + return this.id; + } + + @Override + public String toString() { + return "StreamisDatasourceFields{" + + ", id=" + id + + ", streamisTableMetaId=" + streamisTableMetaId + + ", fieldName=" + fieldName + + ", fieldType=" + fieldType + + ", fieldIsPrimary=" + fieldIsPrimary + + ", fieldIsPartition=" + fieldIsPartition + + ", verifyRule=" + verifyRule + + ", fieldAlias=" + fieldAlias + + ", fieldDescription=" + fieldDescription + + "}"; + } +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisTableMeta.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisTableMeta.java new file mode 100644 index 000000000..422e55329 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisTableMeta.java @@ -0,0 +1,224 @@ +package com.webank.wedatasphere.streamis.datasource.manager.domain; + +import java.util.Date; + +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.activerecord.Model; + +import java.io.Serializable; + +/** + *

+ * StreamTableMeta表 + *

+ * + * @author c01013 + * @since 2021-04-03 + */ +@TableName("streamis_table_meta") +public class StreamisTableMeta extends Model { + + private static final long serialVersionUID = 1L; + + /** + * 自增主键 + */ + private Long id; + /** + * 名字，Streamis节点的名字 + */ + private String name; + /** + * 标签，逗号分隔 + */ + private String tags; + /** + * 基本描述信息 + */ + private String description; + /** + * 表示是DWD,DWS或者维表等 + */ + private String layer; + /** + * 作用域，表示是工程级别还是任务级别等 + */ + private String scope; + /** + * json信息保存额外信息 + */ + private String params; + /** + * 工作空间名 + */ + @TableField("workspace_name") + private String workspaceName; + /** + * 绑定的linkis数据源的名字 + */ + @TableField("linkis_datasource_name") + private String linkisDatasourceName; + + @TableField("table_name") + private String tableName; + + @TableField("node_name") + private String nodeName; + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + @TableField("create_by") + private String createBy; + @TableField("create_time") + private Date createTime; + @TableField("update_by") + private String updateBy; + @TableField("update_time") + private Date updateTime; + + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getTags() { + return tags; + } + + public void setTags(String tags) { + this.tags = tags; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getLayer() { + return layer; + } + + public void setLayer(String layer) { + this.layer = layer; + } + + public String getScope() { + return scope; + } + + public void setScope(String scope) { + this.scope = scope; + } + + public String getParams() { + return params; + } + + public void setParams(String params) { + this.params = params; + } + + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + public String getLinkisDatasourceName() { + return linkisDatasourceName; + } + + public void setLinkisDatasourceName(String linkisDatasourceName) { + this.linkisDatasourceName = linkisDatasourceName; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public String getUpdateBy() { + return updateBy; + } + + public void setUpdateBy(String updateBy) { + this.updateBy = updateBy; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + @Override + protected Serializable pkVal() { + return this.id; + } + + public String getNodeName() { + return nodeName; + } + + public void setNodeName(String nodeName) { + this.nodeName = nodeName; + } + + @Override + public String toString() { + return "StreamisTableMeta{" + + "id=" + id + + ", name='" + name + '\'' + + ", tags='" + tags + '\'' + + ", description='" + description + '\'' + + ", layer='" + layer + '\'' + + ", scope='" + scope + '\'' + + ", params='" + params + '\'' + + ", workspaceName='" + workspaceName + '\'' + + ", linkisDatasourceName='" + linkisDatasourceName + '\'' + + ", tableName='" + tableName + '\'' + + ", nodeName='" + nodeName + '\'' + + ", createBy='" + createBy + '\'' + + ", createTime=" + createTime + + ", updateBy='" + updateBy + '\'' + + ", updateTime=" + updateTime + + '}'; + } +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java new file mode 100644 index 000000000..81409604c --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java @@ -0,0 +1,244 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedatasphere.streamis.datasource.manager.restful.api; + + +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceFields; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisTableMeta; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceFieldsService; +import com.webank.wedatasphere.linkis.server.Message; +import com.webank.wedatasphere.linkis.server.security.SecurityFilter; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisTableMetaService; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.type.TypeReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import java.util.*; + +@Service +public class StreamisTableMetaApi { + + private static final Logger logger = LoggerFactory.getLogger(StreamisTableMetaApi.class); + + @Autowired + private StreamisTableMetaService streamisTableMetaService; + + @Autowired + private StreamisDatasourceFieldsService streamisDatasourceFieldsService; + ObjectMapper mapper = new ObjectMapper(); + + + /** + * 获取左侧数据源树，需要调用API获取linkis数据源，还需获取streamis数据源 + */ + public Response getStreamisDataSourceList( HttpServletRequest req, Map json + ) throws ErrorException { + Message message = null; + try{ + String userName = SecurityFilter.getLoginUsername(req); + String workSpaceName = (String) json.get("workSpaceName"); + String projectName = (String) json.get("projectName"); + String task = (String) json.get("task"); + String dataSourceType = (String) json.get("dataSourceType"); + //TODO: invoke rpc interface to get linkisDataSource Information + + + StreamisTableMeta streamisTableMeta = streamisTableMetaService.getById(1); + logger.warn(streamisTableMeta.toString()); + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.eq("id",1); + StreamisTableMeta one = streamisTableMetaService.getOne(wrapper); + }catch (Throwable e){ + logger.error("",e); + throw new ErrorException(30001, "抱歉，后台获取数据源列表失败"); + } + return Message.messageToResponse(message); + + } + + + public Response getStreamisTableMeta( HttpServletRequest req, Long streamisTableMetaId + ) throws ErrorException { + Message message = null; + try{ + StreamisTableMeta streamisTableMeta = streamisTableMetaService.getById(streamisTableMetaId); + QueryWrapper streamisDatasourceFieldsWrapper = new QueryWrapper<>(); + streamisDatasourceFieldsWrapper.eq("streamis_table_meta_id",streamisTableMetaId); + List streamisDatasourceFields = streamisDatasourceFieldsService.list(streamisDatasourceFieldsWrapper); + message=Message.ok().data("streamisTableMeta",streamisTableMeta).data("streamisDatasourceFields",streamisDatasourceFields); + }catch (Throwable e){ + logger.error("Failed to get streamisTableMetaInfo",e); + throw new ErrorException(30002, "抱歉，后台获取数据源详细信息失败"); + } + return Message.messageToResponse(message); + + } + + public Response getStreamisTableMetaByNodeNames( HttpServletRequest req,JsonNode json + ) throws ErrorException { + Message message = null; + QueryWrapper wrapper = null; + Map> nodeTablesMap = new HashMap<>(); + List list = null; + try{ + List nodeNames = mapper.readValue(json.get("nodeNames"), new TypeReference>() {}); + + for (String nodeName : nodeNames) { + wrapper = new QueryWrapper<>(); + wrapper.eq("node_name",nodeName); + list = streamisTableMetaService.list(wrapper); + nodeTablesMap.put(nodeName,list); + } + message = Message.ok().data("nodeTables", nodeTablesMap); + + }catch (Throwable e){ + logger.error("Failed to get streamisDataSource Tree",e); + throw new ErrorException(30011, "抱歉，后台获取左侧数据源树失败"); + } + return Message.messageToResponse(message); + + } + + + public Response addStreamisTableMeta( HttpServletRequest req, JsonNode json) throws ErrorException { + Message message = null; + try { + String userName = SecurityFilter.getLoginUsername(req); + StreamisTableMeta streamisTableMeta = mapper.readValue(json.get("streamisTableMeta"), StreamisTableMeta.class); + //第一次保存linkisDataSource数据源 + streamisTableMeta.setCreateBy(userName); + boolean result = streamisTableMetaService.save(streamisTableMeta); + if(!result){ + throw new ErrorException(30003, "抱歉，添加streamis数据源失败"); + } + Long streamisTableMetaId = streamisTableMeta.getId(); + + List fieldsList = mapper.readValue(json.get("streamisTableFields"), new TypeReference>() { + }); + + fieldsList.forEach(obj ->obj.setStreamisTableMetaId(streamisTableMetaId)); + boolean b = streamisDatasourceFieldsService.saveBatch(fieldsList); + message = Message.ok().data("streamisTableMetaId",streamisTableMetaId); + } catch (Exception e) { + logger.error("Failed to add StreamisTableMeta: ", e); + throw new ErrorException(30003, "抱歉，添加streamis数据源失败"); + } + return Message.messageToResponse(message); + } + + + public Response updateStreamisTableMeta( HttpServletRequest req, JsonNode json) throws ErrorException { + Message message = null; + try { + String userName = SecurityFilter.getLoginUsername(req); + StreamisTableMeta streamisTableMeta = mapper.readValue(json.get("streamisTableMeta"), StreamisTableMeta.class); + Long streamisTableMetaId = streamisTableMeta.getId(); + if(null == streamisTableMetaId ){ + throw new ErrorException(30004, "抱歉，更新streamis数据源信息失败，streamisTableMetaId is null"); + } + streamisTableMeta.setUpdateBy(userName); + // update StreamisTableMeta + streamisTableMetaService.updateById(streamisTableMeta); + message = Message.ok(); + + } catch (Exception e) { + logger.error("Failed to update StreamisTableMeta: ", e); + throw new ErrorException(30004, "抱歉，更新streamis数据源信息失败"); + } + return Message.messageToResponse(message); + } + + public Response deleteStreamisTableMeta(HttpServletRequest req, Long id) throws ErrorException { + Message message = null; + try { + //delete TableFields + QueryWrapper streamisDatasourceFieldsQueryWrapper = new QueryWrapper<>(); + streamisDatasourceFieldsQueryWrapper.eq("streamis_table_meta_id",id); + boolean remove = streamisDatasourceFieldsService.remove(streamisDatasourceFieldsQueryWrapper); + if (!remove){ + throw new ErrorException(30008, String.format("抱歉，删除streamis数据源 表[%s]字段信息失败",id)); + } + // delete StreamisTableMeta + boolean result = streamisDatasourceFieldsService.removeById(id); + if (!result){ + throw new ErrorException(30009, String.format("抱歉，删除streamis数据源 表[%s]失败",id)); + } + message = Message.ok(); + + } catch (Exception e) { + logger.error("Failed to delete StreamisTableMeta: ", e); + throw new ErrorException(30010, "抱歉，删除streamis数据源信息失败"); + } + return Message.messageToResponse(message); + } + + + public Response addStreamisTableFields( HttpServletRequest req, JsonNode json) throws ErrorException { + Message message = null; + try { + StreamisDatasourceFields streamisDatasourceFields = mapper.readValue(json.get("streamisTableFields"), StreamisDatasourceFields.class); + boolean result = streamisDatasourceFieldsService.save(streamisDatasourceFields); + if(!result){ + throw new ErrorException(30005, "抱歉，添加StreamisTableFields表字段信息失败"); + } + message = Message.ok(); + } catch (Exception e) { + logger.error("Failed to add streamisDatasourceFields: ", e); + throw new ErrorException(30005, "抱歉，添加StreamisTableFields表字段信息失败"); + } + return Message.messageToResponse(message); + } + + public Response updateStreamisTableFields( HttpServletRequest req, JsonNode json) throws ErrorException { + Message message = null; + try { + StreamisDatasourceFields streamisDatasourceFields = mapper.readValue(json.get("streamisTableFields"), StreamisDatasourceFields.class); + streamisDatasourceFieldsService.updateById(streamisDatasourceFields); + message = Message.ok(); + } catch (Exception e) { + logger.error("Failed to update streamisDatasourceFields: ", e); + throw new ErrorException(30006, "抱歉，更新StreamisTableFields表字段信息失败"); + } + return Message.messageToResponse(message); + } + + public Response deleteStreamisTableFields( HttpServletRequest req, Long id) throws ErrorException { + Message message = null; + String userName = SecurityFilter.getLoginUsername(req); + try { + logger.info(userName+" delete StreamisTableFields："+id); + streamisDatasourceFieldsService.removeById(id); + message = Message.ok(); + } catch (Exception e) { + logger.error("Failed to delete StreamisTableFields: ", e); + throw new ErrorException(30007, "抱歉，删除StreamisTableFields表字段信息失败"); + } + return Message.messageToResponse(message); + } + + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceAuthorityService.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceAuthorityService.java new file mode 100644 index 000000000..265710dfe --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceAuthorityService.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.streamis.datasource.manager.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceAuthority; + +/** + *

+ * 数据源授权表 服务类 + *

+ * + * @author c01013 + * @since 2021-04-02 + */ +public interface StreamisDatasourceAuthorityService extends IService { + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceFieldsService.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceFieldsService.java new file mode 100644 index 000000000..dfc7d86c5 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceFieldsService.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.streamis.datasource.manager.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceFields; + +/** + *

+ * 数据源字段表 服务类 + *

+ * + * @author c01013 + * @since 2021-04-02 + */ +public interface StreamisDatasourceFieldsService extends IService { + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisTableMetaService.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisTableMetaService.java new file mode 100644 index 000000000..d2388f493 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisTableMetaService.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.streamis.datasource.manager.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisTableMeta; + +/** + *

+ * StreamTableMeta表 服务类 + *

+ * + * @author c01013 + * @since 2021-04-02 + */ +public interface StreamisTableMetaService extends IService { + + +} + diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceAuthorityServiceImpl.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceAuthorityServiceImpl.java new file mode 100644 index 000000000..eea74fbef --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceAuthorityServiceImpl.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.streamis.datasource.manager.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.streamis.datasource.manager.dao.StreamisDatasourceAuthorityDao; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceAuthority; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceAuthorityService; +import org.springframework.stereotype.Service; + + +/** + *

+ * 数据源授权表 服务实现类 + *

+ * + * @author c01013 + * @since 2021-04-02 + */ +@Service +public class StreamisDatasourceAuthorityServiceImpl extends ServiceImpl implements StreamisDatasourceAuthorityService { + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceFieldsServiceImpl.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceFieldsServiceImpl.java new file mode 100644 index 000000000..43828dc6f --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceFieldsServiceImpl.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.streamis.datasource.manager.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.streamis.datasource.manager.dao.StreamisDatasourceFieldsDao; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceFields; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceFieldsService; + +import org.springframework.stereotype.Service; + +/** + *

+ * 数据源字段表 服务实现类 + *

+ * + * @author c01013 + * @since 2021-04-02 + */ +@Service +public class StreamisDatasourceFieldsServiceImpl extends ServiceImpl implements StreamisDatasourceFieldsService { + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisTableMetaServiceImpl.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisTableMetaServiceImpl.java new file mode 100644 index 000000000..7dcb7a643 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisTableMetaServiceImpl.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.streamis.datasource.manager.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.streamis.datasource.manager.dao.StreamisTableMetaDao; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisTableMeta; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisTableMetaService; +import org.springframework.stereotype.Service; + +/** + *

+ * StreamTableMeta表 服务实现类 + *

+ * + * @author c01013 + * @since 2021-04-02 + */ +@Service +public class StreamisTableMetaServiceImpl extends ServiceImpl implements StreamisTableMetaService { + +} diff --git a/streamis-datasource/streamis-datasource-server/pom.xml b/streamis-datasource/streamis-datasource-server/pom.xml index e07073317..e3bda5bfd 100644 --- a/streamis-datasource/streamis-datasource-server/pom.xml +++ b/streamis-datasource/streamis-datasource-server/pom.xml @@ -11,5 +11,33 @@ streamis-datasource-server + + + com.webank.wedatasphere.linkis + linkis-rpc + ${linkis.version} + + + com.webank.wedatasphere.linkis + linkis-module + ${linkis.version} + + + com.webank.wedatasphere.linkis + linkis-mybatis + ${linkis.version} + + + com.webank.wedatasphere.streamis + streamis-datasource-transfer + 0.1.0-SNAPSHOT + + + com.webank.wedatasphere.streamis + streamis-datasource-manager + 0.1.0-SNAPSHOT + compile + + \ No newline at end of file diff --git a/streamis-datasource/streamis-datasource-server/src/bin/start-streamis-datasource.sh b/streamis-datasource/streamis-datasource-server/src/bin/start-streamis-datasource.sh new file mode 100644 index 000000000..56cab6df0 --- /dev/null +++ b/streamis-datasource/streamis-datasource-server/src/bin/start-streamis-datasource.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +cd `dirname $0` +cd .. +HOME=`pwd` + +export SERVER_PID=$HOME/bin/linkis.pid +export SERVER_LOG_PATH=$HOME/logs +export SERVER_CLASS=com.webank.wedatasphere.linkis.DataWorkCloudApplication + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS=" -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$HOME/logs/linkis-gc.log" +fi + +if [[ -f "${SERVER_PID}" ]]; then + pid=$(cat ${SERVER_PID}) + if kill -0 ${pid} >/dev/null 2>&1; then + echo "Server is already running." + exit 1 + fi +fi + +nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & +pid=$! +if [[ -z "${pid}" ]]; then + echo "server $SERVER_NAME start failed!" + exit 1 +else + echo "server $SERVER_NAME start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi \ No newline at end of file diff --git a/streamis-datasource/streamis-datasource-server/src/bin/stop-streamis-datasource.sh b/streamis-datasource/streamis-datasource-server/src/bin/stop-streamis-datasource.sh new file mode 100644 index 000000000..f03288711 --- /dev/null +++ b/streamis-datasource/streamis-datasource-server/src/bin/stop-streamis-datasource.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +cd `dirname $0` +cd .. +HOME=`pwd` + +export SERVER_PID=$HOME/bin/linkis.pid + +function wait_for_server_to_die() { + local pid + local count + pid=$1 + timeout=$2 + count=0 + timeoutTime=$(date "+%s") + let "timeoutTime+=$timeout" + currentTime=$(date "+%s") + forceKill=1 + + while [[ $currentTime -lt $timeoutTime ]]; do + $(kill ${pid} > /dev/null 2> /dev/null) + if kill -0 ${pid} > /dev/null 2>&1; then + sleep 3 + else + forceKill=0 + break + fi + currentTime=$(date "+%s") + done + + if [[ forceKill -ne 0 ]]; then + $(kill -9 ${pid} > /dev/null 2> /dev/null) + fi +} + +if [[ ! -f "${SERVER_PID}" ]]; then + echo "server $SERVER_NAME is not running" +else + pid=$(cat ${SERVER_PID}) + if [[ -z "${pid}" ]]; then + echo "server $SERVER_NAME is not running" + else + wait_for_server_to_die $pid 40 + $(rm -f ${SERVER_PID}) + echo "server $SERVER_NAME is stopped." + fi +fi \ No newline at end of file diff --git a/streamis-datasource/streamis-datasource-server/src/conf/application.yml b/streamis-datasource/streamis-datasource-server/src/conf/application.yml new file mode 100644 index 000000000..52f5701c5 --- /dev/null +++ b/streamis-datasource/streamis-datasource-server/src/conf/application.yml @@ -0,0 +1,28 @@ +server: + port: 9203 +spring: + application: + name: streamis-datasource + + +eureka: + client: + serviceUrl: + defaultZone: http://192.168.2.242/:20303/eureka/ + instance: + metadata-map: + test: wedatasphere + +management: + endpoints: + web: + exposure: + include: refresh,info +logging: + config: classpath:log4j2.xml + +pagehelper: + helper-dialect: mysql + reasonable: true + support-methods-arguments: true + params: countSql \ No newline at end of file diff --git a/streamis-datasource/streamis-datasource-server/src/conf/linkis.properties b/streamis-datasource/streamis-datasource-server/src/conf/linkis.properties new file mode 100644 index 000000000..5015b276c --- /dev/null +++ b/streamis-datasource/streamis-datasource-server/src/conf/linkis.properties @@ -0,0 +1,32 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#wds.linkis.test.mode=true + +wds.linkis.server.version=v1 + +wds.linkis.server.mybatis.datasource.url=jdbc:mysql://***REMOVED***:3306/***REMOVED***?characterEncoding=UTF-8 +wds.linkis.server.mybatis.datasource.username=root +wds.linkis.server.mybatis.datasource.password=***REMOVED*** + +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/streamis/datasource/manager/dao/impl/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.streamis.datasource.manager.domain +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.streamis.datasource.manager.dao + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.streamis.datasource.server.restful.api + + diff --git a/streamis-datasource/streamis-datasource-server/src/conf/log4j.properties b/streamis-datasource/streamis-datasource-server/src/conf/log4j.properties new file mode 100644 index 000000000..178f8dfa2 --- /dev/null +++ b/streamis-datasource/streamis-datasource-server/src/conf/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/streamis-datasource/streamis-datasource-server/src/conf/log4j2.xml b/streamis-datasource/streamis-datasource-server/src/conf/log4j2.xml new file mode 100644 index 000000000..ad88ea570 --- /dev/null +++ b/streamis-datasource/streamis-datasource-server/src/conf/log4j2.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java new file mode 100644 index 000000000..aa54a3026 --- /dev/null +++ b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java @@ -0,0 +1,88 @@ +package com.webank.wedatasphere.streamis.datasource.server.restful.api; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.streamis.datasource.manager.restful.api.StreamisTableMetaApi; +import org.codehaus.jackson.JsonNode; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.Map; + +@Component +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +@Path("streamis") +public class DataSourceServer2managerRestfulApi { + + @Autowired + private StreamisTableMetaApi streamisTableMetaApi; + + + /** + * 获取左侧数据源树，需要调用API获取linkis数据源，还需获取streamis数据源 + */ + @POST + @Path("streamisDataSourceList") + public Response getStreamisDataSourceList(@Context HttpServletRequest req, Map json + ) throws ErrorException { + return streamisTableMetaApi.getStreamisDataSourceList(req, json); + } + + /** + * 获取数据源的详细信息，包括数据源信息、表信息和字段信息 + */ + @GET + @Path("streamisTableMetaInfo/{streamis_table_meta_Id}") + public Response getStreamisTableMeta(@Context HttpServletRequest req, @PathParam("streamis_table_meta_Id")Long streamisTableMetaId + ) throws ErrorException { + return streamisTableMetaApi.getStreamisTableMeta(req,streamisTableMetaId); + } + + @POST + @Path("streamisDataSourceTree") + public Response getStreamisTableMetaByNodeNames(@Context HttpServletRequest req,JsonNode json + ) throws ErrorException { + return streamisTableMetaApi.getStreamisTableMetaByNodeNames(req,json); + } + + @POST + @Path("addStreamisTableMeta") + public Response addStreamisTableMeta(@Context HttpServletRequest req, JsonNode json) throws ErrorException { + return streamisTableMetaApi.addStreamisTableMeta(req,json); + } + + @POST + @Path("updateStreamisTableMeta") + public Response updateStreamisTableMeta(@Context HttpServletRequest req, JsonNode json) throws ErrorException { + return streamisTableMetaApi.updateStreamisTableMeta(req, json); + + } + + @GET + @Path("streamisTableMetaInfo/{streamis_table_meta_Id}") + public Response deleteStreamisTableMeta(@Context HttpServletRequest req, @PathParam("streamis_table_meta_Id")Long id) throws ErrorException { + return streamisTableMetaApi.deleteStreamisTableMeta(req,id); + } + + @POST + @Path("addFields") + public Response addStreamisTableFields(@Context HttpServletRequest req, JsonNode json) throws ErrorException { + return streamisTableMetaApi.addStreamisTableFields(req,json); + } + + @POST + @Path("updateFields") + public Response updateStreamisTableFields(@Context HttpServletRequest req, JsonNode json) throws ErrorException { + return streamisTableMetaApi.updateStreamisTableFields(req, json); + } + @GET + @Path("deleteFields/{id}") + public Response deleteStreamisTableFields(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + return streamisTableMetaApi.deleteStreamisTableFields(req,id); + } +} diff --git a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java new file mode 100644 index 000000000..1972a0749 --- /dev/null +++ b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java @@ -0,0 +1,59 @@ +package com.webank.wedatasphere.streamis.datasource.server.restful.api; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.server.Message; +import com.webank.wedatasphere.streamis.datasource.transfer.entity.StreamisDataSourceCode; +import com.webank.wedatasphere.streamis.datasource.transfer.entity.StreamisTableEntity; +import com.webank.wedatasphere.streamis.datasource.transfer.service.DataSourceTransfer; +import org.apache.commons.lang.StringUtils; +import org.glassfish.jersey.media.multipart.FormDataParam; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import java.util.HashMap; +import java.util.Map; + +public class DataSourceTransferRestfulApi { + private static final Logger logger = LoggerFactory.getLogger(DataSourceTransferRestfulApi.class); + + @Autowired + private DataSourceTransfer dataSourceTransfer; + + @GET + @Path("/transfer") + public Response transfer(@Context HttpServletRequest request, @QueryParam("streamisTableMetaId") String streamisTableMetaId, @FormDataParam("labels") Map labels) throws ErrorException { + Message message; + if(StringUtils.isBlank(streamisTableMetaId)){ + throw new ErrorException(30201,"streamisTableMetaId is null."); + } + try { + //通过streamisTableMetaId拿到StreamisTableMeta实体 + StreamisTableEntity streamisTableEntity = dataSourceTransfer.getStreamisTableMetaById(streamisTableMetaId); + //转换为FlinkSql + StreamisDataSourceCode streamisDataSourceCode = dataSourceTransfer.transfer(streamisTableEntity, labels, new HashMap()); + message = Message.ok().data("streamisDataSourceCode",streamisDataSourceCode); + } catch (Exception e) { + logger.error("传入streamisTableMetaId {} 数据源转换失败：", streamisTableMetaId, e); + throw new ErrorException(30202, "抱歉，后台数据源转换失败"); + } + + return Message.messageToResponse(message); + } + + @POST + @Path("/exportStreamisTableMeta") + public Response exportStreamisTableMeta(@Context HttpServletRequest request, @FormDataParam("streamisTableMetaId") String streamisTableMetaId){ + Message message; + + return Message.messageToResponse(Message.ok()); + } + +} diff --git a/streamis-datasource/streamis-datasource-transfer/pom.xml b/streamis-datasource/streamis-datasource-transfer/pom.xml index 5350180f9..d5c82d43e 100644 --- a/streamis-datasource/streamis-datasource-transfer/pom.xml +++ b/streamis-datasource/streamis-datasource-transfer/pom.xml @@ -11,5 +11,56 @@ streamis-datasource-transfer + + + com.webank.wedatasphere.linkis + linkis-label-common + ${linkis.version} + + + com.webank.wedatasphere.linkis + linkis-datasource-client + ${linkis.version} + + + com.webank.wedatasphere.streamis + streamis-datasource-manager + 0.1.0-SNAPSHOT + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + src/main/java + + **/*.xml + + + + src/main/resources + + **/*.properties + **/application.yml + **/bootstrap.yml + **/log4j2.xml + + + + \ No newline at end of file diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/conf/TransferConfiguration.java b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/conf/TransferConfiguration.java new file mode 100644 index 000000000..01ed6194a --- /dev/null +++ b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/conf/TransferConfiguration.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.streamis.datasource.transfer.conf; + +import com.webank.wedatasphere.linkis.common.conf.CommonVars; + +public class TransferConfiguration { + public static final CommonVars SERVER_URL = CommonVars.apply("wds.streamis.datasource.client.serverurl", ""); + public static final CommonVars CONNECTION_TIMEOUT = CommonVars.apply("wds.streamis.datasource.client.connection.timeout", 30000L); + public static final CommonVars DISCOVERY_ENABLED = CommonVars.apply("wds.streamis.datasource.client.discovery.enabled", true); + public static final CommonVars DISCOVERY_FREQUENCY_PERIOD = CommonVars.apply("wds.streamis.datasource.client.discoveryfrequency.period", 1L); + public static final CommonVars LOAD_BALANCER_ENABLED = CommonVars.apply("wds.streamis.datasource.client.loadbalancer.enabled", true); + public static final CommonVars MAX_CONNECTION_SIZE = CommonVars.apply("wds.streamis.datasource.client.maxconnection.size", 5); + public static final CommonVars RETRY_ENABLED = CommonVars.apply("wds.streamis.datasource.client.retryenabled", false); + public static final CommonVars READ_TIMEOUT = CommonVars.apply("wds.streamis.datasource.client.readtimeout", 30000L); + + public static final CommonVars AUTHTOKEN_KEY = CommonVars.apply("wds.streamis.datasource.client.authtoken.key", ""); + public static final CommonVars AUTHTOKEN_VALUE = CommonVars.apply("wds.streamis.datasource.client.authtoken.value", ""); + public static final CommonVars DWS_VERSION = CommonVars.apply("wds.streamis.datasource.client.dws.version", ""); + + + + + + +} diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisDataSourceCode.java b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisDataSourceCode.java new file mode 100644 index 000000000..c265083e1 --- /dev/null +++ b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisDataSourceCode.java @@ -0,0 +1,44 @@ +package com.webank.wedatasphere.streamis.datasource.transfer.entity; + +import com.webank.wedatasphere.linkis.manager.label.entity.Label; + +import java.util.List; +import java.util.Map; + +public class StreamisDataSourceCode { + String executionCode; //执行代码内容 + Map labels; //linkis的标签,按照linkis1.0 rc-1的标签体系 + Map params; //预留参数 + + public StreamisDataSourceCode(){} + + public StreamisDataSourceCode(String executionCode, Map linkisLabels, Map params) { + this.executionCode = executionCode; + this.labels = labels; + this.params = params; + } + + public String getExecutionCode() { + return executionCode; + } + + public void setExecutionCode(String executionCode) { + this.executionCode = executionCode; + } + + public Map getLabels() { + return labels; + } + + public void setLabels(Map labels) { + this.labels = labels; + } + + public Map getParams() { + return params; + } + + public void setParams(Map params) { + this.params = params; + } +} diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisField.java b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisField.java new file mode 100644 index 000000000..b3aba9fbd --- /dev/null +++ b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisField.java @@ -0,0 +1,77 @@ +package com.webank.wedatasphere.streamis.datasource.transfer.entity; + +public class StreamisField { + private Long id; + private String fieldName; + private Long streamisTableMetaId; + private String fieldType; + private Boolean fieldIsPrimary; + private Boolean fieldIsPartition; + private String description; + + public StreamisField(Long id, String fieldName, Long streamisTableMetaId, String fieldType, Boolean fieldIsPrimary, Boolean fieldIsPartition, String description) { + this.id = id; + this.fieldName = fieldName; + this.streamisTableMetaId = streamisTableMetaId; + this.fieldType = fieldType; + this.fieldIsPrimary = fieldIsPrimary; + this.fieldIsPartition = fieldIsPartition; + this.description = description; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getFieldName() { + return fieldName; + } + + public void setFieldName(String fieldName) { + this.fieldName = fieldName; + } + + public Long getStreamisTableMetaId() { + return streamisTableMetaId; + } + + public void setStreamisTableMetaId(Long streamisTableMetaId) { + this.streamisTableMetaId = streamisTableMetaId; + } + + public String getFieldType() { + return fieldType; + } + + public void setFieldType(String fieldType) { + this.fieldType = fieldType; + } + + public Boolean getFieldIsPrimary() { + return fieldIsPrimary; + } + + public void setFieldIsPrimary(Boolean fieldIsPrimary) { + this.fieldIsPrimary = fieldIsPrimary; + } + + public Boolean getFieldIsPartition() { + return fieldIsPartition; + } + + public void setFieldIsPartition(Boolean fieldIsPartition) { + this.fieldIsPartition = fieldIsPartition; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } +} diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisTableEntity.java b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisTableEntity.java new file mode 100644 index 000000000..831556180 --- /dev/null +++ b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisTableEntity.java @@ -0,0 +1,45 @@ +package com.webank.wedatasphere.streamis.datasource.transfer.entity; + +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceFields; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisTableMeta; +import com.webank.wedatasphere.streamis.datasource.transfer.service.LinkisDataSource; + +import java.util.List; + +public class StreamisTableEntity { + private LinkisDataSource linkisDatasource; //和linkis的相关信息，数据源的连接信息，数据源的元信息 + private List fields; //字段信息 + private StreamisTableMeta tableInfo; // 名字，创建人等信息 + + public StreamisTableEntity(){} + + public StreamisTableEntity(LinkisDataSource linkisDatasource, List fields, StreamisTableMeta tableInfo) { + this.linkisDatasource = linkisDatasource; + this.fields = fields; + this.tableInfo = tableInfo; + } + + public LinkisDataSource getLinkisDatasource() { + return linkisDatasource; + } + + public void setLinkisDatasource(LinkisDataSource linkisDatasource) { + this.linkisDatasource = linkisDatasource; + } + + public List getFields() { + return fields; + } + + public void setFields(List fields) { + this.fields = fields; + } + + public StreamisTableMeta getTableInfo() { + return tableInfo; + } + + public void setTableInfo(StreamisTableMeta tableInfo) { + this.tableInfo = tableInfo; + } +} diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/TableInfo.java b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/TableInfo.java new file mode 100644 index 000000000..98630bcd6 --- /dev/null +++ b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/TableInfo.java @@ -0,0 +1,71 @@ +package com.webank.wedatasphere.streamis.datasource.transfer.entity; + +import java.util.List; + +public class TableInfo { + private String tableName; + private String alias; + private List tags; + //作用域 任务级或者工程级等 + private String scope; + //层级，表示 DWS 或者 DWD层 + private String layer; + private String description; + + public TableInfo(String tableName, String alias, List tags, String scope, String layer, String description) { + this.tableName = tableName; + this.alias = alias; + this.tags = tags; + this.scope = scope; + this.layer = layer; + this.description = description; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + public List getTags() { + return tags; + } + + public void setTags(List tags) { + this.tags = tags; + } + + public String getScope() { + return scope; + } + + public void setScope(String scope) { + this.scope = scope; + } + + public String getLayer() { + return layer; + } + + public void setLayer(String layer) { + this.layer = layer; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } +} diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/DataSourceTransfer.java b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/DataSourceTransfer.java new file mode 100644 index 000000000..da8906832 --- /dev/null +++ b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/DataSourceTransfer.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.streamis.datasource.transfer.service; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.streamis.datasource.transfer.entity.StreamisDataSourceCode; +import com.webank.wedatasphere.streamis.datasource.transfer.entity.StreamisTableEntity; + +import java.util.Map; + +public interface DataSourceTransfer { + StreamisDataSourceCode transfer(StreamisTableEntity streamisTableEntity, Map labels, Map params); + + StreamisTableEntity getStreamisTableMetaById(String streamisTableMetaId) throws ErrorException; + +} diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/LinkisDataSource.java b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/LinkisDataSource.java new file mode 100644 index 000000000..e32d4f0e3 --- /dev/null +++ b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/LinkisDataSource.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.streamis.datasource.transfer.service; + +import java.util.Map; + +public class LinkisDataSource { + private String linkisDataSourceUniqueId; + private Map linkisDataSourceContent; + + public LinkisDataSource(String linkisDataSourceUniqueId, Map linkisDataSourceContent) { + this.linkisDataSourceUniqueId = linkisDataSourceUniqueId; + this.linkisDataSourceContent = linkisDataSourceContent; + } + + public String getLinkisDataSourceUniqueId() { + return linkisDataSourceUniqueId; + } + + public void setLinkisDataSourceUniqueId(String linkisDataSourceUniqueId) { + this.linkisDataSourceUniqueId = linkisDataSourceUniqueId; + } + + public Map getLinkisDataSourceContent() { + return linkisDataSourceContent; + } + + public void setLinkisDataSourceContent(Map linkisDataSourceContent) { + this.linkisDataSourceContent = linkisDataSourceContent; + } +} diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/AbstractLinkisDataSource.java b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/AbstractLinkisDataSource.java new file mode 100644 index 000000000..6a91eac95 --- /dev/null +++ b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/AbstractLinkisDataSource.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.streamis.datasource.transfer.service.impl; + +import java.util.Map; + +public abstract class AbstractLinkisDataSource{ + protected String workspaceName; + + protected String linkisDataSourceName; + + protected String linkisDataSourceUniqueId; + + protected Map linkisDataSourceContent; + + public String getLinkisDataSourceUniqueId() { + linkisDataSourceUniqueId = this.workspaceName + "_" + this.linkisDataSourceName; + return linkisDataSourceUniqueId; + } + + public Map getLinkisDataSourceContent(Map connectParams) { + //子类的字段信息封装到此处 + return this.linkisDataSourceContent; + } +} diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java new file mode 100644 index 000000000..bb7ffb3f4 --- /dev/null +++ b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java @@ -0,0 +1,114 @@ +package com.webank.wedatasphere.streamis.datasource.transfer.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.datasource.client.response.QueryDataSourceResult; +import com.webank.wedatasphere.linkis.manager.label.entity.Label; +import com.webank.wedatasphere.linkis.manager.label.entity.engine.EngineType; +import com.webank.wedatasphere.linkis.manager.label.entity.engine.RunType; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceFields; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisTableMeta; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceFieldsService; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisTableMetaService; +import com.webank.wedatasphere.streamis.datasource.transfer.client.LinkisDataSourceClient; +import com.webank.wedatasphere.streamis.datasource.transfer.entity.StreamisDataSourceCode; +import com.webank.wedatasphere.streamis.datasource.transfer.entity.StreamisTableEntity; +import com.webank.wedatasphere.streamis.datasource.transfer.service.DataSourceTransfer; +import com.webank.wedatasphere.streamis.datasource.transfer.service.LinkisDataSource; +import com.webank.wedatasphere.streamis.datasource.transfer.util.DataSourceTransferFlinksqlUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +public class DataSourceTransferImpl implements DataSourceTransfer { + private static final Logger logger = LoggerFactory.getLogger(DataSourceTransferImpl.class); + + @Autowired + StreamisTableMetaService streamisTableMetaService; + @Autowired + StreamisDatasourceFieldsService streamisDatasourceFieldsService; + + @Override + public StreamisDataSourceCode transfer(StreamisTableEntity streamisTableEntity, Map labels, Map params) { + String runType = (String)params.get("runType"); + String engineType = (String)params.get("engingType"); + String version = (String)params.get("version"); + + //获取表名，列名 + StreamisDataSourceCode streamisDataSourceCode = new StreamisDataSourceCode(); + String tableName = streamisTableEntity.getTableInfo().getTableName(); + List streamisFields = streamisTableEntity.getFields(); + String cols = ""; + for(StreamisDatasourceFields streamisField : streamisFields) { + cols = streamisField.getFieldName() +" "+ streamisField.getFieldType() +","; + } + + if(EngineType.FLINK().equals(engineType) && version.equals("1.12.2") && RunType.SQL().equals(runType)){ + Map linkisDataSourceContent = streamisTableEntity.getLinkisDatasource().getLinkisDataSourceContent(); + String dataSourceType = linkisDataSourceContent.get("dataSourceType").toString(); + if(dataSourceType.equals("kafka")){ + streamisDataSourceCode = transferKafkaSql(tableName, cols, labels, linkisDataSourceContent); + }else if(dataSourceType.equals("mysql")){ + streamisDataSourceCode = transferMysql(tableName, cols, labels, linkisDataSourceContent); + } + } + return streamisDataSourceCode; + } + + public StreamisDataSourceCode transferKafkaSql(String tableName, String cols, Map labels, Map linkisDataSourceContent) { + String servers = (String)linkisDataSourceContent.get("servers"); + String topicName = (String)linkisDataSourceContent.get("topicName"); + String groupId = (String)linkisDataSourceContent.get("groupId"); + String offsetMode = (String)linkisDataSourceContent.getOrDefault("offsetMode","earliest-offset"); + String formatMode = (String)linkisDataSourceContent.getOrDefault("formatMode","formatMode"); + String executionCode = DataSourceTransferFlinksqlUtils.kafkaDataSourceTransfer(tableName , cols , topicName , servers , groupId , offsetMode , formatMode); + StreamisDataSourceCode streamisDataSourceCode = new StreamisDataSourceCode(executionCode, labels, new HashMap()); + return streamisDataSourceCode; + } + + public StreamisDataSourceCode transferMysql(String tableName, String cols, Map labels, Map linkisDataSourceContent){ + String serverIp = (String)linkisDataSourceContent.get("serverIp"); + String dbName = (String)linkisDataSourceContent.get("dbName"); + String executionCode = DataSourceTransferFlinksqlUtils.mysqlDataSourceTransfer(tableName,cols,serverIp,dbName); + StreamisDataSourceCode streamisDataSourceCode = new StreamisDataSourceCode(executionCode, labels, new HashMap()); + return streamisDataSourceCode; + } + + + @Override + public StreamisTableEntity getStreamisTableMetaById(String streamisTableMetaId) throws ErrorException{ + StreamisTableEntity streamisTableEntity = new StreamisTableEntity(); + try { + //查询表信息 + StreamisTableMeta tableInfo = streamisTableMetaService.getById(streamisTableMetaId); + //查询列信息 + QueryWrapper streamisDatasourceFieldsWrapper = new QueryWrapper<>(); + streamisDatasourceFieldsWrapper.eq("streamis_table_meta_id",streamisTableMetaId); + List fields = streamisDatasourceFieldsService.list(streamisDatasourceFieldsWrapper); + //查询linkis数据源信息 + String linkisDatasourceName = tableInfo.getLinkisDatasourceName(); + String workspaceName = tableInfo.getWorkspaceName(); + String linkisDataSourceUniqueId = workspaceName + "_" + linkisDatasourceName; + //通过linkisDatasourceName查询linkis数据源信息 + QueryDataSourceResult queryDataSourceResult = LinkisDataSourceClient.queryDataSource(linkisDatasourceName); + //拿到linkis数据源里的连接信息 + Map linkisDataSourceContent = queryDataSourceResult.getQuery_list().get(0).getConnectParams(); + LinkisDataSource linkisDatasource = new LinkisDataSource(linkisDataSourceUniqueId, linkisDataSourceContent); + + //返回实体 + streamisTableEntity.setTableInfo(tableInfo); + streamisTableEntity.setFields(fields); + streamisTableEntity.setLinkisDatasource(linkisDatasource); + } catch (Exception e) { + logger.error("传入streamisTableMetaId {} 查询元数据失败：", streamisTableMetaId, e); + throw new ErrorException(30203, "抱歉，后台查询streamis元数据失败"); + } + return streamisTableEntity; + } +} diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/KafkaLinkisDataSource.java b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/KafkaLinkisDataSource.java new file mode 100644 index 000000000..3d61313b4 --- /dev/null +++ b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/KafkaLinkisDataSource.java @@ -0,0 +1,26 @@ +package com.webank.wedatasphere.streamis.datasource.transfer.service.impl; + +import org.springframework.stereotype.Service; +import java.util.Map; + +@Service +public class KafkaLinkisDataSource extends AbstractLinkisDataSource { + private String workspaceName; + private String linkisDataSourceName; + private String linkisDataSourceUniqueId; + private Map linkisDataSourceContent; + + + public Map getLinkisDataSourceContent(Map connectParams) { + this.linkisDataSourceContent = connectParams; + return linkisDataSourceContent; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + public void setLinkisDataSourceName(String linkisDataSourceName) { + this.linkisDataSourceName = linkisDataSourceName; + } +} diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala b/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala new file mode 100644 index 000000000..9f6e6d063 --- /dev/null +++ b/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala @@ -0,0 +1,55 @@ +package com.webank.wedatasphere.streamis.datasource.transfer.client + +import java.util.concurrent.TimeUnit + +import com.webank.wedatasphere.linkis.datasource.client.impl.LinkisDataSourceRemoteClient +import com.webank.wedatasphere.linkis.datasource.client.request.QueryDataSourceAction +import com.webank.wedatasphere.linkis.datasource.client.response.QueryDataSourceResult +import com.webank.wedatasphere.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy +import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfigBuilder +import com.webank.wedatasphere.streamis.datasource.transfer.conf.TransferConfiguration + +object LinkisDataSourceClient { + //Linkis Datasource Client Config + val serverUrl = TransferConfiguration.SERVER_URL.getValue + val connectionTimeout = TransferConfiguration.CONNECTION_TIMEOUT.getValue + val discoveryEnabled = TransferConfiguration.DISCOVERY_ENABLED.getValue + val discoveryFrequencyPeriod = TransferConfiguration.DISCOVERY_FREQUENCY_PERIOD.getValue + val loadbalancerEnabled = TransferConfiguration.LOAD_BALANCER_ENABLED.getValue + val maxConnectionSize = TransferConfiguration.MAX_CONNECTION_SIZE.getValue + val retryEnabled = TransferConfiguration.RETRY_ENABLED.getValue + val readTimeout = TransferConfiguration.READ_TIMEOUT.getValue + val authTokenKey = TransferConfiguration.AUTHTOKEN_KEY.getValue + val authTokenValue = TransferConfiguration.AUTHTOKEN_VALUE.getValue + val dwsVersion = TransferConfiguration.DWS_VERSION.getValue + + + val clientConfig = DWSClientConfigBuilder.newBuilder() + .addServerUrl(serverUrl) + .connectionTimeout(connectionTimeout) + .discoveryEnabled(discoveryEnabled) + .discoveryFrequency(1,TimeUnit.MINUTES) + .loadbalancerEnabled(loadbalancerEnabled) + .maxConnectionSize(maxConnectionSize) + .retryEnabled(retryEnabled) + .readTimeout(readTimeout) + .setAuthenticationStrategy(new StaticAuthenticationStrategy()) + .setAuthTokenKey(authTokenKey) + .setAuthTokenValue(authTokenValue) + .setDWSVersion(dwsVersion) + .build() + + val dataSourceClient = new LinkisDataSourceRemoteClient(clientConfig) + + def queryDataSource(linkisDatasourceName:String) : QueryDataSourceResult = { + dataSourceClient.queryDataSource(QueryDataSourceAction.builder() + .setSystem("") + .setName(linkisDatasourceName) + .setTypeId(1) + .setIdentifies("") + .setCurrentPage(1) + .setPageSize(1).build() + ) + } + +} diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/util/DataSourceTransferFlinksqlUtils.scala b/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/util/DataSourceTransferFlinksqlUtils.scala new file mode 100644 index 000000000..d42f193c5 --- /dev/null +++ b/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/util/DataSourceTransferFlinksqlUtils.scala @@ -0,0 +1,38 @@ +package com.webank.wedatasphere.streamis.datasource.transfer.util + +object DataSourceTransferFlinksqlUtils { + + val kafkaTable = + s""" + |CREATE TABLE %s ( + | %s + | ts TIMESTAMP(3) METADATA FROM 'timestamp' + |) WITH ( + | 'connector' = 'kafka', + | 'topic' = '%s', + | 'properties.bootstrap.servers' = '%s', + | 'properties.group.id' = '%s', + | 'scan.startup.mode' = '%s', + | 'format' = '%s' + |); + """.stripMargin + def kafkaDataSourceTransfer(tableName:String,cols:String,topicName:String,servers:String,groupId:String,offsetMode:String,formatMode:String) : String = { + kafkaTable.format(tableName , cols , topicName , servers , groupId , offsetMode , formatMode) + } + + val mysqlTable = + s""" + |CREATE TABLE %s ( + | %s + | ts TIMESTAMP(3) METADATA FROM 'timestamp' + |) WITH ( + | 'connector' = 'jdbc', + | 'url' = 'jdbc:mysql://%s:3306/%s', + | 'table-name' = '%s' + |); + """.stripMargin + def mysqlDataSourceTransfer(tableName:String, cols:String, serverIp:String, dbName:String) : String = { + mysqlTable.format(tableName, cols, serverIp, dbName, tableName) + } + +} From f48b21922ebd07e0a3d65e57e1add2536f5b429d Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Sun, 25 Apr 2021 13:11:20 +0800 Subject: [PATCH 002/434] stash to pull codes from github --- .../project/server/dao/ProjectMapper.java | 17 ++++++++++++ .../entity/request/DeleteProjectRequest.java | 27 +++++++++++++++++++ .../restful/StreamisProjectRestful.java | 7 +++++ .../server/service/ProjectService.java | 3 +++ .../service/impl/ProjectServiceImpl.java | 6 +++++ .../receiver/ProjectServerReceiver.scala | 9 +++++++ 6 files changed, 69 insertions(+) create mode 100644 streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/dao/ProjectMapper.java create mode 100644 streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/entity/request/DeleteProjectRequest.java create mode 100644 streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/ProjectServerReceiver.scala diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/dao/ProjectMapper.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/dao/ProjectMapper.java new file mode 100644 index 000000000..53a84c1c2 --- /dev/null +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/dao/ProjectMapper.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.streamis.project.server.dao; + +import com.webank.wedatasphere.streamis.project.server.entity.StreamisProject; +import org.apache.ibatis.annotations.Mapper; + +/** + * created by yangzhiyue on 2021/4/23 + * Description: + */ +@Mapper +public interface ProjectMapper { + + Long insertProject(StreamisProject streamisProject); + + + +} diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/entity/request/DeleteProjectRequest.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/entity/request/DeleteProjectRequest.java new file mode 100644 index 000000000..f1d65bbd3 --- /dev/null +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/entity/request/DeleteProjectRequest.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.streamis.project.server.entity.request; + +import javax.validation.constraints.NotNull; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * created by yangzhiyue on 2021/4/23 + * Description: + */ +@XmlRootElement +public class DeleteProjectRequest { + + /** + * streamis工程是全局唯一的 + */ + @NotNull(message = "streamis projetName can not null") + private String projectName; + + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } +} diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/restful/StreamisProjectRestful.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/restful/StreamisProjectRestful.java index 0c90118fc..5e4ede465 100644 --- a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/restful/StreamisProjectRestful.java +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/restful/StreamisProjectRestful.java @@ -4,6 +4,7 @@ import com.webank.wedatasphere.linkis.server.security.SecurityFilter; import com.webank.wedatasphere.streamis.project.server.entity.StreamisProject; import com.webank.wedatasphere.streamis.project.server.entity.request.CreateProjectRequest; +import com.webank.wedatasphere.streamis.project.server.entity.request.DeleteProjectRequest; import com.webank.wedatasphere.streamis.project.server.entity.request.UpdateProjectRequest; import com.webank.wedatasphere.streamis.project.server.service.ProjectService; import com.webank.wedatasphere.streamis.project.server.utils.RestfulUtils; @@ -69,6 +70,12 @@ public Response updateProject(@Context HttpServletRequest request, @Valid Update } + @POST + @Path("deleteProject") + public Response deleteProject(@Context HttpServletRequest request, @Valid DeleteProjectRequest deleteProjectRequest){ + return null; + } + diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/ProjectService.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/ProjectService.java index 208fe1df6..a20ebfd83 100644 --- a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/ProjectService.java +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/ProjectService.java @@ -18,4 +18,7 @@ public interface ProjectService { void updateProject() throws StreamisProjectErrorException; + void deleteProject() throws StreamisProjectErrorException; + + } diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/impl/ProjectServiceImpl.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/impl/ProjectServiceImpl.java index 9e1166ff6..22d195ef6 100644 --- a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/impl/ProjectServiceImpl.java +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/impl/ProjectServiceImpl.java @@ -34,4 +34,10 @@ public StreamisProject createProject(String username, CreateProjectRequest creat public void updateProject() throws StreamisProjectErrorException { } + + + @Override + public void deleteProject() throws StreamisProjectErrorException { + + } } diff --git a/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/ProjectServerReceiver.scala b/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/ProjectServerReceiver.scala new file mode 100644 index 000000000..75ec5f520 --- /dev/null +++ b/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/ProjectServerReceiver.scala @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.streamis.project.server.receiver + +/** + * created by yangzhiyue on 2021/4/23 + * Description: + */ +class ProjectServerReceiver extends Receiver{ + +} From fcab3891d648a0a5d1ae19f2c7bfed954ed9a305 Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Sun, 25 Apr 2021 22:16:12 +0800 Subject: [PATCH 003/434] commit some server and appconn codes --- pom.xml | 1 + streamis-plugins/streamis-appconn/pom.xml | 6 ++ .../exception/StreamisExternalException.java | 21 ++++ .../StreamFlowCreationOperation.java | 7 ++ .../operation/StreamFlowExportOperation.java | 7 +- .../operation/StreamFlowQueryOperation.java | 18 ++++ .../StreamisProjectCreationOperation.java | 43 +++++++- .../StreamisProjectDeleteOperation.java | 31 ++++++ .../StreamisProjectUpdateOperation.java | 1 + .../ref/StreamisProjectResponseRef.java | 6 ++ .../streamis-project-common/pom.xml | 61 +++++++++++ .../common/StreamisProjectProtocol.scala | 47 ++++++++ .../streamis-project-server/pom.xml | 8 +- .../server/service/ProjectService.java | 2 + .../service/impl/ProjectServiceImpl.java | 7 ++ .../receiver/ProjectServerReceiver.scala | 42 +++++++- .../common/protocol/StreamisProtocol.scala | 63 +++++++++-- .../streamis-workflow-server/pom.xml | 12 +++ ...n.java => StreamisFlowErrorException.java} | 6 +- .../server/service/StreamFlowService.java | 24 +++-- .../service/impl/StreamFlowServiceImpl.java | 102 ++++++++++++++++-- .../receiver/StreamisWorkflowReceiver.scala | 32 +++++- 22 files changed, 515 insertions(+), 32 deletions(-) create mode 100644 streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/exception/StreamisExternalException.java create mode 100644 streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamisProjectDeleteOperation.java create mode 100644 streamis-project/streamis-project-common/pom.xml create mode 100644 streamis-project/streamis-project-common/src/main/scala/com/webank/wedatasphere/streamis/project/common/StreamisProjectProtocol.scala rename streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/exception/{StreamisErrorException.java => StreamisFlowErrorException.java} (59%) diff --git a/pom.xml b/pom.xml index fdc324178..6c46b3dbd 100644 --- a/pom.xml +++ b/pom.xml @@ -30,6 +30,7 @@ streamis-plugins/streamis-plugins-common streamis-project streamis-project/streamis-project-server + streamis-project/streamis-project-common diff --git a/streamis-plugins/streamis-appconn/pom.xml b/streamis-plugins/streamis-appconn/pom.xml index 98a5c421f..c99218d3e 100644 --- a/streamis-plugins/streamis-appconn/pom.xml +++ b/streamis-plugins/streamis-appconn/pom.xml @@ -113,6 +113,12 @@ ${streamis.version} + + com.webank.wedatasphere.streamis + streamis-project-common + ${streamis.version} + + diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/exception/StreamisExternalException.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/exception/StreamisExternalException.java new file mode 100644 index 000000000..f84c88b3e --- /dev/null +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/exception/StreamisExternalException.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.streamis.appconn.exception; + +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +/** + * created by yangzhiyue on 2021/4/25 + * Description: + */ +public class StreamisExternalException extends ExternalOperationFailedException { + + + + public StreamisExternalException(int errorCode, String message) { + super(errorCode, message); + } + + public StreamisExternalException(int errorCode, String message, Throwable cause) { + super(errorCode, message, cause); + this.initCause(cause); + } +} diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowCreationOperation.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowCreationOperation.java index ee55331a6..d20b6f9b0 100644 --- a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowCreationOperation.java +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowCreationOperation.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.streamis.appconn.operation; +import com.google.common.collect.Lists; import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; import com.webank.wedatasphere.dss.standard.app.development.DevelopmentService; import com.webank.wedatasphere.dss.standard.app.development.crud.RefCreationOperation; @@ -15,6 +16,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + /** * created by yangzhiyue on 2021/4/20 * Description:用于创建流式工作流的operation @@ -51,12 +54,16 @@ private void init(){ public StreamFlowCreateResponseRef createRef(StreamFlowCreateRequestRef streamFlowCreateRequestRef) throws ExternalOperationFailedException { LOGGER.info("begin to create stream flow for user {}, ref is {}", streamFlowCreateRequestRef.getUserName(), StreamisUtils.COMMON_GSON.toJson(streamFlowCreateRequestRef)); + //todo linkedAppConnName先写死吧 + List linkedAppConnNames = Lists.newArrayList("source","sink"); StreamFlowCreateRequest streamFlowCreateRequest = new StreamFlowCreateRequest(streamFlowCreateRequestRef.getDSSOrchestratorInfo().getName(), streamFlowCreateRequestRef.getUserName(), streamFlowCreateRequestRef.getDSSOrchestratorInfo().getDesc(), streamFlowCreateRequestRef.getWorkspaceName(), streamFlowCreateRequestRef.getParentFlowId(), streamFlowCreateRequestRef.getDSSOrchestratorInfo().getUses(), + linkedAppConnNames, + streamFlowCreateRequestRef.getContextIDStr(), streamFlowCreateRequestRef.getProjectId(), streamFlowCreateRequestRef.getProjectName()); StreamFlowCreateResponse response = null; diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowExportOperation.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowExportOperation.java index 1bc0288c1..e39c4cafb 100644 --- a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowExportOperation.java +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowExportOperation.java @@ -48,8 +48,11 @@ private void init() { @Override public StreamFlowExportResponseRef exportRef(StreamFlowExportRequestRef streamFlowExportRequestRef) throws ExternalOperationFailedException { LOGGER.info("begin to export stream flow {} from streamis", streamFlowExportRequestRef.getStreamFlowId()); - StreamFlowExportRequest streamFlowExportRequest = new StreamFlowExportRequest(streamFlowExportRequestRef.getStreamFlowId(), - streamFlowExportRequestRef.getName()); + StreamFlowExportRequest streamFlowExportRequest = new StreamFlowExportRequest(streamFlowExportRequestRef.getUserName(), + streamFlowExportRequestRef.getStreamFlowId(), + streamFlowExportRequestRef.getProjectId(), + streamFlowExportRequestRef.getProjectName(), + streamFlowExportRequestRef.getWorkspaceName()); StreamFlowExportResponse streamFlowExportResponse = null; try{ streamFlowExportResponse = (StreamFlowExportResponse) sender.ask(streamFlowExportRequest); diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowQueryOperation.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowQueryOperation.java index 721421bb4..f2145cbf2 100644 --- a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowQueryOperation.java +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowQueryOperation.java @@ -2,9 +2,12 @@ import com.webank.wedatasphere.dss.standard.app.development.DevelopmentService; import com.webank.wedatasphere.dss.standard.app.development.query.RefQueryOperation; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.linkis.rpc.Sender; import com.webank.wedatasphere.streamis.appconn.ref.StreamFlowQueryRequestRef; import com.webank.wedatasphere.streamis.appconn.ref.StreamFlowQueryResponseRef; +import com.webank.wedatasphere.streamis.plugins.common.StreamisPluginUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,6 +20,21 @@ public class StreamFlowQueryOperation implements RefQueryOperation + + + streamis + com.webank.wedatasphere.streamis + 0.1.0-SNAPSHOT + ../../pom.xml + + 4.0.0 + + streamis-project-common + + + 8 + 8 + + + + + + + com.webank.wedatasphere.linkis + linkis-common + + + + com.webank.wedatasphere.linkis + linkis-protocol + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + src/main/java + + **/*.xml + + + + + + \ No newline at end of file diff --git a/streamis-project/streamis-project-common/src/main/scala/com/webank/wedatasphere/streamis/project/common/StreamisProjectProtocol.scala b/streamis-project/streamis-project-common/src/main/scala/com/webank/wedatasphere/streamis/project/common/StreamisProjectProtocol.scala new file mode 100644 index 000000000..98ea6c2b5 --- /dev/null +++ b/streamis-project/streamis-project-common/src/main/scala/com/webank/wedatasphere/streamis/project/common/StreamisProjectProtocol.scala @@ -0,0 +1,47 @@ +package com.webank.wedatasphere.streamis.project.common + +import com.webank.wedatasphere.linkis.protocol.Protocol + +/** + * created by yangzhiyue on 2021/4/25 + * Description: + */ +trait StreamisProjectProtocol extends Protocol + + + + +case class CreateStreamProjectRequest(projectName:String, + description:String, + createBy:String) extends StreamisProjectProtocol + + +case class CreateStreamProjectResponse(status:Int, + projectName:String, + streamisProjectId:Long, + errorMessage:String) extends StreamisProjectProtocol + + + + +case class UpdateStreamProjectRequest(streamisProjectId:Long, + projectName:String, + description:String, + updateBy:String) extends StreamisProjectProtocol + + +case class UpdateStreamProjectResponse(status:Int, + streamisProjectId:Long, + errorMessage:String)extends StreamisProjectProtocol + + +case class DeleteStreamProjectRequest(streamisProjectId:Long, + projectName:String) extends StreamisProjectProtocol + + +case class DeleteStreamProjectResponse(status:Long, + projectName:String, + errorMessage:String) extends StreamisProjectProtocol + + + diff --git a/streamis-project/streamis-project-server/pom.xml b/streamis-project/streamis-project-server/pom.xml index 532cb9b19..f8a872af5 100644 --- a/streamis-project/streamis-project-server/pom.xml +++ b/streamis-project/streamis-project-server/pom.xml @@ -41,10 +41,16 @@ com.webank.wedatasphere.streamis - streamis-workflow-common + streamis-project-common ${streamis.version} + + com.webank.wedatasphere.linkis + linkis-rpc + ${linkis.version} + + org.apache.commons diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/ProjectService.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/ProjectService.java index a20ebfd83..f692f69e3 100644 --- a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/ProjectService.java +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/ProjectService.java @@ -1,6 +1,7 @@ package com.webank.wedatasphere.streamis.project.server.service; +import com.webank.wedatasphere.streamis.project.common.CreateStreamProjectRequest; import com.webank.wedatasphere.streamis.project.server.entity.StreamisProject; import com.webank.wedatasphere.streamis.project.server.entity.request.CreateProjectRequest; import com.webank.wedatasphere.streamis.project.server.exception.StreamisProjectErrorException; @@ -13,6 +14,7 @@ public interface ProjectService { StreamisProject createProject(String username, CreateProjectRequest createProjectRequest) throws StreamisProjectErrorException; + StreamisProject createProject(CreateStreamProjectRequest createStreamProjectRequest) throws StreamisProjectErrorException; void updateProject() throws StreamisProjectErrorException; diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/impl/ProjectServiceImpl.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/impl/ProjectServiceImpl.java index 22d195ef6..b294c8711 100644 --- a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/impl/ProjectServiceImpl.java +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/impl/ProjectServiceImpl.java @@ -1,6 +1,7 @@ package com.webank.wedatasphere.streamis.project.server.service.impl; +import com.webank.wedatasphere.streamis.project.common.CreateStreamProjectRequest; import com.webank.wedatasphere.streamis.project.server.entity.StreamisProject; import com.webank.wedatasphere.streamis.project.server.entity.request.CreateProjectRequest; import com.webank.wedatasphere.streamis.project.server.exception.StreamisProjectErrorException; @@ -30,6 +31,12 @@ public StreamisProject createProject(String username, CreateProjectRequest creat return null; } + @Override + public StreamisProject createProject(CreateStreamProjectRequest createStreamProjectRequest) throws StreamisProjectErrorException { + LOGGER.info("user {} starts to create project {}", createStreamProjectRequest.createBy(), createStreamProjectRequest.projectName()); + return null; + } + @Override public void updateProject() throws StreamisProjectErrorException { diff --git a/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/ProjectServerReceiver.scala b/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/ProjectServerReceiver.scala index 75ec5f520..e970c0b38 100644 --- a/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/ProjectServerReceiver.scala +++ b/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/ProjectServerReceiver.scala @@ -1,9 +1,47 @@ package com.webank.wedatasphere.streamis.project.server.receiver +import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} +import com.webank.wedatasphere.linkis.rpc.{Receiver, Sender} +import com.webank.wedatasphere.streamis.project.common.{CreateStreamProjectRequest, CreateStreamProjectResponse} +import com.webank.wedatasphere.streamis.project.server.entity.request.CreateProjectRequest +import com.webank.wedatasphere.streamis.project.server.service.ProjectService +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +import scala.concurrent.duration.Duration + /** * created by yangzhiyue on 2021/4/23 - * Description: + * Description: streamis project 支持rest 和 rpc两种方式 + * 本receiver是rpc的方式 */ -class ProjectServerReceiver extends Receiver{ +@Component +class ProjectServerReceiver extends Receiver with Logging{ + + + + @Autowired + private var projectService:ProjectService = _ + + + override def receive(message: Any, sender: Sender): Unit = { + + } + + override def receiveAndReply(message: Any, sender: Sender): Any = receiveAndReply(message, Duration.create(300, "seconds"), sender) + + override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = message match { + case createStreamProjectRequest: CreateStreamProjectRequest => + Utils.tryCatch{ + val streamisProject = projectService.createProject(createStreamProjectRequest) + CreateStreamProjectResponse(0, streamisProject.getName, streamisProject.getId, "") + }{ + t => error("failed to create project in streamis", t) + CreateStreamProjectResponse(-1, createStreamProjectRequest.projectName, -1, t.getCause.getMessage) + } + case _ => + } + + } diff --git a/streamis-workflow/streamis-workflow-common/src/main/scala/com/webank/wedatasphere/streamis/workflow/common/protocol/StreamisProtocol.scala b/streamis-workflow/streamis-workflow-common/src/main/scala/com/webank/wedatasphere/streamis/workflow/common/protocol/StreamisProtocol.scala index ed7f74016..9e148db00 100644 --- a/streamis-workflow/streamis-workflow-common/src/main/scala/com/webank/wedatasphere/streamis/workflow/common/protocol/StreamisProtocol.scala +++ b/streamis-workflow/streamis-workflow-common/src/main/scala/com/webank/wedatasphere/streamis/workflow/common/protocol/StreamisProtocol.scala @@ -28,24 +28,75 @@ case class StreamFlowCreateRequest(flowName:String, workspaceName:String, parentFlowId:Long, uses:String, + linkedAppConnNames:java.util.List[String], + contextId:String, projectId:Long, projectName:String ) extends StreamisProtocol -case class StreamFlowCreateResponse(flowId:Long, +case class StreamFlowCreateResponse(status:Int, + flowId:Long, flowVersion:String, - flowName:String) extends StreamisProtocol + flowName:String, + errorMessage:String) extends StreamisProtocol -case class StreamFlowExportRequest(streamFlowId:Long, - StreamFlowName:String) extends StreamisProtocol +case class StreamFlowExportRequest(username:String, + streamFlowId:Long, + projectId:Long, + projectName:String, + workspaceStr:String + ) extends StreamisProtocol + + +case class StreamFlowCopyRequest(username:String, + workspaceName:String, + streamFlowId:Long, + contextId:String, + projectName:String, + orchestratorVersionId:Long, + version:String, + description:String, + orchestratorId:Long) extends StreamisProtocol + + +case class StreamFlowCopyResponse(status:Int, + oldStreamFlowId:Long, + newStreamFlowId:Long, + orchestratorId:Long, + errorMessage:String) extends StreamisProtocol -case class StreamFlowExportResponse(streamFlowId:Long, + + + +case class StreamFlowExportResponse(status:Int, + streamFlowId:Long, streamFlowName:String, bmlResourceId:String, - bmlVersion:String) extends StreamisProtocol \ No newline at end of file + bmlVersion:String, + errorMessage:String) extends StreamisProtocol + + +case class StreamFlowImportRequest(streamFlowName:String, + username:String, + projectName:String, + projectId:Long, + bmlResourceId:String, + bmlVersion:String, + orchestratorVersion:String, + workspaceName:String, + workspaceStr:String + ) extends StreamisProtocol + + +case class StreamFlowImportResponse(status:Int, + streamFlowName:String, + errorMessage:String) extends StreamisProtocol + + +case class StreamFlowQueryRequest() \ No newline at end of file diff --git a/streamis-workflow/streamis-workflow-server/pom.xml b/streamis-workflow/streamis-workflow-server/pom.xml index 918399394..97202f9e0 100644 --- a/streamis-workflow/streamis-workflow-server/pom.xml +++ b/streamis-workflow/streamis-workflow-server/pom.xml @@ -37,12 +37,24 @@ ${linkis.version} + + com.webank.wedatasphere.linkis + linkis-rpc + ${linkis.version} + + com.webank.wedatasphere.streamis streamis-workflow-common ${streamis.version} + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + + org.apache.commons diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/exception/StreamisErrorException.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/exception/StreamisFlowErrorException.java similarity index 59% rename from streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/exception/StreamisErrorException.java rename to streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/exception/StreamisFlowErrorException.java index ce38c07d7..65a080d6b 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/exception/StreamisErrorException.java +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/exception/StreamisFlowErrorException.java @@ -6,17 +6,17 @@ * created by yangzhiyue on 2021/4/19 * Description: */ -public class StreamisErrorException extends ErrorException { +public class StreamisFlowErrorException extends ErrorException { - public StreamisErrorException(int errCode, String desc) { + public StreamisFlowErrorException(int errCode, String desc) { super(errCode, desc); } - public StreamisErrorException(int errCode, String desc, Throwable t){ + public StreamisFlowErrorException(int errCode, String desc, Throwable t){ super(errCode, desc); this.initCause(t); } diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java index 54b50bf54..867461ab6 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java @@ -1,7 +1,13 @@ package com.webank.wedatasphere.streamis.workflow.server.service; -import com.webank.wedatasphere.streamis.workflow.server.entity.StreamisFlow; -import com.webank.wedatasphere.streamis.workflow.server.exception.StreamisErrorException; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowCopyRequest; +import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowCreateRequest; +import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowExportRequest; +import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowImportRequest; +import com.webank.wedatasphere.streamis.workflow.server.exception.StreamisFlowErrorException; + +import java.util.List; /** * created by yangzhiyue on 2021/4/19 @@ -9,14 +15,18 @@ */ public interface StreamFlowService { - StreamisFlow createStreamFlow(); + DSSFlow createStreamFlow(StreamFlowCreateRequest streamFlowCreateRequest) throws StreamisFlowErrorException; + + void exportStreamFlow(StreamFlowExportRequest streamFlowExportRequest) throws StreamisFlowErrorException; + + List importStreamFlow(StreamFlowImportRequest streamFlowImportRequest) throws StreamisFlowErrorException; - void updateStreamFlow(); + void updateStreamFlow() throws StreamisFlowErrorException ; - void publishStreamFlow(); + void publishStreamFlow() throws StreamisFlowErrorException; - void copyStreamFlow(); + DSSFlow copyStreamFlow(StreamFlowCopyRequest streamFlowCopyRequest) throws StreamisFlowErrorException; - void deleteStreamFlow() throws StreamisErrorException; + void deleteStreamFlow() throws StreamisFlowErrorException; } diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java index d34a08642..00b497bb8 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java @@ -1,22 +1,99 @@ package com.webank.wedatasphere.streamis.workflow.server.service.impl; -import com.webank.wedatasphere.streamis.workflow.server.entity.StreamisFlow; -import com.webank.wedatasphere.streamis.workflow.server.exception.StreamisErrorException; +import com.google.common.collect.Lists; +import com.webank.wedatasphere.dss.common.entity.DSSLabel; +import com.webank.wedatasphere.dss.common.entity.IOEnv; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.WorkFlowManager; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.entity.DSSFlowImportParam; +import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowCopyRequest; +import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowCreateRequest; +import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowExportRequest; +import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowImportRequest; +import com.webank.wedatasphere.streamis.workflow.server.exception.StreamisFlowErrorException; import com.webank.wedatasphere.streamis.workflow.server.service.StreamFlowService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; /** * created by yangzhiyue on 2021/4/19 * Description: */ +@Service public class StreamFlowServiceImpl implements StreamFlowService { + private static final Logger LOGGER = LoggerFactory.getLogger(StreamFlowServiceImpl.class); + + @Autowired + private WorkFlowManager workFlowManager; + + @Override + public DSSFlow createStreamFlow(StreamFlowCreateRequest streamFlowCreateRequest) throws StreamisFlowErrorException { + try{ + LOGGER.info("begin to create stream flow {} for user {}", streamFlowCreateRequest.flowName(), streamFlowCreateRequest.createBy()); + //todo 以后可以通过orchestrator传过来，但是创建一般都是在dev + List dssLabels = Lists.newArrayList(new DSSLabel("dev")); + DSSFlow dssFlow = workFlowManager.createWorkflow(streamFlowCreateRequest.createBy(), streamFlowCreateRequest.flowName(), + streamFlowCreateRequest.contextId(), streamFlowCreateRequest.description(), streamFlowCreateRequest.parentFlowId(), + streamFlowCreateRequest.uses(), streamFlowCreateRequest.linkedAppConnNames(),dssLabels); + LOGGER.info("end to create stream flow {} and id is {}", streamFlowCreateRequest.flowName(), dssFlow.getId()); + return dssFlow; + }catch(final Throwable t){ + LOGGER.error("Failed to create flow {} for user {} ", streamFlowCreateRequest.flowName(), streamFlowCreateRequest.createBy()); + throw new StreamisFlowErrorException(600401, "Failed to create streamflow", t); + } + } + @Override + public void exportStreamFlow(StreamFlowExportRequest streamFlowExportRequest) throws StreamisFlowErrorException{ + try{ + LOGGER.info("begin to export stream flow {} for user {}", streamFlowExportRequest.streamFlowId(), streamFlowExportRequest.streamFlowId()); + Map exportResponse = workFlowManager.exportWorkflow(streamFlowExportRequest.username(), streamFlowExportRequest.streamFlowId(), + streamFlowExportRequest.projectId(), streamFlowExportRequest.projectName(), + DSSCommonUtils.COMMON_GSON.fromJson(streamFlowExportRequest.workspaceStr(), Workspace.class)); + String resourceId = exportResponse.get("resourceId").toString(); + String version = exportResponse.get("version").toString(); + LOGGER.info("End to export stream flow, resourceId is {}, version is {}", resourceId, version); + }catch(final Throwable t){ + LOGGER.error("failed to export stream flow {} for user {}", streamFlowExportRequest.streamFlowId(), streamFlowExportRequest.username()); + throw new StreamisFlowErrorException(600402, "Failed to export streamflow", t); + } + } @Override - public StreamisFlow createStreamFlow() { - return null; + public List importStreamFlow(StreamFlowImportRequest streamFlowImportRequest) throws StreamisFlowErrorException { + try{ + LOGGER.info("begin to import flow {} for user {}, resourceId is {}, version is {}", streamFlowImportRequest.streamFlowName(), + streamFlowImportRequest.username(), streamFlowImportRequest.bmlResourceId(), streamFlowImportRequest.bmlVersion()); + DSSFlowImportParam dssFlowImportParam = new DSSFlowImportParam(); + dssFlowImportParam.setProjectID(streamFlowImportRequest.projectId()); + dssFlowImportParam.setProjectName(streamFlowImportRequest.projectName()); + //todo 这个env先这么写吧 + dssFlowImportParam.setSourceEnv(IOEnv.BDAP_PRODUCTION); + dssFlowImportParam.setUserName(streamFlowImportRequest.username()); + dssFlowImportParam.setOrcVersion(streamFlowImportRequest.orchestratorVersion()); + dssFlowImportParam.setWorkspaceName(streamFlowImportRequest.workspaceName()); + dssFlowImportParam.setWorkspace(DSSCommonUtils.COMMON_GSON.fromJson(streamFlowImportRequest.workspaceStr(),Workspace.class)); + List dssFlows = workFlowManager.importWorkflow(streamFlowImportRequest.username(), + streamFlowImportRequest.bmlResourceId(), + streamFlowImportRequest.bmlVersion(), + dssFlowImportParam); + LOGGER.info("end to import stream flow and dssFlowId are is {}", Arrays.toString(dssFlows.stream().map(DSSFlow::getId).toArray())); + return dssFlows; + }catch(final Throwable t){ + LOGGER.error("failed to import import stream flow {} for user {}", streamFlowImportRequest.streamFlowName(), streamFlowImportRequest.username()); + throw new StreamisFlowErrorException(600405, "failed to import stream flow", t); + } } @Override @@ -30,12 +107,23 @@ public void publishStreamFlow() { } @Override - public void copyStreamFlow() { - + public DSSFlow copyStreamFlow(StreamFlowCopyRequest streamFlowCopyRequest) throws StreamisFlowErrorException{ + try{ + LOGGER.info("begin to copy stream flow {} for user {}", streamFlowCopyRequest.streamFlowId(), streamFlowCopyRequest.username()); + DSSFlow dssFlow = workFlowManager.copyRootflowWithSubflows(streamFlowCopyRequest.username(), + streamFlowCopyRequest.streamFlowId(),streamFlowCopyRequest.workspaceName(), + streamFlowCopyRequest.projectName(), streamFlowCopyRequest.contextId(), + streamFlowCopyRequest.version(), streamFlowCopyRequest.description()); + LOGGER.error("end to copy stream flow {} for user {}, and result flow is {}", streamFlowCopyRequest.streamFlowId(), streamFlowCopyRequest.username(), dssFlow.getId()); + return dssFlow; + }catch(final Throwable t){ + LOGGER.error("failed to copy stream flow {} for user{}", streamFlowCopyRequest.streamFlowId(), streamFlowCopyRequest.username()); + throw new StreamisFlowErrorException(600406, "failed to stream flow", t); + } } @Override - public void deleteStreamFlow() throws StreamisErrorException { + public void deleteStreamFlow() throws StreamisFlowErrorException { } } diff --git a/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala b/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala index 465944d99..9946b022e 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala +++ b/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala @@ -1,9 +1,39 @@ package com.webank.wedatasphere.streamis.worflow.server.receiver +import com.webank.wedatasphere.linkis.common.utils.Logging +import com.webank.wedatasphere.linkis.rpc.{Receiver, Sender} +import com.webank.wedatasphere.streamis.workflow.common.protocol.{StreamFlowCreateRequest, StreamFlowExportRequest} +import com.webank.wedatasphere.streamis.workflow.server.service.StreamFlowService +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +import scala.concurrent.duration.Duration + /** * created by yangzhiyue on 2021/4/20 * Description: */ -class StreamisWorkflowReceiver { +@Component +class StreamisWorkflowReceiver extends Receiver with Logging{ + + @Autowired + var streamFlowService:StreamFlowService = _ + + + + + + override def receive(message: Any, sender: Sender): Unit = { + + } + + override def receiveAndReply(message: Any, sender: Sender): Any = message match { + case streamFlowCreateRequest: StreamFlowCreateRequest => streamFlowService.createStreamFlow(streamFlowCreateRequest) + case streamFlowExportRequest: StreamFlowExportRequest => streamFlowService.exportStreamFlow(streamFlowExportRequest) + + } + + override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = ??? + } From f43a18873cc61c3fbc10d8a994d0a4f26a9b70df Mon Sep 17 00:00:00 2001 From: Dlimeng <77954309@qq.com> Date: Mon, 26 Apr 2021 11:09:37 +0800 Subject: [PATCH 004/434] jobmanager all query --- .../db/streamis-jobmanager.sql | 335 ++++++++++++++++++ pom.xml | 21 ++ .../streamis-job-launcher/pom.xml | 6 +- .../jobmanager/launcher/JobLauncher.java | 4 - .../streamis-job-launcher-base/pom.xml | 54 +++ .../jobmanager/launcher/dao/ConfigMapper.java | 23 ++ .../launcher/dao/JobUserRoleMapper.java | 27 ++ .../launcher/dao/impl/ConfigMapper.xml | 128 +++++++ .../launcher/dao/impl/JobUserRoleMapper.xml | 105 ++++++ .../jobmanager/launcher/entity/ConfigKey.java | 133 +++++++ .../launcher/entity/ConfigKeyValue.java | 166 +++++++++ .../jobmanager/launcher/entity/JobRole.java | 51 +++ .../jobmanager/launcher/entity/JobUser.java | 40 +++ .../launcher/entity/JobUserRole.java | 52 +++ .../jobmanager/launcher/entity/User.java | 34 ++ .../entity/dto/ConfigKeyValueDTO.java | 79 +++++ .../launcher/entity/vo/ConfigKeyVO.java | 70 ++++ .../launcher/entity/vo/ConfigRelationVO.java | 83 +++++ .../streamis-job-launcher-service/pom.xml | 50 +++ .../jobmanager/launcher/conf/ConfigConf.scala | 30 ++ .../service/ConfigurationService.scala | 202 +++++++++++ .../streamis-job-manager/pom.xml | 6 +- .../streamis-job-manager-base/pom.xml | 54 +++ .../manager/dao/StreamBmlMapper.java | 23 ++ .../manager/dao/StreamJobMapper.java | 40 +++ .../manager/dao/StreamProjectMapper.java | 19 + .../manager/dao/StreamTaskMapper.java | 20 ++ .../manager/dao/impl/StreamBmlMapper.xml | 104 ++++++ .../manager/dao/impl/StreamJobMapper.xml | 164 +++++++++ .../manager/dao/impl/StreamProjectMapper.xml | 56 +++ .../manager/dao/impl/StreamTaskMapper.xml | 86 +++++ .../jobmanager/manager/entity/StreamBml.java | 49 +++ .../manager/entity/StreamBmlVersion.java | 52 +++ .../jobmanager/manager/entity/StreamJob.java | 146 ++++++++ .../manager/entity/StreamJobSqlResource.java | 34 ++ .../manager/entity/StreamJobVersion.java | 61 ++++ .../manager/entity/StreamProject.java | 43 +++ .../jobmanager/manager/entity/StreamTask.java | 90 +++++ .../manager/entity/vo/JobDetailsVO.java | 179 ++++++++++ .../manager/entity/vo/JobFlowVO.java | 126 +++++++ .../manager/entity/vo/JobHistoryVO.java | 85 +++++ .../manager/entity/vo/JobProgressVO.java | 22 ++ .../manager/entity/vo/JobUploadDetailsVO.java | 106 ++++++ .../manager/entity/vo/QueryJobListVO.java | 76 ++++ .../manager/entity/vo/StreamJobVO.java | 97 +++++ .../manager/entity/vo/StreamTaskListVO.java | 99 ++++++ .../manager/entity/vo/TaskCoreNumVO.java | 77 ++++ .../jobmanager/manager/util/DateUtils.scala | 54 +++ .../streamis-job-manager-service/pom.xml | 51 +++ .../jobmanager/manager/conf/JobConf.scala | 23 ++ .../manager/service/JobService.scala | 99 ++++++ .../manager/service/ProjectService.scala | 12 + .../manager/service/TaskService.scala | 147 ++++++++ .../streamis-jobmanager-server/pom.xml | 69 ++++ .../exception/ConfigurationException.java | 13 + .../jobmanager/exception/JobException.java | 14 + .../exception/JobExceptionManager.java | 18 + .../exception/ProjectException.java | 10 + .../restful/api/ConfigurationRestfulApi.java | 66 ++++ .../jobmanager/restful/api/JobRestfulApi.java | 231 ++++++++++++ .../restful/api/ProjectRestfulApi.java | 38 ++ .../src/main/resources/application.yml | 20 ++ .../src/main/resources/linkis.properties | 40 +++ .../src/main/resources/log4j.properties | 36 ++ .../src/main/resources/log4j2.xml | 38 ++ 65 files changed, 4580 insertions(+), 6 deletions(-) create mode 100644 docs/zh_CN/0.1.0/development/Table_Structure_documentation/db/streamis-jobmanager.sql mode change 100644 => 100755 streamis-jobmanager/streamis-job-launcher/pom.xml delete mode 100644 streamis-jobmanager/streamis-job-launcher/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/JobLauncher.java create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/ConfigMapper.java create mode 100644 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/JobUserRoleMapper.java create mode 100644 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/ConfigMapper.xml create mode 100644 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/JobUserRoleMapper.xml create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/ConfigKey.java create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/ConfigKeyValue.java create mode 100644 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobRole.java create mode 100644 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUser.java create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUserRole.java create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/User.java create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/dto/ConfigKeyValueDTO.java create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/vo/ConfigKeyVO.java create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/vo/ConfigRelationVO.java create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml create mode 100644 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/ConfigConf.scala create mode 100644 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/service/ConfigurationService.scala mode change 100644 => 100755 streamis-jobmanager/streamis-job-manager/pom.xml create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamBmlMapper.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamProjectMapper.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamTaskMapper.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamBmlMapper.xml create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamProjectMapper.xml create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamBml.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamBmlVersion.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJob.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobSqlResource.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersion.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamProject.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamTask.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobDetailsVO.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobFlowVO.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobHistoryVO.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobProgressVO.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobUploadDetailsVO.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/QueryJobListVO.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/StreamJobVO.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/StreamTaskListVO.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/TaskCoreNumVO.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/DateUtils.scala create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/ProjectService.scala create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskService.scala create mode 100755 streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/ConfigurationException.java create mode 100644 streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobException.java create mode 100644 streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobExceptionManager.java create mode 100644 streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/ProjectException.java create mode 100644 streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java create mode 100644 streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java create mode 100644 streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java create mode 100755 streamis-jobmanager/streamis-jobmanager-server/src/main/resources/application.yml create mode 100755 streamis-jobmanager/streamis-jobmanager-server/src/main/resources/linkis.properties create mode 100755 streamis-jobmanager/streamis-jobmanager-server/src/main/resources/log4j.properties create mode 100755 streamis-jobmanager/streamis-jobmanager-server/src/main/resources/log4j2.xml diff --git a/docs/zh_CN/0.1.0/development/Table_Structure_documentation/db/streamis-jobmanager.sql b/docs/zh_CN/0.1.0/development/Table_Structure_documentation/db/streamis-jobmanager.sql new file mode 100644 index 000000000..b7ca41221 --- /dev/null +++ b/docs/zh_CN/0.1.0/development/Table_Structure_documentation/db/streamis-jobmanager.sql @@ -0,0 +1,335 @@ + +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for linkis_stream_bml +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_bml`; +CREATE TABLE `linkis_stream_bml` ( + `id` bigint(20) NOT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `bml_type` tinyint(1) NULL DEFAULT NULL, + `org_identification` bigint(20) NULL DEFAULT NULL, + ` latest_version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_bml +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_bml_version +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_bml_version`; +CREATE TABLE `linkis_stream_bml_version` ( + `id` bigint(20) NOT NULL, + `bml_id` bigint(20) NULL DEFAULT NULL, + `version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `storage_path` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + ` attribute` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '物料版本' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_bml_version +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_cluster +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_cluster`; +CREATE TABLE `linkis_stream_cluster` ( + `id` int(11) NOT NULL, + `yarn_conf_dir` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `hdfs_conf_dir` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `resource_manager_url` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `savepoint_dir` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = 'flink 集群信息' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_cluster +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_configuration_config_key +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_configuration_config_key`; +CREATE TABLE `linkis_stream_configuration_config_key` ( + `id` bigint(20) NOT NULL, + `key` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `description` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `default_value` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `validate_type` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `validate_range` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `is_hidden` tinyint(1) NULL DEFAULT NULL, + `is_advanced` tinyint(1) NULL DEFAULT NULL, + `level` tinyint(1) NULL DEFAULT NULL, + `treename` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `type` int(10) NULL DEFAULT NULL, + `sort` int(10) NULL DEFAULT NULL, + `status` tinyint(10) NULL DEFAULT NULL COMMENT '1 custom , 2 selected ', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE INDEX `key_index`(`key`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '配置信息' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_configuration_config_key +-- ---------------------------- +INSERT INTO `linkis_stream_configuration_config_key` VALUES (1, 'wds.linkis.flink.resource', '资源配置', '资源配置', NULL, 'None', NULL, 0, 0, 1, '资源配置', 1, 0, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (2, 'wds.linkis.flink.taskmanager.num', 'Task Managers数量', 'Task Managers数量', '4', 'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|128)$', 0, 0, 2, '资源配置', 1, 1, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (3, 'wds.linkis.flink.jobmanager.memory', 'JobManager Memory', 'JobManager Memory', '1.5', 'Regex', '^([1-9]\\d{0,2}|1000)(G|g)$', 0, 0, 2, '资源配置', 1, 2, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (4, 'wds.linkis.flink.taskmanager.memory', 'TaskManager Memory', 'TaskManager Memory', '1.5', 'Regex', '^([1-9]\\d{0,2}|1000)(G|g)$', 0, 0, 2, '资源配置', 1, 3, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (5, 'wds.linkis.flink.jobmanager.cpus', 'JobManager CPUs', 'JobManager CPUs', '1', 'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|128)$', 0, 0, 2, '资源配置', 1, 4, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (6, 'wds.linkis.flink.taskManager.cpus', 'TaskManager CPUs', 'TaskManager CPUs', '1', 'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|128)$', 0, 0, 2, '资源配置', 1, 5, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (7, 'wds.linkis.flink.custom', '自定义参数', '自定义参数', NULL, 'None', NULL, 0, 0, 1, '自定义参数', 2, 0, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (8, 'wds.linkis.flink.produce', '生产配置', '生产配置', NULL, 'None', NULL, 0, 0, 1, '生产配置', 3, 0, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (9, 'wds.linkis.flink.checkpoint.interval', 'Checkpoint间隔', 'Checkpoint间隔', NULL, NULL, NULL, 0, 0, 2, '生产配置', 3, 1, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (10, 'wds.linkis.flink.reboot.strategy', '重启策略', '重启策略', '不重启,基于Checkpoint自动重启,无Checkpoint不重启', 'None', NULL, 0, 0, 2, '重启策略', 3, 2, 2); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (11, 'wds.linkis.flink.alert', '告警设置', '告警设置', NULL, 'None', NULL, 0, 0, 1, '告警设置', 4, 0, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (12, 'wds.linkis.flink.alert.rule', '告警规则', '告警规则', '任务日志中出现ERROR/EXCEPTION,任务核心指标出现异常', 'None', NULL, 0, 0, 2, '告警规则', 4, 1, 2); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (13, 'wds.linkis.flink.alert.user', '告警用户', '告警用户', NULL, NULL, NULL, 0, 0, 2, '告警用户', 4, 3, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (14, 'wds.linkis.flink.alert.leve', '告警级别', '告警级别', 'CLEARED,INDETERMINATE,WARNING,MINOR,MAJOR,CRITICAL', 'None', NULL, 0, 0, 2, '告警级别', 4, 2, 2); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (15, 'wds.linkis.flink.alert.failure.level', '失败时告警级别', '失败时告警级别', 'CLEARED,INDETERMINATE,WARNING,MINOR,MAJOR,CRITICAL', 'None', NULL, 0, 0, 2, '失败时告警级别', 4, 4, 2); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (16, 'wds.linkis.flink.alert.failure.user', '失败时告警用户', '失败时告警用户', NULL, 'None', NULL, 0, 0, 2, '失败时告警用户', 4, 5, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (17, 'wds.linkis.flink.authority', '权限设置', '权限设置', NULL, 'None', NULL, 0, 0, 1, '权限设置', 5, 0, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (18, 'wds.linkis.flink.authority.author', '授权模式', '授权模式', '私密,指定全员可见,指定人员可见', 'None', NULL, 0, 0, 2, '授权模式', 5, 1, 2); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (19, 'wds.linkis.flink.authority.visible', '可见人员', '可见人员', NULL, 'None', NULL, 0, 0, 2, '可见人员', 5, 2, 1); + +-- ---------------------------- +-- Table structure for linkis_stream_configuration_config_value +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_configuration_config_value`; +CREATE TABLE `linkis_stream_configuration_config_value` ( + `id` bigint(20) NOT NULL, + `configkey_id` bigint(20) NULL DEFAULT NULL, + `config_value` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `type` int(10) NULL DEFAULT NULL, + `job_id` bigint(20) NULL DEFAULT NULL, + `job_name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `config_key` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE, + INDEX `key`(`config_key`) USING BTREE, + INDEX `keyid`(`configkey_id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '配置信息' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_configuration_config_value +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_frame_version +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_frame_version`; +CREATE TABLE `linkis_stream_frame_version` ( + `id` bigint(20) NOT NULL, + `frame` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `java_version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '框架信息' ROW_FORMAT = COMPACT; + +-- ---------------------------- +-- Records of linkis_stream_frame_version +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_job +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_job`; +CREATE TABLE `linkis_stream_job` ( + `id` bigint(20) NOT NULL, + `project_id` bigint(20) NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `type` tinyint(1) NULL DEFAULT NULL, + `current_task_id` bigint(20) NULL DEFAULT NULL, + `current_version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `current_release_time` datetime NULL DEFAULT NULL, + `status` tinyint(1) NULL DEFAULT NULL COMMENT '1:已完成 ，2:等待重启 ，3:告警 ，4:慢任务 ，5:运行中 ，6:失败任务', + `org_identification` bigint(20) NULL DEFAULT NULL, + `create_by` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `label` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `current_released` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `description` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '作业表' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_job +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_job_alarm_send_history +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_job_alarm_send_history`; +CREATE TABLE `linkis_stream_job_alarm_send_history` ( + `id` bigint(20) NOT NULL, + `job_id` bigint(20) NULL DEFAULT NULL, + `task_id` bigint(20) NULL DEFAULT NULL, + `create_by` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `type` tinyint(1) NULL DEFAULT NULL, + `rule_type` tinyint(1) NULL DEFAULT NULL, + `content` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '报警历史信息' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_job_alarm_send_history +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_job_checkpoints +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_job_checkpoints`; +CREATE TABLE `linkis_stream_job_checkpoints` ( + `id` bigint(20) NOT NULL, + `config_value_id` bigint(20) NULL DEFAULT NULL, + `path` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `size` int(20) NULL DEFAULT NULL, + `status` tinyint(1) NULL DEFAULT NULL, + `trigger_timestamp` datetime NULL DEFAULT NULL, + `latest_ack_timestamp` datetime NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_job_checkpoints +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_job_code_resource +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_job_code_resource`; +CREATE TABLE `linkis_stream_job_code_resource` ( + `id` bigint(20) NOT NULL, + `job_version_id` bigint(20) NULL DEFAULT NULL, + `bml_version_id` bigint(20) NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '其他代码' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_job_code_resource +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_job_role +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_job_role`; +CREATE TABLE `linkis_stream_job_role` ( + `id` bigint(20) NOT NULL, + `job_id` bigint(20) NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `front_name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `update_time` datetime NULL DEFAULT NULL, + `description` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_job_role +-- ---------------------------- +INSERT INTO `linkis_stream_job_role` VALUES (1, -1, '管理员', '管理员', '2021-04-07 20:57:09', NULL); + +-- ---------------------------- +-- Table structure for linkis_stream_job_sql_resource +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_job_sql_resource`; +CREATE TABLE `linkis_stream_job_sql_resource` ( + `id` bigint(20) NOT NULL, + `job_version_id` bigint(20) NULL DEFAULT NULL, + `execute_sql` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_job_sql_resource +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_job_user_role +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_job_user_role`; +CREATE TABLE `linkis_stream_job_user_role` ( + `id` bigint(20) NOT NULL, + `job_id` bigint(20) NULL DEFAULT NULL, + `user_id` bigint(20) NULL DEFAULT NULL, + `role_id` bigint(20) NULL DEFAULT NULL, + `type` tinyint(1) NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '作业角色关系' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_job_user_role +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_job_version +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_job_version`; +CREATE TABLE `linkis_stream_job_version` ( + `id` bigint(20) NOT NULL, + ` job_id` bigint(20) NULL DEFAULT NULL, + `version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `program_arguments` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `bml_version` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `resource_id` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_job_version +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_project +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_project`; +CREATE TABLE `linkis_stream_project` ( + `id` bigint(20) NOT NULL, + `workspace_id` bigint(20) NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `create_by` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '项目表' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_project +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_task +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_task`; +CREATE TABLE `linkis_stream_task` ( + `id` bigint(20) NOT NULL, + `job_version_id` bigint(20) NULL DEFAULT NULL, + `version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `stream_task_identification` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `status` tinyint(10) NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_task +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_user +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_user`; +CREATE TABLE `linkis_stream_user` ( + `id` bigint(20) NOT NULL, + `username` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '用户表' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_user +-- ---------------------------- +INSERT INTO `linkis_stream_user` VALUES (1, 'hdfs', 'hdfs'); + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/pom.xml b/pom.xml index 6c46b3dbd..4bb0b383a 100644 --- a/pom.xml +++ b/pom.xml @@ -59,6 +59,7 @@ 2.21 1.9.5 1.4.15 + 0.1.0-SNAPSHOT @@ -90,6 +91,26 @@ commons-lang ${commons.lang.version} + + com.webank.wedatasphere.linkis + linkis-mybatis + ${linkis.version} + + + com.webank.wedatasphere.linkis + linkis-module + + + org.springframework.boot + spring-boot-starter-tomcat + + + hibernate-validator + org.hibernate.validator + + + ${linkis.version} + com.webank.wedatasphere.linkis linkis-common diff --git a/streamis-jobmanager/streamis-job-launcher/pom.xml b/streamis-jobmanager/streamis-job-launcher/pom.xml old mode 100644 new mode 100755 index 77944f06e..d880c2b22 --- a/streamis-jobmanager/streamis-job-launcher/pom.xml +++ b/streamis-jobmanager/streamis-job-launcher/pom.xml @@ -10,6 +10,10 @@ 4.0.0 streamis-job-launcher + pom - + + streamis-job-launcher-base + streamis-job-launcher-service + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-launcher/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/JobLauncher.java b/streamis-jobmanager/streamis-job-launcher/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/JobLauncher.java deleted file mode 100644 index 33105093b..000000000 --- a/streamis-jobmanager/streamis-job-launcher/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/JobLauncher.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.webank.wedatasphere.streamis.jobmanager.launcher; - -public interface JobLauncher { -} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml new file mode 100755 index 000000000..740efecd8 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml @@ -0,0 +1,54 @@ + + + + streamis-jobmanager + com.webank.wedatasphere.streamis + 0.1.0-SNAPSHOT + ../../pom.xml + + 4.0.0 + + streamis-job-launcher-base + + + + 8 + 8 + + + + + com.webank.wedatasphere.linkis + linkis-mybatis + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + src/main/java + + **/*.xml + + + + ${project.artifactId}-${project.version} + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/ConfigMapper.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/ConfigMapper.java new file mode 100755 index 000000000..8b8161d28 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/ConfigMapper.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.dao; + +import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.ConfigKey; +import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.ConfigKeyValue; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +public interface ConfigMapper { + + List getConfigKeyValues(@Param("type") Integer type,@Param("jobId") Long jobId); + + ConfigKeyValue getConfigKeyValue(@Param("jobId") Long jobId,@Param("configkeyId") Long configkeyId); + + void deleteKeyValue(@Param("jobId") Long jobId,@Param("type") Integer type,@Param("configkeyId") Long configkeyId); + + List getConfigKey(); + + void insertValue(ConfigKeyValue configValue); + + + void updateUserValue(ConfigKeyValue configValue); +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/JobUserRoleMapper.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/JobUserRoleMapper.java new file mode 100644 index 000000000..66e7a4d7b --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/JobUserRoleMapper.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.dao; + +import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.JobUserRole; +import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.User; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +/** + * @author limeng + */ +public interface JobUserRoleMapper { + + void insertUser(User user); + + void insertJobUserRole(JobUserRole jobUserRole); + + void updateJobUserRole(JobUserRole jobUserRole); + + void deleteByJobUserRole(@Param("id") Long id,@Param("jobId") Long jobId); + + + List getUsersByUserName(@Param("username") String username); + + List getUserRoleById(@Param("jobId") Long jobId,@Param("username") String usernam); + +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/ConfigMapper.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/ConfigMapper.xml new file mode 100644 index 000000000..4a2c668b9 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/ConfigMapper.xml @@ -0,0 +1,128 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + `id`, `key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`,`type`,`sort`,`status` + + + + `id` ,`configkey_id`,`config_value`,`type`,`job_id`,`job_name`,`config_key` + + + + k.id,k.key,k.name,k.description, k.default_value, k.validate_type, k.validate_range, k.is_hidden, k.is_advanced,k.level, k.treeName,k.type,k.sort,k.`status`,v.id as value_id ,v.config_key,v.configkey_id,v.config_value,v.job_id,v.job_name + + + + + + + + + + + + + delete from linkis_stream_configuration_config_value + + 1=1 + + AND job_id=#{jobId} + + + AND `type`=#{type} + + + AND configkey_id=#{configkeyId} + + + + + + + INSERT INTO linkis_stream_configuration_config_value(`configkey_id`,`config_value`,`type`,`job_id`,`job_name`,`config_key`) VALUES(#{configkeyId},#{configValue},#{type},#{jobId},#{jobName},#{configKey}) + + + + UPDATE linkis_stream_configuration_config_value + + + config_value=#{configValue}, + + + WHERE id=#{id} + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/JobUserRoleMapper.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/JobUserRoleMapper.xml new file mode 100644 index 000000000..3889b31a7 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/JobUserRoleMapper.xml @@ -0,0 +1,105 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + `id`,`username`,`name` + + + + `id`,`job_id`,`user_id`,`role_id`,`username` + + + + INSERT INTO linkis_stream_user() VALUES(#{id},#{username},#{name}) + + + + INSERT INTO linkis_stream_job_user_role(`job_id`,`user_id`,`role_id`,`username`) VALUES(#{jobId},#{userId},#{roleId},#{username}) + + + + UPDATE linkis_stream_job_user_role + + + job_id=#{jobId}, + + + user_id=#{userId}, + + + username=#{username}, + + + where id=#{id} + + + + DELETE FROM linkis_stream_job_user_role + + 1=1 + AND id=#{id} + AND job_id=#{jobId} + + + + + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/ConfigKey.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/ConfigKey.java new file mode 100755 index 000000000..9f3d58dbe --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/ConfigKey.java @@ -0,0 +1,133 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.entity; + +public class ConfigKey { + private Long id; + private String key; + private String description; + private String name; + private String defaultValue; + private String validateType; + private String validateRange; + private Boolean isAdvanced; + private Boolean isHidden; + private Integer level; + private Integer type; + private Integer sort; + private String treeName; + private Integer status; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + + public String getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(String defaultValue) { + this.defaultValue = defaultValue; + } + + public String getValidateType() { + return validateType; + } + + public void setValidateType(String validateType) { + this.validateType = validateType; + } + + public String getValidateRange() { + return validateRange; + } + + public void setValidateRange(String validateRange) { + this.validateRange = validateRange; + } + + public Boolean getAdvanced() { + return isAdvanced; + } + + public void setAdvanced(Boolean advanced) { + isAdvanced = advanced; + } + + public Boolean getHidden() { + return isHidden; + } + + public void setHidden(Boolean hidden) { + isHidden = hidden; + } + + public Integer getLevel() { + return level; + } + + public void setLevel(Integer level) { + this.level = level; + } + + public Integer getType() { + return type; + } + + public void setType(Integer type) { + this.type = type; + } + + public Integer getSort() { + return sort; + } + + public void setSort(Integer sort) { + this.sort = sort; + } + + public String getTreeName() { + return treeName; + } + + public void setTreeName(String treeName) { + this.treeName = treeName; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/ConfigKeyValue.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/ConfigKeyValue.java new file mode 100755 index 000000000..d55ffdb1e --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/ConfigKeyValue.java @@ -0,0 +1,166 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.entity; + +public class ConfigKeyValue { + private Long id; + private String key; + private String description; + private String name; + private String defaultValue; + private String validateType; + private String validateRange; + private Integer level; + private String treeName; + private Long configkeyId; + private String configValue; + private Integer type; + private Long jobId; + private String jobName; + private Integer status; + private Integer sort; + private Long valueId; + private String configKey; + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(String defaultValue) { + this.defaultValue = defaultValue; + } + + public String getValidateType() { + return validateType; + } + + public void setValidateType(String validateType) { + this.validateType = validateType; + } + + public String getValidateRange() { + return validateRange; + } + + public void setValidateRange(String validateRange) { + this.validateRange = validateRange; + } + + public Integer getLevel() { + return level; + } + + public void setLevel(Integer level) { + this.level = level; + } + + public String getTreeName() { + return treeName; + } + + public void setTreeName(String treeName) { + this.treeName = treeName; + } + + public Long getConfigkeyId() { + return configkeyId; + } + + public void setConfigkeyId(Long configkeyId) { + this.configkeyId = configkeyId; + } + + public String getConfigValue() { + return configValue; + } + + public void setConfigValue(String configValue) { + this.configValue = configValue; + } + + public Integer getType() { + return type; + } + + public void setType(Integer type) { + this.type = type; + } + + public Long getJobId() { + return jobId; + } + + public void setJobId(Long jobId) { + this.jobId = jobId; + } + + public String getJobName() { + return jobName; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public Integer getSort() { + return sort; + } + + public void setSort(Integer sort) { + this.sort = sort; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getValueId() { + return valueId; + } + + public void setValueId(Long valueId) { + this.valueId = valueId; + } + + public String getConfigKey() { + return configKey; + } + + public void setConfigKey(String configKey) { + this.configKey = configKey; + } +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobRole.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobRole.java new file mode 100644 index 000000000..16695dcb6 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobRole.java @@ -0,0 +1,51 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.entity; + +import java.util.Date; + +public class JobRole { + private Long id; + private Long jobId; + private Long roleId; + private Date updateTime; + private String description; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getJobId() { + return jobId; + } + + public void setJobId(Long jobId) { + this.jobId = jobId; + } + + public Long getRoleId() { + return roleId; + } + + public void setRoleId(Long roleId) { + this.roleId = roleId; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUser.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUser.java new file mode 100644 index 000000000..b6860511f --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUser.java @@ -0,0 +1,40 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.entity; + +public class JobUser { + private Long id; + private Long jobId; + private Long userId; + private String username; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getJobId() { + return jobId; + } + + public void setJobId(Long jobId) { + this.jobId = jobId; + } + + public Long getUserId() { + return userId; + } + + public void setUserId(Long userId) { + this.userId = userId; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUserRole.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUserRole.java new file mode 100755 index 000000000..f736e62f9 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUserRole.java @@ -0,0 +1,52 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.entity; + +/** + * @author limeng + */ +public class JobUserRole { + private Long id; + private Long jobId; + private Long userId; + private Long roleId; + private String username; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getJobId() { + return jobId; + } + + public void setJobId(Long jobId) { + this.jobId = jobId; + } + + public Long getUserId() { + return userId; + } + + public void setUserId(Long userId) { + this.userId = userId; + } + + public Long getRoleId() { + return roleId; + } + + public void setRoleId(Long roleId) { + this.roleId = roleId; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/User.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/User.java new file mode 100755 index 000000000..fd300f5b6 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/User.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.entity; + +/** + * @author limeng + */ +public class User { + private Long id; + private String username; + private String name; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/dto/ConfigKeyValueDTO.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/dto/ConfigKeyValueDTO.java new file mode 100755 index 000000000..82c533fa7 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/dto/ConfigKeyValueDTO.java @@ -0,0 +1,79 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.entity.dto; + +/** + * @author limeng + */ +public class ConfigKeyValueDTO { + private Long configkeyId; + private String name; + private String key; + private String defaultValue; + private String value; + private Integer type; + private Integer status; + private Integer sort; + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public Integer getType() { + return type; + } + + public void setType(Integer type) { + this.type = type; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public Integer getSort() { + return sort; + } + + public void setSort(Integer sort) { + this.sort = sort; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(String defaultValue) { + this.defaultValue = defaultValue; + } + + public Long getConfigkeyId() { + return configkeyId; + } + + public void setConfigkeyId(Long configkeyId) { + this.configkeyId = configkeyId; + } +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/vo/ConfigKeyVO.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/vo/ConfigKeyVO.java new file mode 100755 index 000000000..004578922 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/vo/ConfigKeyVO.java @@ -0,0 +1,70 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.entity.vo; + +import java.util.List; + +/** + * @author limeng + */ +public class ConfigKeyVO { + + private Long jobId; + + private List resourceConfig; + + private List produceConfig; + + private List parameterConfig; + + private List alarmConfig; + + private List permissionConfig; + + + public Long getJobId() { + return jobId; + } + + public void setJobId(Long jobId) { + this.jobId = jobId; + } + + public List getResourceConfig() { + return resourceConfig; + } + + public void setResourceConfig(List resourceConfig) { + this.resourceConfig = resourceConfig; + } + + public List getProduceConfig() { + return produceConfig; + } + + public void setProduceConfig(List produceConfig) { + this.produceConfig = produceConfig; + } + + public List getParameterConfig() { + return parameterConfig; + } + + public void setParameterConfig(List parameterConfig) { + this.parameterConfig = parameterConfig; + } + + public List getAlarmConfig() { + return alarmConfig; + } + + public void setAlarmConfig(List alarmConfig) { + this.alarmConfig = alarmConfig; + } + + public List getPermissionConfig() { + return permissionConfig; + } + + public void setPermissionConfig(List permissionConfig) { + this.permissionConfig = permissionConfig; + } +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/vo/ConfigRelationVO.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/vo/ConfigRelationVO.java new file mode 100755 index 000000000..8bc9635b2 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/vo/ConfigRelationVO.java @@ -0,0 +1,83 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.entity.vo; + +import java.util.List; + +/** + * @author limeng + */ +public class ConfigRelationVO { + private Long configkeyId; + private String key; + private String name; + private String value; + private List valueLists; + + public static class ValueList{ + private String value; + private Boolean selected; + + public ValueList() { + } + + public ValueList(String value, Boolean selected) { + this.value = value; + this.selected = selected; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public Boolean getSelected() { + return selected; + } + + public void setSelected(Boolean selected) { + this.selected = selected; + } + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public List getValueLists() { + return valueLists; + } + + public void setValueLists(List valueLists) { + this.valueLists = valueLists; + } + + public Long getConfigkeyId() { + return configkeyId; + } + + public void setConfigkeyId(Long configkeyId) { + this.configkeyId = configkeyId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml new file mode 100755 index 000000000..773a5c78d --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml @@ -0,0 +1,50 @@ + + + + streamis-jobmanager + com.webank.wedatasphere.streamis + 0.1.0-SNAPSHOT + ../../pom.xml + + 4.0.0 + + streamis-job-launcher-service + + + 8 + 8 + + + + + com.webank.wedatasphere.streamis + streamis-job-launcher-base + ${jobmanager.version} + + + com.webank.wedatasphere.linkis + linkis-module + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + ${project.artifactId}-${project.version} + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/ConfigConf.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/ConfigConf.scala new file mode 100644 index 000000000..65a68317f --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/ConfigConf.scala @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.conf + +import com.webank.wedatasphere.linkis.common.conf.{CommonVars, TimeType} + +/** + * @author limeng + */ +object ConfigConf { + + val JOBMANAGER_FLINK_RESOURCE = CommonVars("wds.linkis.flink.resource", 1) + + val JOBMANAGER_FLINK_CUSTOM = CommonVars("wds.linkis.flink.custom", 2) + + val JOBMANAGER_FLINK_PRODUCE = CommonVars("wds.linkis.flink.produce", 3) + + val JOBMANAGER_FLINK_ALERT = CommonVars("wds.linkis.flink.alert", 4) + + val JOBMANAGER_FLINK_AUTHORITY = CommonVars("wds.linkis.flink.authority", 5) + + val JOBMANAGER_FLINK_CUSTOM_STATUS_ONE = CommonVars("wds.linkis.flink.custom.one", 1) + + val JOBMANAGER_FLINK_CUSTOM_STATUS_TWO = CommonVars("wds.linkis.flink.custom.two", 2) + + val JOBMANAGER_FLINK_AUTHORITY_VISIBLE = CommonVars("wds.linkis.flink.authority.visible", "wds.linkis.flink.authority.visible") + + val JOBMANAGER_FLINK_AUTHORITY_AUTHOR = CommonVars("wds.linkis.flink.authority.author", "wds.linkis.flink.authority.author") + + val JOBMANAGER_FLINK_ALERT_RULE = CommonVars("wds.linkis.flink.alert.rule", "wds.linkis.flink.alert.rule") + +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/service/ConfigurationService.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/service/ConfigurationService.scala new file mode 100644 index 000000000..f36a486e0 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/service/ConfigurationService.scala @@ -0,0 +1,202 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.service + +import com.webank.wedatasphere.linkis.common.utils.Logging +import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.ConfigConf +import com.webank.wedatasphere.streamis.jobmanager.launcher.dao.{ConfigMapper, JobUserRoleMapper} +import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.dto.ConfigKeyValueDTO +import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.{ConfigKeyValue, JobUserRole} +import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.vo.{ConfigKeyVO, ConfigRelationVO} +import org.springframework.beans.BeanUtils +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service + +import java.util +import scala.collection.JavaConverters._ +/** + * @author limeng + */ +@Service +class ConfigurationService extends Logging{ + + @Autowired private var configMapper:ConfigMapper = _ + + @Autowired private var jobUserRoleMapper:JobUserRoleMapper= _ + + def addKeyValue(vo:ConfigKeyVO): Unit ={ + + val baseList = vo.getResourceConfig.asScala.++=(vo.getAlarmConfig.asScala).++=(vo.getProduceConfig.asScala) + + baseList.foreach(f=>{ + val configKeyValue = configMapper.getConfigKeyValue(vo.getJobId, f.getConfigkeyId) + if(configKeyValue != null){ + configMapper.deleteKeyValue(vo.getJobId,null,f.getConfigkeyId) + } + + val value = new ConfigKeyValue() + value.setConfigkeyId(f.getConfigkeyId) + value.setKey(f.getKey) + value.setConfigValue(f.getValue) + value.setJobId(vo.getJobId) + configMapper.insertValue(value) + + }) + + val jobKeyTypes = configMapper.getConfigKeyValues(ConfigConf.JOBMANAGER_FLINK_CUSTOM.getValue, vo.getJobId) + if(jobKeyTypes!=null && jobKeyTypes.size()>0){ + configMapper.deleteKeyValue(vo.getJobId,ConfigConf.JOBMANAGER_FLINK_CUSTOM.getValue,null) + } + vo.getParameterConfig.asScala.foreach(f=>{ + val keyValue = new ConfigKeyValue() + keyValue.setKey(f.getKey) + keyValue.setJobId(vo.getJobId) + keyValue.setConfigValue(f.getValue) + keyValue.setType(ConfigConf.JOBMANAGER_FLINK_CUSTOM.getValue) + configMapper.insertValue(keyValue) + }) + + //权限 + vo.getPermissionConfig.asScala.filter(f=>f.getKey.equals(ConfigConf.JOBMANAGER_FLINK_AUTHORITY_AUTHOR.getValue)).foreach(f=>{ + val value = configMapper.getConfigKeyValue(vo.getJobId, f.getConfigkeyId) + if(value != null){ + configMapper.deleteKeyValue(vo.getJobId,null,f.getConfigkeyId) + } + + val keyValue = new ConfigKeyValue() + keyValue.setKey(f.getKey) + keyValue.setJobId(vo.getJobId) + keyValue.setConfigValue(f.getValue) + keyValue.setType(ConfigConf.JOBMANAGER_FLINK_AUTHORITY.getValue) + keyValue.setConfigkeyId(f.getConfigkeyId) + configMapper.insertValue(keyValue) + }) + + jobUserRoleMapper.deleteByJobUserRole(null,vo.getJobId) + vo.getPermissionConfig.asScala.filter(f=>f.getKey.equals(ConfigConf.JOBMANAGER_FLINK_AUTHORITY_VISIBLE.getValue)).foreach(f=>{ + f.getValueLists.asScala.foreach(f=>{ + val users = jobUserRoleMapper.getUsersByUserName(f.getValue) + if(users != null && users.size() > 0){ + users.asScala.foreach(f2=>{ + val role = new JobUserRole() + role.setJobId(vo.getJobId) + role.setUserId(f2.getId) + role.setUsername(f2.getUsername) + jobUserRoleMapper.insertJobUserRole(role) + }) + } + }) + }) + + } + + + def getFullTree(jobId:Long): ConfigKeyVO ={ + + val configValues = configMapper.getConfigKeyValues(null,jobId).asScala + + var dtos:List[ConfigKeyValueDTO] = null + if(configValues == null || configValues.isEmpty){ + val configKeys = configMapper.getConfigKey.asScala + dtos = configKeys.filter(f=>f.getSort != 0).map(m=>{ + val dto = new ConfigKeyValueDTO() + BeanUtils.copyProperties(m,dto) + dto.setConfigkeyId(m.getId) + dto + }).toList + }else{ + dtos = configValues.filter(f=>f.getSort != 0).map(m=>{ + val dto = new ConfigKeyValueDTO() + BeanUtils.copyProperties(m,dto) + dto.setValue(m.getConfigValue) + dto.setConfigkeyId(m.getId) + dto + }).toList + } + + val vo = configKeyValueToVO(jobId,dtos) + val jobKeyTypes = configMapper.getConfigKeyValues(ConfigConf.JOBMANAGER_FLINK_CUSTOM.getValue, jobId) + if(jobKeyTypes!=null && jobKeyTypes.size()>0){ + vo.setParameterConfig(jobKeyTypes.asScala.map(m=>{ + val vo = new ConfigRelationVO() + vo.setConfigkeyId(m.getConfigkeyId) + vo.setKey(m.getConfigKey) + vo.setName(m.getName) + vo.setValue(m.getConfigValue) + vo + }).asJava) + } + + vo.setJobId(jobId) + vo + } + + + private def configKeyValueToVO(jobId:Long,dtos:List[ConfigKeyValueDTO]): ConfigKeyVO ={ + val statusTwo = ConfigConf.JOBMANAGER_FLINK_CUSTOM_STATUS_TWO.getValue + val configVO = new ConfigKeyVO() + + val groupDDs = dtos.groupBy(_.getType).map(m=>{ + (m._1,m._2.sortBy(_.getSort).map(p => { + val vo = new ConfigRelationVO() + BeanUtils.copyProperties(p,vo) + + if (p.getStatus.equals(statusTwo)) { + val lists = p.getDefaultValue.split(",").map(f => { + if(p.getKey.equals(ConfigConf.JOBMANAGER_FLINK_ALERT_RULE.getValue) && p.getValue != null) { + val s = p.getValue.split(",") + if(s.contains(f)) new ConfigRelationVO.ValueList(f, true) + else new ConfigRelationVO.ValueList(f, false) + } + else if (p.getValue != null && f.trim.equals(p.getValue)) new ConfigRelationVO.ValueList(f, true) + else new ConfigRelationVO.ValueList(f, false) + }) + vo.setValueLists(lists.toList.asJava) + } + vo + })) + }) + + configVO.setResourceConfig(groupDDs.filter(f => f._1.equals(ConfigConf.JOBMANAGER_FLINK_RESOURCE.getValue)).flatMap(_._2).toList.asJava) + configVO.setProduceConfig(groupDDs.filter(f => f._1.equals(ConfigConf.JOBMANAGER_FLINK_PRODUCE.getValue)).flatMap(_._2).toList.asJava) + configVO.setAlarmConfig(groupDDs.filter(f => f._1.equals(ConfigConf.JOBMANAGER_FLINK_ALERT.getValue)).flatMap(_._2).toList.asJava) + + configVO.setPermissionConfig(groupDDs.filter(f => f._1.equals(ConfigConf.JOBMANAGER_FLINK_AUTHORITY.getValue)).flatMap(_._2).toList.asJava) + + //权限 + val users = jobUserRoleMapper.getUsersByUserName(null) + val roles = jobUserRoleMapper.getUserRoleById(jobId,null) + val list = new util.ArrayList[ConfigRelationVO.ValueList]() + val configRelationVO = new ConfigRelationVO() + + if(users != null && users.size() > 0){ + val lists = users.asScala.map(f => { + val value = new ConfigRelationVO.ValueList() + value.setValue(f.getUsername) + value.setSelected(false) + value + }) + if(roles != null && roles.size() > 0){ + lists.foreach(f=>{ + roles.asScala.foreach(f2=>{ + if(f2.getUsername.equals(f.getValue)){ + f.setSelected(true) + } + }) + }) + } + configRelationVO.setKey(ConfigConf.JOBMANAGER_FLINK_AUTHORITY_VISIBLE.getValue) + configRelationVO.setValueLists(list) + + } + + val permissionConfigs =new util.ArrayList[ConfigRelationVO]() + permissionConfigs.add(configVO.getPermissionConfig.asScala.filter(f=>f.getKey.equals(ConfigConf.JOBMANAGER_FLINK_AUTHORITY_AUTHOR.getValue)).head) + permissionConfigs.add(configRelationVO) + configVO.setPermissionConfig(permissionConfigs) + + configVO + } + + + + +} diff --git a/streamis-jobmanager/streamis-job-manager/pom.xml b/streamis-jobmanager/streamis-job-manager/pom.xml old mode 100644 new mode 100755 index a2bd21064..4a5af7de7 --- a/streamis-jobmanager/streamis-job-manager/pom.xml +++ b/streamis-jobmanager/streamis-job-manager/pom.xml @@ -10,6 +10,10 @@ 4.0.0 streamis-job-manager - + pom + + streamis-job-manager-base + streamis-job-manager-service + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml new file mode 100755 index 000000000..93e5148be --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml @@ -0,0 +1,54 @@ + + + + streamis-jobmanager + com.webank.wedatasphere.streamis + 0.1.0-SNAPSHOT + ../../pom.xml + + 4.0.0 + + streamis-job-manager-base + + + + 8 + 8 + + + + + com.webank.wedatasphere.linkis + linkis-mybatis + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + src/main/java + + **/*.xml + + + + ${project.artifactId}-${project.version} + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamBmlMapper.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamBmlMapper.java new file mode 100644 index 000000000..d39cf8ebd --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamBmlMapper.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.dao; + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamBml; +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamBmlVersion; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +public interface StreamBmlMapper { + + void insertBml(StreamBml streamBml); + + void updateBml(StreamBml streamBml); + + List getStreamBmlById(@Param("id") Long id); + + void insertBmlVersion(StreamBmlVersion streamBmlVersion); + + void updateBmlVersion(StreamBmlVersion streamBmlVersion); + + List getStreamBmlVersionById(@Param("bmlId") Long bmlId,@Param("version") String version); + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java new file mode 100644 index 000000000..07d885720 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java @@ -0,0 +1,40 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.dao; + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob; +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJobSqlResource; +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJobVersion; +import org.apache.ibatis.annotations.Param; + +import java.util.List; +import java.util.Map; + +/** + * @author limeng + */ +public interface StreamJobMapper { + + List getJobLists(@Param("projectId") Long projectId,@Param("name") String name,@Param("status") Integer status,@Param("createBy") String createBy); + + StreamJob getJobById(@Param("jobId") Long jobId); + + List getByJobCount(@Param("projectId") Long projectId); + + List getJobSQLResourcesByJobId(@Param("jobVersionId") Long jobVersionId); + + List getJobVersionsById(@Param("jobId") Long jobId,@Param("version") String version); + + void insertJob(StreamJob streamJob); + + void insertJobVersion(StreamJobVersion streamJobVersion); + + void insertJobSqlResource(StreamJobSqlResource streamJobSqlResource); + + void updateJob(StreamJob streamJob); + + void updateJobVersion(StreamJobVersion streamJobVersion); + + void updateJobSqlResource(StreamJobSqlResource streamJobSqlResource); + + + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamProjectMapper.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamProjectMapper.java new file mode 100755 index 000000000..553251cf0 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamProjectMapper.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.dao; + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamProject; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +/** + * @author limeng + */ +public interface StreamProjectMapper { + + void instertProject(StreamProject streamProject); + + void updateProject(StreamProject streamProject); + + List getByProjects(@Param("id") Long id,@Param("workspaceId") Long workspaceId); + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamTaskMapper.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamTaskMapper.java new file mode 100755 index 000000000..a90c3f42b --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamTaskMapper.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.dao; + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamTask; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +/** + * @author limeng + */ +public interface StreamTaskMapper { + + void insertTask(StreamTask streamTask); + + void updateTask(StreamTask streamTask); + + List getByJobIds(@Param("jobId") Long jobId,@Param("version") String version); + + StreamTask getTaskById(@Param("id") Long id); +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamBmlMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamBmlMapper.xml new file mode 100755 index 000000000..19b875c62 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamBmlMapper.xml @@ -0,0 +1,104 @@ + + + + + + + + + + + + + + + + + + + + + + + + + `id`,`name`,`bml_type`,`org_identification`,`latest_version` + + + + `id`,`bml_id`,`version`,`storage_path`,`attribute` + + + + INSERT INTO linkis_stream_bml() VALUES(#{id},#{name},#{bmlType},#{orgIdentification},#{latestVersion}) + + + + UPDATE linkis_stream_bml + + + name=#{name}, + + + bml_type=#{bmlType}, + + + latest_version=#{latestVersion} + + + WHERE id=#{id} + + + + + + INSERT INTO linkis_stream_bml_version() VALUES(#{id},#{bmlId},#{version},#{storagePath},#{attribute}) + + + + UPDATE linkis_stream_bml_version + + + storage_path=#{storagePath}, + + + `attribute`=#{attribute}, + + + WHERE id=#{id} + + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml new file mode 100644 index 000000000..96a89e105 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml @@ -0,0 +1,164 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + `id`,`project_id`,`name`,`current_task_id`,`current_version`,`current_release_time`,`org_identification`,`create_by`,`label`,`description` + + + + `id`,`job_id`,`version`,`program_arguments`,`bml_version`,`bml_id` + + + + `id`,`job_version_id`,`execute_sql` + + + + + + + + + + + + + + + INSERT INTO linkis_stream_job(`project_id`,`name`,`current_task_id`,`current_version`,`current_release_time`,`org_identification`,`create_by`,`label`,`description`,`job_type`) VALUES(##{projectId},#{name},#{currentTaskId},#{currentVersion},#{currentReleaseTime},#{orgIdentification},#{createBy},#{label},#{description},#{jobType}) + + + + INSERT INTO linkis_stream_job_version(`job_id`,`version`,`program_arguments`,`bml_version`,`bml_id`) VALUES(#{jobId},#{version},#{programArguments},#{bmlVersion},#{bmlId}) + + + + INSERT INTO linkis_stream_job_sql_resource(`job_version_id`,`execute_sql`) VALUES(#{jobVersionId},#{executeSql}) + + + + UPDATE linkis_stream_job + + + name=#{name}, + + + current_task_id=#{currentTaskId}, + + + current_version=#{currentVersion}, + + + current_release_time=#{currentReleaseTime}, + + + create_by=#{createBy}, + + + label=#{label}, + + + WHERE id=#{id} + + + + UPDATE linkis_stream_job_version + + + program_arguments={programArguments}, + + + WHERE id=#{id} + + + + UPDATE linkis_stream_job_sql_resource + + + execute_sql=#{executeSql}, + + + WHERE id=#{id} + + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamProjectMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamProjectMapper.xml new file mode 100755 index 000000000..5bc274a3a --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamProjectMapper.xml @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + `id`,`workspace_id`,`name`,`create_by` + + + + + + INSERT INTO linkis_stream_project() VALUES(#{id},#{workspaceId},#{name},#{createBy}) + + + + UPDATE linkis_stream_project + + + name=#{name}, + + + WHERE id=#{id} + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml new file mode 100644 index 000000000..e7564e0a5 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + + + `id`,`job_version_id`,`version`,`status`,`start_time`,`end_time`,`err_desc`,`submit_user` + + + + INSERT INTO linkis_stream_task(`job_version_id`,`version`,`status`,`start_time`,`end_time`,`err_desc`,`submit_user`) VALUES(#{jobVersionId},#{version},#{status},#{startTime},#{endTime},#{errDesc},#{submitUser}) + + + + UPDATE linkis_stream_task + + + job_version_id=#{jobVersionId}, + + + version=#{version}, + + + status=#{status}, + + + start_time=#{startTime}, + + + end_time=#{endTime}, + + + WHERE id=#{id} + + + + + + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamBml.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamBml.java new file mode 100644 index 000000000..c1a56552a --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamBml.java @@ -0,0 +1,49 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity; + +public class StreamBml { + private Long id; + private String name; + private Integer bmlType; + private String orgIdentification; + private String latestVersion; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Integer getBmlType() { + return bmlType; + } + + public void setBmlType(Integer bmlType) { + this.bmlType = bmlType; + } + + public String getOrgIdentification() { + return orgIdentification; + } + + public void setOrgIdentification(String orgIdentification) { + this.orgIdentification = orgIdentification; + } + + public String getLatestVersion() { + return latestVersion; + } + + public void setLatestVersion(String latestVersion) { + this.latestVersion = latestVersion; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamBmlVersion.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamBmlVersion.java new file mode 100755 index 000000000..0e1e114ac --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamBmlVersion.java @@ -0,0 +1,52 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity; + +/** + * @author limeng + */ +public class StreamBmlVersion { + private Long id; + private Long bmlId; + private String version; + private String storagePath; + private String attribute; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getBmlId() { + return bmlId; + } + + public void setBmlId(Long bmlId) { + this.bmlId = bmlId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getStoragePath() { + return storagePath; + } + + public void setStoragePath(String storagePath) { + this.storagePath = storagePath; + } + + public String getAttribute() { + return attribute; + } + + public void setAttribute(String attribute) { + this.attribute = attribute; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJob.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJob.java new file mode 100644 index 000000000..d6b82c91e --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJob.java @@ -0,0 +1,146 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity; + +import java.util.Date; + +/** + * @author limeng + */ +public class StreamJob { + private Long id; + private Long workspaceId; + private Long projectId; + private String name; + private Integer jobType; + private Long currentTaskId; + private String currentVersion; + private Date currentReleaseTime; + private Integer status; + private Long orgIdentification; + private String createBy; + private String label; + private String currentReleased; + private String description; + private String submitUser; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + + public Long getCurrentTaskId() { + return currentTaskId; + } + + public void setCurrentTaskId(Long currentTaskId) { + this.currentTaskId = currentTaskId; + } + + public String getCurrentVersion() { + return currentVersion; + } + + public void setCurrentVersion(String currentVersion) { + this.currentVersion = currentVersion; + } + + public Date getCurrentReleaseTime() { + return currentReleaseTime; + } + + public void setCurrentReleaseTime(Date currentReleaseTime) { + this.currentReleaseTime = currentReleaseTime; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public Long getOrgIdentification() { + return orgIdentification; + } + + public void setOrgIdentification(Long orgIdentification) { + this.orgIdentification = orgIdentification; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public String getCurrentReleased() { + return currentReleased; + } + + public void setCurrentReleased(String currentReleased) { + this.currentReleased = currentReleased; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public String getSubmitUser() { + return submitUser; + } + + public void setSubmitUser(String submitUser) { + this.submitUser = submitUser; + } + + public Integer getJobType() { + return jobType; + } + + public void setJobType(Integer jobType) { + this.jobType = jobType; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobSqlResource.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobSqlResource.java new file mode 100755 index 000000000..ad4101676 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobSqlResource.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity; + +/** + * @author limeng + */ +public class StreamJobSqlResource { + private Long id; + private Long jobVersionId; + private String executeSql; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getJobVersionId() { + return jobVersionId; + } + + public void setJobVersionId(Long jobVersionId) { + this.jobVersionId = jobVersionId; + } + + public String getExecuteSql() { + return executeSql; + } + + public void setExecuteSql(String executeSql) { + this.executeSql = executeSql; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersion.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersion.java new file mode 100755 index 000000000..6cea14739 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersion.java @@ -0,0 +1,61 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity; + +/** + * @author limeng + */ +public class StreamJobVersion { + private Long id; + private Long jobId; + private String version; + private String programArguments; + private String bmlVersion; + private Long bmlId; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getJobId() { + return jobId; + } + + public void setJobId(Long jobId) { + this.jobId = jobId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getProgramArguments() { + return programArguments; + } + + public void setProgramArguments(String programArguments) { + this.programArguments = programArguments; + } + + public String getBmlVersion() { + return bmlVersion; + } + + public void setBmlVersion(String bmlVersion) { + this.bmlVersion = bmlVersion; + } + + public Long getBmlId() { + return bmlId; + } + + public void setBmlId(Long bmlId) { + this.bmlId = bmlId; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamProject.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamProject.java new file mode 100755 index 000000000..b4ee37923 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamProject.java @@ -0,0 +1,43 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity; + +/** + * @author limeng + */ +public class StreamProject { + private Long id; + private Long workspaceId; + private String name; + private String createBy; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamTask.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamTask.java new file mode 100755 index 000000000..2e85be5fa --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamTask.java @@ -0,0 +1,90 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity; + +import java.util.Date; + +/** + * @author limeng + */ +public class StreamTask { + private Long id; + private Long jobVersionId; + private String jobName; + private String submitUser; + private String version; + private Date startTime; + private Date endTime; + private String errDesc; + private Integer status; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getJobName() { + return jobName; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public Date getStartTime() { + return startTime; + } + + public void setStartTime(Date startTime) { + this.startTime = startTime; + } + + public Date getEndTime() { + return endTime; + } + + public void setEndTime(Date endTime) { + this.endTime = endTime; + } + + public String getErrDesc() { + return errDesc; + } + + public void setErrDesc(String errDesc) { + this.errDesc = errDesc; + } + + public Long getJobVersionId() { + return jobVersionId; + } + + public void setJobVersionId(Long jobVersionId) { + this.jobVersionId = jobVersionId; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public String getSubmitUser() { + return submitUser; + } + + public void setSubmitUser(String submitUser) { + this.submitUser = submitUser; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobDetailsVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobDetailsVO.java new file mode 100644 index 000000000..9ca623ea0 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobDetailsVO.java @@ -0,0 +1,179 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +import java.util.List; + +public class JobDetailsVO { + + private List realTimeTraffic; + private List dataNumber; + private List loadCondition; + + public List getRealTimeTraffic() { + return realTimeTraffic; + } + + public void setRealTimeTraffic(List realTimeTraffic) { + this.realTimeTraffic = realTimeTraffic; + } + + public List getDataNumber() { + return dataNumber; + } + + public void setDataNumber(List dataNumber) { + this.dataNumber = dataNumber; + } + + public List getLoadCondition() { + return loadCondition; + } + + public void setLoadCondition(List loadCondition) { + this.loadCondition = loadCondition; + } + + public static class RealTimeTrafficDTO{ + private String sourceKey; + private String sourceSpeed; + private String transformKey; + private String transformSpeed; + private String sinkKey; + private String sinkSpeed; + + public String getSourceKey() { + return sourceKey; + } + + public void setSourceKey(String sourceKey) { + this.sourceKey = sourceKey; + } + + public String getSourceSpeed() { + return sourceSpeed; + } + + public void setSourceSpeed(String sourceSpeed) { + this.sourceSpeed = sourceSpeed; + } + + public String getTransformKey() { + return transformKey; + } + + public void setTransformKey(String transformKey) { + this.transformKey = transformKey; + } + + public String getTransformSpeed() { + return transformSpeed; + } + + public void setTransformSpeed(String transformSpeed) { + this.transformSpeed = transformSpeed; + } + + public String getSinkKey() { + return sinkKey; + } + + public void setSinkKey(String sinkKey) { + this.sinkKey = sinkKey; + } + + public String getSinkSpeed() { + return sinkSpeed; + } + + public void setSinkSpeed(String sinkSpeed) { + this.sinkSpeed = sinkSpeed; + } + } + + public static class DataNumberDTO{ + private String dataName; + private Integer dataNumber; + + public String getDataName() { + return dataName; + } + + public void setDataName(String dataName) { + this.dataName = dataName; + } + + public Integer getDataNumber() { + return dataNumber; + } + + public void setDataNumber(Integer dataNumber) { + this.dataNumber = dataNumber; + } + } + + public static class LoadConditionDTO{ + private String type; + private String host; + private String memory; + private String totalMemory; + private String gcTotalTime; + private String gcLastTime; + private String gcLastConsume; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + + public String getGcTotalTime() { + return gcTotalTime; + } + + public void setGcTotalTime(String gcTotalTime) { + this.gcTotalTime = gcTotalTime; + } + + public String getGcLastTime() { + return gcLastTime; + } + + public void setGcLastTime(String gcLastTime) { + this.gcLastTime = gcLastTime; + } + + public String getGcLastConsume() { + return gcLastConsume; + } + + public void setGcLastConsume(String gcLastConsume) { + this.gcLastConsume = gcLastConsume; + } + + public String getMemory() { + return memory; + } + + public void setMemory(String memory) { + this.memory = memory; + } + + public String getTotalMemory() { + return totalMemory; + } + + public void setTotalMemory(String totalMemory) { + this.totalMemory = totalMemory; + } + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobFlowVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobFlowVO.java new file mode 100644 index 000000000..06580a507 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobFlowVO.java @@ -0,0 +1,126 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +/** + * 工作流程 + */ +public class JobFlowVO { + private Long jobId; + private String name; + private Boolean state; + private String source; + private String description; + private String createTime; + private String creator; + private Long projectID; + private String dssLabels; + private Long resourceId; + private String bmlVersion; + private String flowJson; + private Boolean rootFlow; + + public Long getJobId() { + return jobId; + } + + public void setJobId(Long jobId) { + this.jobId = jobId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Boolean getState() { + return state; + } + + public void setState(Boolean state) { + this.state = state; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getCreateTime() { + return createTime; + } + + public void setCreateTime(String createTime) { + this.createTime = createTime; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public Long getProjectID() { + return projectID; + } + + public void setProjectID(Long projectID) { + this.projectID = projectID; + } + + public String getDssLabels() { + return dssLabels; + } + + public void setDssLabels(String dssLabels) { + this.dssLabels = dssLabels; + } + + + + public String getBmlVersion() { + return bmlVersion; + } + + public void setBmlVersion(String bmlVersion) { + this.bmlVersion = bmlVersion; + } + + public String getFlowJson() { + return flowJson; + } + + public void setFlowJson(String flowJson) { + this.flowJson = flowJson; + } + + public Boolean getRootFlow() { + return rootFlow; + } + + public void setRootFlow(Boolean rootFlow) { + this.rootFlow = rootFlow; + } + + public Long getResourceId() { + return resourceId; + } + + public void setResourceId(Long resourceId) { + this.resourceId = resourceId; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobHistoryVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobHistoryVO.java new file mode 100644 index 000000000..359c75e3b --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobHistoryVO.java @@ -0,0 +1,85 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +public class JobHistoryVO { + private String taskId; + private String jobName; + private String creator; + private String version; + private String status; + private String startTime; + private String endTime; + private String runTime; + private String stopCause; + + public String getTaskId() { + return taskId; + } + + public void setTaskId(String taskId) { + this.taskId = taskId; + } + + public String getJobName() { + return jobName; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getStartTime() { + return startTime; + } + + public void setStartTime(String startTime) { + this.startTime = startTime; + } + + public String getEndTime() { + return endTime; + } + + public void setEndTime(String endTime) { + this.endTime = endTime; + } + + public String getRunTime() { + return runTime; + } + + public void setRunTime(String runTime) { + this.runTime = runTime; + } + + public String getStopCause() { + return stopCause; + } + + public void setStopCause(String stopCause) { + this.stopCause = stopCause; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobProgressVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobProgressVO.java new file mode 100644 index 000000000..199424739 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobProgressVO.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +public class JobProgressVO { + private Long taskId; + private Integer progress; + + public Long getTaskId() { + return taskId; + } + + public void setTaskId(Long taskId) { + this.taskId = taskId; + } + + public Integer getProgress() { + return progress; + } + + public void setProgress(Integer progress) { + this.progress = progress; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobUploadDetailsVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobUploadDetailsVO.java new file mode 100644 index 000000000..7f0513007 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobUploadDetailsVO.java @@ -0,0 +1,106 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +import java.util.List; + +/** + * jar 基本信息 + */ +public class JobUploadDetailsVO { + + private List mainJars; + private List dependentList; + private List userList; + private String programArguement; + + public List getMainJars() { + return mainJars; + } + + public void setMainJars(List mainJars) { + this.mainJars = mainJars; + } + + public List getDependentList() { + return dependentList; + } + + public void setDependentList(List dependentList) { + this.dependentList = dependentList; + } + + + public String getProgramArguement() { + return programArguement; + } + + public void setProgramArguement(String programArguement) { + this.programArguement = programArguement; + } + + public List getUserList() { + return userList; + } + + public void setUserList(List userList) { + this.userList = userList; + } + + public static class JarDependentDTO{ + private Long id; + private String name; + private String version; + private String description; + private String entryClass; + private String updateTime; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getEntryClass() { + return entryClass; + } + + public void setEntryClass(String entryClass) { + this.entryClass = entryClass; + } + + public String getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(String updateTime) { + this.updateTime = updateTime; + } + } + + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/QueryJobListVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/QueryJobListVO.java new file mode 100644 index 000000000..34ed8a2a7 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/QueryJobListVO.java @@ -0,0 +1,76 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +public class QueryJobListVO { + private Long jobId; + private String jobName; + private Integer taskStatus; + private String lastReleaseTime; + private String label; + private String version; + private String lastRelease; + private String description; + + public Long getJobId() { + return jobId; + } + + public void setJobId(Long jobId) { + this.jobId = jobId; + } + + public String getJobName() { + return jobName; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public Integer getTaskStatus() { + return taskStatus; + } + + public void setTaskStatus(Integer taskStatus) { + this.taskStatus = taskStatus; + } + + public String getLastReleaseTime() { + return lastReleaseTime; + } + + public void setLastReleaseTime(String lastReleaseTime) { + this.lastReleaseTime = lastReleaseTime; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getLastRelease() { + return lastRelease; + } + + public void setLastRelease(String lastRelease) { + this.lastRelease = lastRelease; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/StreamJobVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/StreamJobVO.java new file mode 100755 index 000000000..b4d9b96bb --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/StreamJobVO.java @@ -0,0 +1,97 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +/** + * @author limeng + */ +public class StreamJobVO { + private Long projectId; + private String name; + private Integer type; + private String orgIdentification; + private String createBy; + private String label; + private String description; + private String bmlVersion; + private String programArguments; + private String resourceId; + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Integer getType() { + return type; + } + + public void setType(Integer type) { + this.type = type; + } + + public String getOrgIdentification() { + return orgIdentification; + } + + public void setOrgIdentification(String orgIdentification) { + this.orgIdentification = orgIdentification; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getBmlVersion() { + return bmlVersion; + } + + public void setBmlVersion(String bmlVersion) { + this.bmlVersion = bmlVersion; + } + + public String getProgramArguments() { + return programArguments; + } + + public void setProgramArguments(String programArguments) { + this.programArguments = programArguments; + } + + public String getResourceId() { + return resourceId; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/StreamTaskListVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/StreamTaskListVO.java new file mode 100755 index 000000000..ecb7b4324 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/StreamTaskListVO.java @@ -0,0 +1,99 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +import java.util.Date; + +/** + * @author limeng + */ +public class StreamTaskListVO { + private Long taskId; + private Long jobVersionId; + private String jobName; + private String creator; + private String version; + private Integer status; + private String startTime; + private String endTime; + private String runTime; + private String stopCause; + + public Long getTaskId() { + return taskId; + } + + public void setTaskId(Long taskId) { + this.taskId = taskId; + } + + public Long getJobVersionId() { + return jobVersionId; + } + + public void setJobVersionId(Long jobVersionId) { + this.jobVersionId = jobVersionId; + } + + public String getJobName() { + return jobName; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public String getStartTime() { + return startTime; + } + + public void setStartTime(String startTime) { + this.startTime = startTime; + } + + public String getEndTime() { + return endTime; + } + + public void setEndTime(String endTime) { + this.endTime = endTime; + } + + public String getRunTime() { + return runTime; + } + + public void setRunTime(String runTime) { + this.runTime = runTime; + } + + public String getStopCause() { + return stopCause; + } + + public void setStopCause(String stopCause) { + this.stopCause = stopCause; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/TaskCoreNumVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/TaskCoreNumVO.java new file mode 100644 index 000000000..8c41b79d5 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/TaskCoreNumVO.java @@ -0,0 +1,77 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +/** + * job核心指标 + */ +public class TaskCoreNumVO { + private Long projectId; + //失败任务数目 + private Integer failureNum; + //运行数目 + private Integer runningNum; + //慢任务数目 + private Integer slowTaskNum; + //告警任务 + private Integer alertNum; + //等待重启数目 + private Integer waitRestartNum; + //已完成数目 + private Integer successNum; + + + public Integer getFailureNum() { + return failureNum; + } + + public void setFailureNum(Integer failureNum) { + this.failureNum = failureNum; + } + + public Integer getRunningNum() { + return runningNum; + } + + public void setRunningNum(Integer runningNum) { + this.runningNum = runningNum; + } + + public Integer getSlowTaskNum() { + return slowTaskNum; + } + + public void setSlowTaskNum(Integer slowTaskNum) { + this.slowTaskNum = slowTaskNum; + } + + public Integer getAlertNum() { + return alertNum; + } + + public void setAlertNum(Integer alertNum) { + this.alertNum = alertNum; + } + + public Integer getWaitRestartNum() { + return waitRestartNum; + } + + public void setWaitRestartNum(Integer waitRestartNum) { + this.waitRestartNum = waitRestartNum; + } + + public Integer getSuccessNum() { + return successNum; + } + + public void setSuccessNum(Integer successNum) { + this.successNum = successNum; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/DateUtils.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/DateUtils.scala new file mode 100644 index 000000000..5feb3910c --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/DateUtils.scala @@ -0,0 +1,54 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.util + +import org.slf4j.LoggerFactory + +import java.text.SimpleDateFormat +import java.util.Date + +object DateUtils extends Serializable { + + private val logger = LoggerFactory.getLogger(DateUtils.getClass) + private val serialVersionUID = 1L + /** + * 默认短日期格式 + * yyyy-MM-dd + */ + val DATE_DEFAULT_FORMAT = "yyyy-MM-dd" + + /** + * 订单号前缀 yyyyMMddHHmmss + */ + val DATETIME_ORDER_FORMAT = "yyyyMMddHHmmss" + /** + * 默认日期时间格式 + * yyyy-MM-dd HH:mm:ss + */ + val DATETIME_DEFAULT_FORMAT = "yyyy-MM-dd HH:mm:ss" + /** + * 默认时间格式 + * HH:mm:ss + */ + val TIME_DEFAULT_FORMAT = "HH:mm:ss" + /** + * 默认日期短格式 + * yyyyMMdd + */ + val DATE_DEFAULT_SHORT_FORMAT = "yyyyMMdd" + /** + * 默认日期时间格式化 + */ + val dateTimeFormat = new SimpleDateFormat(DATETIME_DEFAULT_FORMAT) + + def intervals(start:Date,end:Date): String ={ + if(start == null || end == null) return "" + val nm = 1000 * 60 + val diff = end.getTime - start.getTime + (diff/nm)+"分钟" + } + + def formatDate(dateTime:Date):String = { + if(dateTime == null) return "" + dateTimeFormat.format(dateTime) + } + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml new file mode 100755 index 000000000..a5e2990b3 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml @@ -0,0 +1,51 @@ + + + + streamis-jobmanager + com.webank.wedatasphere.streamis + 0.1.0-SNAPSHOT + ../../pom.xml + + 4.0.0 + + streamis-job-manager-service + + + + 8 + 8 + + + + + com.webank.wedatasphere.streamis + streamis-job-manager-base + ${jobmanager.version} + + + com.webank.wedatasphere.linkis + linkis-module + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + ${project.artifactId}-${project.version} + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala new file mode 100755 index 000000000..c7d243579 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.conf + +import com.webank.wedatasphere.linkis.common.conf.CommonVars + +/** + * @author limeng + */ +object JobConf { + val JOBMANAGER_FLINK_JOB_STATUS_ONE = CommonVars("wds.linkis.flink.job.status.one", 1,"complete") + + val JOBMANAGER_FLINK_JOB_STATUS_TWO = CommonVars("wds.linkis.flink.job.status.two", 2,"Wait for restart") + + val JOBMANAGER_FLINK_JOB_STATUS_THREE = CommonVars("wds.linkis.flink.job.status.three", 3,"Alert") + + val JOBMANAGER_FLINK_JOB_STATUS_FOUR = CommonVars("wds.linkis.flink.job.status.four", 4,"Slow task") + + val JOBMANAGER_FLINK_JOB_STATUS_FIVE = CommonVars("wds.linkis.flink.job.status.five", 5,"run") + + val JOBMANAGER_FLINK_JOB_STATUS_SIX = CommonVars("wds.linkis.flink.job.status.six", 6,"Failed task") + + val JOBMANAGER_FLINK_JOB_STATUS_SEVEN = CommonVars("wds.linkis.flink.job.status.seven", 7,"Stop task") + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala new file mode 100755 index 000000000..1755652a5 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala @@ -0,0 +1,99 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.service + + +import com.webank.wedatasphere.linkis.common.utils.Logging +import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf +import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamBmlMapper, StreamJobMapper} +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{JobFlowVO, JobProgressVO, QueryJobListVO, TaskCoreNumVO} +import com.webank.wedatasphere.streamis.jobmanager.manager.util.DateUtils +import org.slf4j.Logger +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ListBuffer +/** + * @author limeng + */ +@Service +class JobService extends Logging{ + + @Autowired private var streamJobMapper:StreamJobMapper=_ + @Autowired private var streamBmlMapper:StreamBmlMapper=_ + + def getByProList(projectId:Long,jobName:String,jobStatus:Integer,jobCreator:String):java.util.List[QueryJobListVO] = { + val jobLists = streamJobMapper.getJobLists(projectId, jobName, jobStatus, jobCreator).asScala + if(jobLists != null && jobLists.nonEmpty){ + val list = new ListBuffer[QueryJobListVO]() + jobLists.foreach(f=>{ + val queryJobList = new QueryJobListVO() + queryJobList.setJobId(f.getId) + queryJobList.setJobName(f.getName) + queryJobList.setLabel(f.getLabel) + queryJobList.setVersion(f.getCurrentVersion) + queryJobList.setDescription(f.getDescription) + queryJobList.setLastRelease(f.getCreateBy) + queryJobList.setLastReleaseTime(DateUtils.formatDate(f.getCurrentReleaseTime)) + queryJobList.setTaskStatus(f.getStatus) + list.append(queryJobList) + }) + return list.toList.asJava + } + null + } + + /** + * 核心指标 + */ + def countByCores(proId: Long): TaskCoreNumVO ={ + val jobs = streamJobMapper.getByJobCount(proId).asScala + if(jobs != null && jobs.nonEmpty){ + val taskNum = new TaskCoreNumVO() + taskNum.setProjectId(proId) + jobs.filter(f=> f.getStatus!= null).groupBy(_.getStatus).map(m=>(m._1,m._2.size)).foreach(fo=>{ + if(fo._1.equals(JobConf.JOBMANAGER_FLINK_JOB_STATUS_ONE.getValue)) taskNum.setSuccessNum(fo._2) + if(fo._1.equals(JobConf.JOBMANAGER_FLINK_JOB_STATUS_TWO.getValue)) taskNum.setWaitRestartNum(fo._2) + if(fo._1.equals(JobConf.JOBMANAGER_FLINK_JOB_STATUS_THREE.getValue)) taskNum.setAlertNum(fo._2) + if(fo._1.equals(JobConf.JOBMANAGER_FLINK_JOB_STATUS_FOUR.getValue)) taskNum.setSlowTaskNum(fo._2) + if(fo._1.equals(JobConf.JOBMANAGER_FLINK_JOB_STATUS_FIVE.getValue)) taskNum.setRunningNum(fo._2) + if(fo._1.equals(JobConf.JOBMANAGER_FLINK_JOB_STATUS_SIX.getValue)) taskNum.setFailureNum(fo._2) + }) + + return taskNum + } + null + } + + + def getJobFlow(jobId:Long,version:String): JobFlowVO ={ + val job = streamJobMapper.getJobById(jobId) + if(job == null) return null + val jobVersions = streamJobMapper.getJobVersionsById(jobId, version) + if(jobVersions == null || jobVersions.isEmpty) return null + val jobVersionHead = jobVersions.asScala.head + val jobFlow = new JobFlowVO() + jobFlow.setName(job.getName) + jobFlow.setJobId(jobId) + jobFlow.setCreator(job.getCreateBy) + jobFlow.setBmlVersion(jobVersionHead.getBmlVersion) + jobFlow.setResourceId(jobVersionHead.getBmlId) + jobFlow.setDescription(job.getDescription) + jobFlow + } + + + + /** + * 任务版本 + * @param jobId + */ + def getByJobVersion(jobId:Long): Unit ={ + val job = streamJobMapper.getJobById(jobId) + if(job == null) return null + val jobVersions = streamJobMapper.getJobVersionsById(jobId, job.getCurrentVersion) + if(jobVersions == null || jobVersions.isEmpty) return null + val jobVersionHead = jobVersions.asScala.head + + } + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/ProjectService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/ProjectService.scala new file mode 100755 index 000000000..e927f4f7d --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/ProjectService.scala @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.service + +import com.webank.wedatasphere.linkis.common.utils.Logging +import org.springframework.stereotype.Service + +/** + * @author limeng + */ +@Service +class ProjectService extends Logging{ + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskService.scala new file mode 100644 index 000000000..3a1ebcc08 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskService.scala @@ -0,0 +1,147 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.service + + +import com.webank.wedatasphere.linkis.common.utils.Logging +import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf +import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamJobMapper, StreamTaskMapper} +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{StreamJob, StreamJobVersion, StreamTask} +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{JobProgressVO, StreamTaskListVO} +import com.webank.wedatasphere.streamis.jobmanager.manager.util.DateUtils +import org.apache.commons.lang.StringUtils +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service + +import java.util.Date +import scala.collection.JavaConverters._ +import scala.collection.mutable.ListBuffer +/** + * @author limeng + */ +@Service +class TaskService extends Logging{ + + @Autowired private var streamTaskMapper:StreamTaskMapper=_ + @Autowired private var streamJobMapper:StreamJobMapper=_ + + + def saveJob(streamJob:StreamJob): Unit ={ + streamJobMapper.insertJob(streamJob) + } + + def updateJob(streamJob:StreamJob): Unit ={ + streamJobMapper.updateJob(streamJob) + } + + def saveJobVersion(): Unit ={ + + } + + def updateJobVersion(): Unit ={ + + } + + /** + * 执行job + * @param jobId + */ + def executeJob(jobId: Long,userName:String): Unit ={ + val jobs = streamJobMapper.getJobById(jobId) + if(jobs == null || StringUtils.isEmpty(jobs.getCurrentVersion)) return + val versionsJob: StreamJobVersion = streamJobMapper.getJobVersionsById(jobId, jobs.getCurrentVersion).asScala.head + var taskId:Long = 0 + if(jobs.getCurrentTaskId == null){ + taskId = saveTask(versionsJob.getId,userName,new Date(),"v1") + }else{ + val task = streamTaskMapper.getTaskById(jobs.getCurrentTaskId) + val versionVal = task.getVersion.split("v").last.toInt+1 + taskId = saveTask(versionsJob.getId,userName,new Date(),"v"+versionVal) + } + val streamJob = new StreamJob() + streamJob.setId(jobId) + streamJob.setCurrentTaskId(taskId) + streamJobMapper.updateJob(streamJob) + //todo 执行linkis + + } + + def stopJob(jobId: Long,userName:String): Unit ={ + //todo 执行linkis + val jobs = streamJobMapper.getJobById(jobId) + if(jobs == null) return + val task = streamTaskMapper.getTaskById(jobs.getCurrentTaskId) + if(task==null) return + val taskModel = new StreamTask() + taskModel.setId(task.getId) + taskModel.setStatus(JobConf.JOBMANAGER_FLINK_JOB_STATUS_SEVEN.getValue) + streamTaskMapper.updateTask(taskModel) + + val versionsJob: StreamJobVersion = streamJobMapper.getJobVersionsById(jobId, jobs.getCurrentVersion).asScala.head + val taskId = saveTask(versionsJob.getId,userName,new Date(),"v1") + val streamJob = new StreamJob() + streamJob.setId(jobId) + streamJob.setCurrentTaskId(taskId) + streamJobMapper.updateJob(streamJob) + } + + /** + * 查询运行历史 + * @param jobId + * @param version + * @return + */ + def executeHistory(jobId:Long,version:String): java.util.List[StreamTaskListVO] ={ + if(version == null || version =="") return null + val jobVersions = streamJobMapper.getJobVersionsById(jobId, version) + if(jobVersions==null || jobVersions.isEmpty) return null + val jobVersionId = jobVersions.asScala.head.getId + val tasks = streamTaskMapper.getByJobIds(jobVersionId, null) + if(tasks == null || tasks.isEmpty) return null + val list = new ListBuffer[StreamTaskListVO] + tasks.asScala.foreach(f=>{ + val svo = new StreamTaskListVO() + svo.setTaskId(f.getId) + svo.setStatus(f.getStatus) + svo.setCreator(f.getSubmitUser) + svo.setVersion(f.getVersion) + svo.setJobName(f.getJobName) + svo.setStartTime(DateUtils.formatDate(f.getStartTime)) + svo.setEndTime(DateUtils.formatDate(f.getEndTime)) + svo.setJobVersionId(f.getJobVersionId) + svo.setRunTime(DateUtils.intervals(f.getStartTime,f.getEndTime)) + svo.setStopCause(f.getErrDesc) + list.append(svo) + }) + list.asJava + } + + /** + * 每次任务状态 + * @param jobId + * @return + */ + def getByJobStatus(jobId:Long): JobProgressVO ={ + val job = streamJobMapper.getJobById(jobId) + if(job == null) return null + if(job.getCurrentTaskId == null) return null + val task = streamTaskMapper.getTaskById(job.getCurrentTaskId) + if(task == null) return null + val jobProgressVO = new JobProgressVO() + jobProgressVO.setTaskId(job.getCurrentTaskId) + jobProgressVO.setProgress(task.getStatus) + jobProgressVO + } + + private def saveTask(jobVersionId:Long,userName:String,date:Date,version:String): Long ={ + val task = new StreamTask() + task.setVersion(version) + task.setStatus(JobConf.JOBMANAGER_FLINK_JOB_STATUS_FIVE.getValue) + task.setJobVersionId(jobVersionId) + task.setStartTime(date) + task.setEndTime(date) + task.setSubmitUser(userName) + streamTaskMapper.insertTask(task) + task.getId + } + + +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/pom.xml b/streamis-jobmanager/streamis-jobmanager-server/pom.xml index c67d6d121..2c6f8d811 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/pom.xml +++ b/streamis-jobmanager/streamis-jobmanager-server/pom.xml @@ -11,5 +11,74 @@ streamis-jobmanager-server + + + com.webank.wedatasphere.linkis + linkis-module + + + com.webank.wedatasphere.streamis + streamis-job-launcher-service + ${jobmanager.version} + + + com.webank.wedatasphere.linkis + linkis-module + + + + + com.webank.wedatasphere.streamis + streamis-job-manager-service + ${jobmanager.version} + + + com.webank.wedatasphere.linkis + linkis-module + + + + + org.springframework.boot + spring-boot-starter-test + ${spring.boot.version} + test + + + junit + junit + 4.11 + test + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + ${basedir}/src/main/resources + + **/*.properties + **/*.xml + **/*.yml + + + + + ${project.artifactId}-${project.version} + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/ConfigurationException.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/ConfigurationException.java new file mode 100755 index 000000000..5947dc563 --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/ConfigurationException.java @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.streamis.jobmanager.exception; + +/** + * @author limeng + */ +public class ConfigurationException extends Exception { + public ConfigurationException() { + } + + public ConfigurationException(String message) { + super(message); + } +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobException.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobException.java new file mode 100644 index 000000000..d9bf3deff --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobException.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.streamis.jobmanager.exception; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; + +public class JobException extends ErrorException { + + public JobException(int errCode, String desc) { + super(errCode, desc); + } + + public JobException(int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobExceptionManager.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobExceptionManager.java new file mode 100644 index 000000000..4f52b9a9e --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobExceptionManager.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.streamis.jobmanager.exception; + +import java.util.HashMap; +import java.util.Map; + +public class JobExceptionManager { + //30300-30599 + private static Map desc = new HashMap(32) { + { + put("30300", "上传失败"); + put("30301","%s cannot be empty!"); + } + }; + + public static JobException createException(int errorCode, Object... format) throws JobException { + return new JobException(errorCode, String.format(desc.get(String.valueOf(errorCode)), format)); + } +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/ProjectException.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/ProjectException.java new file mode 100644 index 000000000..32855c48b --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/ProjectException.java @@ -0,0 +1,10 @@ +package com.webank.wedatasphere.streamis.jobmanager.exception; + +public class ProjectException extends Exception{ + public ProjectException() { + } + + public ProjectException(String message) { + super(message); + } +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java new file mode 100644 index 000000000..6b0e9602c --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java @@ -0,0 +1,66 @@ +package com.webank.wedatasphere.streamis.jobmanager.restful.api; + +import com.webank.wedatasphere.linkis.server.Message; +import com.webank.wedatasphere.linkis.server.security.SecurityFilter; +import com.webank.wedatasphere.streamis.jobmanager.exception.ConfigurationException; +import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.vo.ConfigKeyVO; +import com.webank.wedatasphere.streamis.jobmanager.launcher.service.ConfigurationService; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.util.List; + +/** + * @author limeng + */ +@Component +@Path("/streamis/streamJobManager/config") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +public class ConfigurationRestfulApi { + @Autowired + ConfigurationService configurationService; + + ObjectMapper mapper = new ObjectMapper(); + + + @GET + @Path("/view") + public Response getView(@Context HttpServletRequest req,@QueryParam("jobId") Long jobId) throws IOException, ConfigurationException { + String username = SecurityFilter.getLoginUsername(req); + if(jobId==null){ + throw new ConfigurationException("params cannot be empty!"); + } + ConfigKeyVO fullTree = configurationService.getFullTree(jobId); + return Message.messageToResponse(Message.ok().data("fullTree",fullTree)); + } + + @POST + @Path("/update") + public Response updateFullTree(@Context HttpServletRequest req, JsonNode json) throws IOException,ConfigurationException { + ConfigKeyVO fullTrees = mapper.readValue(json.get("fullTree"), ConfigKeyVO.class); + + configurationService.addKeyValue(fullTrees); + return Message.messageToResponse(Message.ok()); + } + + @POST + @Path("/add") + public Response saveFullTree(@Context HttpServletRequest req, JsonNode json) throws IOException,ConfigurationException { + ConfigKeyVO fullTrees = mapper.readValue(json.get("fullTree"), ConfigKeyVO.class); + + configurationService.addKeyValue(fullTrees); + return Message.messageToResponse(Message.ok()); + } + + +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java new file mode 100644 index 000000000..448789452 --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -0,0 +1,231 @@ +package com.webank.wedatasphere.streamis.jobmanager.restful.api; + +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; + +import com.webank.wedatasphere.linkis.server.Message; +import com.webank.wedatasphere.linkis.server.security.SecurityFilter; +import com.webank.wedatasphere.streamis.jobmanager.exception.JobException; +import com.webank.wedatasphere.streamis.jobmanager.exception.JobExceptionManager; +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.*; +import com.webank.wedatasphere.streamis.jobmanager.manager.service.JobService; +import com.webank.wedatasphere.streamis.jobmanager.manager.service.TaskService; +import org.glassfish.jersey.media.multipart.FormDataMultiPart; +import org.glassfish.jersey.media.multipart.FormDataParam; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Component +@Path("/streamis/streamJobManager/job") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +public class JobRestfulApi { + @Autowired + JobService jobService; + @Autowired + TaskService taskService; + + + @GET + @Path("/list") + public Response getJobList(@Context HttpServletRequest req, + @QueryParam("projectId") Long projectId, + @QueryParam("workspaceId") Long workspaceId, + @QueryParam("pageNow") Integer pageNow, + @QueryParam("pageSize") Integer pageSize, + @QueryParam("jobName") String jobName, + @QueryParam("jobStatus") Integer jobStatus, + @QueryParam("jobCreator") String jobCreator) throws IOException, JobException { + if (StringUtils.isEmpty(pageNow)) { + pageNow = 1; + } + if (StringUtils.isEmpty(pageSize)) { + pageSize = 20; + } + if(projectId == null){ + JobExceptionManager.createException(30301,"projectId"); + } + List jobList = null; + PageHelper.startPage(pageNow, pageSize); + try { + jobList = jobService.getByProList(projectId,jobName,jobStatus,jobCreator); + }finally { + PageHelper.clearPage(); + } + PageInfo pageInfo = new PageInfo<>(jobList); + long total = pageInfo.getTotal(); + + return Message.messageToResponse(Message.ok().data("tasks", pageInfo.getList()).data("totalPage", total)); + } + + @POST + @Path("/upload") + public Response uploadJar(@Context HttpServletRequest req, + @FormDataParam("jobName") String jobName, + @FormDataParam("entrypointClass") String entrypointClass, + @FormDataParam("label") String label, + @FormDataParam("entrypointMainArgs") String entrypointMainArgs, + @FormDataParam("parallelism") Integer parallelism, + @FormDataParam("workspaceId") Long workspaceId, + @FormDataParam("projectId") Long projectId, + @FormDataParam("path") String path, + FormDataMultiPart form) throws IOException, JobException { + String userName = SecurityFilter.getLoginUsername(req); + if (StringUtils.isEmpty(path)) { + JobExceptionManager.createException(30300,path); + } + + return Message.messageToResponse(Message.ok()); + } + + @POST + @Path("/execute") + public Response executeJob(@Context HttpServletRequest req, Map json) throws IOException, JobException { + String userName = SecurityFilter.getLoginUsername(req); + if(!json.containsKey("jobId") || json.get("jobId")==null){ + JobExceptionManager.createException(30301,"jobId"); + } + long jobId = Long.parseLong(json.get("jobId").toString()); + taskService.executeJob(jobId,userName); + //todo 缺少linkis + return Message.messageToResponse(Message.ok()); + } + + @GET + @Path("/stop") + public Response killJob(@Context HttpServletRequest req,@QueryParam("jobId") Long jobId) throws IOException, JobException{ + String userName = SecurityFilter.getLoginUsername(req); + if(jobId == null){ + JobExceptionManager.createException(30301,"jobId"); + } + taskService.stopJob(jobId,userName); + //todo 缺少linkis + return Message.messageToResponse(Message.ok()); + } + + @GET + @Path("/details") + public Response detailsJob(@Context HttpServletRequest req,@QueryParam("jobId") Long jobId) throws IOException, JobException{ + if(jobId == null){ + JobExceptionManager.createException(30301,"jobId"); + } + JobDetailsVO jobDetailsVO = new JobDetailsVO(); + List dataNumberDTOS =new ArrayList<>(); + JobDetailsVO.DataNumberDTO dataNumberDTO = new JobDetailsVO.DataNumberDTO(); + dataNumberDTO.setDataName("kafka topic"); + dataNumberDTO.setDataNumber(109345); + dataNumberDTOS.add(dataNumberDTO); + + List loadConditionDTOs =new ArrayList<>(); + JobDetailsVO.LoadConditionDTO loadConditionDTO = new JobDetailsVO.LoadConditionDTO(); + loadConditionDTO.setType("jobManager"); + loadConditionDTO.setHost("localhost"); + loadConditionDTO.setMemory("1.5"); + loadConditionDTO.setTotalMemory("2.0"); + loadConditionDTO.setGcLastTime("2020-08-01"); + loadConditionDTO.setGcLastConsume("1"); + loadConditionDTO.setGcTotalTime("2min"); + loadConditionDTOs.add(loadConditionDTO); + + List realTimeTrafficDTOS = new ArrayList<>(); + JobDetailsVO.RealTimeTrafficDTO realTimeTrafficDTO = new JobDetailsVO.RealTimeTrafficDTO(); + realTimeTrafficDTO.setSourceKey("kafka topic"); + realTimeTrafficDTO.setSourceSpeed("100 Records/S"); + realTimeTrafficDTO.setTransformKey("transform"); + realTimeTrafficDTO.setSinkKey("hbase key"); + realTimeTrafficDTO.setSinkSpeed("10 Records/S"); + realTimeTrafficDTOS.add(realTimeTrafficDTO); + + jobDetailsVO.setDataNumber(dataNumberDTOS); + jobDetailsVO.setLoadCondition(loadConditionDTOs); + jobDetailsVO.setRealTimeTraffic(realTimeTrafficDTOS); + + return Message.messageToResponse(Message.ok().data("details",jobDetailsVO)); + } + @GET + @Path("/execute/history") + public Response executeHistoryJob(@Context HttpServletRequest req,@QueryParam("jobId") Long jobId,@QueryParam("version") String version) throws IOException, JobException{ + if(jobId == null){ + JobExceptionManager.createException(30301,"jobId"); + } + if(StringUtils.isEmpty(version)){ + JobExceptionManager.createException(30301,"version"); + } + List streamTaskListVOS = taskService.executeHistory(jobId, version); + return Message.messageToResponse(Message.ok().data("details",streamTaskListVOS)); + } + + @POST + @Path("/release") + public Response releaseJob(@Context HttpServletRequest req, Map json) throws IOException, JobException{ + + return Message.messageToResponse(Message.ok()); + } + @GET + @Path("/flow") + public Response flowJob(@Context HttpServletRequest req,@QueryParam("jobId") Long jobId,@QueryParam("version") String version) throws IOException, JobException{ + if(jobId == null){ + JobExceptionManager.createException(30301,"jobId"); + } + if(StringUtils.isEmpty(version)){ + JobExceptionManager.createException(30301,"version"); + } + + JobFlowVO jobFlow = jobService.getJobFlow(jobId, version); + + return Message.messageToResponse(Message.ok().data("flowData",jobFlow)); + } + @GET + @Path("/progress") + public Response progressJob(@Context HttpServletRequest req,@QueryParam("jobId") Long jobId) throws IOException, JobException{ + if(jobId == null){ + JobExceptionManager.createException(30301,"jobId"); + } + JobProgressVO jobProgressVO = taskService.getByJobStatus(jobId); + return Message.messageToResponse(Message.ok().data("taskId",jobProgressVO.getTaskId()).data("progress",jobProgressVO.getProgress())); + } + + @GET + @Path("/upload/details") + public Response uploadDetailsJob(@Context HttpServletRequest req,@QueryParam("jobId") Long jobId) throws IOException, JobException{ + + JobUploadDetailsVO jobUploadDetailsVO = new JobUploadDetailsVO(); + List mainJars = new ArrayList<>(); + List dependentList = new ArrayList<>(); + List userList = new ArrayList<>(); + JobUploadDetailsVO.JarDependentDTO jarDependentDTO = new JobUploadDetailsVO.JarDependentDTO(); + jarDependentDTO.setId(1L); + jarDependentDTO.setName("main jar"); + + mainJars.add(jarDependentDTO); + + jarDependentDTO = new JobUploadDetailsVO.JarDependentDTO(); + jarDependentDTO.setId(1L); + jarDependentDTO.setName("jar 依赖"); + dependentList.add(jarDependentDTO); + + jarDependentDTO = new JobUploadDetailsVO.JarDependentDTO(); + jarDependentDTO.setId(1L); + jarDependentDTO.setName("user 资源"); + userList.add(jarDependentDTO); + + jobUploadDetailsVO.setProgramArguement("--hostname localhost --port 12345"); + jobUploadDetailsVO.setMainJars(mainJars); + jobUploadDetailsVO.setDependentList(dependentList); + jobUploadDetailsVO.setUserList(userList); + + return Message.messageToResponse(Message.ok().data("details",jobUploadDetailsVO)); + } + +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java new file mode 100644 index 000000000..6de09b319 --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java @@ -0,0 +1,38 @@ +package com.webank.wedatasphere.streamis.jobmanager.restful.api; + +import com.webank.wedatasphere.linkis.server.Message; +import com.webank.wedatasphere.linkis.server.security.SecurityFilter; +import com.webank.wedatasphere.streamis.jobmanager.exception.ProjectException; +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.TaskCoreNumVO; +import com.webank.wedatasphere.streamis.jobmanager.manager.service.JobService; +import com.webank.wedatasphere.streamis.jobmanager.manager.service.ProjectService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; + +@Component +@Path("/streamis/streamJobManager/project") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +public class ProjectRestfulApi { + @Autowired + ProjectService projectService; + @Autowired + JobService jobService; + @GET + @Path("/core/target") + public Response getView(@Context HttpServletRequest req, @QueryParam("projectId") Long projectId) throws IOException, ProjectException { + String username = SecurityFilter.getLoginUsername(req); + if(projectId==null){ + throw new ProjectException("params cannot be empty!"); + } + TaskCoreNumVO taskCoreNumVO = jobService.countByCores(projectId); + return Message.messageToResponse(Message.ok().data("taskCore",taskCoreNumVO)); + } +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/application.yml b/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/application.yml new file mode 100755 index 000000000..4e39747a5 --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/application.yml @@ -0,0 +1,20 @@ +server: + port: 9009 +spring: + application: + name: cloud-streammanager +eureka: + client: + serviceUrl: + defaultZone: http://ip:port/eureka/ + instance: + metadata-map: + test: wedatasphere + +management: + endpoints: + web: + exposure: + include: refresh,info +logging: + config: classpath:log4j2.xml \ No newline at end of file diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/linkis.properties b/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/linkis.properties new file mode 100755 index 000000000..9470062ed --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/linkis.properties @@ -0,0 +1,40 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +wds.linkis.test.mode=true +wds.linkis.server.mybatis.datasource.url=jdbc:mysql://xx:3306/streamis?characterEncoding=UTF-8 +wds.linkis.server.mybatis.datasource.username=xx +wds.linkis.server.mybatis.datasource.password=xxx + +##restful +wds.linkis.log.clear=true +wds.linkis.server.version=v1 +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.streamis.jobmanager.restful.api + + + +##mybatis +wds.linkis.server.mybatis.mapperLocations=\ + classpath:com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/*.xml,\ + classpath:com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=\ + com.webank.wedatasphere.streamis.jobmanager.launcher.entity,\ + com.webank.wedatasphere.streamis.jobmanager.manager.entity,\ + com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo,\ + com.webank.wedatasphere.streamis.jobmanager.launcher.entity.vo,\ + com.webank.wedatasphere.streamis.jobmanager.launcher.entity.dto +wds.linkis.server.mybatis.BasePackage=\ + com.webank.wedatasphere.streamis.jobmanager.launcher.dao,\ + com.webank.wedatasphere.streamis.jobmanager.manager.dao \ No newline at end of file diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/log4j.properties b/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/log4j.properties new file mode 100755 index 000000000..178f8dfa2 --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/log4j2.xml b/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/log4j2.xml new file mode 100755 index 000000000..ad88ea570 --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/log4j2.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + From f702e2adcb5ea2da3787cf9cad565f25cb81c387 Mon Sep 17 00:00:00 2001 From: "77954309@qq.com" Date: Mon, 26 Apr 2021 11:09:37 +0800 Subject: [PATCH 005/434] jobmanager all query --- .../db/streamis-jobmanager.sql | 335 ++++++++++++++++++ pom.xml | 21 ++ .../streamis-job-launcher/pom.xml | 6 +- .../jobmanager/launcher/JobLauncher.java | 4 - .../streamis-job-launcher-base/pom.xml | 54 +++ .../jobmanager/launcher/dao/ConfigMapper.java | 23 ++ .../launcher/dao/JobUserRoleMapper.java | 27 ++ .../launcher/dao/impl/ConfigMapper.xml | 128 +++++++ .../launcher/dao/impl/JobUserRoleMapper.xml | 105 ++++++ .../jobmanager/launcher/entity/ConfigKey.java | 133 +++++++ .../launcher/entity/ConfigKeyValue.java | 166 +++++++++ .../jobmanager/launcher/entity/JobRole.java | 51 +++ .../jobmanager/launcher/entity/JobUser.java | 40 +++ .../launcher/entity/JobUserRole.java | 52 +++ .../jobmanager/launcher/entity/User.java | 34 ++ .../entity/dto/ConfigKeyValueDTO.java | 79 +++++ .../launcher/entity/vo/ConfigKeyVO.java | 70 ++++ .../launcher/entity/vo/ConfigRelationVO.java | 83 +++++ .../streamis-job-launcher-service/pom.xml | 50 +++ .../jobmanager/launcher/conf/ConfigConf.scala | 30 ++ .../service/ConfigurationService.scala | 202 +++++++++++ .../streamis-job-manager/pom.xml | 6 +- .../streamis-job-manager-base/pom.xml | 54 +++ .../manager/dao/StreamBmlMapper.java | 23 ++ .../manager/dao/StreamJobMapper.java | 40 +++ .../manager/dao/StreamProjectMapper.java | 19 + .../manager/dao/StreamTaskMapper.java | 20 ++ .../manager/dao/impl/StreamBmlMapper.xml | 104 ++++++ .../manager/dao/impl/StreamJobMapper.xml | 164 +++++++++ .../manager/dao/impl/StreamProjectMapper.xml | 56 +++ .../manager/dao/impl/StreamTaskMapper.xml | 86 +++++ .../jobmanager/manager/entity/StreamBml.java | 49 +++ .../manager/entity/StreamBmlVersion.java | 52 +++ .../jobmanager/manager/entity/StreamJob.java | 146 ++++++++ .../manager/entity/StreamJobSqlResource.java | 34 ++ .../manager/entity/StreamJobVersion.java | 61 ++++ .../manager/entity/StreamProject.java | 43 +++ .../jobmanager/manager/entity/StreamTask.java | 90 +++++ .../manager/entity/vo/JobDetailsVO.java | 179 ++++++++++ .../manager/entity/vo/JobFlowVO.java | 126 +++++++ .../manager/entity/vo/JobHistoryVO.java | 85 +++++ .../manager/entity/vo/JobProgressVO.java | 22 ++ .../manager/entity/vo/JobUploadDetailsVO.java | 106 ++++++ .../manager/entity/vo/QueryJobListVO.java | 76 ++++ .../manager/entity/vo/StreamJobVO.java | 97 +++++ .../manager/entity/vo/StreamTaskListVO.java | 99 ++++++ .../manager/entity/vo/TaskCoreNumVO.java | 77 ++++ .../jobmanager/manager/util/DateUtils.scala | 54 +++ .../streamis-job-manager-service/pom.xml | 51 +++ .../jobmanager/manager/conf/JobConf.scala | 23 ++ .../manager/service/JobService.scala | 99 ++++++ .../manager/service/ProjectService.scala | 12 + .../manager/service/TaskService.scala | 147 ++++++++ .../streamis-jobmanager-server/pom.xml | 69 ++++ .../exception/ConfigurationException.java | 13 + .../jobmanager/exception/JobException.java | 14 + .../exception/JobExceptionManager.java | 18 + .../exception/ProjectException.java | 10 + .../restful/api/ConfigurationRestfulApi.java | 66 ++++ .../jobmanager/restful/api/JobRestfulApi.java | 231 ++++++++++++ .../restful/api/ProjectRestfulApi.java | 38 ++ .../src/main/resources/application.yml | 20 ++ .../src/main/resources/linkis.properties | 40 +++ .../src/main/resources/log4j.properties | 36 ++ .../src/main/resources/log4j2.xml | 38 ++ 65 files changed, 4580 insertions(+), 6 deletions(-) create mode 100644 docs/zh_CN/0.1.0/development/Table_Structure_documentation/db/streamis-jobmanager.sql mode change 100644 => 100755 streamis-jobmanager/streamis-job-launcher/pom.xml delete mode 100644 streamis-jobmanager/streamis-job-launcher/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/JobLauncher.java create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/ConfigMapper.java create mode 100644 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/JobUserRoleMapper.java create mode 100644 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/ConfigMapper.xml create mode 100644 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/JobUserRoleMapper.xml create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/ConfigKey.java create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/ConfigKeyValue.java create mode 100644 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobRole.java create mode 100644 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUser.java create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUserRole.java create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/User.java create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/dto/ConfigKeyValueDTO.java create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/vo/ConfigKeyVO.java create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/vo/ConfigRelationVO.java create mode 100755 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml create mode 100644 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/ConfigConf.scala create mode 100644 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/service/ConfigurationService.scala mode change 100644 => 100755 streamis-jobmanager/streamis-job-manager/pom.xml create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamBmlMapper.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamProjectMapper.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamTaskMapper.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamBmlMapper.xml create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamProjectMapper.xml create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamBml.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamBmlVersion.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJob.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobSqlResource.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersion.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamProject.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamTask.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobDetailsVO.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobFlowVO.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobHistoryVO.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobProgressVO.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobUploadDetailsVO.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/QueryJobListVO.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/StreamJobVO.java create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/StreamTaskListVO.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/TaskCoreNumVO.java create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/DateUtils.scala create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala create mode 100755 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/ProjectService.scala create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskService.scala create mode 100755 streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/ConfigurationException.java create mode 100644 streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobException.java create mode 100644 streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobExceptionManager.java create mode 100644 streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/ProjectException.java create mode 100644 streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java create mode 100644 streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java create mode 100644 streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java create mode 100755 streamis-jobmanager/streamis-jobmanager-server/src/main/resources/application.yml create mode 100755 streamis-jobmanager/streamis-jobmanager-server/src/main/resources/linkis.properties create mode 100755 streamis-jobmanager/streamis-jobmanager-server/src/main/resources/log4j.properties create mode 100755 streamis-jobmanager/streamis-jobmanager-server/src/main/resources/log4j2.xml diff --git a/docs/zh_CN/0.1.0/development/Table_Structure_documentation/db/streamis-jobmanager.sql b/docs/zh_CN/0.1.0/development/Table_Structure_documentation/db/streamis-jobmanager.sql new file mode 100644 index 000000000..b7ca41221 --- /dev/null +++ b/docs/zh_CN/0.1.0/development/Table_Structure_documentation/db/streamis-jobmanager.sql @@ -0,0 +1,335 @@ + +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for linkis_stream_bml +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_bml`; +CREATE TABLE `linkis_stream_bml` ( + `id` bigint(20) NOT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `bml_type` tinyint(1) NULL DEFAULT NULL, + `org_identification` bigint(20) NULL DEFAULT NULL, + ` latest_version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_bml +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_bml_version +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_bml_version`; +CREATE TABLE `linkis_stream_bml_version` ( + `id` bigint(20) NOT NULL, + `bml_id` bigint(20) NULL DEFAULT NULL, + `version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `storage_path` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + ` attribute` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '物料版本' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_bml_version +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_cluster +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_cluster`; +CREATE TABLE `linkis_stream_cluster` ( + `id` int(11) NOT NULL, + `yarn_conf_dir` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `hdfs_conf_dir` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `resource_manager_url` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `savepoint_dir` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = 'flink 集群信息' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_cluster +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_configuration_config_key +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_configuration_config_key`; +CREATE TABLE `linkis_stream_configuration_config_key` ( + `id` bigint(20) NOT NULL, + `key` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `description` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `default_value` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `validate_type` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `validate_range` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `is_hidden` tinyint(1) NULL DEFAULT NULL, + `is_advanced` tinyint(1) NULL DEFAULT NULL, + `level` tinyint(1) NULL DEFAULT NULL, + `treename` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `type` int(10) NULL DEFAULT NULL, + `sort` int(10) NULL DEFAULT NULL, + `status` tinyint(10) NULL DEFAULT NULL COMMENT '1 custom , 2 selected ', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE INDEX `key_index`(`key`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '配置信息' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_configuration_config_key +-- ---------------------------- +INSERT INTO `linkis_stream_configuration_config_key` VALUES (1, 'wds.linkis.flink.resource', '资源配置', '资源配置', NULL, 'None', NULL, 0, 0, 1, '资源配置', 1, 0, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (2, 'wds.linkis.flink.taskmanager.num', 'Task Managers数量', 'Task Managers数量', '4', 'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|128)$', 0, 0, 2, '资源配置', 1, 1, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (3, 'wds.linkis.flink.jobmanager.memory', 'JobManager Memory', 'JobManager Memory', '1.5', 'Regex', '^([1-9]\\d{0,2}|1000)(G|g)$', 0, 0, 2, '资源配置', 1, 2, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (4, 'wds.linkis.flink.taskmanager.memory', 'TaskManager Memory', 'TaskManager Memory', '1.5', 'Regex', '^([1-9]\\d{0,2}|1000)(G|g)$', 0, 0, 2, '资源配置', 1, 3, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (5, 'wds.linkis.flink.jobmanager.cpus', 'JobManager CPUs', 'JobManager CPUs', '1', 'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|128)$', 0, 0, 2, '资源配置', 1, 4, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (6, 'wds.linkis.flink.taskManager.cpus', 'TaskManager CPUs', 'TaskManager CPUs', '1', 'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|128)$', 0, 0, 2, '资源配置', 1, 5, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (7, 'wds.linkis.flink.custom', '自定义参数', '自定义参数', NULL, 'None', NULL, 0, 0, 1, '自定义参数', 2, 0, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (8, 'wds.linkis.flink.produce', '生产配置', '生产配置', NULL, 'None', NULL, 0, 0, 1, '生产配置', 3, 0, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (9, 'wds.linkis.flink.checkpoint.interval', 'Checkpoint间隔', 'Checkpoint间隔', NULL, NULL, NULL, 0, 0, 2, '生产配置', 3, 1, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (10, 'wds.linkis.flink.reboot.strategy', '重启策略', '重启策略', '不重启,基于Checkpoint自动重启,无Checkpoint不重启', 'None', NULL, 0, 0, 2, '重启策略', 3, 2, 2); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (11, 'wds.linkis.flink.alert', '告警设置', '告警设置', NULL, 'None', NULL, 0, 0, 1, '告警设置', 4, 0, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (12, 'wds.linkis.flink.alert.rule', '告警规则', '告警规则', '任务日志中出现ERROR/EXCEPTION,任务核心指标出现异常', 'None', NULL, 0, 0, 2, '告警规则', 4, 1, 2); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (13, 'wds.linkis.flink.alert.user', '告警用户', '告警用户', NULL, NULL, NULL, 0, 0, 2, '告警用户', 4, 3, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (14, 'wds.linkis.flink.alert.leve', '告警级别', '告警级别', 'CLEARED,INDETERMINATE,WARNING,MINOR,MAJOR,CRITICAL', 'None', NULL, 0, 0, 2, '告警级别', 4, 2, 2); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (15, 'wds.linkis.flink.alert.failure.level', '失败时告警级别', '失败时告警级别', 'CLEARED,INDETERMINATE,WARNING,MINOR,MAJOR,CRITICAL', 'None', NULL, 0, 0, 2, '失败时告警级别', 4, 4, 2); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (16, 'wds.linkis.flink.alert.failure.user', '失败时告警用户', '失败时告警用户', NULL, 'None', NULL, 0, 0, 2, '失败时告警用户', 4, 5, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (17, 'wds.linkis.flink.authority', '权限设置', '权限设置', NULL, 'None', NULL, 0, 0, 1, '权限设置', 5, 0, 1); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (18, 'wds.linkis.flink.authority.author', '授权模式', '授权模式', '私密,指定全员可见,指定人员可见', 'None', NULL, 0, 0, 2, '授权模式', 5, 1, 2); +INSERT INTO `linkis_stream_configuration_config_key` VALUES (19, 'wds.linkis.flink.authority.visible', '可见人员', '可见人员', NULL, 'None', NULL, 0, 0, 2, '可见人员', 5, 2, 1); + +-- ---------------------------- +-- Table structure for linkis_stream_configuration_config_value +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_configuration_config_value`; +CREATE TABLE `linkis_stream_configuration_config_value` ( + `id` bigint(20) NOT NULL, + `configkey_id` bigint(20) NULL DEFAULT NULL, + `config_value` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `type` int(10) NULL DEFAULT NULL, + `job_id` bigint(20) NULL DEFAULT NULL, + `job_name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `config_key` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE, + INDEX `key`(`config_key`) USING BTREE, + INDEX `keyid`(`configkey_id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '配置信息' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_configuration_config_value +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_frame_version +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_frame_version`; +CREATE TABLE `linkis_stream_frame_version` ( + `id` bigint(20) NOT NULL, + `frame` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `java_version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '框架信息' ROW_FORMAT = COMPACT; + +-- ---------------------------- +-- Records of linkis_stream_frame_version +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_job +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_job`; +CREATE TABLE `linkis_stream_job` ( + `id` bigint(20) NOT NULL, + `project_id` bigint(20) NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `type` tinyint(1) NULL DEFAULT NULL, + `current_task_id` bigint(20) NULL DEFAULT NULL, + `current_version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `current_release_time` datetime NULL DEFAULT NULL, + `status` tinyint(1) NULL DEFAULT NULL COMMENT '1:已完成 ，2:等待重启 ，3:告警 ，4:慢任务 ，5:运行中 ，6:失败任务', + `org_identification` bigint(20) NULL DEFAULT NULL, + `create_by` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `label` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `current_released` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `description` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '作业表' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_job +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_job_alarm_send_history +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_job_alarm_send_history`; +CREATE TABLE `linkis_stream_job_alarm_send_history` ( + `id` bigint(20) NOT NULL, + `job_id` bigint(20) NULL DEFAULT NULL, + `task_id` bigint(20) NULL DEFAULT NULL, + `create_by` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `type` tinyint(1) NULL DEFAULT NULL, + `rule_type` tinyint(1) NULL DEFAULT NULL, + `content` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '报警历史信息' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_job_alarm_send_history +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_job_checkpoints +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_job_checkpoints`; +CREATE TABLE `linkis_stream_job_checkpoints` ( + `id` bigint(20) NOT NULL, + `config_value_id` bigint(20) NULL DEFAULT NULL, + `path` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `size` int(20) NULL DEFAULT NULL, + `status` tinyint(1) NULL DEFAULT NULL, + `trigger_timestamp` datetime NULL DEFAULT NULL, + `latest_ack_timestamp` datetime NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_job_checkpoints +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_job_code_resource +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_job_code_resource`; +CREATE TABLE `linkis_stream_job_code_resource` ( + `id` bigint(20) NOT NULL, + `job_version_id` bigint(20) NULL DEFAULT NULL, + `bml_version_id` bigint(20) NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '其他代码' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_job_code_resource +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_job_role +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_job_role`; +CREATE TABLE `linkis_stream_job_role` ( + `id` bigint(20) NOT NULL, + `job_id` bigint(20) NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `front_name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `update_time` datetime NULL DEFAULT NULL, + `description` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_job_role +-- ---------------------------- +INSERT INTO `linkis_stream_job_role` VALUES (1, -1, '管理员', '管理员', '2021-04-07 20:57:09', NULL); + +-- ---------------------------- +-- Table structure for linkis_stream_job_sql_resource +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_job_sql_resource`; +CREATE TABLE `linkis_stream_job_sql_resource` ( + `id` bigint(20) NOT NULL, + `job_version_id` bigint(20) NULL DEFAULT NULL, + `execute_sql` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_job_sql_resource +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_job_user_role +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_job_user_role`; +CREATE TABLE `linkis_stream_job_user_role` ( + `id` bigint(20) NOT NULL, + `job_id` bigint(20) NULL DEFAULT NULL, + `user_id` bigint(20) NULL DEFAULT NULL, + `role_id` bigint(20) NULL DEFAULT NULL, + `type` tinyint(1) NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '作业角色关系' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_job_user_role +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_job_version +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_job_version`; +CREATE TABLE `linkis_stream_job_version` ( + `id` bigint(20) NOT NULL, + ` job_id` bigint(20) NULL DEFAULT NULL, + `version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `program_arguments` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `bml_version` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `resource_id` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_job_version +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_project +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_project`; +CREATE TABLE `linkis_stream_project` ( + `id` bigint(20) NOT NULL, + `workspace_id` bigint(20) NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `create_by` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '项目表' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_project +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_task +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_task`; +CREATE TABLE `linkis_stream_task` ( + `id` bigint(20) NOT NULL, + `job_version_id` bigint(20) NULL DEFAULT NULL, + `version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `stream_task_identification` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `status` tinyint(10) NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_task +-- ---------------------------- + +-- ---------------------------- +-- Table structure for linkis_stream_user +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_stream_user`; +CREATE TABLE `linkis_stream_user` ( + `id` bigint(20) NOT NULL, + `username` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '用户表' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of linkis_stream_user +-- ---------------------------- +INSERT INTO `linkis_stream_user` VALUES (1, 'hdfs', 'hdfs'); + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/pom.xml b/pom.xml index 6c46b3dbd..4bb0b383a 100644 --- a/pom.xml +++ b/pom.xml @@ -59,6 +59,7 @@ 2.21 1.9.5 1.4.15 + 0.1.0-SNAPSHOT @@ -90,6 +91,26 @@ commons-lang ${commons.lang.version} + + com.webank.wedatasphere.linkis + linkis-mybatis + ${linkis.version} + + + com.webank.wedatasphere.linkis + linkis-module + + + org.springframework.boot + spring-boot-starter-tomcat + + + hibernate-validator + org.hibernate.validator + + + ${linkis.version} + com.webank.wedatasphere.linkis linkis-common diff --git a/streamis-jobmanager/streamis-job-launcher/pom.xml b/streamis-jobmanager/streamis-job-launcher/pom.xml old mode 100644 new mode 100755 index 77944f06e..d880c2b22 --- a/streamis-jobmanager/streamis-job-launcher/pom.xml +++ b/streamis-jobmanager/streamis-job-launcher/pom.xml @@ -10,6 +10,10 @@ 4.0.0 streamis-job-launcher + pom - + + streamis-job-launcher-base + streamis-job-launcher-service + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-launcher/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/JobLauncher.java b/streamis-jobmanager/streamis-job-launcher/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/JobLauncher.java deleted file mode 100644 index 33105093b..000000000 --- a/streamis-jobmanager/streamis-job-launcher/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/JobLauncher.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.webank.wedatasphere.streamis.jobmanager.launcher; - -public interface JobLauncher { -} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml new file mode 100755 index 000000000..740efecd8 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml @@ -0,0 +1,54 @@ + + + + streamis-jobmanager + com.webank.wedatasphere.streamis + 0.1.0-SNAPSHOT + ../../pom.xml + + 4.0.0 + + streamis-job-launcher-base + + + + 8 + 8 + + + + + com.webank.wedatasphere.linkis + linkis-mybatis + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + src/main/java + + **/*.xml + + + + ${project.artifactId}-${project.version} + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/ConfigMapper.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/ConfigMapper.java new file mode 100755 index 000000000..8b8161d28 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/ConfigMapper.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.dao; + +import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.ConfigKey; +import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.ConfigKeyValue; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +public interface ConfigMapper { + + List getConfigKeyValues(@Param("type") Integer type,@Param("jobId") Long jobId); + + ConfigKeyValue getConfigKeyValue(@Param("jobId") Long jobId,@Param("configkeyId") Long configkeyId); + + void deleteKeyValue(@Param("jobId") Long jobId,@Param("type") Integer type,@Param("configkeyId") Long configkeyId); + + List getConfigKey(); + + void insertValue(ConfigKeyValue configValue); + + + void updateUserValue(ConfigKeyValue configValue); +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/JobUserRoleMapper.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/JobUserRoleMapper.java new file mode 100644 index 000000000..66e7a4d7b --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/JobUserRoleMapper.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.dao; + +import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.JobUserRole; +import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.User; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +/** + * @author limeng + */ +public interface JobUserRoleMapper { + + void insertUser(User user); + + void insertJobUserRole(JobUserRole jobUserRole); + + void updateJobUserRole(JobUserRole jobUserRole); + + void deleteByJobUserRole(@Param("id") Long id,@Param("jobId") Long jobId); + + + List getUsersByUserName(@Param("username") String username); + + List getUserRoleById(@Param("jobId") Long jobId,@Param("username") String usernam); + +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/ConfigMapper.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/ConfigMapper.xml new file mode 100644 index 000000000..4a2c668b9 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/ConfigMapper.xml @@ -0,0 +1,128 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + `id`, `key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`,`type`,`sort`,`status` + + + + `id` ,`configkey_id`,`config_value`,`type`,`job_id`,`job_name`,`config_key` + + + + k.id,k.key,k.name,k.description, k.default_value, k.validate_type, k.validate_range, k.is_hidden, k.is_advanced,k.level, k.treeName,k.type,k.sort,k.`status`,v.id as value_id ,v.config_key,v.configkey_id,v.config_value,v.job_id,v.job_name + + + + + + + + + + + + + delete from linkis_stream_configuration_config_value + + 1=1 + + AND job_id=#{jobId} + + + AND `type`=#{type} + + + AND configkey_id=#{configkeyId} + + + + + + + INSERT INTO linkis_stream_configuration_config_value(`configkey_id`,`config_value`,`type`,`job_id`,`job_name`,`config_key`) VALUES(#{configkeyId},#{configValue},#{type},#{jobId},#{jobName},#{configKey}) + + + + UPDATE linkis_stream_configuration_config_value + + + config_value=#{configValue}, + + + WHERE id=#{id} + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/JobUserRoleMapper.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/JobUserRoleMapper.xml new file mode 100644 index 000000000..3889b31a7 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/JobUserRoleMapper.xml @@ -0,0 +1,105 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + `id`,`username`,`name` + + + + `id`,`job_id`,`user_id`,`role_id`,`username` + + + + INSERT INTO linkis_stream_user() VALUES(#{id},#{username},#{name}) + + + + INSERT INTO linkis_stream_job_user_role(`job_id`,`user_id`,`role_id`,`username`) VALUES(#{jobId},#{userId},#{roleId},#{username}) + + + + UPDATE linkis_stream_job_user_role + + + job_id=#{jobId}, + + + user_id=#{userId}, + + + username=#{username}, + + + where id=#{id} + + + + DELETE FROM linkis_stream_job_user_role + + 1=1 + AND id=#{id} + AND job_id=#{jobId} + + + + + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/ConfigKey.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/ConfigKey.java new file mode 100755 index 000000000..9f3d58dbe --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/ConfigKey.java @@ -0,0 +1,133 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.entity; + +public class ConfigKey { + private Long id; + private String key; + private String description; + private String name; + private String defaultValue; + private String validateType; + private String validateRange; + private Boolean isAdvanced; + private Boolean isHidden; + private Integer level; + private Integer type; + private Integer sort; + private String treeName; + private Integer status; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + + public String getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(String defaultValue) { + this.defaultValue = defaultValue; + } + + public String getValidateType() { + return validateType; + } + + public void setValidateType(String validateType) { + this.validateType = validateType; + } + + public String getValidateRange() { + return validateRange; + } + + public void setValidateRange(String validateRange) { + this.validateRange = validateRange; + } + + public Boolean getAdvanced() { + return isAdvanced; + } + + public void setAdvanced(Boolean advanced) { + isAdvanced = advanced; + } + + public Boolean getHidden() { + return isHidden; + } + + public void setHidden(Boolean hidden) { + isHidden = hidden; + } + + public Integer getLevel() { + return level; + } + + public void setLevel(Integer level) { + this.level = level; + } + + public Integer getType() { + return type; + } + + public void setType(Integer type) { + this.type = type; + } + + public Integer getSort() { + return sort; + } + + public void setSort(Integer sort) { + this.sort = sort; + } + + public String getTreeName() { + return treeName; + } + + public void setTreeName(String treeName) { + this.treeName = treeName; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/ConfigKeyValue.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/ConfigKeyValue.java new file mode 100755 index 000000000..d55ffdb1e --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/ConfigKeyValue.java @@ -0,0 +1,166 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.entity; + +public class ConfigKeyValue { + private Long id; + private String key; + private String description; + private String name; + private String defaultValue; + private String validateType; + private String validateRange; + private Integer level; + private String treeName; + private Long configkeyId; + private String configValue; + private Integer type; + private Long jobId; + private String jobName; + private Integer status; + private Integer sort; + private Long valueId; + private String configKey; + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(String defaultValue) { + this.defaultValue = defaultValue; + } + + public String getValidateType() { + return validateType; + } + + public void setValidateType(String validateType) { + this.validateType = validateType; + } + + public String getValidateRange() { + return validateRange; + } + + public void setValidateRange(String validateRange) { + this.validateRange = validateRange; + } + + public Integer getLevel() { + return level; + } + + public void setLevel(Integer level) { + this.level = level; + } + + public String getTreeName() { + return treeName; + } + + public void setTreeName(String treeName) { + this.treeName = treeName; + } + + public Long getConfigkeyId() { + return configkeyId; + } + + public void setConfigkeyId(Long configkeyId) { + this.configkeyId = configkeyId; + } + + public String getConfigValue() { + return configValue; + } + + public void setConfigValue(String configValue) { + this.configValue = configValue; + } + + public Integer getType() { + return type; + } + + public void setType(Integer type) { + this.type = type; + } + + public Long getJobId() { + return jobId; + } + + public void setJobId(Long jobId) { + this.jobId = jobId; + } + + public String getJobName() { + return jobName; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public Integer getSort() { + return sort; + } + + public void setSort(Integer sort) { + this.sort = sort; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getValueId() { + return valueId; + } + + public void setValueId(Long valueId) { + this.valueId = valueId; + } + + public String getConfigKey() { + return configKey; + } + + public void setConfigKey(String configKey) { + this.configKey = configKey; + } +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobRole.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobRole.java new file mode 100644 index 000000000..16695dcb6 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobRole.java @@ -0,0 +1,51 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.entity; + +import java.util.Date; + +public class JobRole { + private Long id; + private Long jobId; + private Long roleId; + private Date updateTime; + private String description; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getJobId() { + return jobId; + } + + public void setJobId(Long jobId) { + this.jobId = jobId; + } + + public Long getRoleId() { + return roleId; + } + + public void setRoleId(Long roleId) { + this.roleId = roleId; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUser.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUser.java new file mode 100644 index 000000000..b6860511f --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUser.java @@ -0,0 +1,40 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.entity; + +public class JobUser { + private Long id; + private Long jobId; + private Long userId; + private String username; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getJobId() { + return jobId; + } + + public void setJobId(Long jobId) { + this.jobId = jobId; + } + + public Long getUserId() { + return userId; + } + + public void setUserId(Long userId) { + this.userId = userId; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUserRole.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUserRole.java new file mode 100755 index 000000000..f736e62f9 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/JobUserRole.java @@ -0,0 +1,52 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.entity; + +/** + * @author limeng + */ +public class JobUserRole { + private Long id; + private Long jobId; + private Long userId; + private Long roleId; + private String username; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getJobId() { + return jobId; + } + + public void setJobId(Long jobId) { + this.jobId = jobId; + } + + public Long getUserId() { + return userId; + } + + public void setUserId(Long userId) { + this.userId = userId; + } + + public Long getRoleId() { + return roleId; + } + + public void setRoleId(Long roleId) { + this.roleId = roleId; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/User.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/User.java new file mode 100755 index 000000000..fd300f5b6 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/User.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.entity; + +/** + * @author limeng + */ +public class User { + private Long id; + private String username; + private String name; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/dto/ConfigKeyValueDTO.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/dto/ConfigKeyValueDTO.java new file mode 100755 index 000000000..82c533fa7 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/dto/ConfigKeyValueDTO.java @@ -0,0 +1,79 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.entity.dto; + +/** + * @author limeng + */ +public class ConfigKeyValueDTO { + private Long configkeyId; + private String name; + private String key; + private String defaultValue; + private String value; + private Integer type; + private Integer status; + private Integer sort; + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public Integer getType() { + return type; + } + + public void setType(Integer type) { + this.type = type; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public Integer getSort() { + return sort; + } + + public void setSort(Integer sort) { + this.sort = sort; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(String defaultValue) { + this.defaultValue = defaultValue; + } + + public Long getConfigkeyId() { + return configkeyId; + } + + public void setConfigkeyId(Long configkeyId) { + this.configkeyId = configkeyId; + } +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/vo/ConfigKeyVO.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/vo/ConfigKeyVO.java new file mode 100755 index 000000000..004578922 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/vo/ConfigKeyVO.java @@ -0,0 +1,70 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.entity.vo; + +import java.util.List; + +/** + * @author limeng + */ +public class ConfigKeyVO { + + private Long jobId; + + private List resourceConfig; + + private List produceConfig; + + private List parameterConfig; + + private List alarmConfig; + + private List permissionConfig; + + + public Long getJobId() { + return jobId; + } + + public void setJobId(Long jobId) { + this.jobId = jobId; + } + + public List getResourceConfig() { + return resourceConfig; + } + + public void setResourceConfig(List resourceConfig) { + this.resourceConfig = resourceConfig; + } + + public List getProduceConfig() { + return produceConfig; + } + + public void setProduceConfig(List produceConfig) { + this.produceConfig = produceConfig; + } + + public List getParameterConfig() { + return parameterConfig; + } + + public void setParameterConfig(List parameterConfig) { + this.parameterConfig = parameterConfig; + } + + public List getAlarmConfig() { + return alarmConfig; + } + + public void setAlarmConfig(List alarmConfig) { + this.alarmConfig = alarmConfig; + } + + public List getPermissionConfig() { + return permissionConfig; + } + + public void setPermissionConfig(List permissionConfig) { + this.permissionConfig = permissionConfig; + } +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/vo/ConfigRelationVO.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/vo/ConfigRelationVO.java new file mode 100755 index 000000000..8bc9635b2 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/entity/vo/ConfigRelationVO.java @@ -0,0 +1,83 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.entity.vo; + +import java.util.List; + +/** + * @author limeng + */ +public class ConfigRelationVO { + private Long configkeyId; + private String key; + private String name; + private String value; + private List valueLists; + + public static class ValueList{ + private String value; + private Boolean selected; + + public ValueList() { + } + + public ValueList(String value, Boolean selected) { + this.value = value; + this.selected = selected; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public Boolean getSelected() { + return selected; + } + + public void setSelected(Boolean selected) { + this.selected = selected; + } + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public List getValueLists() { + return valueLists; + } + + public void setValueLists(List valueLists) { + this.valueLists = valueLists; + } + + public Long getConfigkeyId() { + return configkeyId; + } + + public void setConfigkeyId(Long configkeyId) { + this.configkeyId = configkeyId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml new file mode 100755 index 000000000..773a5c78d --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml @@ -0,0 +1,50 @@ + + + + streamis-jobmanager + com.webank.wedatasphere.streamis + 0.1.0-SNAPSHOT + ../../pom.xml + + 4.0.0 + + streamis-job-launcher-service + + + 8 + 8 + + + + + com.webank.wedatasphere.streamis + streamis-job-launcher-base + ${jobmanager.version} + + + com.webank.wedatasphere.linkis + linkis-module + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + ${project.artifactId}-${project.version} + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/ConfigConf.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/ConfigConf.scala new file mode 100644 index 000000000..65a68317f --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/ConfigConf.scala @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.conf + +import com.webank.wedatasphere.linkis.common.conf.{CommonVars, TimeType} + +/** + * @author limeng + */ +object ConfigConf { + + val JOBMANAGER_FLINK_RESOURCE = CommonVars("wds.linkis.flink.resource", 1) + + val JOBMANAGER_FLINK_CUSTOM = CommonVars("wds.linkis.flink.custom", 2) + + val JOBMANAGER_FLINK_PRODUCE = CommonVars("wds.linkis.flink.produce", 3) + + val JOBMANAGER_FLINK_ALERT = CommonVars("wds.linkis.flink.alert", 4) + + val JOBMANAGER_FLINK_AUTHORITY = CommonVars("wds.linkis.flink.authority", 5) + + val JOBMANAGER_FLINK_CUSTOM_STATUS_ONE = CommonVars("wds.linkis.flink.custom.one", 1) + + val JOBMANAGER_FLINK_CUSTOM_STATUS_TWO = CommonVars("wds.linkis.flink.custom.two", 2) + + val JOBMANAGER_FLINK_AUTHORITY_VISIBLE = CommonVars("wds.linkis.flink.authority.visible", "wds.linkis.flink.authority.visible") + + val JOBMANAGER_FLINK_AUTHORITY_AUTHOR = CommonVars("wds.linkis.flink.authority.author", "wds.linkis.flink.authority.author") + + val JOBMANAGER_FLINK_ALERT_RULE = CommonVars("wds.linkis.flink.alert.rule", "wds.linkis.flink.alert.rule") + +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/service/ConfigurationService.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/service/ConfigurationService.scala new file mode 100644 index 000000000..f36a486e0 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/service/ConfigurationService.scala @@ -0,0 +1,202 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.service + +import com.webank.wedatasphere.linkis.common.utils.Logging +import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.ConfigConf +import com.webank.wedatasphere.streamis.jobmanager.launcher.dao.{ConfigMapper, JobUserRoleMapper} +import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.dto.ConfigKeyValueDTO +import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.{ConfigKeyValue, JobUserRole} +import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.vo.{ConfigKeyVO, ConfigRelationVO} +import org.springframework.beans.BeanUtils +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service + +import java.util +import scala.collection.JavaConverters._ +/** + * @author limeng + */ +@Service +class ConfigurationService extends Logging{ + + @Autowired private var configMapper:ConfigMapper = _ + + @Autowired private var jobUserRoleMapper:JobUserRoleMapper= _ + + def addKeyValue(vo:ConfigKeyVO): Unit ={ + + val baseList = vo.getResourceConfig.asScala.++=(vo.getAlarmConfig.asScala).++=(vo.getProduceConfig.asScala) + + baseList.foreach(f=>{ + val configKeyValue = configMapper.getConfigKeyValue(vo.getJobId, f.getConfigkeyId) + if(configKeyValue != null){ + configMapper.deleteKeyValue(vo.getJobId,null,f.getConfigkeyId) + } + + val value = new ConfigKeyValue() + value.setConfigkeyId(f.getConfigkeyId) + value.setKey(f.getKey) + value.setConfigValue(f.getValue) + value.setJobId(vo.getJobId) + configMapper.insertValue(value) + + }) + + val jobKeyTypes = configMapper.getConfigKeyValues(ConfigConf.JOBMANAGER_FLINK_CUSTOM.getValue, vo.getJobId) + if(jobKeyTypes!=null && jobKeyTypes.size()>0){ + configMapper.deleteKeyValue(vo.getJobId,ConfigConf.JOBMANAGER_FLINK_CUSTOM.getValue,null) + } + vo.getParameterConfig.asScala.foreach(f=>{ + val keyValue = new ConfigKeyValue() + keyValue.setKey(f.getKey) + keyValue.setJobId(vo.getJobId) + keyValue.setConfigValue(f.getValue) + keyValue.setType(ConfigConf.JOBMANAGER_FLINK_CUSTOM.getValue) + configMapper.insertValue(keyValue) + }) + + //权限 + vo.getPermissionConfig.asScala.filter(f=>f.getKey.equals(ConfigConf.JOBMANAGER_FLINK_AUTHORITY_AUTHOR.getValue)).foreach(f=>{ + val value = configMapper.getConfigKeyValue(vo.getJobId, f.getConfigkeyId) + if(value != null){ + configMapper.deleteKeyValue(vo.getJobId,null,f.getConfigkeyId) + } + + val keyValue = new ConfigKeyValue() + keyValue.setKey(f.getKey) + keyValue.setJobId(vo.getJobId) + keyValue.setConfigValue(f.getValue) + keyValue.setType(ConfigConf.JOBMANAGER_FLINK_AUTHORITY.getValue) + keyValue.setConfigkeyId(f.getConfigkeyId) + configMapper.insertValue(keyValue) + }) + + jobUserRoleMapper.deleteByJobUserRole(null,vo.getJobId) + vo.getPermissionConfig.asScala.filter(f=>f.getKey.equals(ConfigConf.JOBMANAGER_FLINK_AUTHORITY_VISIBLE.getValue)).foreach(f=>{ + f.getValueLists.asScala.foreach(f=>{ + val users = jobUserRoleMapper.getUsersByUserName(f.getValue) + if(users != null && users.size() > 0){ + users.asScala.foreach(f2=>{ + val role = new JobUserRole() + role.setJobId(vo.getJobId) + role.setUserId(f2.getId) + role.setUsername(f2.getUsername) + jobUserRoleMapper.insertJobUserRole(role) + }) + } + }) + }) + + } + + + def getFullTree(jobId:Long): ConfigKeyVO ={ + + val configValues = configMapper.getConfigKeyValues(null,jobId).asScala + + var dtos:List[ConfigKeyValueDTO] = null + if(configValues == null || configValues.isEmpty){ + val configKeys = configMapper.getConfigKey.asScala + dtos = configKeys.filter(f=>f.getSort != 0).map(m=>{ + val dto = new ConfigKeyValueDTO() + BeanUtils.copyProperties(m,dto) + dto.setConfigkeyId(m.getId) + dto + }).toList + }else{ + dtos = configValues.filter(f=>f.getSort != 0).map(m=>{ + val dto = new ConfigKeyValueDTO() + BeanUtils.copyProperties(m,dto) + dto.setValue(m.getConfigValue) + dto.setConfigkeyId(m.getId) + dto + }).toList + } + + val vo = configKeyValueToVO(jobId,dtos) + val jobKeyTypes = configMapper.getConfigKeyValues(ConfigConf.JOBMANAGER_FLINK_CUSTOM.getValue, jobId) + if(jobKeyTypes!=null && jobKeyTypes.size()>0){ + vo.setParameterConfig(jobKeyTypes.asScala.map(m=>{ + val vo = new ConfigRelationVO() + vo.setConfigkeyId(m.getConfigkeyId) + vo.setKey(m.getConfigKey) + vo.setName(m.getName) + vo.setValue(m.getConfigValue) + vo + }).asJava) + } + + vo.setJobId(jobId) + vo + } + + + private def configKeyValueToVO(jobId:Long,dtos:List[ConfigKeyValueDTO]): ConfigKeyVO ={ + val statusTwo = ConfigConf.JOBMANAGER_FLINK_CUSTOM_STATUS_TWO.getValue + val configVO = new ConfigKeyVO() + + val groupDDs = dtos.groupBy(_.getType).map(m=>{ + (m._1,m._2.sortBy(_.getSort).map(p => { + val vo = new ConfigRelationVO() + BeanUtils.copyProperties(p,vo) + + if (p.getStatus.equals(statusTwo)) { + val lists = p.getDefaultValue.split(",").map(f => { + if(p.getKey.equals(ConfigConf.JOBMANAGER_FLINK_ALERT_RULE.getValue) && p.getValue != null) { + val s = p.getValue.split(",") + if(s.contains(f)) new ConfigRelationVO.ValueList(f, true) + else new ConfigRelationVO.ValueList(f, false) + } + else if (p.getValue != null && f.trim.equals(p.getValue)) new ConfigRelationVO.ValueList(f, true) + else new ConfigRelationVO.ValueList(f, false) + }) + vo.setValueLists(lists.toList.asJava) + } + vo + })) + }) + + configVO.setResourceConfig(groupDDs.filter(f => f._1.equals(ConfigConf.JOBMANAGER_FLINK_RESOURCE.getValue)).flatMap(_._2).toList.asJava) + configVO.setProduceConfig(groupDDs.filter(f => f._1.equals(ConfigConf.JOBMANAGER_FLINK_PRODUCE.getValue)).flatMap(_._2).toList.asJava) + configVO.setAlarmConfig(groupDDs.filter(f => f._1.equals(ConfigConf.JOBMANAGER_FLINK_ALERT.getValue)).flatMap(_._2).toList.asJava) + + configVO.setPermissionConfig(groupDDs.filter(f => f._1.equals(ConfigConf.JOBMANAGER_FLINK_AUTHORITY.getValue)).flatMap(_._2).toList.asJava) + + //权限 + val users = jobUserRoleMapper.getUsersByUserName(null) + val roles = jobUserRoleMapper.getUserRoleById(jobId,null) + val list = new util.ArrayList[ConfigRelationVO.ValueList]() + val configRelationVO = new ConfigRelationVO() + + if(users != null && users.size() > 0){ + val lists = users.asScala.map(f => { + val value = new ConfigRelationVO.ValueList() + value.setValue(f.getUsername) + value.setSelected(false) + value + }) + if(roles != null && roles.size() > 0){ + lists.foreach(f=>{ + roles.asScala.foreach(f2=>{ + if(f2.getUsername.equals(f.getValue)){ + f.setSelected(true) + } + }) + }) + } + configRelationVO.setKey(ConfigConf.JOBMANAGER_FLINK_AUTHORITY_VISIBLE.getValue) + configRelationVO.setValueLists(list) + + } + + val permissionConfigs =new util.ArrayList[ConfigRelationVO]() + permissionConfigs.add(configVO.getPermissionConfig.asScala.filter(f=>f.getKey.equals(ConfigConf.JOBMANAGER_FLINK_AUTHORITY_AUTHOR.getValue)).head) + permissionConfigs.add(configRelationVO) + configVO.setPermissionConfig(permissionConfigs) + + configVO + } + + + + +} diff --git a/streamis-jobmanager/streamis-job-manager/pom.xml b/streamis-jobmanager/streamis-job-manager/pom.xml old mode 100644 new mode 100755 index a2bd21064..4a5af7de7 --- a/streamis-jobmanager/streamis-job-manager/pom.xml +++ b/streamis-jobmanager/streamis-job-manager/pom.xml @@ -10,6 +10,10 @@ 4.0.0 streamis-job-manager - + pom + + streamis-job-manager-base + streamis-job-manager-service + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml new file mode 100755 index 000000000..93e5148be --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml @@ -0,0 +1,54 @@ + + + + streamis-jobmanager + com.webank.wedatasphere.streamis + 0.1.0-SNAPSHOT + ../../pom.xml + + 4.0.0 + + streamis-job-manager-base + + + + 8 + 8 + + + + + com.webank.wedatasphere.linkis + linkis-mybatis + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + src/main/java + + **/*.xml + + + + ${project.artifactId}-${project.version} + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamBmlMapper.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamBmlMapper.java new file mode 100644 index 000000000..d39cf8ebd --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamBmlMapper.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.dao; + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamBml; +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamBmlVersion; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +public interface StreamBmlMapper { + + void insertBml(StreamBml streamBml); + + void updateBml(StreamBml streamBml); + + List getStreamBmlById(@Param("id") Long id); + + void insertBmlVersion(StreamBmlVersion streamBmlVersion); + + void updateBmlVersion(StreamBmlVersion streamBmlVersion); + + List getStreamBmlVersionById(@Param("bmlId") Long bmlId,@Param("version") String version); + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java new file mode 100644 index 000000000..07d885720 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java @@ -0,0 +1,40 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.dao; + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob; +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJobSqlResource; +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJobVersion; +import org.apache.ibatis.annotations.Param; + +import java.util.List; +import java.util.Map; + +/** + * @author limeng + */ +public interface StreamJobMapper { + + List getJobLists(@Param("projectId") Long projectId,@Param("name") String name,@Param("status") Integer status,@Param("createBy") String createBy); + + StreamJob getJobById(@Param("jobId") Long jobId); + + List getByJobCount(@Param("projectId") Long projectId); + + List getJobSQLResourcesByJobId(@Param("jobVersionId") Long jobVersionId); + + List getJobVersionsById(@Param("jobId") Long jobId,@Param("version") String version); + + void insertJob(StreamJob streamJob); + + void insertJobVersion(StreamJobVersion streamJobVersion); + + void insertJobSqlResource(StreamJobSqlResource streamJobSqlResource); + + void updateJob(StreamJob streamJob); + + void updateJobVersion(StreamJobVersion streamJobVersion); + + void updateJobSqlResource(StreamJobSqlResource streamJobSqlResource); + + + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamProjectMapper.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamProjectMapper.java new file mode 100755 index 000000000..553251cf0 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamProjectMapper.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.dao; + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamProject; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +/** + * @author limeng + */ +public interface StreamProjectMapper { + + void instertProject(StreamProject streamProject); + + void updateProject(StreamProject streamProject); + + List getByProjects(@Param("id") Long id,@Param("workspaceId") Long workspaceId); + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamTaskMapper.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamTaskMapper.java new file mode 100755 index 000000000..a90c3f42b --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamTaskMapper.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.dao; + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamTask; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +/** + * @author limeng + */ +public interface StreamTaskMapper { + + void insertTask(StreamTask streamTask); + + void updateTask(StreamTask streamTask); + + List getByJobIds(@Param("jobId") Long jobId,@Param("version") String version); + + StreamTask getTaskById(@Param("id") Long id); +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamBmlMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamBmlMapper.xml new file mode 100755 index 000000000..19b875c62 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamBmlMapper.xml @@ -0,0 +1,104 @@ + + + + + + + + + + + + + + + + + + + + + + + + + `id`,`name`,`bml_type`,`org_identification`,`latest_version` + + + + `id`,`bml_id`,`version`,`storage_path`,`attribute` + + + + INSERT INTO linkis_stream_bml() VALUES(#{id},#{name},#{bmlType},#{orgIdentification},#{latestVersion}) + + + + UPDATE linkis_stream_bml + + + name=#{name}, + + + bml_type=#{bmlType}, + + + latest_version=#{latestVersion} + + + WHERE id=#{id} + + + + + + INSERT INTO linkis_stream_bml_version() VALUES(#{id},#{bmlId},#{version},#{storagePath},#{attribute}) + + + + UPDATE linkis_stream_bml_version + + + storage_path=#{storagePath}, + + + `attribute`=#{attribute}, + + + WHERE id=#{id} + + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml new file mode 100644 index 000000000..96a89e105 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml @@ -0,0 +1,164 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + `id`,`project_id`,`name`,`current_task_id`,`current_version`,`current_release_time`,`org_identification`,`create_by`,`label`,`description` + + + + `id`,`job_id`,`version`,`program_arguments`,`bml_version`,`bml_id` + + + + `id`,`job_version_id`,`execute_sql` + + + + + + + + + + + + + + + INSERT INTO linkis_stream_job(`project_id`,`name`,`current_task_id`,`current_version`,`current_release_time`,`org_identification`,`create_by`,`label`,`description`,`job_type`) VALUES(##{projectId},#{name},#{currentTaskId},#{currentVersion},#{currentReleaseTime},#{orgIdentification},#{createBy},#{label},#{description},#{jobType}) + + + + INSERT INTO linkis_stream_job_version(`job_id`,`version`,`program_arguments`,`bml_version`,`bml_id`) VALUES(#{jobId},#{version},#{programArguments},#{bmlVersion},#{bmlId}) + + + + INSERT INTO linkis_stream_job_sql_resource(`job_version_id`,`execute_sql`) VALUES(#{jobVersionId},#{executeSql}) + + + + UPDATE linkis_stream_job + + + name=#{name}, + + + current_task_id=#{currentTaskId}, + + + current_version=#{currentVersion}, + + + current_release_time=#{currentReleaseTime}, + + + create_by=#{createBy}, + + + label=#{label}, + + + WHERE id=#{id} + + + + UPDATE linkis_stream_job_version + + + program_arguments={programArguments}, + + + WHERE id=#{id} + + + + UPDATE linkis_stream_job_sql_resource + + + execute_sql=#{executeSql}, + + + WHERE id=#{id} + + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamProjectMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamProjectMapper.xml new file mode 100755 index 000000000..5bc274a3a --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamProjectMapper.xml @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + `id`,`workspace_id`,`name`,`create_by` + + + + + + INSERT INTO linkis_stream_project() VALUES(#{id},#{workspaceId},#{name},#{createBy}) + + + + UPDATE linkis_stream_project + + + name=#{name}, + + + WHERE id=#{id} + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml new file mode 100644 index 000000000..e7564e0a5 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + + + `id`,`job_version_id`,`version`,`status`,`start_time`,`end_time`,`err_desc`,`submit_user` + + + + INSERT INTO linkis_stream_task(`job_version_id`,`version`,`status`,`start_time`,`end_time`,`err_desc`,`submit_user`) VALUES(#{jobVersionId},#{version},#{status},#{startTime},#{endTime},#{errDesc},#{submitUser}) + + + + UPDATE linkis_stream_task + + + job_version_id=#{jobVersionId}, + + + version=#{version}, + + + status=#{status}, + + + start_time=#{startTime}, + + + end_time=#{endTime}, + + + WHERE id=#{id} + + + + + + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamBml.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamBml.java new file mode 100644 index 000000000..c1a56552a --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamBml.java @@ -0,0 +1,49 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity; + +public class StreamBml { + private Long id; + private String name; + private Integer bmlType; + private String orgIdentification; + private String latestVersion; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Integer getBmlType() { + return bmlType; + } + + public void setBmlType(Integer bmlType) { + this.bmlType = bmlType; + } + + public String getOrgIdentification() { + return orgIdentification; + } + + public void setOrgIdentification(String orgIdentification) { + this.orgIdentification = orgIdentification; + } + + public String getLatestVersion() { + return latestVersion; + } + + public void setLatestVersion(String latestVersion) { + this.latestVersion = latestVersion; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamBmlVersion.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamBmlVersion.java new file mode 100755 index 000000000..0e1e114ac --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamBmlVersion.java @@ -0,0 +1,52 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity; + +/** + * @author limeng + */ +public class StreamBmlVersion { + private Long id; + private Long bmlId; + private String version; + private String storagePath; + private String attribute; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getBmlId() { + return bmlId; + } + + public void setBmlId(Long bmlId) { + this.bmlId = bmlId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getStoragePath() { + return storagePath; + } + + public void setStoragePath(String storagePath) { + this.storagePath = storagePath; + } + + public String getAttribute() { + return attribute; + } + + public void setAttribute(String attribute) { + this.attribute = attribute; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJob.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJob.java new file mode 100644 index 000000000..d6b82c91e --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJob.java @@ -0,0 +1,146 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity; + +import java.util.Date; + +/** + * @author limeng + */ +public class StreamJob { + private Long id; + private Long workspaceId; + private Long projectId; + private String name; + private Integer jobType; + private Long currentTaskId; + private String currentVersion; + private Date currentReleaseTime; + private Integer status; + private Long orgIdentification; + private String createBy; + private String label; + private String currentReleased; + private String description; + private String submitUser; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + + public Long getCurrentTaskId() { + return currentTaskId; + } + + public void setCurrentTaskId(Long currentTaskId) { + this.currentTaskId = currentTaskId; + } + + public String getCurrentVersion() { + return currentVersion; + } + + public void setCurrentVersion(String currentVersion) { + this.currentVersion = currentVersion; + } + + public Date getCurrentReleaseTime() { + return currentReleaseTime; + } + + public void setCurrentReleaseTime(Date currentReleaseTime) { + this.currentReleaseTime = currentReleaseTime; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public Long getOrgIdentification() { + return orgIdentification; + } + + public void setOrgIdentification(Long orgIdentification) { + this.orgIdentification = orgIdentification; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public String getCurrentReleased() { + return currentReleased; + } + + public void setCurrentReleased(String currentReleased) { + this.currentReleased = currentReleased; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public String getSubmitUser() { + return submitUser; + } + + public void setSubmitUser(String submitUser) { + this.submitUser = submitUser; + } + + public Integer getJobType() { + return jobType; + } + + public void setJobType(Integer jobType) { + this.jobType = jobType; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobSqlResource.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobSqlResource.java new file mode 100755 index 000000000..ad4101676 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobSqlResource.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity; + +/** + * @author limeng + */ +public class StreamJobSqlResource { + private Long id; + private Long jobVersionId; + private String executeSql; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getJobVersionId() { + return jobVersionId; + } + + public void setJobVersionId(Long jobVersionId) { + this.jobVersionId = jobVersionId; + } + + public String getExecuteSql() { + return executeSql; + } + + public void setExecuteSql(String executeSql) { + this.executeSql = executeSql; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersion.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersion.java new file mode 100755 index 000000000..6cea14739 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersion.java @@ -0,0 +1,61 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity; + +/** + * @author limeng + */ +public class StreamJobVersion { + private Long id; + private Long jobId; + private String version; + private String programArguments; + private String bmlVersion; + private Long bmlId; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getJobId() { + return jobId; + } + + public void setJobId(Long jobId) { + this.jobId = jobId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getProgramArguments() { + return programArguments; + } + + public void setProgramArguments(String programArguments) { + this.programArguments = programArguments; + } + + public String getBmlVersion() { + return bmlVersion; + } + + public void setBmlVersion(String bmlVersion) { + this.bmlVersion = bmlVersion; + } + + public Long getBmlId() { + return bmlId; + } + + public void setBmlId(Long bmlId) { + this.bmlId = bmlId; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamProject.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamProject.java new file mode 100755 index 000000000..b4ee37923 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamProject.java @@ -0,0 +1,43 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity; + +/** + * @author limeng + */ +public class StreamProject { + private Long id; + private Long workspaceId; + private String name; + private String createBy; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamTask.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamTask.java new file mode 100755 index 000000000..2e85be5fa --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamTask.java @@ -0,0 +1,90 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity; + +import java.util.Date; + +/** + * @author limeng + */ +public class StreamTask { + private Long id; + private Long jobVersionId; + private String jobName; + private String submitUser; + private String version; + private Date startTime; + private Date endTime; + private String errDesc; + private Integer status; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getJobName() { + return jobName; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public Date getStartTime() { + return startTime; + } + + public void setStartTime(Date startTime) { + this.startTime = startTime; + } + + public Date getEndTime() { + return endTime; + } + + public void setEndTime(Date endTime) { + this.endTime = endTime; + } + + public String getErrDesc() { + return errDesc; + } + + public void setErrDesc(String errDesc) { + this.errDesc = errDesc; + } + + public Long getJobVersionId() { + return jobVersionId; + } + + public void setJobVersionId(Long jobVersionId) { + this.jobVersionId = jobVersionId; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public String getSubmitUser() { + return submitUser; + } + + public void setSubmitUser(String submitUser) { + this.submitUser = submitUser; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobDetailsVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobDetailsVO.java new file mode 100644 index 000000000..9ca623ea0 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobDetailsVO.java @@ -0,0 +1,179 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +import java.util.List; + +public class JobDetailsVO { + + private List realTimeTraffic; + private List dataNumber; + private List loadCondition; + + public List getRealTimeTraffic() { + return realTimeTraffic; + } + + public void setRealTimeTraffic(List realTimeTraffic) { + this.realTimeTraffic = realTimeTraffic; + } + + public List getDataNumber() { + return dataNumber; + } + + public void setDataNumber(List dataNumber) { + this.dataNumber = dataNumber; + } + + public List getLoadCondition() { + return loadCondition; + } + + public void setLoadCondition(List loadCondition) { + this.loadCondition = loadCondition; + } + + public static class RealTimeTrafficDTO{ + private String sourceKey; + private String sourceSpeed; + private String transformKey; + private String transformSpeed; + private String sinkKey; + private String sinkSpeed; + + public String getSourceKey() { + return sourceKey; + } + + public void setSourceKey(String sourceKey) { + this.sourceKey = sourceKey; + } + + public String getSourceSpeed() { + return sourceSpeed; + } + + public void setSourceSpeed(String sourceSpeed) { + this.sourceSpeed = sourceSpeed; + } + + public String getTransformKey() { + return transformKey; + } + + public void setTransformKey(String transformKey) { + this.transformKey = transformKey; + } + + public String getTransformSpeed() { + return transformSpeed; + } + + public void setTransformSpeed(String transformSpeed) { + this.transformSpeed = transformSpeed; + } + + public String getSinkKey() { + return sinkKey; + } + + public void setSinkKey(String sinkKey) { + this.sinkKey = sinkKey; + } + + public String getSinkSpeed() { + return sinkSpeed; + } + + public void setSinkSpeed(String sinkSpeed) { + this.sinkSpeed = sinkSpeed; + } + } + + public static class DataNumberDTO{ + private String dataName; + private Integer dataNumber; + + public String getDataName() { + return dataName; + } + + public void setDataName(String dataName) { + this.dataName = dataName; + } + + public Integer getDataNumber() { + return dataNumber; + } + + public void setDataNumber(Integer dataNumber) { + this.dataNumber = dataNumber; + } + } + + public static class LoadConditionDTO{ + private String type; + private String host; + private String memory; + private String totalMemory; + private String gcTotalTime; + private String gcLastTime; + private String gcLastConsume; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + + public String getGcTotalTime() { + return gcTotalTime; + } + + public void setGcTotalTime(String gcTotalTime) { + this.gcTotalTime = gcTotalTime; + } + + public String getGcLastTime() { + return gcLastTime; + } + + public void setGcLastTime(String gcLastTime) { + this.gcLastTime = gcLastTime; + } + + public String getGcLastConsume() { + return gcLastConsume; + } + + public void setGcLastConsume(String gcLastConsume) { + this.gcLastConsume = gcLastConsume; + } + + public String getMemory() { + return memory; + } + + public void setMemory(String memory) { + this.memory = memory; + } + + public String getTotalMemory() { + return totalMemory; + } + + public void setTotalMemory(String totalMemory) { + this.totalMemory = totalMemory; + } + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobFlowVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobFlowVO.java new file mode 100644 index 000000000..06580a507 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobFlowVO.java @@ -0,0 +1,126 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +/** + * 工作流程 + */ +public class JobFlowVO { + private Long jobId; + private String name; + private Boolean state; + private String source; + private String description; + private String createTime; + private String creator; + private Long projectID; + private String dssLabels; + private Long resourceId; + private String bmlVersion; + private String flowJson; + private Boolean rootFlow; + + public Long getJobId() { + return jobId; + } + + public void setJobId(Long jobId) { + this.jobId = jobId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Boolean getState() { + return state; + } + + public void setState(Boolean state) { + this.state = state; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getCreateTime() { + return createTime; + } + + public void setCreateTime(String createTime) { + this.createTime = createTime; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public Long getProjectID() { + return projectID; + } + + public void setProjectID(Long projectID) { + this.projectID = projectID; + } + + public String getDssLabels() { + return dssLabels; + } + + public void setDssLabels(String dssLabels) { + this.dssLabels = dssLabels; + } + + + + public String getBmlVersion() { + return bmlVersion; + } + + public void setBmlVersion(String bmlVersion) { + this.bmlVersion = bmlVersion; + } + + public String getFlowJson() { + return flowJson; + } + + public void setFlowJson(String flowJson) { + this.flowJson = flowJson; + } + + public Boolean getRootFlow() { + return rootFlow; + } + + public void setRootFlow(Boolean rootFlow) { + this.rootFlow = rootFlow; + } + + public Long getResourceId() { + return resourceId; + } + + public void setResourceId(Long resourceId) { + this.resourceId = resourceId; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobHistoryVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobHistoryVO.java new file mode 100644 index 000000000..359c75e3b --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobHistoryVO.java @@ -0,0 +1,85 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +public class JobHistoryVO { + private String taskId; + private String jobName; + private String creator; + private String version; + private String status; + private String startTime; + private String endTime; + private String runTime; + private String stopCause; + + public String getTaskId() { + return taskId; + } + + public void setTaskId(String taskId) { + this.taskId = taskId; + } + + public String getJobName() { + return jobName; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getStartTime() { + return startTime; + } + + public void setStartTime(String startTime) { + this.startTime = startTime; + } + + public String getEndTime() { + return endTime; + } + + public void setEndTime(String endTime) { + this.endTime = endTime; + } + + public String getRunTime() { + return runTime; + } + + public void setRunTime(String runTime) { + this.runTime = runTime; + } + + public String getStopCause() { + return stopCause; + } + + public void setStopCause(String stopCause) { + this.stopCause = stopCause; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobProgressVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobProgressVO.java new file mode 100644 index 000000000..199424739 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobProgressVO.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +public class JobProgressVO { + private Long taskId; + private Integer progress; + + public Long getTaskId() { + return taskId; + } + + public void setTaskId(Long taskId) { + this.taskId = taskId; + } + + public Integer getProgress() { + return progress; + } + + public void setProgress(Integer progress) { + this.progress = progress; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobUploadDetailsVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobUploadDetailsVO.java new file mode 100644 index 000000000..7f0513007 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobUploadDetailsVO.java @@ -0,0 +1,106 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +import java.util.List; + +/** + * jar 基本信息 + */ +public class JobUploadDetailsVO { + + private List mainJars; + private List dependentList; + private List userList; + private String programArguement; + + public List getMainJars() { + return mainJars; + } + + public void setMainJars(List mainJars) { + this.mainJars = mainJars; + } + + public List getDependentList() { + return dependentList; + } + + public void setDependentList(List dependentList) { + this.dependentList = dependentList; + } + + + public String getProgramArguement() { + return programArguement; + } + + public void setProgramArguement(String programArguement) { + this.programArguement = programArguement; + } + + public List getUserList() { + return userList; + } + + public void setUserList(List userList) { + this.userList = userList; + } + + public static class JarDependentDTO{ + private Long id; + private String name; + private String version; + private String description; + private String entryClass; + private String updateTime; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getEntryClass() { + return entryClass; + } + + public void setEntryClass(String entryClass) { + this.entryClass = entryClass; + } + + public String getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(String updateTime) { + this.updateTime = updateTime; + } + } + + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/QueryJobListVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/QueryJobListVO.java new file mode 100644 index 000000000..34ed8a2a7 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/QueryJobListVO.java @@ -0,0 +1,76 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +public class QueryJobListVO { + private Long jobId; + private String jobName; + private Integer taskStatus; + private String lastReleaseTime; + private String label; + private String version; + private String lastRelease; + private String description; + + public Long getJobId() { + return jobId; + } + + public void setJobId(Long jobId) { + this.jobId = jobId; + } + + public String getJobName() { + return jobName; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public Integer getTaskStatus() { + return taskStatus; + } + + public void setTaskStatus(Integer taskStatus) { + this.taskStatus = taskStatus; + } + + public String getLastReleaseTime() { + return lastReleaseTime; + } + + public void setLastReleaseTime(String lastReleaseTime) { + this.lastReleaseTime = lastReleaseTime; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getLastRelease() { + return lastRelease; + } + + public void setLastRelease(String lastRelease) { + this.lastRelease = lastRelease; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/StreamJobVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/StreamJobVO.java new file mode 100755 index 000000000..b4d9b96bb --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/StreamJobVO.java @@ -0,0 +1,97 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +/** + * @author limeng + */ +public class StreamJobVO { + private Long projectId; + private String name; + private Integer type; + private String orgIdentification; + private String createBy; + private String label; + private String description; + private String bmlVersion; + private String programArguments; + private String resourceId; + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Integer getType() { + return type; + } + + public void setType(Integer type) { + this.type = type; + } + + public String getOrgIdentification() { + return orgIdentification; + } + + public void setOrgIdentification(String orgIdentification) { + this.orgIdentification = orgIdentification; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getBmlVersion() { + return bmlVersion; + } + + public void setBmlVersion(String bmlVersion) { + this.bmlVersion = bmlVersion; + } + + public String getProgramArguments() { + return programArguments; + } + + public void setProgramArguments(String programArguments) { + this.programArguments = programArguments; + } + + public String getResourceId() { + return resourceId; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/StreamTaskListVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/StreamTaskListVO.java new file mode 100755 index 000000000..ecb7b4324 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/StreamTaskListVO.java @@ -0,0 +1,99 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +import java.util.Date; + +/** + * @author limeng + */ +public class StreamTaskListVO { + private Long taskId; + private Long jobVersionId; + private String jobName; + private String creator; + private String version; + private Integer status; + private String startTime; + private String endTime; + private String runTime; + private String stopCause; + + public Long getTaskId() { + return taskId; + } + + public void setTaskId(Long taskId) { + this.taskId = taskId; + } + + public Long getJobVersionId() { + return jobVersionId; + } + + public void setJobVersionId(Long jobVersionId) { + this.jobVersionId = jobVersionId; + } + + public String getJobName() { + return jobName; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public String getStartTime() { + return startTime; + } + + public void setStartTime(String startTime) { + this.startTime = startTime; + } + + public String getEndTime() { + return endTime; + } + + public void setEndTime(String endTime) { + this.endTime = endTime; + } + + public String getRunTime() { + return runTime; + } + + public void setRunTime(String runTime) { + this.runTime = runTime; + } + + public String getStopCause() { + return stopCause; + } + + public void setStopCause(String stopCause) { + this.stopCause = stopCause; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/TaskCoreNumVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/TaskCoreNumVO.java new file mode 100644 index 000000000..8c41b79d5 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/TaskCoreNumVO.java @@ -0,0 +1,77 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +/** + * job核心指标 + */ +public class TaskCoreNumVO { + private Long projectId; + //失败任务数目 + private Integer failureNum; + //运行数目 + private Integer runningNum; + //慢任务数目 + private Integer slowTaskNum; + //告警任务 + private Integer alertNum; + //等待重启数目 + private Integer waitRestartNum; + //已完成数目 + private Integer successNum; + + + public Integer getFailureNum() { + return failureNum; + } + + public void setFailureNum(Integer failureNum) { + this.failureNum = failureNum; + } + + public Integer getRunningNum() { + return runningNum; + } + + public void setRunningNum(Integer runningNum) { + this.runningNum = runningNum; + } + + public Integer getSlowTaskNum() { + return slowTaskNum; + } + + public void setSlowTaskNum(Integer slowTaskNum) { + this.slowTaskNum = slowTaskNum; + } + + public Integer getAlertNum() { + return alertNum; + } + + public void setAlertNum(Integer alertNum) { + this.alertNum = alertNum; + } + + public Integer getWaitRestartNum() { + return waitRestartNum; + } + + public void setWaitRestartNum(Integer waitRestartNum) { + this.waitRestartNum = waitRestartNum; + } + + public Integer getSuccessNum() { + return successNum; + } + + public void setSuccessNum(Integer successNum) { + this.successNum = successNum; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/DateUtils.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/DateUtils.scala new file mode 100644 index 000000000..5feb3910c --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/DateUtils.scala @@ -0,0 +1,54 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.util + +import org.slf4j.LoggerFactory + +import java.text.SimpleDateFormat +import java.util.Date + +object DateUtils extends Serializable { + + private val logger = LoggerFactory.getLogger(DateUtils.getClass) + private val serialVersionUID = 1L + /** + * 默认短日期格式 + * yyyy-MM-dd + */ + val DATE_DEFAULT_FORMAT = "yyyy-MM-dd" + + /** + * 订单号前缀 yyyyMMddHHmmss + */ + val DATETIME_ORDER_FORMAT = "yyyyMMddHHmmss" + /** + * 默认日期时间格式 + * yyyy-MM-dd HH:mm:ss + */ + val DATETIME_DEFAULT_FORMAT = "yyyy-MM-dd HH:mm:ss" + /** + * 默认时间格式 + * HH:mm:ss + */ + val TIME_DEFAULT_FORMAT = "HH:mm:ss" + /** + * 默认日期短格式 + * yyyyMMdd + */ + val DATE_DEFAULT_SHORT_FORMAT = "yyyyMMdd" + /** + * 默认日期时间格式化 + */ + val dateTimeFormat = new SimpleDateFormat(DATETIME_DEFAULT_FORMAT) + + def intervals(start:Date,end:Date): String ={ + if(start == null || end == null) return "" + val nm = 1000 * 60 + val diff = end.getTime - start.getTime + (diff/nm)+"分钟" + } + + def formatDate(dateTime:Date):String = { + if(dateTime == null) return "" + dateTimeFormat.format(dateTime) + } + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml new file mode 100755 index 000000000..a5e2990b3 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml @@ -0,0 +1,51 @@ + + + + streamis-jobmanager + com.webank.wedatasphere.streamis + 0.1.0-SNAPSHOT + ../../pom.xml + + 4.0.0 + + streamis-job-manager-service + + + + 8 + 8 + + + + + com.webank.wedatasphere.streamis + streamis-job-manager-base + ${jobmanager.version} + + + com.webank.wedatasphere.linkis + linkis-module + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + ${project.artifactId}-${project.version} + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala new file mode 100755 index 000000000..c7d243579 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.conf + +import com.webank.wedatasphere.linkis.common.conf.CommonVars + +/** + * @author limeng + */ +object JobConf { + val JOBMANAGER_FLINK_JOB_STATUS_ONE = CommonVars("wds.linkis.flink.job.status.one", 1,"complete") + + val JOBMANAGER_FLINK_JOB_STATUS_TWO = CommonVars("wds.linkis.flink.job.status.two", 2,"Wait for restart") + + val JOBMANAGER_FLINK_JOB_STATUS_THREE = CommonVars("wds.linkis.flink.job.status.three", 3,"Alert") + + val JOBMANAGER_FLINK_JOB_STATUS_FOUR = CommonVars("wds.linkis.flink.job.status.four", 4,"Slow task") + + val JOBMANAGER_FLINK_JOB_STATUS_FIVE = CommonVars("wds.linkis.flink.job.status.five", 5,"run") + + val JOBMANAGER_FLINK_JOB_STATUS_SIX = CommonVars("wds.linkis.flink.job.status.six", 6,"Failed task") + + val JOBMANAGER_FLINK_JOB_STATUS_SEVEN = CommonVars("wds.linkis.flink.job.status.seven", 7,"Stop task") + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala new file mode 100755 index 000000000..1755652a5 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala @@ -0,0 +1,99 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.service + + +import com.webank.wedatasphere.linkis.common.utils.Logging +import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf +import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamBmlMapper, StreamJobMapper} +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{JobFlowVO, JobProgressVO, QueryJobListVO, TaskCoreNumVO} +import com.webank.wedatasphere.streamis.jobmanager.manager.util.DateUtils +import org.slf4j.Logger +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ListBuffer +/** + * @author limeng + */ +@Service +class JobService extends Logging{ + + @Autowired private var streamJobMapper:StreamJobMapper=_ + @Autowired private var streamBmlMapper:StreamBmlMapper=_ + + def getByProList(projectId:Long,jobName:String,jobStatus:Integer,jobCreator:String):java.util.List[QueryJobListVO] = { + val jobLists = streamJobMapper.getJobLists(projectId, jobName, jobStatus, jobCreator).asScala + if(jobLists != null && jobLists.nonEmpty){ + val list = new ListBuffer[QueryJobListVO]() + jobLists.foreach(f=>{ + val queryJobList = new QueryJobListVO() + queryJobList.setJobId(f.getId) + queryJobList.setJobName(f.getName) + queryJobList.setLabel(f.getLabel) + queryJobList.setVersion(f.getCurrentVersion) + queryJobList.setDescription(f.getDescription) + queryJobList.setLastRelease(f.getCreateBy) + queryJobList.setLastReleaseTime(DateUtils.formatDate(f.getCurrentReleaseTime)) + queryJobList.setTaskStatus(f.getStatus) + list.append(queryJobList) + }) + return list.toList.asJava + } + null + } + + /** + * 核心指标 + */ + def countByCores(proId: Long): TaskCoreNumVO ={ + val jobs = streamJobMapper.getByJobCount(proId).asScala + if(jobs != null && jobs.nonEmpty){ + val taskNum = new TaskCoreNumVO() + taskNum.setProjectId(proId) + jobs.filter(f=> f.getStatus!= null).groupBy(_.getStatus).map(m=>(m._1,m._2.size)).foreach(fo=>{ + if(fo._1.equals(JobConf.JOBMANAGER_FLINK_JOB_STATUS_ONE.getValue)) taskNum.setSuccessNum(fo._2) + if(fo._1.equals(JobConf.JOBMANAGER_FLINK_JOB_STATUS_TWO.getValue)) taskNum.setWaitRestartNum(fo._2) + if(fo._1.equals(JobConf.JOBMANAGER_FLINK_JOB_STATUS_THREE.getValue)) taskNum.setAlertNum(fo._2) + if(fo._1.equals(JobConf.JOBMANAGER_FLINK_JOB_STATUS_FOUR.getValue)) taskNum.setSlowTaskNum(fo._2) + if(fo._1.equals(JobConf.JOBMANAGER_FLINK_JOB_STATUS_FIVE.getValue)) taskNum.setRunningNum(fo._2) + if(fo._1.equals(JobConf.JOBMANAGER_FLINK_JOB_STATUS_SIX.getValue)) taskNum.setFailureNum(fo._2) + }) + + return taskNum + } + null + } + + + def getJobFlow(jobId:Long,version:String): JobFlowVO ={ + val job = streamJobMapper.getJobById(jobId) + if(job == null) return null + val jobVersions = streamJobMapper.getJobVersionsById(jobId, version) + if(jobVersions == null || jobVersions.isEmpty) return null + val jobVersionHead = jobVersions.asScala.head + val jobFlow = new JobFlowVO() + jobFlow.setName(job.getName) + jobFlow.setJobId(jobId) + jobFlow.setCreator(job.getCreateBy) + jobFlow.setBmlVersion(jobVersionHead.getBmlVersion) + jobFlow.setResourceId(jobVersionHead.getBmlId) + jobFlow.setDescription(job.getDescription) + jobFlow + } + + + + /** + * 任务版本 + * @param jobId + */ + def getByJobVersion(jobId:Long): Unit ={ + val job = streamJobMapper.getJobById(jobId) + if(job == null) return null + val jobVersions = streamJobMapper.getJobVersionsById(jobId, job.getCurrentVersion) + if(jobVersions == null || jobVersions.isEmpty) return null + val jobVersionHead = jobVersions.asScala.head + + } + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/ProjectService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/ProjectService.scala new file mode 100755 index 000000000..e927f4f7d --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/ProjectService.scala @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.service + +import com.webank.wedatasphere.linkis.common.utils.Logging +import org.springframework.stereotype.Service + +/** + * @author limeng + */ +@Service +class ProjectService extends Logging{ + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskService.scala new file mode 100644 index 000000000..3a1ebcc08 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskService.scala @@ -0,0 +1,147 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.service + + +import com.webank.wedatasphere.linkis.common.utils.Logging +import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf +import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamJobMapper, StreamTaskMapper} +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{StreamJob, StreamJobVersion, StreamTask} +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{JobProgressVO, StreamTaskListVO} +import com.webank.wedatasphere.streamis.jobmanager.manager.util.DateUtils +import org.apache.commons.lang.StringUtils +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service + +import java.util.Date +import scala.collection.JavaConverters._ +import scala.collection.mutable.ListBuffer +/** + * @author limeng + */ +@Service +class TaskService extends Logging{ + + @Autowired private var streamTaskMapper:StreamTaskMapper=_ + @Autowired private var streamJobMapper:StreamJobMapper=_ + + + def saveJob(streamJob:StreamJob): Unit ={ + streamJobMapper.insertJob(streamJob) + } + + def updateJob(streamJob:StreamJob): Unit ={ + streamJobMapper.updateJob(streamJob) + } + + def saveJobVersion(): Unit ={ + + } + + def updateJobVersion(): Unit ={ + + } + + /** + * 执行job + * @param jobId + */ + def executeJob(jobId: Long,userName:String): Unit ={ + val jobs = streamJobMapper.getJobById(jobId) + if(jobs == null || StringUtils.isEmpty(jobs.getCurrentVersion)) return + val versionsJob: StreamJobVersion = streamJobMapper.getJobVersionsById(jobId, jobs.getCurrentVersion).asScala.head + var taskId:Long = 0 + if(jobs.getCurrentTaskId == null){ + taskId = saveTask(versionsJob.getId,userName,new Date(),"v1") + }else{ + val task = streamTaskMapper.getTaskById(jobs.getCurrentTaskId) + val versionVal = task.getVersion.split("v").last.toInt+1 + taskId = saveTask(versionsJob.getId,userName,new Date(),"v"+versionVal) + } + val streamJob = new StreamJob() + streamJob.setId(jobId) + streamJob.setCurrentTaskId(taskId) + streamJobMapper.updateJob(streamJob) + //todo 执行linkis + + } + + def stopJob(jobId: Long,userName:String): Unit ={ + //todo 执行linkis + val jobs = streamJobMapper.getJobById(jobId) + if(jobs == null) return + val task = streamTaskMapper.getTaskById(jobs.getCurrentTaskId) + if(task==null) return + val taskModel = new StreamTask() + taskModel.setId(task.getId) + taskModel.setStatus(JobConf.JOBMANAGER_FLINK_JOB_STATUS_SEVEN.getValue) + streamTaskMapper.updateTask(taskModel) + + val versionsJob: StreamJobVersion = streamJobMapper.getJobVersionsById(jobId, jobs.getCurrentVersion).asScala.head + val taskId = saveTask(versionsJob.getId,userName,new Date(),"v1") + val streamJob = new StreamJob() + streamJob.setId(jobId) + streamJob.setCurrentTaskId(taskId) + streamJobMapper.updateJob(streamJob) + } + + /** + * 查询运行历史 + * @param jobId + * @param version + * @return + */ + def executeHistory(jobId:Long,version:String): java.util.List[StreamTaskListVO] ={ + if(version == null || version =="") return null + val jobVersions = streamJobMapper.getJobVersionsById(jobId, version) + if(jobVersions==null || jobVersions.isEmpty) return null + val jobVersionId = jobVersions.asScala.head.getId + val tasks = streamTaskMapper.getByJobIds(jobVersionId, null) + if(tasks == null || tasks.isEmpty) return null + val list = new ListBuffer[StreamTaskListVO] + tasks.asScala.foreach(f=>{ + val svo = new StreamTaskListVO() + svo.setTaskId(f.getId) + svo.setStatus(f.getStatus) + svo.setCreator(f.getSubmitUser) + svo.setVersion(f.getVersion) + svo.setJobName(f.getJobName) + svo.setStartTime(DateUtils.formatDate(f.getStartTime)) + svo.setEndTime(DateUtils.formatDate(f.getEndTime)) + svo.setJobVersionId(f.getJobVersionId) + svo.setRunTime(DateUtils.intervals(f.getStartTime,f.getEndTime)) + svo.setStopCause(f.getErrDesc) + list.append(svo) + }) + list.asJava + } + + /** + * 每次任务状态 + * @param jobId + * @return + */ + def getByJobStatus(jobId:Long): JobProgressVO ={ + val job = streamJobMapper.getJobById(jobId) + if(job == null) return null + if(job.getCurrentTaskId == null) return null + val task = streamTaskMapper.getTaskById(job.getCurrentTaskId) + if(task == null) return null + val jobProgressVO = new JobProgressVO() + jobProgressVO.setTaskId(job.getCurrentTaskId) + jobProgressVO.setProgress(task.getStatus) + jobProgressVO + } + + private def saveTask(jobVersionId:Long,userName:String,date:Date,version:String): Long ={ + val task = new StreamTask() + task.setVersion(version) + task.setStatus(JobConf.JOBMANAGER_FLINK_JOB_STATUS_FIVE.getValue) + task.setJobVersionId(jobVersionId) + task.setStartTime(date) + task.setEndTime(date) + task.setSubmitUser(userName) + streamTaskMapper.insertTask(task) + task.getId + } + + +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/pom.xml b/streamis-jobmanager/streamis-jobmanager-server/pom.xml index c67d6d121..2c6f8d811 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/pom.xml +++ b/streamis-jobmanager/streamis-jobmanager-server/pom.xml @@ -11,5 +11,74 @@ streamis-jobmanager-server + + + com.webank.wedatasphere.linkis + linkis-module + + + com.webank.wedatasphere.streamis + streamis-job-launcher-service + ${jobmanager.version} + + + com.webank.wedatasphere.linkis + linkis-module + + + + + com.webank.wedatasphere.streamis + streamis-job-manager-service + ${jobmanager.version} + + + com.webank.wedatasphere.linkis + linkis-module + + + + + org.springframework.boot + spring-boot-starter-test + ${spring.boot.version} + test + + + junit + junit + 4.11 + test + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + ${basedir}/src/main/resources + + **/*.properties + **/*.xml + **/*.yml + + + + + ${project.artifactId}-${project.version} + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/ConfigurationException.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/ConfigurationException.java new file mode 100755 index 000000000..5947dc563 --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/ConfigurationException.java @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.streamis.jobmanager.exception; + +/** + * @author limeng + */ +public class ConfigurationException extends Exception { + public ConfigurationException() { + } + + public ConfigurationException(String message) { + super(message); + } +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobException.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobException.java new file mode 100644 index 000000000..d9bf3deff --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobException.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.streamis.jobmanager.exception; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; + +public class JobException extends ErrorException { + + public JobException(int errCode, String desc) { + super(errCode, desc); + } + + public JobException(int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobExceptionManager.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobExceptionManager.java new file mode 100644 index 000000000..4f52b9a9e --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobExceptionManager.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.streamis.jobmanager.exception; + +import java.util.HashMap; +import java.util.Map; + +public class JobExceptionManager { + //30300-30599 + private static Map desc = new HashMap(32) { + { + put("30300", "上传失败"); + put("30301","%s cannot be empty!"); + } + }; + + public static JobException createException(int errorCode, Object... format) throws JobException { + return new JobException(errorCode, String.format(desc.get(String.valueOf(errorCode)), format)); + } +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/ProjectException.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/ProjectException.java new file mode 100644 index 000000000..32855c48b --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/ProjectException.java @@ -0,0 +1,10 @@ +package com.webank.wedatasphere.streamis.jobmanager.exception; + +public class ProjectException extends Exception{ + public ProjectException() { + } + + public ProjectException(String message) { + super(message); + } +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java new file mode 100644 index 000000000..6b0e9602c --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java @@ -0,0 +1,66 @@ +package com.webank.wedatasphere.streamis.jobmanager.restful.api; + +import com.webank.wedatasphere.linkis.server.Message; +import com.webank.wedatasphere.linkis.server.security.SecurityFilter; +import com.webank.wedatasphere.streamis.jobmanager.exception.ConfigurationException; +import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.vo.ConfigKeyVO; +import com.webank.wedatasphere.streamis.jobmanager.launcher.service.ConfigurationService; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.util.List; + +/** + * @author limeng + */ +@Component +@Path("/streamis/streamJobManager/config") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +public class ConfigurationRestfulApi { + @Autowired + ConfigurationService configurationService; + + ObjectMapper mapper = new ObjectMapper(); + + + @GET + @Path("/view") + public Response getView(@Context HttpServletRequest req,@QueryParam("jobId") Long jobId) throws IOException, ConfigurationException { + String username = SecurityFilter.getLoginUsername(req); + if(jobId==null){ + throw new ConfigurationException("params cannot be empty!"); + } + ConfigKeyVO fullTree = configurationService.getFullTree(jobId); + return Message.messageToResponse(Message.ok().data("fullTree",fullTree)); + } + + @POST + @Path("/update") + public Response updateFullTree(@Context HttpServletRequest req, JsonNode json) throws IOException,ConfigurationException { + ConfigKeyVO fullTrees = mapper.readValue(json.get("fullTree"), ConfigKeyVO.class); + + configurationService.addKeyValue(fullTrees); + return Message.messageToResponse(Message.ok()); + } + + @POST + @Path("/add") + public Response saveFullTree(@Context HttpServletRequest req, JsonNode json) throws IOException,ConfigurationException { + ConfigKeyVO fullTrees = mapper.readValue(json.get("fullTree"), ConfigKeyVO.class); + + configurationService.addKeyValue(fullTrees); + return Message.messageToResponse(Message.ok()); + } + + +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java new file mode 100644 index 000000000..448789452 --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -0,0 +1,231 @@ +package com.webank.wedatasphere.streamis.jobmanager.restful.api; + +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; + +import com.webank.wedatasphere.linkis.server.Message; +import com.webank.wedatasphere.linkis.server.security.SecurityFilter; +import com.webank.wedatasphere.streamis.jobmanager.exception.JobException; +import com.webank.wedatasphere.streamis.jobmanager.exception.JobExceptionManager; +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.*; +import com.webank.wedatasphere.streamis.jobmanager.manager.service.JobService; +import com.webank.wedatasphere.streamis.jobmanager.manager.service.TaskService; +import org.glassfish.jersey.media.multipart.FormDataMultiPart; +import org.glassfish.jersey.media.multipart.FormDataParam; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Component +@Path("/streamis/streamJobManager/job") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +public class JobRestfulApi { + @Autowired + JobService jobService; + @Autowired + TaskService taskService; + + + @GET + @Path("/list") + public Response getJobList(@Context HttpServletRequest req, + @QueryParam("projectId") Long projectId, + @QueryParam("workspaceId") Long workspaceId, + @QueryParam("pageNow") Integer pageNow, + @QueryParam("pageSize") Integer pageSize, + @QueryParam("jobName") String jobName, + @QueryParam("jobStatus") Integer jobStatus, + @QueryParam("jobCreator") String jobCreator) throws IOException, JobException { + if (StringUtils.isEmpty(pageNow)) { + pageNow = 1; + } + if (StringUtils.isEmpty(pageSize)) { + pageSize = 20; + } + if(projectId == null){ + JobExceptionManager.createException(30301,"projectId"); + } + List jobList = null; + PageHelper.startPage(pageNow, pageSize); + try { + jobList = jobService.getByProList(projectId,jobName,jobStatus,jobCreator); + }finally { + PageHelper.clearPage(); + } + PageInfo pageInfo = new PageInfo<>(jobList); + long total = pageInfo.getTotal(); + + return Message.messageToResponse(Message.ok().data("tasks", pageInfo.getList()).data("totalPage", total)); + } + + @POST + @Path("/upload") + public Response uploadJar(@Context HttpServletRequest req, + @FormDataParam("jobName") String jobName, + @FormDataParam("entrypointClass") String entrypointClass, + @FormDataParam("label") String label, + @FormDataParam("entrypointMainArgs") String entrypointMainArgs, + @FormDataParam("parallelism") Integer parallelism, + @FormDataParam("workspaceId") Long workspaceId, + @FormDataParam("projectId") Long projectId, + @FormDataParam("path") String path, + FormDataMultiPart form) throws IOException, JobException { + String userName = SecurityFilter.getLoginUsername(req); + if (StringUtils.isEmpty(path)) { + JobExceptionManager.createException(30300,path); + } + + return Message.messageToResponse(Message.ok()); + } + + @POST + @Path("/execute") + public Response executeJob(@Context HttpServletRequest req, Map json) throws IOException, JobException { + String userName = SecurityFilter.getLoginUsername(req); + if(!json.containsKey("jobId") || json.get("jobId")==null){ + JobExceptionManager.createException(30301,"jobId"); + } + long jobId = Long.parseLong(json.get("jobId").toString()); + taskService.executeJob(jobId,userName); + //todo 缺少linkis + return Message.messageToResponse(Message.ok()); + } + + @GET + @Path("/stop") + public Response killJob(@Context HttpServletRequest req,@QueryParam("jobId") Long jobId) throws IOException, JobException{ + String userName = SecurityFilter.getLoginUsername(req); + if(jobId == null){ + JobExceptionManager.createException(30301,"jobId"); + } + taskService.stopJob(jobId,userName); + //todo 缺少linkis + return Message.messageToResponse(Message.ok()); + } + + @GET + @Path("/details") + public Response detailsJob(@Context HttpServletRequest req,@QueryParam("jobId") Long jobId) throws IOException, JobException{ + if(jobId == null){ + JobExceptionManager.createException(30301,"jobId"); + } + JobDetailsVO jobDetailsVO = new JobDetailsVO(); + List dataNumberDTOS =new ArrayList<>(); + JobDetailsVO.DataNumberDTO dataNumberDTO = new JobDetailsVO.DataNumberDTO(); + dataNumberDTO.setDataName("kafka topic"); + dataNumberDTO.setDataNumber(109345); + dataNumberDTOS.add(dataNumberDTO); + + List loadConditionDTOs =new ArrayList<>(); + JobDetailsVO.LoadConditionDTO loadConditionDTO = new JobDetailsVO.LoadConditionDTO(); + loadConditionDTO.setType("jobManager"); + loadConditionDTO.setHost("localhost"); + loadConditionDTO.setMemory("1.5"); + loadConditionDTO.setTotalMemory("2.0"); + loadConditionDTO.setGcLastTime("2020-08-01"); + loadConditionDTO.setGcLastConsume("1"); + loadConditionDTO.setGcTotalTime("2min"); + loadConditionDTOs.add(loadConditionDTO); + + List realTimeTrafficDTOS = new ArrayList<>(); + JobDetailsVO.RealTimeTrafficDTO realTimeTrafficDTO = new JobDetailsVO.RealTimeTrafficDTO(); + realTimeTrafficDTO.setSourceKey("kafka topic"); + realTimeTrafficDTO.setSourceSpeed("100 Records/S"); + realTimeTrafficDTO.setTransformKey("transform"); + realTimeTrafficDTO.setSinkKey("hbase key"); + realTimeTrafficDTO.setSinkSpeed("10 Records/S"); + realTimeTrafficDTOS.add(realTimeTrafficDTO); + + jobDetailsVO.setDataNumber(dataNumberDTOS); + jobDetailsVO.setLoadCondition(loadConditionDTOs); + jobDetailsVO.setRealTimeTraffic(realTimeTrafficDTOS); + + return Message.messageToResponse(Message.ok().data("details",jobDetailsVO)); + } + @GET + @Path("/execute/history") + public Response executeHistoryJob(@Context HttpServletRequest req,@QueryParam("jobId") Long jobId,@QueryParam("version") String version) throws IOException, JobException{ + if(jobId == null){ + JobExceptionManager.createException(30301,"jobId"); + } + if(StringUtils.isEmpty(version)){ + JobExceptionManager.createException(30301,"version"); + } + List streamTaskListVOS = taskService.executeHistory(jobId, version); + return Message.messageToResponse(Message.ok().data("details",streamTaskListVOS)); + } + + @POST + @Path("/release") + public Response releaseJob(@Context HttpServletRequest req, Map json) throws IOException, JobException{ + + return Message.messageToResponse(Message.ok()); + } + @GET + @Path("/flow") + public Response flowJob(@Context HttpServletRequest req,@QueryParam("jobId") Long jobId,@QueryParam("version") String version) throws IOException, JobException{ + if(jobId == null){ + JobExceptionManager.createException(30301,"jobId"); + } + if(StringUtils.isEmpty(version)){ + JobExceptionManager.createException(30301,"version"); + } + + JobFlowVO jobFlow = jobService.getJobFlow(jobId, version); + + return Message.messageToResponse(Message.ok().data("flowData",jobFlow)); + } + @GET + @Path("/progress") + public Response progressJob(@Context HttpServletRequest req,@QueryParam("jobId") Long jobId) throws IOException, JobException{ + if(jobId == null){ + JobExceptionManager.createException(30301,"jobId"); + } + JobProgressVO jobProgressVO = taskService.getByJobStatus(jobId); + return Message.messageToResponse(Message.ok().data("taskId",jobProgressVO.getTaskId()).data("progress",jobProgressVO.getProgress())); + } + + @GET + @Path("/upload/details") + public Response uploadDetailsJob(@Context HttpServletRequest req,@QueryParam("jobId") Long jobId) throws IOException, JobException{ + + JobUploadDetailsVO jobUploadDetailsVO = new JobUploadDetailsVO(); + List mainJars = new ArrayList<>(); + List dependentList = new ArrayList<>(); + List userList = new ArrayList<>(); + JobUploadDetailsVO.JarDependentDTO jarDependentDTO = new JobUploadDetailsVO.JarDependentDTO(); + jarDependentDTO.setId(1L); + jarDependentDTO.setName("main jar"); + + mainJars.add(jarDependentDTO); + + jarDependentDTO = new JobUploadDetailsVO.JarDependentDTO(); + jarDependentDTO.setId(1L); + jarDependentDTO.setName("jar 依赖"); + dependentList.add(jarDependentDTO); + + jarDependentDTO = new JobUploadDetailsVO.JarDependentDTO(); + jarDependentDTO.setId(1L); + jarDependentDTO.setName("user 资源"); + userList.add(jarDependentDTO); + + jobUploadDetailsVO.setProgramArguement("--hostname localhost --port 12345"); + jobUploadDetailsVO.setMainJars(mainJars); + jobUploadDetailsVO.setDependentList(dependentList); + jobUploadDetailsVO.setUserList(userList); + + return Message.messageToResponse(Message.ok().data("details",jobUploadDetailsVO)); + } + +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java new file mode 100644 index 000000000..6de09b319 --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java @@ -0,0 +1,38 @@ +package com.webank.wedatasphere.streamis.jobmanager.restful.api; + +import com.webank.wedatasphere.linkis.server.Message; +import com.webank.wedatasphere.linkis.server.security.SecurityFilter; +import com.webank.wedatasphere.streamis.jobmanager.exception.ProjectException; +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.TaskCoreNumVO; +import com.webank.wedatasphere.streamis.jobmanager.manager.service.JobService; +import com.webank.wedatasphere.streamis.jobmanager.manager.service.ProjectService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; + +@Component +@Path("/streamis/streamJobManager/project") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +public class ProjectRestfulApi { + @Autowired + ProjectService projectService; + @Autowired + JobService jobService; + @GET + @Path("/core/target") + public Response getView(@Context HttpServletRequest req, @QueryParam("projectId") Long projectId) throws IOException, ProjectException { + String username = SecurityFilter.getLoginUsername(req); + if(projectId==null){ + throw new ProjectException("params cannot be empty!"); + } + TaskCoreNumVO taskCoreNumVO = jobService.countByCores(projectId); + return Message.messageToResponse(Message.ok().data("taskCore",taskCoreNumVO)); + } +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/application.yml b/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/application.yml new file mode 100755 index 000000000..4e39747a5 --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/application.yml @@ -0,0 +1,20 @@ +server: + port: 9009 +spring: + application: + name: cloud-streammanager +eureka: + client: + serviceUrl: + defaultZone: http://ip:port/eureka/ + instance: + metadata-map: + test: wedatasphere + +management: + endpoints: + web: + exposure: + include: refresh,info +logging: + config: classpath:log4j2.xml \ No newline at end of file diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/linkis.properties b/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/linkis.properties new file mode 100755 index 000000000..9470062ed --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/linkis.properties @@ -0,0 +1,40 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +wds.linkis.test.mode=true +wds.linkis.server.mybatis.datasource.url=jdbc:mysql://xx:3306/streamis?characterEncoding=UTF-8 +wds.linkis.server.mybatis.datasource.username=xx +wds.linkis.server.mybatis.datasource.password=xxx + +##restful +wds.linkis.log.clear=true +wds.linkis.server.version=v1 +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.streamis.jobmanager.restful.api + + + +##mybatis +wds.linkis.server.mybatis.mapperLocations=\ + classpath:com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/*.xml,\ + classpath:com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=\ + com.webank.wedatasphere.streamis.jobmanager.launcher.entity,\ + com.webank.wedatasphere.streamis.jobmanager.manager.entity,\ + com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo,\ + com.webank.wedatasphere.streamis.jobmanager.launcher.entity.vo,\ + com.webank.wedatasphere.streamis.jobmanager.launcher.entity.dto +wds.linkis.server.mybatis.BasePackage=\ + com.webank.wedatasphere.streamis.jobmanager.launcher.dao,\ + com.webank.wedatasphere.streamis.jobmanager.manager.dao \ No newline at end of file diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/log4j.properties b/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/log4j.properties new file mode 100755 index 000000000..178f8dfa2 --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/log4j2.xml b/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/log4j2.xml new file mode 100755 index 000000000..ad88ea570 --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/resources/log4j2.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + From 6e0aa475a768db387874c022cf7fd4beae9e3159 Mon Sep 17 00:00:00 2001 From: ryanlei Date: Mon, 26 Apr 2021 19:47:44 +0800 Subject: [PATCH 006/434] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/package.json | 2 +- web/vue.config.js | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/web/package.json b/web/package.json index 98e76d28f..b6c427645 100644 --- a/web/package.json +++ b/web/package.json @@ -1,5 +1,5 @@ { - "name": "dataspherestudio", + "name": "streamis", "version": "1.0.0", "private": true, "scripts": { diff --git a/web/vue.config.js b/web/vue.config.js index f3e4635e4..e60c40d97 100644 --- a/web/vue.config.js +++ b/web/vue.config.js @@ -53,8 +53,7 @@ if (process.env.NODE_ENV !== 'dev') { "worker-src": "'self'", "connect-src": [ "'self'", - "ws:", - "http://adm.webank.io" + "ws:" ], "img-src": [ "data:", From 09b3be4c3e5e5a548b1ab22418273e67f168a5c3 Mon Sep 17 00:00:00 2001 From: ryanlei Date: Mon, 26 Apr 2021 19:47:44 +0800 Subject: [PATCH 007/434] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/package.json | 2 +- web/vue.config.js | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/web/package.json b/web/package.json index 98e76d28f..b6c427645 100644 --- a/web/package.json +++ b/web/package.json @@ -1,5 +1,5 @@ { - "name": "dataspherestudio", + "name": "streamis", "version": "1.0.0", "private": true, "scripts": { diff --git a/web/vue.config.js b/web/vue.config.js index f3e4635e4..e60c40d97 100644 --- a/web/vue.config.js +++ b/web/vue.config.js @@ -53,8 +53,7 @@ if (process.env.NODE_ENV !== 'dev') { "worker-src": "'self'", "connect-src": [ "'self'", - "ws:", - "http://adm.webank.io" + "ws:" ], "img-src": [ "data:", From 0ff99af12e973fadea203430c822711ec11a6ccd Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Mon, 26 Apr 2021 22:53:45 +0800 Subject: [PATCH 008/434] =?UTF-8?q?=E5=AE=8C=E5=96=84appConn=E7=9A=84?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../streamis/appconn/StreamisAppConn.java | 16 +- .../operation/StreamFlowImportOperation.java | 83 ++++++++++ .../ref/StreamFlowImportRequestRef.java | 150 ++++++++++++++++++ .../ref/StreamFlowImportResponseRef.java | 37 +++++ ...vice.java => StreamisFlowCRUDService.java} | 12 +- .../service/StreamisFlowExportService.java | 45 ++++++ .../service/StreamisFlowImportService.java | 51 ++++++ .../service/StreamisFlowPublishService.java | 47 ++++++ .../service/StreamisFlowQueryService.java | 52 ++++++ .../appconn/service/StreamisQueryService.java | 41 ----- ...treamisDevelopmentIntegrationStandard.java | 101 ++++++++---- .../StreamisStructureIntegrationStandard.java | 9 +- streamis-server/pom.xml | 7 + .../common/protocol/StreamisProtocol.scala | 27 ++-- .../server/conf/StreamisFlowConf.java | 2 +- .../server/entity/ExportResponse.java | 34 ++++ .../server/service/StreamFlowService.java | 3 +- .../service/impl/StreamFlowServiceImpl.java | 4 +- .../receiver/StreamisWorkflowReceiver.scala | 45 ++++-- 19 files changed, 644 insertions(+), 122 deletions(-) create mode 100644 streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowImportOperation.java create mode 100644 streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/ref/StreamFlowImportRequestRef.java create mode 100644 streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/ref/StreamFlowImportResponseRef.java rename streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/{StreamisCRUDService.java => StreamisFlowCRUDService.java} (82%) create mode 100644 streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowExportService.java create mode 100644 streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowImportService.java create mode 100644 streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowPublishService.java create mode 100644 streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowQueryService.java delete mode 100644 streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisQueryService.java create mode 100644 streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/entity/ExportResponse.java diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/StreamisAppConn.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/StreamisAppConn.java index a0034d2ec..5d63b1c85 100644 --- a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/StreamisAppConn.java +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/StreamisAppConn.java @@ -1,14 +1,14 @@ package com.webank.wedatasphere.streamis.appconn; -import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.google.common.collect.Lists; import com.webank.wedatasphere.dss.appconn.core.ext.AlmightyAppConn; import com.webank.wedatasphere.dss.standard.common.core.AppStandard; import com.webank.wedatasphere.dss.standard.common.desc.AppDesc; +import com.webank.wedatasphere.streamis.appconn.standard.StreamisDevelopmentIntegrationStandard; import com.webank.wedatasphere.streamis.appconn.standard.StreamisStructureIntegrationStandard; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Arrays; import java.util.List; @@ -23,12 +23,22 @@ public class StreamisAppConn implements AlmightyAppConn { private final StreamisStructureIntegrationStandard streamisStructureIntegrationStandard = StreamisStructureIntegrationStandard.getInstance(this); + private final StreamisDevelopmentIntegrationStandard developmentIntegrationStandard = StreamisDevelopmentIntegrationStandard.getInstance(this); + private List appStandards; + @Override public List getAppStandards() { - return Arrays.asList(streamisStructureIntegrationStandard); + if (appStandards == null) { + synchronized (StreamisAppConn.class) { + if (appStandards == null) { + appStandards = Lists.newArrayList(streamisStructureIntegrationStandard, developmentIntegrationStandard); + } + } + } + return appStandards; } @Override diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowImportOperation.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowImportOperation.java new file mode 100644 index 000000000..8e655c4d2 --- /dev/null +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowImportOperation.java @@ -0,0 +1,83 @@ +package com.webank.wedatasphere.streamis.appconn.operation; + +import com.webank.wedatasphere.dss.standard.app.development.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.development.publish.RefImportOperation; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.linkis.rpc.Sender; +import com.webank.wedatasphere.streamis.appconn.exception.StreamisExternalException; +import com.webank.wedatasphere.streamis.appconn.ref.StreamFlowImportRequestRef; +import com.webank.wedatasphere.streamis.appconn.ref.StreamFlowImportResponseRef; +import com.webank.wedatasphere.streamis.plugins.common.StreamisPluginUtils; +import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowImportRequest; +import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowImportResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class StreamFlowImportOperation implements RefImportOperation { + + + private static final Logger LOGGER = LoggerFactory.getLogger(StreamFlowImportOperation.class); + + private AppInstance appInstance; + + private Sender sender; + + private DevelopmentService developmentService; + + public StreamFlowImportOperation(AppInstance appInstance) { + this.appInstance = appInstance; + init(); + } + + private void init() { + LOGGER.info("init stream flow import operation"); + sender = Sender.getSender(StreamisPluginUtils.STREAMIS_RPC_SERVER_NAME + "-" + StreamisPluginUtils.getLabel(appInstance)); + } + + + @Override + public StreamFlowImportResponseRef importRef(StreamFlowImportRequestRef streamFlowImportRequestRef) throws ExternalOperationFailedException { + LOGGER.info("begin to import stream flow to streamis, bmlResourceId is {}, version is {}", streamFlowImportRequestRef.getResourceId(),streamFlowImportRequestRef.getBmlVersion()); + try{ + StreamFlowImportRequest streamFlowImportRequest = new StreamFlowImportRequest(streamFlowImportRequestRef.getName(), + streamFlowImportRequestRef.getUserName(), + streamFlowImportRequestRef.getProjectName(), + streamFlowImportRequestRef.getProjectId(), + streamFlowImportRequestRef.getResourceId(), + streamFlowImportRequestRef.getBmlVersion(), + streamFlowImportRequestRef.getOrcVersion(), + streamFlowImportRequestRef.getWorkspaceName(), + streamFlowImportRequestRef.getWorkspaceName()); + StreamFlowImportResponse streamFlowImportResponse = (StreamFlowImportResponse) sender.ask(streamFlowImportRequest); + int status = streamFlowImportResponse.status(); + String errorMsg = streamFlowImportResponse.errorMessage(); + List flowIds = streamFlowImportResponse.flowIds().stream().map(o -> (Long)o).collect(Collectors.toList()); + LOGGER.info("end to import stream flow to streamis status is {}, errorMsg is {} response is {}", status, errorMsg, flowIds); + if(status == 0){ + LOGGER.info("status is 0, it means success"); + return new StreamFlowImportResponseRef(flowIds); + } else { + LOGGER.info("status is {}, erroMsg is {}", status, errorMsg); + throw new StreamisExternalException(600503, errorMsg); + } + }catch(StreamisExternalException exception){ + throw exception; + } catch(final Throwable throwable){ + LOGGER.error("failed to import stream flow, errorMsg is {}", throwable.getMessage()); + throw new StreamisExternalException(600501, "failed to import stream flow", throwable); + } + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + this.developmentService = developmentService; + } +} diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/ref/StreamFlowImportRequestRef.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/ref/StreamFlowImportRequestRef.java new file mode 100644 index 000000000..07d9fefdd --- /dev/null +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/ref/StreamFlowImportRequestRef.java @@ -0,0 +1,150 @@ +package com.webank.wedatasphere.streamis.appconn.ref; + +import com.webank.wedatasphere.dss.common.entity.DSSLabel; +import com.webank.wedatasphere.dss.common.entity.IOEnv; +import com.webank.wedatasphere.dss.orchestrator.core.ref.OrchestratorImportRequestRef; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class StreamFlowImportRequestRef implements OrchestratorImportRequestRef { + + + private String bmlResourceId; + + private String bmlVersion; + + private String username; + + private String projectName; + + private Long projectId; + + private String workspaceName; + + private Map params = new HashMap<>(); + + private String orchestratorVersion; + + private String streamFlowName; + + @Override + public void setUserName(String s) { + this.username = s; + } + + @Override + public String getUserName() { + return this.username; + } + + @Override + public void setResourceId(String s) { + this.bmlResourceId = s; + } + + @Override + public String getResourceId() { + return this.bmlResourceId; + } + + @Override + public void setBmlVersion(String s) { + this.bmlVersion = s; + } + + @Override + public String getBmlVersion() { + return this.bmlVersion; + } + + @Override + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + @Override + public Long getProjectId() { + return this.projectId; + } + + @Override + public void setProjectName(String s) { + this.projectName = s; + } + + @Override + public String getProjectName() { + return this.projectName; + } + + @Override + public void setSourceEnv(IOEnv ioEnv) { + + } + + @Override + public IOEnv getSourceEnv() { + return null; + } + + @Override + public void setOrcVersion(String s) { + this.orchestratorVersion = s; + } + + @Override + public String getOrcVersion() { + return this.orchestratorVersion; + } + + @Override + public void setWorkspaceName(String s) { + this.workspaceName = s; + } + + @Override + public String getWorkspaceName() { + return this.workspaceName; + } + + @Override + public List getDSSLabels() { + return null; + } + + @Override + public void setDSSLabels(List list) { + + } + + @Override + public Object getParameter(String s) { + return this.params.get(s); + } + + @Override + public void setParameter(String s, Object o) { + this.params.put(s, o); + } + + @Override + public Map getParameters() { + return this.params; + } + + @Override + public String getName() { + return this.streamFlowName; + } + + @Override + public String getType() { + return "import"; + } +} diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/ref/StreamFlowImportResponseRef.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/ref/StreamFlowImportResponseRef.java new file mode 100644 index 000000000..6e4651eec --- /dev/null +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/ref/StreamFlowImportResponseRef.java @@ -0,0 +1,37 @@ +package com.webank.wedatasphere.streamis.appconn.ref; + +import com.webank.wedatasphere.dss.standard.common.entity.ref.AbstractResponseRef; + +import java.util.List; +import java.util.Map; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class StreamFlowImportResponseRef extends AbstractResponseRef { + + + private List streamFlowIds; + + public StreamFlowImportResponseRef(String responseBody, int status){ + super(responseBody, status); + } + + public StreamFlowImportResponseRef(List streamFlowIds){ + super("" , 0); + this.streamFlowIds = streamFlowIds; + } + + @Override + public Map toMap() { + return null; + } + + @Override + public String getErrorMsg() { + return null; + } + + +} diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisCRUDService.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowCRUDService.java similarity index 82% rename from streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisCRUDService.java rename to streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowCRUDService.java index b2c748543..71a365a16 100644 --- a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisCRUDService.java +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowCRUDService.java @@ -1,8 +1,8 @@ package com.webank.wedatasphere.streamis.appconn.service; -import com.webank.wedatasphere.dss.appconn.core.AppConn; import com.webank.wedatasphere.dss.standard.app.development.DevelopmentService; import com.webank.wedatasphere.dss.standard.app.development.crud.*; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; import org.slf4j.Logger; @@ -12,16 +12,16 @@ * created by yangzhiyue on 2021/4/8 * Description: streamis crud service是通过http的方式进行的 */ -public class StreamisCRUDService implements RefCRUDService { +public class StreamisFlowCRUDService implements RefCRUDService { - private static final Logger LOGGER = LoggerFactory.getLogger(StreamisCRUDService.class); + private static final Logger LOGGER = LoggerFactory.getLogger(StreamisFlowCRUDService.class); - private AppConn appConn; + private AppInstance appInstance; - public StreamisCRUDService(AppConn appConn){ - this.appConn = appConn; + public StreamisFlowCRUDService(AppInstance appInstance){ + this.appInstance = appInstance; } diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowExportService.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowExportService.java new file mode 100644 index 000000000..c61ca387d --- /dev/null +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowExportService.java @@ -0,0 +1,45 @@ +package com.webank.wedatasphere.streamis.appconn.service; + +import com.webank.wedatasphere.dss.standard.app.development.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.development.publish.RefExportService; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.streamis.appconn.operation.StreamFlowExportOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class StreamisFlowExportService implements RefExportService { + + + private static final Logger LOGGER = LoggerFactory.getLogger(StreamisFlowExportService.class); + + private AppInstance appInstance; + + private DevelopmentService developmentService; + + private StreamFlowExportOperation streamFlowExportOperation; + + public StreamisFlowExportService(AppInstance appInstance) { + this.appInstance = appInstance; + this.streamFlowExportOperation = new StreamFlowExportOperation(appInstance); + } + + @Override + @SuppressWarnings("unchecked") + public StreamFlowExportOperation createRefExportOperation() { + return this.streamFlowExportOperation; + } + + @Override + public DevelopmentService getDevelopmentService() { + return this.developmentService; + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + this.developmentService = developmentService; + } +} diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowImportService.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowImportService.java new file mode 100644 index 000000000..959d4de0f --- /dev/null +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowImportService.java @@ -0,0 +1,51 @@ +package com.webank.wedatasphere.streamis.appconn.service; + +import com.webank.wedatasphere.dss.standard.app.development.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.development.publish.RefImportService; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.streamis.appconn.operation.StreamFlowImportOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class StreamisFlowImportService implements RefImportService { + + + private static final Logger LOGGER = LoggerFactory.getLogger(StreamisFlowImportService.class); + + private final AppInstance appInstance; + + private DevelopmentService developmentService; + + private StreamFlowImportOperation streamFlowImportOperation; + + + public StreamisFlowImportService(AppInstance appInstance){ + this.appInstance = appInstance; + init(); + } + + private void init() { + LOGGER.info("init in Steamis flow import service"); + this.streamFlowImportOperation = new StreamFlowImportOperation(appInstance); + } + + @Override + @SuppressWarnings("unchecked") + public StreamFlowImportOperation createRefImportOperation() { + return this.streamFlowImportOperation; + } + + @Override + public DevelopmentService getDevelopmentService() { + return this.developmentService; + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + this.developmentService = developmentService; + } +} diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowPublishService.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowPublishService.java new file mode 100644 index 000000000..70d1469fd --- /dev/null +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowPublishService.java @@ -0,0 +1,47 @@ +package com.webank.wedatasphere.streamis.appconn.service; + +import com.webank.wedatasphere.dss.standard.app.development.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.development.publish.RefPublishToSchedulerService; +import com.webank.wedatasphere.dss.standard.app.development.publish.scheduler.OperationalRefScheduleOperation; +import com.webank.wedatasphere.dss.standard.app.development.publish.scheduler.RefScheduleOperation; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class StreamisFlowPublishService implements RefPublishToSchedulerService { + + + private DevelopmentService developmentService; + + private AppInstance appInstance; + + + public StreamisFlowPublishService(AppInstance appInstance){ + this.appInstance = appInstance; + } + + + + @Override + public RefScheduleOperation createRefScheduleOperation() { + return null; + } + + @Override + public OperationalRefScheduleOperation createOperationalRefScheduleOperation(RequestRef requestRef) { + return null; + } + + @Override + public DevelopmentService getDevelopmentService() { + return null; + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + + } +} diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowQueryService.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowQueryService.java new file mode 100644 index 000000000..965541308 --- /dev/null +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowQueryService.java @@ -0,0 +1,52 @@ +package com.webank.wedatasphere.streamis.appconn.service; + +import com.webank.wedatasphere.dss.standard.app.development.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.development.query.RefQueryService; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.streamis.appconn.operation.StreamFlowQueryOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * created by yangzhiyue on 2021/4/8 + * Description: + */ +public class StreamisFlowQueryService implements RefQueryService { + + + private static final Logger LOGGER = LoggerFactory.getLogger(StreamisFlowQueryService.class); + + + private AppInstance appInstance; + + private DevelopmentService developmentService; + + + private StreamFlowQueryOperation streamFlowQueryOperation; + + + public StreamisFlowQueryService(AppInstance appInstance){ + this.appInstance = appInstance; + init(); + } + + private void init() { + this.streamFlowQueryOperation = new StreamFlowQueryOperation(appInstance); + } + + + @Override + public StreamFlowQueryOperation getRefQueryOperation() { + return this.streamFlowQueryOperation; + } + + @Override + public DevelopmentService getDevelopmentService() { + return this.developmentService; + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + this.developmentService = developmentService; + } +} diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisQueryService.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisQueryService.java deleted file mode 100644 index 1ceff39df..000000000 --- a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisQueryService.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.webank.wedatasphere.streamis.appconn.service; - -import com.webank.wedatasphere.dss.appconn.core.AppConn; -import com.webank.wedatasphere.dss.standard.app.development.DevelopmentService; -import com.webank.wedatasphere.dss.standard.app.development.query.RefQueryOperation; -import com.webank.wedatasphere.dss.standard.app.development.query.RefQueryService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * created by yangzhiyue on 2021/4/8 - * Description: - */ -public class StreamisQueryService implements RefQueryService { - - - private static final Logger LOGGER = LoggerFactory.getLogger(StreamisQueryService.class); - - - private AppConn appConn; - - public StreamisQueryService(AppConn appConn){ - this.appConn = appConn; - } - - - @Override - public RefQueryOperation getRefQueryOperation() { - return null; - } - - @Override - public DevelopmentService getDevelopmentService() { - return null; - } - - @Override - public void setDevelopmentService(DevelopmentService developmentService) { - - } -} diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/standard/StreamisDevelopmentIntegrationStandard.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/standard/StreamisDevelopmentIntegrationStandard.java index 2d6dd57b8..d5fae2f7f 100644 --- a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/standard/StreamisDevelopmentIntegrationStandard.java +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/standard/StreamisDevelopmentIntegrationStandard.java @@ -2,16 +2,21 @@ import com.google.common.collect.Lists; import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.common.entity.DSSLabel; import com.webank.wedatasphere.dss.standard.app.development.AbstractLabelDevelopmentIntegrationStandard; import com.webank.wedatasphere.dss.standard.app.development.RefOperationService; +import com.webank.wedatasphere.dss.standard.app.development.process.DevProcessService; +import com.webank.wedatasphere.dss.standard.app.development.process.ProcessService; +import com.webank.wedatasphere.dss.standard.app.development.process.ProdProcessService; import com.webank.wedatasphere.dss.standard.common.desc.AppDesc; -import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException; -import com.webank.wedatasphere.streamis.appconn.service.StreamisCRUDService; -import com.webank.wedatasphere.streamis.appconn.service.StreamisQueryService; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.streamis.appconn.service.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; /** @@ -23,30 +28,66 @@ public class StreamisDevelopmentIntegrationStandard extends AbstractLabelDevelop private static final Logger LOGGER = LoggerFactory.getLogger(StreamisDevelopmentIntegrationStandard.class); - private AppConn appConn; + private final AppConn appConn; private static StreamisDevelopmentIntegrationStandard instance; - private AppDesc appDesc; - - - private StreamisCRUDService streamisCRUDService; - - private StreamisQueryService streamisQueryService; - + private List processServices; private List refOperationServiceList; + private static final List DEV_LABEL = Lists.newArrayList(new DSSLabel("dev")); + + private static final List PROD_LABEL = Lists.newArrayList(new DSSLabel("prod")); - private StreamisDevelopmentIntegrationStandard(AppConn appConn){ + private StreamisDevelopmentIntegrationStandard(AppConn appConn) { this.appConn = appConn; + init(); + } + + @Override + public void init() { + try { + AppInstance devAppInstance = appConn.getAppDesc().getAppInstancesByLabels(DEV_LABEL).get(0); + AppInstance prodAppInstance = appConn.getAppDesc().getAppInstancesByLabels(PROD_LABEL).get(0); + processServices = new ArrayList<>(); + if (devAppInstance != null) { + LOGGER.info("dev appInstance is not null, url is {}", devAppInstance.getBaseUrl()); + StreamisFlowCRUDService streamisFlowCRUDService = new StreamisFlowCRUDService(devAppInstance); + StreamisFlowQueryService streamisFlowQueryService = new StreamisFlowQueryService(devAppInstance); + StreamisFlowExportService streamisFlowExportService = new StreamisFlowExportService(devAppInstance); + DevProcessService devProcessService = new DevProcessService(Lists.newArrayList(streamisFlowCRUDService, streamisFlowQueryService, streamisFlowExportService)); + setService(devProcessService, streamisFlowCRUDService, streamisFlowQueryService, streamisFlowExportService); + processServices.add(devProcessService); + refOperationServiceList.addAll(Lists.newArrayList(streamisFlowCRUDService, streamisFlowQueryService, streamisFlowExportService)); + } else { + LOGGER.info("dev appInstance in Streamis dev standard is null"); + } + if (prodAppInstance != null) { + LOGGER.info("prod instance is not null, url is {}", prodAppInstance.getBaseUrl()); + StreamisFlowQueryService queryService = new StreamisFlowQueryService(prodAppInstance); + StreamisFlowPublishService publishService = new StreamisFlowPublishService(prodAppInstance); + StreamisFlowImportService importService = new StreamisFlowImportService(prodAppInstance); + ProdProcessService prodProcessService = new ProdProcessService(Lists.newArrayList(queryService, + publishService, importService)); + setService(prodProcessService, queryService, publishService, importService); + processServices.add(prodProcessService); + refOperationServiceList.addAll(Lists.newArrayList(queryService, publishService, importService)); + }else{ + LOGGER.info("prod app instance in streamis dev standard is null"); + } + } catch (final Exception e) { + LOGGER.error("failed to init StreamisDevelopmentIntegrationStandard", e); + processServices = new ArrayList<>(); + refOperationServiceList = new ArrayList<>(); + } } - public static StreamisDevelopmentIntegrationStandard getInstance(AppConn appConn){ - if (null == instance){ - synchronized (StreamisDevelopmentIntegrationStandard.class){ - if (null == instance){ + public static StreamisDevelopmentIntegrationStandard getInstance(AppConn appConn) { + if (null == instance) { + synchronized (StreamisDevelopmentIntegrationStandard.class) { + if (null == instance) { instance = new StreamisDevelopmentIntegrationStandard(appConn); } } @@ -55,37 +96,31 @@ public static StreamisDevelopmentIntegrationStandard getInstance(AppConn appConn } + private void setService(ProcessService processService, RefOperationService... refOperationService){ + Arrays.stream(refOperationService).forEach(service -> service.setDevelopmentService(processService)); + } + @Override + public List getProcessServices() { + return this.processServices; + } @Override protected List getRefOperationService() { - if (null == refOperationServiceList){ - synchronized (StreamisDevelopmentIntegrationStandard.class){ - if(null == refOperationServiceList){ - streamisCRUDService = new StreamisCRUDService(appConn); - streamisQueryService = new StreamisQueryService(appConn); - refOperationServiceList = Lists.newArrayList(streamisCRUDService, streamisQueryService); - } - } - } return this.refOperationServiceList; } @Override public AppDesc getAppDesc() { - return null; + return this.appConn.getAppDesc(); } @Override public void setAppDesc(AppDesc appDesc) { - + //nothing } - @Override - public void init() throws AppStandardErrorException { - - } @Override public void close() throws IOException { @@ -94,7 +129,7 @@ public void close() throws IOException { @Override public String getStandardName() { - return null; + return "StreamisDevelopmentIntegrationStandard"; } @Override @@ -104,6 +139,6 @@ public int getGrade() { @Override public boolean isNecessary() { - return false; + return true; } } diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/standard/StreamisStructureIntegrationStandard.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/standard/StreamisStructureIntegrationStandard.java index e942bff30..2e7fc2cd2 100644 --- a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/standard/StreamisStructureIntegrationStandard.java +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/standard/StreamisStructureIntegrationStandard.java @@ -6,7 +6,6 @@ import com.webank.wedatasphere.dss.standard.app.structure.role.RoleService; import com.webank.wedatasphere.dss.standard.app.structure.status.AppStatusService; import com.webank.wedatasphere.dss.standard.common.desc.AppDesc; -import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException; import com.webank.wedatasphere.streamis.appconn.service.StreamisProjectService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,11 +30,7 @@ public class StreamisStructureIntegrationStandard implements StructureIntegratio private StreamisStructureIntegrationStandard(AppConn appConn){ this.appConn = appConn; - try{ - init(); - }catch(final Exception e){ - LOGGER.warn("Failed to init for class {}", this.getClass().getSimpleName(), e); - } + init(); } public static StreamisStructureIntegrationStandard getInstance(AppConn appConn){ @@ -51,7 +46,7 @@ public static StreamisStructureIntegrationStandard getInstance(AppConn appConn){ @Override - public void init() throws AppStandardErrorException { + public void init() { this.projectService = new StreamisProjectService(this.appConn); this.projectService.setAppDesc(getAppDesc()); } diff --git a/streamis-server/pom.xml b/streamis-server/pom.xml index 2fa592869..185f96424 100644 --- a/streamis-server/pom.xml +++ b/streamis-server/pom.xml @@ -41,6 +41,13 @@ ${streamis.version} + + + com.webank.wedatasphere.streamis + streamis-project-server + ${streamis.version} + + diff --git a/streamis-workflow/streamis-workflow-common/src/main/scala/com/webank/wedatasphere/streamis/workflow/common/protocol/StreamisProtocol.scala b/streamis-workflow/streamis-workflow-common/src/main/scala/com/webank/wedatasphere/streamis/workflow/common/protocol/StreamisProtocol.scala index 9e148db00..687756608 100644 --- a/streamis-workflow/streamis-workflow-common/src/main/scala/com/webank/wedatasphere/streamis/workflow/common/protocol/StreamisProtocol.scala +++ b/streamis-workflow/streamis-workflow-common/src/main/scala/com/webank/wedatasphere/streamis/workflow/common/protocol/StreamisProtocol.scala @@ -10,17 +10,6 @@ trait StreamisProtocol extends Protocol -case class ProjectCreateRequest(projectName:String, - description:String, - createBy:String, - tags:String) extends StreamisProtocol - - - -case class ProjectCreateResponse(projectName:String, - projectId:Long) extends StreamisProtocol - - case class StreamFlowCreateRequest(flowName:String, createBy:String, @@ -62,6 +51,14 @@ case class StreamFlowCopyRequest(username:String, description:String, orchestratorId:Long) extends StreamisProtocol +case class StreamFlowExportResponse(status:Int, + streamFlowId:Long, + streamFlowName:String, + bmlResourceId:String, + bmlVersion:String, + errorMessage:String) extends StreamisProtocol + + case class StreamFlowCopyResponse(status:Int, oldStreamFlowId:Long, @@ -74,12 +71,6 @@ case class StreamFlowCopyResponse(status:Int, -case class StreamFlowExportResponse(status:Int, - streamFlowId:Long, - streamFlowName:String, - bmlResourceId:String, - bmlVersion:String, - errorMessage:String) extends StreamisProtocol case class StreamFlowImportRequest(streamFlowName:String, @@ -95,7 +86,7 @@ case class StreamFlowImportRequest(streamFlowName:String, case class StreamFlowImportResponse(status:Int, - streamFlowName:String, + flowIds:java.util.List[Long], errorMessage:String) extends StreamisProtocol diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/conf/StreamisFlowConf.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/conf/StreamisFlowConf.java index df0aec172..13029c96d 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/conf/StreamisFlowConf.java +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/conf/StreamisFlowConf.java @@ -7,7 +7,7 @@ public class StreamisFlowConf { - + public static final String DEFAULT_VERSION = "v00001"; } diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/entity/ExportResponse.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/entity/ExportResponse.java new file mode 100644 index 000000000..fa29743bd --- /dev/null +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/entity/ExportResponse.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.streamis.workflow.server.entity; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class ExportResponse { + + private String resourceId; + + private String version; + + public ExportResponse(String resourceId, String version) { + this.resourceId = resourceId; + this.version = version; + } + + + public String getResourceId() { + return resourceId; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } +} diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java index 867461ab6..4beaf56eb 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java @@ -5,6 +5,7 @@ import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowCreateRequest; import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowExportRequest; import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowImportRequest; +import com.webank.wedatasphere.streamis.workflow.server.entity.ExportResponse; import com.webank.wedatasphere.streamis.workflow.server.exception.StreamisFlowErrorException; import java.util.List; @@ -17,7 +18,7 @@ public interface StreamFlowService { DSSFlow createStreamFlow(StreamFlowCreateRequest streamFlowCreateRequest) throws StreamisFlowErrorException; - void exportStreamFlow(StreamFlowExportRequest streamFlowExportRequest) throws StreamisFlowErrorException; + ExportResponse exportStreamFlow(StreamFlowExportRequest streamFlowExportRequest) throws StreamisFlowErrorException; List importStreamFlow(StreamFlowImportRequest streamFlowImportRequest) throws StreamisFlowErrorException; diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java index 00b497bb8..93b9edf1c 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java @@ -12,6 +12,7 @@ import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowCreateRequest; import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowExportRequest; import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowImportRequest; +import com.webank.wedatasphere.streamis.workflow.server.entity.ExportResponse; import com.webank.wedatasphere.streamis.workflow.server.exception.StreamisFlowErrorException; import com.webank.wedatasphere.streamis.workflow.server.service.StreamFlowService; import org.slf4j.Logger; @@ -55,7 +56,7 @@ public DSSFlow createStreamFlow(StreamFlowCreateRequest streamFlowCreateRequest) } @Override - public void exportStreamFlow(StreamFlowExportRequest streamFlowExportRequest) throws StreamisFlowErrorException{ + public ExportResponse exportStreamFlow(StreamFlowExportRequest streamFlowExportRequest) throws StreamisFlowErrorException{ try{ LOGGER.info("begin to export stream flow {} for user {}", streamFlowExportRequest.streamFlowId(), streamFlowExportRequest.streamFlowId()); Map exportResponse = workFlowManager.exportWorkflow(streamFlowExportRequest.username(), streamFlowExportRequest.streamFlowId(), @@ -64,6 +65,7 @@ public void exportStreamFlow(StreamFlowExportRequest streamFlowExportRequest) th String resourceId = exportResponse.get("resourceId").toString(); String version = exportResponse.get("version").toString(); LOGGER.info("End to export stream flow, resourceId is {}, version is {}", resourceId, version); + return new ExportResponse(resourceId, version); }catch(final Throwable t){ LOGGER.error("failed to export stream flow {} for user {}", streamFlowExportRequest.streamFlowId(), streamFlowExportRequest.username()); throw new StreamisFlowErrorException(600402, "Failed to export streamflow", t); diff --git a/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala b/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala index 9946b022e..c6714262e 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala +++ b/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala @@ -1,39 +1,62 @@ package com.webank.wedatasphere.streamis.worflow.server.receiver -import com.webank.wedatasphere.linkis.common.utils.Logging +import com.google.common.collect.Lists +import com.webank.wedatasphere.linkis.common.conf.CommonVars +import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} import com.webank.wedatasphere.linkis.rpc.{Receiver, Sender} -import com.webank.wedatasphere.streamis.workflow.common.protocol.{StreamFlowCreateRequest, StreamFlowExportRequest} +import com.webank.wedatasphere.streamis.workflow.common.protocol.{StreamFlowCreateRequest, StreamFlowCreateResponse, StreamFlowExportRequest, StreamFlowExportResponse, StreamFlowImportRequest, StreamFlowImportResponse} +import com.webank.wedatasphere.streamis.workflow.server.conf.StreamisFlowConf import com.webank.wedatasphere.streamis.workflow.server.service.StreamFlowService import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component import scala.concurrent.duration.Duration +import scala.collection.JavaConversions._ +import scala.collection.JavaConverters._ /** * created by yangzhiyue on 2021/4/20 * Description: */ @Component -class StreamisWorkflowReceiver extends Receiver with Logging{ +class StreamisWorkflowReceiver extends Receiver with Logging { @Autowired - var streamFlowService:StreamFlowService = _ - + var streamFlowService: StreamFlowService = _ + private val timeout = CommonVars("wds.streamis.workflow.ask.timeout", 300).getValue override def receive(message: Any, sender: Sender): Unit = { } - override def receiveAndReply(message: Any, sender: Sender): Any = message match { - case streamFlowCreateRequest: StreamFlowCreateRequest => streamFlowService.createStreamFlow(streamFlowCreateRequest) - case streamFlowExportRequest: StreamFlowExportRequest => streamFlowService.exportStreamFlow(streamFlowExportRequest) - + override def receiveAndReply(message: Any, sender: Sender): Any = receiveAndReply(message, Duration(timeout, "seconds"), sender) + + override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = message match { + case streamFlowCreateRequest: StreamFlowCreateRequest => Utils.tryCatch { + val flow = streamFlowService.createStreamFlow(streamFlowCreateRequest) + StreamFlowCreateResponse(0, flow.getId, StreamisFlowConf.DEFAULT_VERSION, flow.getName, "") + } { + t => logger.error(s"failed to create stream flow ${streamFlowCreateRequest.parentFlowId} in streamis", t) + StreamFlowCreateResponse(-1, -1, "", streamFlowCreateRequest.flowName, t.getCause.getMessage) + } + case streamFlowExportRequest: StreamFlowExportRequest => Utils.tryCatch{ + val exportResponse = streamFlowService.exportStreamFlow(streamFlowExportRequest) + StreamFlowExportResponse(0, streamFlowExportRequest.streamFlowId, "", exportResponse.getResourceId, exportResponse.getVersion, "") + }{ + t => logger.error(s"failed to export stream flow ${streamFlowExportRequest.streamFlowId}", t) + StreamFlowExportResponse(-1, streamFlowExportRequest.streamFlowId, "" , null, null, t.getCause.getMessage) + } + case streamFlowImportRequest: StreamFlowImportRequest => Utils.tryCatch{ + val dssFlows = streamFlowService.importStreamFlow(streamFlowImportRequest) + StreamFlowImportResponse(0, Lists.newArrayList(dssFlows.map(_.getId).toList.asJavaCollection), "") + }{ + t => logger.error(s"failed to import resourceId: ${streamFlowImportRequest.bmlResourceId} version is ${streamFlowImportRequest.bmlVersion}", t) + StreamFlowImportResponse(-1, null, t.getCause.getMessage) + } } - override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = ??? - } From 096e6772a5c884c059d9b3946592f4a864b0b6cd Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Mon, 26 Apr 2021 22:53:45 +0800 Subject: [PATCH 009/434] =?UTF-8?q?=E5=AE=8C=E5=96=84appConn=E7=9A=84?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../streamis/appconn/StreamisAppConn.java | 16 +- .../operation/StreamFlowImportOperation.java | 83 ++++++++++ .../ref/StreamFlowImportRequestRef.java | 150 ++++++++++++++++++ .../ref/StreamFlowImportResponseRef.java | 37 +++++ ...vice.java => StreamisFlowCRUDService.java} | 12 +- .../service/StreamisFlowExportService.java | 45 ++++++ .../service/StreamisFlowImportService.java | 51 ++++++ .../service/StreamisFlowPublishService.java | 47 ++++++ .../service/StreamisFlowQueryService.java | 52 ++++++ .../appconn/service/StreamisQueryService.java | 41 ----- ...treamisDevelopmentIntegrationStandard.java | 101 ++++++++---- .../StreamisStructureIntegrationStandard.java | 9 +- streamis-server/pom.xml | 7 + .../common/protocol/StreamisProtocol.scala | 27 ++-- .../server/conf/StreamisFlowConf.java | 2 +- .../server/entity/ExportResponse.java | 34 ++++ .../server/service/StreamFlowService.java | 3 +- .../service/impl/StreamFlowServiceImpl.java | 4 +- .../receiver/StreamisWorkflowReceiver.scala | 45 ++++-- 19 files changed, 644 insertions(+), 122 deletions(-) create mode 100644 streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowImportOperation.java create mode 100644 streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/ref/StreamFlowImportRequestRef.java create mode 100644 streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/ref/StreamFlowImportResponseRef.java rename streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/{StreamisCRUDService.java => StreamisFlowCRUDService.java} (82%) create mode 100644 streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowExportService.java create mode 100644 streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowImportService.java create mode 100644 streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowPublishService.java create mode 100644 streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowQueryService.java delete mode 100644 streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisQueryService.java create mode 100644 streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/entity/ExportResponse.java diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/StreamisAppConn.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/StreamisAppConn.java index a0034d2ec..5d63b1c85 100644 --- a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/StreamisAppConn.java +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/StreamisAppConn.java @@ -1,14 +1,14 @@ package com.webank.wedatasphere.streamis.appconn; -import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.google.common.collect.Lists; import com.webank.wedatasphere.dss.appconn.core.ext.AlmightyAppConn; import com.webank.wedatasphere.dss.standard.common.core.AppStandard; import com.webank.wedatasphere.dss.standard.common.desc.AppDesc; +import com.webank.wedatasphere.streamis.appconn.standard.StreamisDevelopmentIntegrationStandard; import com.webank.wedatasphere.streamis.appconn.standard.StreamisStructureIntegrationStandard; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Arrays; import java.util.List; @@ -23,12 +23,22 @@ public class StreamisAppConn implements AlmightyAppConn { private final StreamisStructureIntegrationStandard streamisStructureIntegrationStandard = StreamisStructureIntegrationStandard.getInstance(this); + private final StreamisDevelopmentIntegrationStandard developmentIntegrationStandard = StreamisDevelopmentIntegrationStandard.getInstance(this); + private List appStandards; + @Override public List getAppStandards() { - return Arrays.asList(streamisStructureIntegrationStandard); + if (appStandards == null) { + synchronized (StreamisAppConn.class) { + if (appStandards == null) { + appStandards = Lists.newArrayList(streamisStructureIntegrationStandard, developmentIntegrationStandard); + } + } + } + return appStandards; } @Override diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowImportOperation.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowImportOperation.java new file mode 100644 index 000000000..8e655c4d2 --- /dev/null +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowImportOperation.java @@ -0,0 +1,83 @@ +package com.webank.wedatasphere.streamis.appconn.operation; + +import com.webank.wedatasphere.dss.standard.app.development.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.development.publish.RefImportOperation; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.linkis.rpc.Sender; +import com.webank.wedatasphere.streamis.appconn.exception.StreamisExternalException; +import com.webank.wedatasphere.streamis.appconn.ref.StreamFlowImportRequestRef; +import com.webank.wedatasphere.streamis.appconn.ref.StreamFlowImportResponseRef; +import com.webank.wedatasphere.streamis.plugins.common.StreamisPluginUtils; +import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowImportRequest; +import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowImportResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class StreamFlowImportOperation implements RefImportOperation { + + + private static final Logger LOGGER = LoggerFactory.getLogger(StreamFlowImportOperation.class); + + private AppInstance appInstance; + + private Sender sender; + + private DevelopmentService developmentService; + + public StreamFlowImportOperation(AppInstance appInstance) { + this.appInstance = appInstance; + init(); + } + + private void init() { + LOGGER.info("init stream flow import operation"); + sender = Sender.getSender(StreamisPluginUtils.STREAMIS_RPC_SERVER_NAME + "-" + StreamisPluginUtils.getLabel(appInstance)); + } + + + @Override + public StreamFlowImportResponseRef importRef(StreamFlowImportRequestRef streamFlowImportRequestRef) throws ExternalOperationFailedException { + LOGGER.info("begin to import stream flow to streamis, bmlResourceId is {}, version is {}", streamFlowImportRequestRef.getResourceId(),streamFlowImportRequestRef.getBmlVersion()); + try{ + StreamFlowImportRequest streamFlowImportRequest = new StreamFlowImportRequest(streamFlowImportRequestRef.getName(), + streamFlowImportRequestRef.getUserName(), + streamFlowImportRequestRef.getProjectName(), + streamFlowImportRequestRef.getProjectId(), + streamFlowImportRequestRef.getResourceId(), + streamFlowImportRequestRef.getBmlVersion(), + streamFlowImportRequestRef.getOrcVersion(), + streamFlowImportRequestRef.getWorkspaceName(), + streamFlowImportRequestRef.getWorkspaceName()); + StreamFlowImportResponse streamFlowImportResponse = (StreamFlowImportResponse) sender.ask(streamFlowImportRequest); + int status = streamFlowImportResponse.status(); + String errorMsg = streamFlowImportResponse.errorMessage(); + List flowIds = streamFlowImportResponse.flowIds().stream().map(o -> (Long)o).collect(Collectors.toList()); + LOGGER.info("end to import stream flow to streamis status is {}, errorMsg is {} response is {}", status, errorMsg, flowIds); + if(status == 0){ + LOGGER.info("status is 0, it means success"); + return new StreamFlowImportResponseRef(flowIds); + } else { + LOGGER.info("status is {}, erroMsg is {}", status, errorMsg); + throw new StreamisExternalException(600503, errorMsg); + } + }catch(StreamisExternalException exception){ + throw exception; + } catch(final Throwable throwable){ + LOGGER.error("failed to import stream flow, errorMsg is {}", throwable.getMessage()); + throw new StreamisExternalException(600501, "failed to import stream flow", throwable); + } + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + this.developmentService = developmentService; + } +} diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/ref/StreamFlowImportRequestRef.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/ref/StreamFlowImportRequestRef.java new file mode 100644 index 000000000..07d9fefdd --- /dev/null +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/ref/StreamFlowImportRequestRef.java @@ -0,0 +1,150 @@ +package com.webank.wedatasphere.streamis.appconn.ref; + +import com.webank.wedatasphere.dss.common.entity.DSSLabel; +import com.webank.wedatasphere.dss.common.entity.IOEnv; +import com.webank.wedatasphere.dss.orchestrator.core.ref.OrchestratorImportRequestRef; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class StreamFlowImportRequestRef implements OrchestratorImportRequestRef { + + + private String bmlResourceId; + + private String bmlVersion; + + private String username; + + private String projectName; + + private Long projectId; + + private String workspaceName; + + private Map params = new HashMap<>(); + + private String orchestratorVersion; + + private String streamFlowName; + + @Override + public void setUserName(String s) { + this.username = s; + } + + @Override + public String getUserName() { + return this.username; + } + + @Override + public void setResourceId(String s) { + this.bmlResourceId = s; + } + + @Override + public String getResourceId() { + return this.bmlResourceId; + } + + @Override + public void setBmlVersion(String s) { + this.bmlVersion = s; + } + + @Override + public String getBmlVersion() { + return this.bmlVersion; + } + + @Override + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + @Override + public Long getProjectId() { + return this.projectId; + } + + @Override + public void setProjectName(String s) { + this.projectName = s; + } + + @Override + public String getProjectName() { + return this.projectName; + } + + @Override + public void setSourceEnv(IOEnv ioEnv) { + + } + + @Override + public IOEnv getSourceEnv() { + return null; + } + + @Override + public void setOrcVersion(String s) { + this.orchestratorVersion = s; + } + + @Override + public String getOrcVersion() { + return this.orchestratorVersion; + } + + @Override + public void setWorkspaceName(String s) { + this.workspaceName = s; + } + + @Override + public String getWorkspaceName() { + return this.workspaceName; + } + + @Override + public List getDSSLabels() { + return null; + } + + @Override + public void setDSSLabels(List list) { + + } + + @Override + public Object getParameter(String s) { + return this.params.get(s); + } + + @Override + public void setParameter(String s, Object o) { + this.params.put(s, o); + } + + @Override + public Map getParameters() { + return this.params; + } + + @Override + public String getName() { + return this.streamFlowName; + } + + @Override + public String getType() { + return "import"; + } +} diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/ref/StreamFlowImportResponseRef.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/ref/StreamFlowImportResponseRef.java new file mode 100644 index 000000000..6e4651eec --- /dev/null +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/ref/StreamFlowImportResponseRef.java @@ -0,0 +1,37 @@ +package com.webank.wedatasphere.streamis.appconn.ref; + +import com.webank.wedatasphere.dss.standard.common.entity.ref.AbstractResponseRef; + +import java.util.List; +import java.util.Map; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class StreamFlowImportResponseRef extends AbstractResponseRef { + + + private List streamFlowIds; + + public StreamFlowImportResponseRef(String responseBody, int status){ + super(responseBody, status); + } + + public StreamFlowImportResponseRef(List streamFlowIds){ + super("" , 0); + this.streamFlowIds = streamFlowIds; + } + + @Override + public Map toMap() { + return null; + } + + @Override + public String getErrorMsg() { + return null; + } + + +} diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisCRUDService.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowCRUDService.java similarity index 82% rename from streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisCRUDService.java rename to streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowCRUDService.java index b2c748543..71a365a16 100644 --- a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisCRUDService.java +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowCRUDService.java @@ -1,8 +1,8 @@ package com.webank.wedatasphere.streamis.appconn.service; -import com.webank.wedatasphere.dss.appconn.core.AppConn; import com.webank.wedatasphere.dss.standard.app.development.DevelopmentService; import com.webank.wedatasphere.dss.standard.app.development.crud.*; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; import org.slf4j.Logger; @@ -12,16 +12,16 @@ * created by yangzhiyue on 2021/4/8 * Description: streamis crud service是通过http的方式进行的 */ -public class StreamisCRUDService implements RefCRUDService { +public class StreamisFlowCRUDService implements RefCRUDService { - private static final Logger LOGGER = LoggerFactory.getLogger(StreamisCRUDService.class); + private static final Logger LOGGER = LoggerFactory.getLogger(StreamisFlowCRUDService.class); - private AppConn appConn; + private AppInstance appInstance; - public StreamisCRUDService(AppConn appConn){ - this.appConn = appConn; + public StreamisFlowCRUDService(AppInstance appInstance){ + this.appInstance = appInstance; } diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowExportService.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowExportService.java new file mode 100644 index 000000000..c61ca387d --- /dev/null +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowExportService.java @@ -0,0 +1,45 @@ +package com.webank.wedatasphere.streamis.appconn.service; + +import com.webank.wedatasphere.dss.standard.app.development.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.development.publish.RefExportService; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.streamis.appconn.operation.StreamFlowExportOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class StreamisFlowExportService implements RefExportService { + + + private static final Logger LOGGER = LoggerFactory.getLogger(StreamisFlowExportService.class); + + private AppInstance appInstance; + + private DevelopmentService developmentService; + + private StreamFlowExportOperation streamFlowExportOperation; + + public StreamisFlowExportService(AppInstance appInstance) { + this.appInstance = appInstance; + this.streamFlowExportOperation = new StreamFlowExportOperation(appInstance); + } + + @Override + @SuppressWarnings("unchecked") + public StreamFlowExportOperation createRefExportOperation() { + return this.streamFlowExportOperation; + } + + @Override + public DevelopmentService getDevelopmentService() { + return this.developmentService; + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + this.developmentService = developmentService; + } +} diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowImportService.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowImportService.java new file mode 100644 index 000000000..959d4de0f --- /dev/null +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowImportService.java @@ -0,0 +1,51 @@ +package com.webank.wedatasphere.streamis.appconn.service; + +import com.webank.wedatasphere.dss.standard.app.development.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.development.publish.RefImportService; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.streamis.appconn.operation.StreamFlowImportOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class StreamisFlowImportService implements RefImportService { + + + private static final Logger LOGGER = LoggerFactory.getLogger(StreamisFlowImportService.class); + + private final AppInstance appInstance; + + private DevelopmentService developmentService; + + private StreamFlowImportOperation streamFlowImportOperation; + + + public StreamisFlowImportService(AppInstance appInstance){ + this.appInstance = appInstance; + init(); + } + + private void init() { + LOGGER.info("init in Steamis flow import service"); + this.streamFlowImportOperation = new StreamFlowImportOperation(appInstance); + } + + @Override + @SuppressWarnings("unchecked") + public StreamFlowImportOperation createRefImportOperation() { + return this.streamFlowImportOperation; + } + + @Override + public DevelopmentService getDevelopmentService() { + return this.developmentService; + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + this.developmentService = developmentService; + } +} diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowPublishService.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowPublishService.java new file mode 100644 index 000000000..70d1469fd --- /dev/null +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowPublishService.java @@ -0,0 +1,47 @@ +package com.webank.wedatasphere.streamis.appconn.service; + +import com.webank.wedatasphere.dss.standard.app.development.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.development.publish.RefPublishToSchedulerService; +import com.webank.wedatasphere.dss.standard.app.development.publish.scheduler.OperationalRefScheduleOperation; +import com.webank.wedatasphere.dss.standard.app.development.publish.scheduler.RefScheduleOperation; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class StreamisFlowPublishService implements RefPublishToSchedulerService { + + + private DevelopmentService developmentService; + + private AppInstance appInstance; + + + public StreamisFlowPublishService(AppInstance appInstance){ + this.appInstance = appInstance; + } + + + + @Override + public RefScheduleOperation createRefScheduleOperation() { + return null; + } + + @Override + public OperationalRefScheduleOperation createOperationalRefScheduleOperation(RequestRef requestRef) { + return null; + } + + @Override + public DevelopmentService getDevelopmentService() { + return null; + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + + } +} diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowQueryService.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowQueryService.java new file mode 100644 index 000000000..965541308 --- /dev/null +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisFlowQueryService.java @@ -0,0 +1,52 @@ +package com.webank.wedatasphere.streamis.appconn.service; + +import com.webank.wedatasphere.dss.standard.app.development.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.development.query.RefQueryService; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.streamis.appconn.operation.StreamFlowQueryOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * created by yangzhiyue on 2021/4/8 + * Description: + */ +public class StreamisFlowQueryService implements RefQueryService { + + + private static final Logger LOGGER = LoggerFactory.getLogger(StreamisFlowQueryService.class); + + + private AppInstance appInstance; + + private DevelopmentService developmentService; + + + private StreamFlowQueryOperation streamFlowQueryOperation; + + + public StreamisFlowQueryService(AppInstance appInstance){ + this.appInstance = appInstance; + init(); + } + + private void init() { + this.streamFlowQueryOperation = new StreamFlowQueryOperation(appInstance); + } + + + @Override + public StreamFlowQueryOperation getRefQueryOperation() { + return this.streamFlowQueryOperation; + } + + @Override + public DevelopmentService getDevelopmentService() { + return this.developmentService; + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + this.developmentService = developmentService; + } +} diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisQueryService.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisQueryService.java deleted file mode 100644 index 1ceff39df..000000000 --- a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/service/StreamisQueryService.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.webank.wedatasphere.streamis.appconn.service; - -import com.webank.wedatasphere.dss.appconn.core.AppConn; -import com.webank.wedatasphere.dss.standard.app.development.DevelopmentService; -import com.webank.wedatasphere.dss.standard.app.development.query.RefQueryOperation; -import com.webank.wedatasphere.dss.standard.app.development.query.RefQueryService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * created by yangzhiyue on 2021/4/8 - * Description: - */ -public class StreamisQueryService implements RefQueryService { - - - private static final Logger LOGGER = LoggerFactory.getLogger(StreamisQueryService.class); - - - private AppConn appConn; - - public StreamisQueryService(AppConn appConn){ - this.appConn = appConn; - } - - - @Override - public RefQueryOperation getRefQueryOperation() { - return null; - } - - @Override - public DevelopmentService getDevelopmentService() { - return null; - } - - @Override - public void setDevelopmentService(DevelopmentService developmentService) { - - } -} diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/standard/StreamisDevelopmentIntegrationStandard.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/standard/StreamisDevelopmentIntegrationStandard.java index 2d6dd57b8..d5fae2f7f 100644 --- a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/standard/StreamisDevelopmentIntegrationStandard.java +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/standard/StreamisDevelopmentIntegrationStandard.java @@ -2,16 +2,21 @@ import com.google.common.collect.Lists; import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.common.entity.DSSLabel; import com.webank.wedatasphere.dss.standard.app.development.AbstractLabelDevelopmentIntegrationStandard; import com.webank.wedatasphere.dss.standard.app.development.RefOperationService; +import com.webank.wedatasphere.dss.standard.app.development.process.DevProcessService; +import com.webank.wedatasphere.dss.standard.app.development.process.ProcessService; +import com.webank.wedatasphere.dss.standard.app.development.process.ProdProcessService; import com.webank.wedatasphere.dss.standard.common.desc.AppDesc; -import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException; -import com.webank.wedatasphere.streamis.appconn.service.StreamisCRUDService; -import com.webank.wedatasphere.streamis.appconn.service.StreamisQueryService; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.streamis.appconn.service.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; /** @@ -23,30 +28,66 @@ public class StreamisDevelopmentIntegrationStandard extends AbstractLabelDevelop private static final Logger LOGGER = LoggerFactory.getLogger(StreamisDevelopmentIntegrationStandard.class); - private AppConn appConn; + private final AppConn appConn; private static StreamisDevelopmentIntegrationStandard instance; - private AppDesc appDesc; - - - private StreamisCRUDService streamisCRUDService; - - private StreamisQueryService streamisQueryService; - + private List processServices; private List refOperationServiceList; + private static final List DEV_LABEL = Lists.newArrayList(new DSSLabel("dev")); + + private static final List PROD_LABEL = Lists.newArrayList(new DSSLabel("prod")); - private StreamisDevelopmentIntegrationStandard(AppConn appConn){ + private StreamisDevelopmentIntegrationStandard(AppConn appConn) { this.appConn = appConn; + init(); + } + + @Override + public void init() { + try { + AppInstance devAppInstance = appConn.getAppDesc().getAppInstancesByLabels(DEV_LABEL).get(0); + AppInstance prodAppInstance = appConn.getAppDesc().getAppInstancesByLabels(PROD_LABEL).get(0); + processServices = new ArrayList<>(); + if (devAppInstance != null) { + LOGGER.info("dev appInstance is not null, url is {}", devAppInstance.getBaseUrl()); + StreamisFlowCRUDService streamisFlowCRUDService = new StreamisFlowCRUDService(devAppInstance); + StreamisFlowQueryService streamisFlowQueryService = new StreamisFlowQueryService(devAppInstance); + StreamisFlowExportService streamisFlowExportService = new StreamisFlowExportService(devAppInstance); + DevProcessService devProcessService = new DevProcessService(Lists.newArrayList(streamisFlowCRUDService, streamisFlowQueryService, streamisFlowExportService)); + setService(devProcessService, streamisFlowCRUDService, streamisFlowQueryService, streamisFlowExportService); + processServices.add(devProcessService); + refOperationServiceList.addAll(Lists.newArrayList(streamisFlowCRUDService, streamisFlowQueryService, streamisFlowExportService)); + } else { + LOGGER.info("dev appInstance in Streamis dev standard is null"); + } + if (prodAppInstance != null) { + LOGGER.info("prod instance is not null, url is {}", prodAppInstance.getBaseUrl()); + StreamisFlowQueryService queryService = new StreamisFlowQueryService(prodAppInstance); + StreamisFlowPublishService publishService = new StreamisFlowPublishService(prodAppInstance); + StreamisFlowImportService importService = new StreamisFlowImportService(prodAppInstance); + ProdProcessService prodProcessService = new ProdProcessService(Lists.newArrayList(queryService, + publishService, importService)); + setService(prodProcessService, queryService, publishService, importService); + processServices.add(prodProcessService); + refOperationServiceList.addAll(Lists.newArrayList(queryService, publishService, importService)); + }else{ + LOGGER.info("prod app instance in streamis dev standard is null"); + } + } catch (final Exception e) { + LOGGER.error("failed to init StreamisDevelopmentIntegrationStandard", e); + processServices = new ArrayList<>(); + refOperationServiceList = new ArrayList<>(); + } } - public static StreamisDevelopmentIntegrationStandard getInstance(AppConn appConn){ - if (null == instance){ - synchronized (StreamisDevelopmentIntegrationStandard.class){ - if (null == instance){ + public static StreamisDevelopmentIntegrationStandard getInstance(AppConn appConn) { + if (null == instance) { + synchronized (StreamisDevelopmentIntegrationStandard.class) { + if (null == instance) { instance = new StreamisDevelopmentIntegrationStandard(appConn); } } @@ -55,37 +96,31 @@ public static StreamisDevelopmentIntegrationStandard getInstance(AppConn appConn } + private void setService(ProcessService processService, RefOperationService... refOperationService){ + Arrays.stream(refOperationService).forEach(service -> service.setDevelopmentService(processService)); + } + @Override + public List getProcessServices() { + return this.processServices; + } @Override protected List getRefOperationService() { - if (null == refOperationServiceList){ - synchronized (StreamisDevelopmentIntegrationStandard.class){ - if(null == refOperationServiceList){ - streamisCRUDService = new StreamisCRUDService(appConn); - streamisQueryService = new StreamisQueryService(appConn); - refOperationServiceList = Lists.newArrayList(streamisCRUDService, streamisQueryService); - } - } - } return this.refOperationServiceList; } @Override public AppDesc getAppDesc() { - return null; + return this.appConn.getAppDesc(); } @Override public void setAppDesc(AppDesc appDesc) { - + //nothing } - @Override - public void init() throws AppStandardErrorException { - - } @Override public void close() throws IOException { @@ -94,7 +129,7 @@ public void close() throws IOException { @Override public String getStandardName() { - return null; + return "StreamisDevelopmentIntegrationStandard"; } @Override @@ -104,6 +139,6 @@ public int getGrade() { @Override public boolean isNecessary() { - return false; + return true; } } diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/standard/StreamisStructureIntegrationStandard.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/standard/StreamisStructureIntegrationStandard.java index e942bff30..2e7fc2cd2 100644 --- a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/standard/StreamisStructureIntegrationStandard.java +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/standard/StreamisStructureIntegrationStandard.java @@ -6,7 +6,6 @@ import com.webank.wedatasphere.dss.standard.app.structure.role.RoleService; import com.webank.wedatasphere.dss.standard.app.structure.status.AppStatusService; import com.webank.wedatasphere.dss.standard.common.desc.AppDesc; -import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException; import com.webank.wedatasphere.streamis.appconn.service.StreamisProjectService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,11 +30,7 @@ public class StreamisStructureIntegrationStandard implements StructureIntegratio private StreamisStructureIntegrationStandard(AppConn appConn){ this.appConn = appConn; - try{ - init(); - }catch(final Exception e){ - LOGGER.warn("Failed to init for class {}", this.getClass().getSimpleName(), e); - } + init(); } public static StreamisStructureIntegrationStandard getInstance(AppConn appConn){ @@ -51,7 +46,7 @@ public static StreamisStructureIntegrationStandard getInstance(AppConn appConn){ @Override - public void init() throws AppStandardErrorException { + public void init() { this.projectService = new StreamisProjectService(this.appConn); this.projectService.setAppDesc(getAppDesc()); } diff --git a/streamis-server/pom.xml b/streamis-server/pom.xml index 2fa592869..185f96424 100644 --- a/streamis-server/pom.xml +++ b/streamis-server/pom.xml @@ -41,6 +41,13 @@ ${streamis.version} + + + com.webank.wedatasphere.streamis + streamis-project-server + ${streamis.version} + + diff --git a/streamis-workflow/streamis-workflow-common/src/main/scala/com/webank/wedatasphere/streamis/workflow/common/protocol/StreamisProtocol.scala b/streamis-workflow/streamis-workflow-common/src/main/scala/com/webank/wedatasphere/streamis/workflow/common/protocol/StreamisProtocol.scala index 9e148db00..687756608 100644 --- a/streamis-workflow/streamis-workflow-common/src/main/scala/com/webank/wedatasphere/streamis/workflow/common/protocol/StreamisProtocol.scala +++ b/streamis-workflow/streamis-workflow-common/src/main/scala/com/webank/wedatasphere/streamis/workflow/common/protocol/StreamisProtocol.scala @@ -10,17 +10,6 @@ trait StreamisProtocol extends Protocol -case class ProjectCreateRequest(projectName:String, - description:String, - createBy:String, - tags:String) extends StreamisProtocol - - - -case class ProjectCreateResponse(projectName:String, - projectId:Long) extends StreamisProtocol - - case class StreamFlowCreateRequest(flowName:String, createBy:String, @@ -62,6 +51,14 @@ case class StreamFlowCopyRequest(username:String, description:String, orchestratorId:Long) extends StreamisProtocol +case class StreamFlowExportResponse(status:Int, + streamFlowId:Long, + streamFlowName:String, + bmlResourceId:String, + bmlVersion:String, + errorMessage:String) extends StreamisProtocol + + case class StreamFlowCopyResponse(status:Int, oldStreamFlowId:Long, @@ -74,12 +71,6 @@ case class StreamFlowCopyResponse(status:Int, -case class StreamFlowExportResponse(status:Int, - streamFlowId:Long, - streamFlowName:String, - bmlResourceId:String, - bmlVersion:String, - errorMessage:String) extends StreamisProtocol case class StreamFlowImportRequest(streamFlowName:String, @@ -95,7 +86,7 @@ case class StreamFlowImportRequest(streamFlowName:String, case class StreamFlowImportResponse(status:Int, - streamFlowName:String, + flowIds:java.util.List[Long], errorMessage:String) extends StreamisProtocol diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/conf/StreamisFlowConf.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/conf/StreamisFlowConf.java index df0aec172..13029c96d 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/conf/StreamisFlowConf.java +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/conf/StreamisFlowConf.java @@ -7,7 +7,7 @@ public class StreamisFlowConf { - + public static final String DEFAULT_VERSION = "v00001"; } diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/entity/ExportResponse.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/entity/ExportResponse.java new file mode 100644 index 000000000..fa29743bd --- /dev/null +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/entity/ExportResponse.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.streamis.workflow.server.entity; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class ExportResponse { + + private String resourceId; + + private String version; + + public ExportResponse(String resourceId, String version) { + this.resourceId = resourceId; + this.version = version; + } + + + public String getResourceId() { + return resourceId; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } +} diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java index 867461ab6..4beaf56eb 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java @@ -5,6 +5,7 @@ import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowCreateRequest; import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowExportRequest; import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowImportRequest; +import com.webank.wedatasphere.streamis.workflow.server.entity.ExportResponse; import com.webank.wedatasphere.streamis.workflow.server.exception.StreamisFlowErrorException; import java.util.List; @@ -17,7 +18,7 @@ public interface StreamFlowService { DSSFlow createStreamFlow(StreamFlowCreateRequest streamFlowCreateRequest) throws StreamisFlowErrorException; - void exportStreamFlow(StreamFlowExportRequest streamFlowExportRequest) throws StreamisFlowErrorException; + ExportResponse exportStreamFlow(StreamFlowExportRequest streamFlowExportRequest) throws StreamisFlowErrorException; List importStreamFlow(StreamFlowImportRequest streamFlowImportRequest) throws StreamisFlowErrorException; diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java index 00b497bb8..93b9edf1c 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java @@ -12,6 +12,7 @@ import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowCreateRequest; import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowExportRequest; import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowImportRequest; +import com.webank.wedatasphere.streamis.workflow.server.entity.ExportResponse; import com.webank.wedatasphere.streamis.workflow.server.exception.StreamisFlowErrorException; import com.webank.wedatasphere.streamis.workflow.server.service.StreamFlowService; import org.slf4j.Logger; @@ -55,7 +56,7 @@ public DSSFlow createStreamFlow(StreamFlowCreateRequest streamFlowCreateRequest) } @Override - public void exportStreamFlow(StreamFlowExportRequest streamFlowExportRequest) throws StreamisFlowErrorException{ + public ExportResponse exportStreamFlow(StreamFlowExportRequest streamFlowExportRequest) throws StreamisFlowErrorException{ try{ LOGGER.info("begin to export stream flow {} for user {}", streamFlowExportRequest.streamFlowId(), streamFlowExportRequest.streamFlowId()); Map exportResponse = workFlowManager.exportWorkflow(streamFlowExportRequest.username(), streamFlowExportRequest.streamFlowId(), @@ -64,6 +65,7 @@ public void exportStreamFlow(StreamFlowExportRequest streamFlowExportRequest) th String resourceId = exportResponse.get("resourceId").toString(); String version = exportResponse.get("version").toString(); LOGGER.info("End to export stream flow, resourceId is {}, version is {}", resourceId, version); + return new ExportResponse(resourceId, version); }catch(final Throwable t){ LOGGER.error("failed to export stream flow {} for user {}", streamFlowExportRequest.streamFlowId(), streamFlowExportRequest.username()); throw new StreamisFlowErrorException(600402, "Failed to export streamflow", t); diff --git a/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala b/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala index 9946b022e..c6714262e 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala +++ b/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala @@ -1,39 +1,62 @@ package com.webank.wedatasphere.streamis.worflow.server.receiver -import com.webank.wedatasphere.linkis.common.utils.Logging +import com.google.common.collect.Lists +import com.webank.wedatasphere.linkis.common.conf.CommonVars +import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} import com.webank.wedatasphere.linkis.rpc.{Receiver, Sender} -import com.webank.wedatasphere.streamis.workflow.common.protocol.{StreamFlowCreateRequest, StreamFlowExportRequest} +import com.webank.wedatasphere.streamis.workflow.common.protocol.{StreamFlowCreateRequest, StreamFlowCreateResponse, StreamFlowExportRequest, StreamFlowExportResponse, StreamFlowImportRequest, StreamFlowImportResponse} +import com.webank.wedatasphere.streamis.workflow.server.conf.StreamisFlowConf import com.webank.wedatasphere.streamis.workflow.server.service.StreamFlowService import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component import scala.concurrent.duration.Duration +import scala.collection.JavaConversions._ +import scala.collection.JavaConverters._ /** * created by yangzhiyue on 2021/4/20 * Description: */ @Component -class StreamisWorkflowReceiver extends Receiver with Logging{ +class StreamisWorkflowReceiver extends Receiver with Logging { @Autowired - var streamFlowService:StreamFlowService = _ - + var streamFlowService: StreamFlowService = _ + private val timeout = CommonVars("wds.streamis.workflow.ask.timeout", 300).getValue override def receive(message: Any, sender: Sender): Unit = { } - override def receiveAndReply(message: Any, sender: Sender): Any = message match { - case streamFlowCreateRequest: StreamFlowCreateRequest => streamFlowService.createStreamFlow(streamFlowCreateRequest) - case streamFlowExportRequest: StreamFlowExportRequest => streamFlowService.exportStreamFlow(streamFlowExportRequest) - + override def receiveAndReply(message: Any, sender: Sender): Any = receiveAndReply(message, Duration(timeout, "seconds"), sender) + + override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = message match { + case streamFlowCreateRequest: StreamFlowCreateRequest => Utils.tryCatch { + val flow = streamFlowService.createStreamFlow(streamFlowCreateRequest) + StreamFlowCreateResponse(0, flow.getId, StreamisFlowConf.DEFAULT_VERSION, flow.getName, "") + } { + t => logger.error(s"failed to create stream flow ${streamFlowCreateRequest.parentFlowId} in streamis", t) + StreamFlowCreateResponse(-1, -1, "", streamFlowCreateRequest.flowName, t.getCause.getMessage) + } + case streamFlowExportRequest: StreamFlowExportRequest => Utils.tryCatch{ + val exportResponse = streamFlowService.exportStreamFlow(streamFlowExportRequest) + StreamFlowExportResponse(0, streamFlowExportRequest.streamFlowId, "", exportResponse.getResourceId, exportResponse.getVersion, "") + }{ + t => logger.error(s"failed to export stream flow ${streamFlowExportRequest.streamFlowId}", t) + StreamFlowExportResponse(-1, streamFlowExportRequest.streamFlowId, "" , null, null, t.getCause.getMessage) + } + case streamFlowImportRequest: StreamFlowImportRequest => Utils.tryCatch{ + val dssFlows = streamFlowService.importStreamFlow(streamFlowImportRequest) + StreamFlowImportResponse(0, Lists.newArrayList(dssFlows.map(_.getId).toList.asJavaCollection), "") + }{ + t => logger.error(s"failed to import resourceId: ${streamFlowImportRequest.bmlResourceId} version is ${streamFlowImportRequest.bmlVersion}", t) + StreamFlowImportResponse(-1, null, t.getCause.getMessage) + } } - override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = ??? - } From 585098aeed804bc5859065bda48054498cd16a06 Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Mon, 26 Apr 2021 23:10:59 +0800 Subject: [PATCH 010/434] =?UTF-8?q?=E4=B8=BAjobamanger=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E4=B8=80=E4=B8=AAprotocol=E6=9D=A5=E8=BF=9B=E8=A1=8C=E4=B8=80?= =?UTF-8?q?=E4=B8=8Brpc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 1 + .../streamis-jobmanager-common/pom.xml | 64 ++++++++++++++++ .../protocol/StreamJobManagerProtocol.scala | 27 +++++++ .../workflow/server/dao/StreamFlowMapper.java | 13 ++++ .../server/entity/po/StreamFlowPo.java | 73 +++++++++++++++++++ .../server/restful/StreamisFlowRestful.java | 59 ++++++++++++++- .../receiver/StreamisWorkflowReceiver.scala | 1 + 7 files changed, 235 insertions(+), 3 deletions(-) create mode 100644 streamis-jobmanager/streamis-jobmanager-common/pom.xml create mode 100644 streamis-jobmanager/streamis-jobmanager-common/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/common/protocol/StreamJobManagerProtocol.scala create mode 100644 streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/dao/StreamFlowMapper.java create mode 100644 streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/entity/po/StreamFlowPo.java diff --git a/pom.xml b/pom.xml index 4bb0b383a..a8685fb22 100644 --- a/pom.xml +++ b/pom.xml @@ -31,6 +31,7 @@ streamis-project streamis-project/streamis-project-server streamis-project/streamis-project-common + streamis-jobmanager/streamis-jobmanager-common diff --git a/streamis-jobmanager/streamis-jobmanager-common/pom.xml b/streamis-jobmanager/streamis-jobmanager-common/pom.xml new file mode 100644 index 000000000..e4ef14f2e --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-common/pom.xml @@ -0,0 +1,64 @@ + + + + streamis + com.webank.wedatasphere.streamis + 0.1.0-SNAPSHOT + ../../pom.xml + + 4.0.0 + + streamis-jobmanager-common + + + 8 + 8 + + + + + + + + com.webank.wedatasphere.linkis + linkis-common + + + + com.webank.wedatasphere.linkis + linkis-protocol + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + src/main/java + + **/*.xml + + + + + + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-jobmanager-common/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/common/protocol/StreamJobManagerProtocol.scala b/streamis-jobmanager/streamis-jobmanager-common/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/common/protocol/StreamJobManagerProtocol.scala new file mode 100644 index 000000000..8c6054a6e --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-common/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/common/protocol/StreamJobManagerProtocol.scala @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.streamis.jobmanager.common.protocol + +import com.webank.wedatasphere.linkis.protocol.Protocol + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +trait StreamJobManagerProtocol extends Protocol + + +case class ImportJobManagerRequest(streamJobName:String, + `type`:String, + executionCode:String, + createBy:String, + updateBy:String, + description:String, + tags:java.util.List[String], + publishUser:String, + workspaceName:String, + version:String, + projectName:String) extends StreamJobManagerProtocol + + +case class ImportJobManagerResponse(status:Int, + streamJobId:Long, + errorMessage:String) extends StreamJobManagerProtocol \ No newline at end of file diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/dao/StreamFlowMapper.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/dao/StreamFlowMapper.java new file mode 100644 index 000000000..b6b84b9f0 --- /dev/null +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/dao/StreamFlowMapper.java @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.streamis.workflow.server.dao; + +import org.apache.ibatis.annotations.Mapper; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + * 按照设计的思路应该是要在streamis这里也存一份streamis flow的相关信息，方便以后进行查看 + * + */ +@Mapper +public interface StreamFlowMapper { +} diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/entity/po/StreamFlowPo.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/entity/po/StreamFlowPo.java new file mode 100644 index 000000000..2f2a2553c --- /dev/null +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/entity/po/StreamFlowPo.java @@ -0,0 +1,73 @@ +package com.webank.wedatasphere.streamis.workflow.server.entity.po; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class StreamFlowPo { + + private Long id; + private Long dssFlowId; + private String flowName; + private String createBy; + private String createdTime; + private String source; + private String lastUpdateBy; + + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getDssFlowId() { + return dssFlowId; + } + + public void setDssFlowId(Long dssFlowId) { + this.dssFlowId = dssFlowId; + } + + public String getFlowName() { + return flowName; + } + + public void setFlowName(String flowName) { + this.flowName = flowName; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public String getCreatedTime() { + return createdTime; + } + + public void setCreatedTime(String createdTime) { + this.createdTime = createdTime; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public String getLastUpdateBy() { + return lastUpdateBy; + } + + public void setLastUpdateBy(String lastUpdateBy) { + this.lastUpdateBy = lastUpdateBy; + } +} diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/restful/StreamisFlowRestful.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/restful/StreamisFlowRestful.java index 2440725d2..7a817fb6d 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/restful/StreamisFlowRestful.java +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/restful/StreamisFlowRestful.java @@ -5,17 +5,25 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.Consumes; import javax.ws.rs.POST; import javax.ws.rs.Path; +import javax.ws.rs.Produces; import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; /** * created by yangzhiyue on 2021/4/19 * Description: */ +@Component +@Path("/streamis") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) public class StreamisFlowRestful { @@ -30,10 +38,55 @@ public class StreamisFlowRestful { @POST - @Path("createNewStreamisOrc") - public Response createNewStreamisOrc(@Context HttpServletRequest request, JsonNode jsonNode){ - //主要是通过workflowserver模块将工作流新建 + @Path("createStreamFlow") + public Response createStreamFlow(@Context HttpServletRequest request, JsonNode jsonNode){ + //todo 先不实现，先通过rpc方式 return null; } + + @POST + @Path("updateStreamFlow") + public Response updateStreamFlow(@Context HttpServletRequest request, JsonNode jsonNode){ + return null; + } + + @POST + @Path("deleteStreamFlow") + public Response deleteStreamFlow(@Context HttpServletRequest request, JsonNode jsonNode){ + return null; + } + + + + @POST + @Path("exportStreamFlow") + public Response exportStreamFlow(@Context HttpServletRequest request, JsonNode jsonNode){ + return null; + } + + + @POST + @Path("importStreamFlow") + public Response importStreamFlow(@Context HttpServletRequest request, JsonNode jsonNode){ + return null; + } + + + @POST + @Path("publishStreamFlow") + public Response publishStreamFlow(@Context HttpServletRequest request, JsonNode jsonNode){ + return null; + } + + + + + + + + + + + } diff --git a/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala b/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala index c6714262e..06fabc9b5 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala +++ b/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala @@ -56,6 +56,7 @@ class StreamisWorkflowReceiver extends Receiver with Logging { t => logger.error(s"failed to import resourceId: ${streamFlowImportRequest.bmlResourceId} version is ${streamFlowImportRequest.bmlVersion}", t) StreamFlowImportResponse(-1, null, t.getCause.getMessage) } + case _ => } From f8bfe70c43558f2456ddfeded340733962ca1fbc Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Mon, 26 Apr 2021 23:10:59 +0800 Subject: [PATCH 011/434] =?UTF-8?q?=E4=B8=BAjobamanger=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E4=B8=80=E4=B8=AAprotocol=E6=9D=A5=E8=BF=9B=E8=A1=8C=E4=B8=80?= =?UTF-8?q?=E4=B8=8Brpc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 1 + .../streamis-jobmanager-common/pom.xml | 64 ++++++++++++++++ .../protocol/StreamJobManagerProtocol.scala | 27 +++++++ .../workflow/server/dao/StreamFlowMapper.java | 13 ++++ .../server/entity/po/StreamFlowPo.java | 73 +++++++++++++++++++ .../server/restful/StreamisFlowRestful.java | 59 ++++++++++++++- .../receiver/StreamisWorkflowReceiver.scala | 1 + 7 files changed, 235 insertions(+), 3 deletions(-) create mode 100644 streamis-jobmanager/streamis-jobmanager-common/pom.xml create mode 100644 streamis-jobmanager/streamis-jobmanager-common/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/common/protocol/StreamJobManagerProtocol.scala create mode 100644 streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/dao/StreamFlowMapper.java create mode 100644 streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/entity/po/StreamFlowPo.java diff --git a/pom.xml b/pom.xml index 4bb0b383a..a8685fb22 100644 --- a/pom.xml +++ b/pom.xml @@ -31,6 +31,7 @@ streamis-project streamis-project/streamis-project-server streamis-project/streamis-project-common + streamis-jobmanager/streamis-jobmanager-common diff --git a/streamis-jobmanager/streamis-jobmanager-common/pom.xml b/streamis-jobmanager/streamis-jobmanager-common/pom.xml new file mode 100644 index 000000000..e4ef14f2e --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-common/pom.xml @@ -0,0 +1,64 @@ + + + + streamis + com.webank.wedatasphere.streamis + 0.1.0-SNAPSHOT + ../../pom.xml + + 4.0.0 + + streamis-jobmanager-common + + + 8 + 8 + + + + + + + + com.webank.wedatasphere.linkis + linkis-common + + + + com.webank.wedatasphere.linkis + linkis-protocol + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + src/main/java + + **/*.xml + + + + + + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-jobmanager-common/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/common/protocol/StreamJobManagerProtocol.scala b/streamis-jobmanager/streamis-jobmanager-common/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/common/protocol/StreamJobManagerProtocol.scala new file mode 100644 index 000000000..8c6054a6e --- /dev/null +++ b/streamis-jobmanager/streamis-jobmanager-common/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/common/protocol/StreamJobManagerProtocol.scala @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.streamis.jobmanager.common.protocol + +import com.webank.wedatasphere.linkis.protocol.Protocol + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +trait StreamJobManagerProtocol extends Protocol + + +case class ImportJobManagerRequest(streamJobName:String, + `type`:String, + executionCode:String, + createBy:String, + updateBy:String, + description:String, + tags:java.util.List[String], + publishUser:String, + workspaceName:String, + version:String, + projectName:String) extends StreamJobManagerProtocol + + +case class ImportJobManagerResponse(status:Int, + streamJobId:Long, + errorMessage:String) extends StreamJobManagerProtocol \ No newline at end of file diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/dao/StreamFlowMapper.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/dao/StreamFlowMapper.java new file mode 100644 index 000000000..b6b84b9f0 --- /dev/null +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/dao/StreamFlowMapper.java @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.streamis.workflow.server.dao; + +import org.apache.ibatis.annotations.Mapper; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + * 按照设计的思路应该是要在streamis这里也存一份streamis flow的相关信息，方便以后进行查看 + * + */ +@Mapper +public interface StreamFlowMapper { +} diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/entity/po/StreamFlowPo.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/entity/po/StreamFlowPo.java new file mode 100644 index 000000000..2f2a2553c --- /dev/null +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/entity/po/StreamFlowPo.java @@ -0,0 +1,73 @@ +package com.webank.wedatasphere.streamis.workflow.server.entity.po; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class StreamFlowPo { + + private Long id; + private Long dssFlowId; + private String flowName; + private String createBy; + private String createdTime; + private String source; + private String lastUpdateBy; + + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getDssFlowId() { + return dssFlowId; + } + + public void setDssFlowId(Long dssFlowId) { + this.dssFlowId = dssFlowId; + } + + public String getFlowName() { + return flowName; + } + + public void setFlowName(String flowName) { + this.flowName = flowName; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public String getCreatedTime() { + return createdTime; + } + + public void setCreatedTime(String createdTime) { + this.createdTime = createdTime; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public String getLastUpdateBy() { + return lastUpdateBy; + } + + public void setLastUpdateBy(String lastUpdateBy) { + this.lastUpdateBy = lastUpdateBy; + } +} diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/restful/StreamisFlowRestful.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/restful/StreamisFlowRestful.java index 2440725d2..7a817fb6d 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/restful/StreamisFlowRestful.java +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/restful/StreamisFlowRestful.java @@ -5,17 +5,25 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.Consumes; import javax.ws.rs.POST; import javax.ws.rs.Path; +import javax.ws.rs.Produces; import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; /** * created by yangzhiyue on 2021/4/19 * Description: */ +@Component +@Path("/streamis") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) public class StreamisFlowRestful { @@ -30,10 +38,55 @@ public class StreamisFlowRestful { @POST - @Path("createNewStreamisOrc") - public Response createNewStreamisOrc(@Context HttpServletRequest request, JsonNode jsonNode){ - //主要是通过workflowserver模块将工作流新建 + @Path("createStreamFlow") + public Response createStreamFlow(@Context HttpServletRequest request, JsonNode jsonNode){ + //todo 先不实现，先通过rpc方式 return null; } + + @POST + @Path("updateStreamFlow") + public Response updateStreamFlow(@Context HttpServletRequest request, JsonNode jsonNode){ + return null; + } + + @POST + @Path("deleteStreamFlow") + public Response deleteStreamFlow(@Context HttpServletRequest request, JsonNode jsonNode){ + return null; + } + + + + @POST + @Path("exportStreamFlow") + public Response exportStreamFlow(@Context HttpServletRequest request, JsonNode jsonNode){ + return null; + } + + + @POST + @Path("importStreamFlow") + public Response importStreamFlow(@Context HttpServletRequest request, JsonNode jsonNode){ + return null; + } + + + @POST + @Path("publishStreamFlow") + public Response publishStreamFlow(@Context HttpServletRequest request, JsonNode jsonNode){ + return null; + } + + + + + + + + + + + } diff --git a/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala b/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala index c6714262e..06fabc9b5 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala +++ b/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala @@ -56,6 +56,7 @@ class StreamisWorkflowReceiver extends Receiver with Logging { t => logger.error(s"failed to import resourceId: ${streamFlowImportRequest.bmlResourceId} version is ${streamFlowImportRequest.bmlVersion}", t) StreamFlowImportResponse(-1, null, t.getCause.getMessage) } + case _ => } From 0c293ac3784b4ec9dd287363a492f924fefc1351 Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Mon, 26 Apr 2021 23:24:49 +0800 Subject: [PATCH 012/434] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=B8=8Bproject?= =?UTF-8?q?=E7=9A=84=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../project/server/dao/ProjectMapper.java | 2 +- .../project/server/service/ProjectService.java | 8 ++++++-- .../service/impl/ProjectServiceImpl.java | 18 +++++++++++++++--- .../receiver/ProjectServerReceiver.scala | 11 +++++++++-- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/dao/ProjectMapper.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/dao/ProjectMapper.java index 53a84c1c2..36f2b762c 100644 --- a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/dao/ProjectMapper.java +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/dao/ProjectMapper.java @@ -10,7 +10,7 @@ @Mapper public interface ProjectMapper { - Long insertProject(StreamisProject streamisProject); + void insertProject(StreamisProject streamisProject); diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/ProjectService.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/ProjectService.java index f692f69e3..cb89b9e16 100644 --- a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/ProjectService.java +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/ProjectService.java @@ -2,6 +2,8 @@ import com.webank.wedatasphere.streamis.project.common.CreateStreamProjectRequest; +import com.webank.wedatasphere.streamis.project.common.DeleteStreamProjectRequest; +import com.webank.wedatasphere.streamis.project.common.UpdateStreamProjectRequest; import com.webank.wedatasphere.streamis.project.server.entity.StreamisProject; import com.webank.wedatasphere.streamis.project.server.entity.request.CreateProjectRequest; import com.webank.wedatasphere.streamis.project.server.exception.StreamisProjectErrorException; @@ -12,15 +14,17 @@ */ public interface ProjectService { + + StreamisProject createProject(String username, CreateProjectRequest createProjectRequest) throws StreamisProjectErrorException; StreamisProject createProject(CreateStreamProjectRequest createStreamProjectRequest) throws StreamisProjectErrorException; - void updateProject() throws StreamisProjectErrorException; + void updateProject(UpdateStreamProjectRequest updateStreamProjectRequest) throws StreamisProjectErrorException; - void deleteProject() throws StreamisProjectErrorException; + void deleteProject(DeleteStreamProjectRequest deleteStreamProjectRequest) throws StreamisProjectErrorException; } diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/impl/ProjectServiceImpl.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/impl/ProjectServiceImpl.java index b294c8711..479a57d6e 100644 --- a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/impl/ProjectServiceImpl.java +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/impl/ProjectServiceImpl.java @@ -2,6 +2,9 @@ import com.webank.wedatasphere.streamis.project.common.CreateStreamProjectRequest; +import com.webank.wedatasphere.streamis.project.common.DeleteStreamProjectRequest; +import com.webank.wedatasphere.streamis.project.common.UpdateStreamProjectRequest; +import com.webank.wedatasphere.streamis.project.server.dao.ProjectMapper; import com.webank.wedatasphere.streamis.project.server.entity.StreamisProject; import com.webank.wedatasphere.streamis.project.server.entity.request.CreateProjectRequest; import com.webank.wedatasphere.streamis.project.server.exception.StreamisProjectErrorException; @@ -9,6 +12,8 @@ import org.jvnet.hk2.annotations.Service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; /** * created by yangzhiyue on 2021/4/19 @@ -23,6 +28,8 @@ public class ProjectServiceImpl implements ProjectService { private static final Logger LOGGER = LoggerFactory.getLogger(ProjectServiceImpl.class); + @Autowired + private ProjectMapper projectMapper; @Override @@ -32,19 +39,24 @@ public StreamisProject createProject(String username, CreateProjectRequest creat } @Override + @Transactional(rollbackFor = Exception.class) public StreamisProject createProject(CreateStreamProjectRequest createStreamProjectRequest) throws StreamisProjectErrorException { LOGGER.info("user {} starts to create project {}", createStreamProjectRequest.createBy(), createStreamProjectRequest.projectName()); - return null; + StreamisProject streamisProject = new StreamisProject(createStreamProjectRequest.projectName(), createStreamProjectRequest.description(), createStreamProjectRequest.createBy()); + projectMapper.insertProject(streamisProject); + LOGGER.info("user {} ends to create project {} and id is {}", createStreamProjectRequest.createBy(), createStreamProjectRequest.projectName(), streamisProject.getId()); + return streamisProject; } @Override - public void updateProject() throws StreamisProjectErrorException { + public void updateProject(UpdateStreamProjectRequest updateStreamProjectRequest) throws StreamisProjectErrorException { + LOGGER.info("User {} begins to update project {}", updateStreamProjectRequest.updateBy(), updateStreamProjectRequest.projectName()); } @Override - public void deleteProject() throws StreamisProjectErrorException { + public void deleteProject(DeleteStreamProjectRequest deleteStreamProjectRequest) throws StreamisProjectErrorException { } } diff --git a/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/ProjectServerReceiver.scala b/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/ProjectServerReceiver.scala index e970c0b38..b93815645 100644 --- a/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/ProjectServerReceiver.scala +++ b/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/ProjectServerReceiver.scala @@ -2,7 +2,7 @@ package com.webank.wedatasphere.streamis.project.server.receiver import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} import com.webank.wedatasphere.linkis.rpc.{Receiver, Sender} -import com.webank.wedatasphere.streamis.project.common.{CreateStreamProjectRequest, CreateStreamProjectResponse} +import com.webank.wedatasphere.streamis.project.common.{CreateStreamProjectRequest, CreateStreamProjectResponse, UpdateStreamProjectRequest, UpdateStreamProjectResponse} import com.webank.wedatasphere.streamis.project.server.entity.request.CreateProjectRequest import com.webank.wedatasphere.streamis.project.server.service.ProjectService import org.springframework.beans.factory.annotation.Autowired @@ -36,9 +36,16 @@ class ProjectServerReceiver extends Receiver with Logging{ val streamisProject = projectService.createProject(createStreamProjectRequest) CreateStreamProjectResponse(0, streamisProject.getName, streamisProject.getId, "") }{ - t => error("failed to create project in streamis", t) + t => logger.error("failed to create project in streamis", t) CreateStreamProjectResponse(-1, createStreamProjectRequest.projectName, -1, t.getCause.getMessage) } + case updateStreamProjectRequest: UpdateStreamProjectRequest => Utils.tryCatch{ + projectService.updateProject(updateStreamProjectRequest) + UpdateStreamProjectResponse(0, updateStreamProjectRequest.streamisProjectId, "") + }{ + t => logger.error(s"failed to update project ${updateStreamProjectRequest.projectName} in streamis",t) + UpdateStreamProjectResponse(-1, updateStreamProjectRequest.streamisProjectId, t.getCause.getMessage) + } case _ => } From 241e19ef529bdc1bf4d17301595f67c5593c5a89 Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Mon, 26 Apr 2021 23:24:49 +0800 Subject: [PATCH 013/434] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=B8=8Bproject?= =?UTF-8?q?=E7=9A=84=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../project/server/dao/ProjectMapper.java | 2 +- .../project/server/service/ProjectService.java | 8 ++++++-- .../service/impl/ProjectServiceImpl.java | 18 +++++++++++++++--- .../receiver/ProjectServerReceiver.scala | 11 +++++++++-- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/dao/ProjectMapper.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/dao/ProjectMapper.java index 53a84c1c2..36f2b762c 100644 --- a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/dao/ProjectMapper.java +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/dao/ProjectMapper.java @@ -10,7 +10,7 @@ @Mapper public interface ProjectMapper { - Long insertProject(StreamisProject streamisProject); + void insertProject(StreamisProject streamisProject); diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/ProjectService.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/ProjectService.java index f692f69e3..cb89b9e16 100644 --- a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/ProjectService.java +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/ProjectService.java @@ -2,6 +2,8 @@ import com.webank.wedatasphere.streamis.project.common.CreateStreamProjectRequest; +import com.webank.wedatasphere.streamis.project.common.DeleteStreamProjectRequest; +import com.webank.wedatasphere.streamis.project.common.UpdateStreamProjectRequest; import com.webank.wedatasphere.streamis.project.server.entity.StreamisProject; import com.webank.wedatasphere.streamis.project.server.entity.request.CreateProjectRequest; import com.webank.wedatasphere.streamis.project.server.exception.StreamisProjectErrorException; @@ -12,15 +14,17 @@ */ public interface ProjectService { + + StreamisProject createProject(String username, CreateProjectRequest createProjectRequest) throws StreamisProjectErrorException; StreamisProject createProject(CreateStreamProjectRequest createStreamProjectRequest) throws StreamisProjectErrorException; - void updateProject() throws StreamisProjectErrorException; + void updateProject(UpdateStreamProjectRequest updateStreamProjectRequest) throws StreamisProjectErrorException; - void deleteProject() throws StreamisProjectErrorException; + void deleteProject(DeleteStreamProjectRequest deleteStreamProjectRequest) throws StreamisProjectErrorException; } diff --git a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/impl/ProjectServiceImpl.java b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/impl/ProjectServiceImpl.java index b294c8711..479a57d6e 100644 --- a/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/impl/ProjectServiceImpl.java +++ b/streamis-project/streamis-project-server/src/main/java/com/webank/wedatasphere/streamis/project/server/service/impl/ProjectServiceImpl.java @@ -2,6 +2,9 @@ import com.webank.wedatasphere.streamis.project.common.CreateStreamProjectRequest; +import com.webank.wedatasphere.streamis.project.common.DeleteStreamProjectRequest; +import com.webank.wedatasphere.streamis.project.common.UpdateStreamProjectRequest; +import com.webank.wedatasphere.streamis.project.server.dao.ProjectMapper; import com.webank.wedatasphere.streamis.project.server.entity.StreamisProject; import com.webank.wedatasphere.streamis.project.server.entity.request.CreateProjectRequest; import com.webank.wedatasphere.streamis.project.server.exception.StreamisProjectErrorException; @@ -9,6 +12,8 @@ import org.jvnet.hk2.annotations.Service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; /** * created by yangzhiyue on 2021/4/19 @@ -23,6 +28,8 @@ public class ProjectServiceImpl implements ProjectService { private static final Logger LOGGER = LoggerFactory.getLogger(ProjectServiceImpl.class); + @Autowired + private ProjectMapper projectMapper; @Override @@ -32,19 +39,24 @@ public StreamisProject createProject(String username, CreateProjectRequest creat } @Override + @Transactional(rollbackFor = Exception.class) public StreamisProject createProject(CreateStreamProjectRequest createStreamProjectRequest) throws StreamisProjectErrorException { LOGGER.info("user {} starts to create project {}", createStreamProjectRequest.createBy(), createStreamProjectRequest.projectName()); - return null; + StreamisProject streamisProject = new StreamisProject(createStreamProjectRequest.projectName(), createStreamProjectRequest.description(), createStreamProjectRequest.createBy()); + projectMapper.insertProject(streamisProject); + LOGGER.info("user {} ends to create project {} and id is {}", createStreamProjectRequest.createBy(), createStreamProjectRequest.projectName(), streamisProject.getId()); + return streamisProject; } @Override - public void updateProject() throws StreamisProjectErrorException { + public void updateProject(UpdateStreamProjectRequest updateStreamProjectRequest) throws StreamisProjectErrorException { + LOGGER.info("User {} begins to update project {}", updateStreamProjectRequest.updateBy(), updateStreamProjectRequest.projectName()); } @Override - public void deleteProject() throws StreamisProjectErrorException { + public void deleteProject(DeleteStreamProjectRequest deleteStreamProjectRequest) throws StreamisProjectErrorException { } } diff --git a/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/ProjectServerReceiver.scala b/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/ProjectServerReceiver.scala index e970c0b38..b93815645 100644 --- a/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/ProjectServerReceiver.scala +++ b/streamis-project/streamis-project-server/src/main/scala/com/webank/wedatasphere/streamis/project/server/receiver/ProjectServerReceiver.scala @@ -2,7 +2,7 @@ package com.webank.wedatasphere.streamis.project.server.receiver import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} import com.webank.wedatasphere.linkis.rpc.{Receiver, Sender} -import com.webank.wedatasphere.streamis.project.common.{CreateStreamProjectRequest, CreateStreamProjectResponse} +import com.webank.wedatasphere.streamis.project.common.{CreateStreamProjectRequest, CreateStreamProjectResponse, UpdateStreamProjectRequest, UpdateStreamProjectResponse} import com.webank.wedatasphere.streamis.project.server.entity.request.CreateProjectRequest import com.webank.wedatasphere.streamis.project.server.service.ProjectService import org.springframework.beans.factory.annotation.Autowired @@ -36,9 +36,16 @@ class ProjectServerReceiver extends Receiver with Logging{ val streamisProject = projectService.createProject(createStreamProjectRequest) CreateStreamProjectResponse(0, streamisProject.getName, streamisProject.getId, "") }{ - t => error("failed to create project in streamis", t) + t => logger.error("failed to create project in streamis", t) CreateStreamProjectResponse(-1, createStreamProjectRequest.projectName, -1, t.getCause.getMessage) } + case updateStreamProjectRequest: UpdateStreamProjectRequest => Utils.tryCatch{ + projectService.updateProject(updateStreamProjectRequest) + UpdateStreamProjectResponse(0, updateStreamProjectRequest.streamisProjectId, "") + }{ + t => logger.error(s"failed to update project ${updateStreamProjectRequest.projectName} in streamis",t) + UpdateStreamProjectResponse(-1, updateStreamProjectRequest.streamisProjectId, t.getCause.getMessage) + } case _ => } From 70e38ba53e0d3434a4bc9d49b18b54fd6307d0ee Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Mon, 26 Apr 2021 23:30:31 +0800 Subject: [PATCH 014/434] =?UTF-8?q?=E8=B0=83=E6=95=B4=E4=B8=8B=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=20=E6=9B=B4=E5=8A=A0=E8=A7=84=E8=8C=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- conf/linkis.properties | 0 db/streamis_ddl.sql | 313 +++++++++++++++++++++++++++++++++++++++++ db/streamis_dml.sql | 22 +++ 3 files changed, 335 insertions(+) create mode 100644 conf/linkis.properties create mode 100644 db/streamis_ddl.sql create mode 100644 db/streamis_dml.sql diff --git a/conf/linkis.properties b/conf/linkis.properties new file mode 100644 index 000000000..e69de29bb diff --git a/db/streamis_ddl.sql b/db/streamis_ddl.sql new file mode 100644 index 000000000..a04dc475d --- /dev/null +++ b/db/streamis_ddl.sql @@ -0,0 +1,313 @@ + +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for streamis_stream_bml +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_bml`; +CREATE TABLE `streamis_stream_bml` ( + `id` bigint(20) NOT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `bml_type` tinyint(1) NULL DEFAULT NULL, + `org_identification` bigint(20) NULL DEFAULT NULL, + ` latest_version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_bml +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_bml_version +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_bml_version`; +CREATE TABLE `streamis_stream_bml_version` ( + `id` bigint(20) NOT NULL, + `bml_id` bigint(20) NULL DEFAULT NULL, + `version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `storage_path` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + ` attribute` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '物料版本' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_bml_version +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_cluster +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_cluster`; +CREATE TABLE `streamis_stream_cluster` ( + `id` int(11) NOT NULL, + `yarn_conf_dir` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `hdfs_conf_dir` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `resource_manager_url` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `savepoint_dir` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = 'flink 集群信息' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_cluster +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_configuration_config_key +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_configuration_config_key`; +CREATE TABLE `streamis_stream_configuration_config_key` ( + `id` bigint(20) NOT NULL, + `key` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `description` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `default_value` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `validate_type` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `validate_range` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `is_hidden` tinyint(1) NULL DEFAULT NULL, + `is_advanced` tinyint(1) NULL DEFAULT NULL, + `level` tinyint(1) NULL DEFAULT NULL, + `treename` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `type` int(10) NULL DEFAULT NULL, + `sort` int(10) NULL DEFAULT NULL, + `status` tinyint(10) NULL DEFAULT NULL COMMENT '1 custom , 2 selected ', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE INDEX `key_index`(`key`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '配置信息' ROW_FORMAT = Compact; + + +-- ---------------------------- +-- Table structure for streamis_stream_configuration_config_value +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_configuration_config_value`; +CREATE TABLE `streamis_stream_configuration_config_value` ( + `id` bigint(20) NOT NULL, + `configkey_id` bigint(20) NULL DEFAULT NULL, + `config_value` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `type` int(10) NULL DEFAULT NULL, + `job_id` bigint(20) NULL DEFAULT NULL, + `job_name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `config_key` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE, + INDEX `key`(`config_key`) USING BTREE, + INDEX `keyid`(`configkey_id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '配置信息' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_configuration_config_value +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_frame_version +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_frame_version`; +CREATE TABLE `streamis_stream_frame_version` ( + `id` bigint(20) NOT NULL, + `frame` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `java_version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '框架信息' ROW_FORMAT = COMPACT; + +-- ---------------------------- +-- Records of streamis_stream_frame_version +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_job +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_job`; +CREATE TABLE `streamis_stream_job` ( + `id` bigint(20) NOT NULL, + `project_id` bigint(20) NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `type` tinyint(1) NULL DEFAULT NULL, + `current_task_id` bigint(20) NULL DEFAULT NULL, + `current_version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `current_release_time` datetime NULL DEFAULT NULL, + `status` tinyint(1) NULL DEFAULT NULL COMMENT '1:已完成 ，2:等待重启 ，3:告警 ，4:慢任务 ，5:运行中 ，6:失败任务', + `org_identification` bigint(20) NULL DEFAULT NULL, + `create_by` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `label` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `current_released` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `description` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '作业表' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_job +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_job_alarm_send_history +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_job_alarm_send_history`; +CREATE TABLE `streamis_stream_job_alarm_send_history` ( + `id` bigint(20) NOT NULL, + `job_id` bigint(20) NULL DEFAULT NULL, + `task_id` bigint(20) NULL DEFAULT NULL, + `create_by` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `type` tinyint(1) NULL DEFAULT NULL, + `rule_type` tinyint(1) NULL DEFAULT NULL, + `content` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '报警历史信息' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_job_alarm_send_history +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_job_checkpoints +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_job_checkpoints`; +CREATE TABLE `streamis_stream_job_checkpoints` ( + `id` bigint(20) NOT NULL, + `config_value_id` bigint(20) NULL DEFAULT NULL, + `path` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `size` int(20) NULL DEFAULT NULL, + `status` tinyint(1) NULL DEFAULT NULL, + `trigger_timestamp` datetime NULL DEFAULT NULL, + `latest_ack_timestamp` datetime NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_job_checkpoints +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_job_code_resource +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_job_code_resource`; +CREATE TABLE `streamis_stream_job_code_resource` ( + `id` bigint(20) NOT NULL, + `job_version_id` bigint(20) NULL DEFAULT NULL, + `bml_version_id` bigint(20) NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '其他代码' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_job_code_resource +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_job_role +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_job_role`; +CREATE TABLE `streamis_stream_job_role` ( + `id` bigint(20) NOT NULL, + `job_id` bigint(20) NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `front_name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `update_time` datetime NULL DEFAULT NULL, + `description` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_job_role +-- ---------------------------- +INSERT INTO `streamis_stream_job_role` VALUES (1, -1, '管理员', '管理员', '2021-04-07 20:57:09', NULL); + +-- ---------------------------- +-- Table structure for streamis_stream_job_sql_resource +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_job_sql_resource`; +CREATE TABLE `streamis_stream_job_sql_resource` ( + `id` bigint(20) NOT NULL, + `job_version_id` bigint(20) NULL DEFAULT NULL, + `execute_sql` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_job_sql_resource +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_job_user_role +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_job_user_role`; +CREATE TABLE `streamis_stream_job_user_role` ( + `id` bigint(20) NOT NULL, + `job_id` bigint(20) NULL DEFAULT NULL, + `user_id` bigint(20) NULL DEFAULT NULL, + `role_id` bigint(20) NULL DEFAULT NULL, + `type` tinyint(1) NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '作业角色关系' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_job_user_role +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_job_version +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_job_version`; +CREATE TABLE `streamis_stream_job_version` ( + `id` bigint(20) NOT NULL, + ` job_id` bigint(20) NULL DEFAULT NULL, + `version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `program_arguments` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `bml_version` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `resource_id` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_job_version +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_project +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_project`; +CREATE TABLE `streamis_stream_project` ( + `id` bigint(20) NOT NULL, + `workspace_id` bigint(20) NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `create_by` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '项目表' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_project +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_task +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_task`; +CREATE TABLE `streamis_stream_task` ( + `id` bigint(20) NOT NULL, + `job_version_id` bigint(20) NULL DEFAULT NULL, + `version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `stream_task_identification` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `status` tinyint(10) NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_task +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_user +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_user`; +CREATE TABLE `streamis_stream_user` ( + `id` bigint(20) NOT NULL, + `username` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '用户表' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_user +-- ---------------------------- +INSERT INTO `streamis_stream_user` VALUES (1, 'hdfs', 'hdfs'); + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/db/streamis_dml.sql b/db/streamis_dml.sql new file mode 100644 index 000000000..f429ea569 --- /dev/null +++ b/db/streamis_dml.sql @@ -0,0 +1,22 @@ +-- ---------------------------- +-- Records of streamis_stream_configuration_config_key +-- ---------------------------- +INSERT INTO `streamis_stream_configuration_config_key` VALUES (1, 'wds.linkis.flink.resource', '资源配置', '资源配置', NULL, 'None', NULL, 0, 0, 1, '资源配置', 1, 0, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (2, 'wds.linkis.flink.taskmanager.num', 'Task Managers数量', 'Task Managers数量', '4', 'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|128)$', 0, 0, 2, '资源配置', 1, 1, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (3, 'wds.linkis.flink.jobmanager.memory', 'JobManager Memory', 'JobManager Memory', '1.5', 'Regex', '^([1-9]\\d{0,2}|1000)(G|g)$', 0, 0, 2, '资源配置', 1, 2, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (4, 'wds.linkis.flink.taskmanager.memory', 'TaskManager Memory', 'TaskManager Memory', '1.5', 'Regex', '^([1-9]\\d{0,2}|1000)(G|g)$', 0, 0, 2, '资源配置', 1, 3, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (5, 'wds.linkis.flink.jobmanager.cpus', 'JobManager CPUs', 'JobManager CPUs', '1', 'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|128)$', 0, 0, 2, '资源配置', 1, 4, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (6, 'wds.linkis.flink.taskManager.cpus', 'TaskManager CPUs', 'TaskManager CPUs', '1', 'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|128)$', 0, 0, 2, '资源配置', 1, 5, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (7, 'wds.linkis.flink.custom', '自定义参数', '自定义参数', NULL, 'None', NULL, 0, 0, 1, '自定义参数', 2, 0, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (8, 'wds.linkis.flink.produce', '生产配置', '生产配置', NULL, 'None', NULL, 0, 0, 1, '生产配置', 3, 0, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (9, 'wds.linkis.flink.checkpoint.interval', 'Checkpoint间隔', 'Checkpoint间隔', NULL, NULL, NULL, 0, 0, 2, '生产配置', 3, 1, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (10, 'wds.linkis.flink.reboot.strategy', '重启策略', '重启策略', '不重启,基于Checkpoint自动重启,无Checkpoint不重启', 'None', NULL, 0, 0, 2, '重启策略', 3, 2, 2); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (11, 'wds.linkis.flink.alert', '告警设置', '告警设置', NULL, 'None', NULL, 0, 0, 1, '告警设置', 4, 0, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (12, 'wds.linkis.flink.alert.rule', '告警规则', '告警规则', '任务日志中出现ERROR/EXCEPTION,任务核心指标出现异常', 'None', NULL, 0, 0, 2, '告警规则', 4, 1, 2); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (13, 'wds.linkis.flink.alert.user', '告警用户', '告警用户', NULL, NULL, NULL, 0, 0, 2, '告警用户', 4, 3, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (14, 'wds.linkis.flink.alert.leve', '告警级别', '告警级别', 'CLEARED,INDETERMINATE,WARNING,MINOR,MAJOR,CRITICAL', 'None', NULL, 0, 0, 2, '告警级别', 4, 2, 2); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (15, 'wds.linkis.flink.alert.failure.level', '失败时告警级别', '失败时告警级别', 'CLEARED,INDETERMINATE,WARNING,MINOR,MAJOR,CRITICAL', 'None', NULL, 0, 0, 2, '失败时告警级别', 4, 4, 2); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (16, 'wds.linkis.flink.alert.failure.user', '失败时告警用户', '失败时告警用户', NULL, 'None', NULL, 0, 0, 2, '失败时告警用户', 4, 5, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (17, 'wds.linkis.flink.authority', '权限设置', '权限设置', NULL, 'None', NULL, 0, 0, 1, '权限设置', 5, 0, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (18, 'wds.linkis.flink.authority.author', '授权模式', '授权模式', '私密,指定全员可见,指定人员可见', 'None', NULL, 0, 0, 2, '授权模式', 5, 1, 2); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (19, 'wds.linkis.flink.authority.visible', '可见人员', '可见人员', NULL, 'None', NULL, 0, 0, 2, '可见人员', 5, 2, 1); From 73a75aab482de0f74358bf29d7a757139a20dd1f Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Mon, 26 Apr 2021 23:30:31 +0800 Subject: [PATCH 015/434] =?UTF-8?q?=E8=B0=83=E6=95=B4=E4=B8=8B=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=20=E6=9B=B4=E5=8A=A0=E8=A7=84=E8=8C=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- conf/linkis.properties | 0 db/streamis_ddl.sql | 313 +++++++++++++++++++++++++++++++++++++++++ db/streamis_dml.sql | 22 +++ 3 files changed, 335 insertions(+) create mode 100644 conf/linkis.properties create mode 100644 db/streamis_ddl.sql create mode 100644 db/streamis_dml.sql diff --git a/conf/linkis.properties b/conf/linkis.properties new file mode 100644 index 000000000..e69de29bb diff --git a/db/streamis_ddl.sql b/db/streamis_ddl.sql new file mode 100644 index 000000000..a04dc475d --- /dev/null +++ b/db/streamis_ddl.sql @@ -0,0 +1,313 @@ + +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for streamis_stream_bml +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_bml`; +CREATE TABLE `streamis_stream_bml` ( + `id` bigint(20) NOT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `bml_type` tinyint(1) NULL DEFAULT NULL, + `org_identification` bigint(20) NULL DEFAULT NULL, + ` latest_version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_bml +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_bml_version +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_bml_version`; +CREATE TABLE `streamis_stream_bml_version` ( + `id` bigint(20) NOT NULL, + `bml_id` bigint(20) NULL DEFAULT NULL, + `version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `storage_path` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + ` attribute` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '物料版本' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_bml_version +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_cluster +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_cluster`; +CREATE TABLE `streamis_stream_cluster` ( + `id` int(11) NOT NULL, + `yarn_conf_dir` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `hdfs_conf_dir` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `resource_manager_url` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `savepoint_dir` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = 'flink 集群信息' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_cluster +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_configuration_config_key +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_configuration_config_key`; +CREATE TABLE `streamis_stream_configuration_config_key` ( + `id` bigint(20) NOT NULL, + `key` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `description` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `default_value` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `validate_type` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `validate_range` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `is_hidden` tinyint(1) NULL DEFAULT NULL, + `is_advanced` tinyint(1) NULL DEFAULT NULL, + `level` tinyint(1) NULL DEFAULT NULL, + `treename` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `type` int(10) NULL DEFAULT NULL, + `sort` int(10) NULL DEFAULT NULL, + `status` tinyint(10) NULL DEFAULT NULL COMMENT '1 custom , 2 selected ', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE INDEX `key_index`(`key`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '配置信息' ROW_FORMAT = Compact; + + +-- ---------------------------- +-- Table structure for streamis_stream_configuration_config_value +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_configuration_config_value`; +CREATE TABLE `streamis_stream_configuration_config_value` ( + `id` bigint(20) NOT NULL, + `configkey_id` bigint(20) NULL DEFAULT NULL, + `config_value` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `type` int(10) NULL DEFAULT NULL, + `job_id` bigint(20) NULL DEFAULT NULL, + `job_name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `config_key` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE, + INDEX `key`(`config_key`) USING BTREE, + INDEX `keyid`(`configkey_id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '配置信息' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_configuration_config_value +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_frame_version +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_frame_version`; +CREATE TABLE `streamis_stream_frame_version` ( + `id` bigint(20) NOT NULL, + `frame` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `java_version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '框架信息' ROW_FORMAT = COMPACT; + +-- ---------------------------- +-- Records of streamis_stream_frame_version +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_job +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_job`; +CREATE TABLE `streamis_stream_job` ( + `id` bigint(20) NOT NULL, + `project_id` bigint(20) NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `type` tinyint(1) NULL DEFAULT NULL, + `current_task_id` bigint(20) NULL DEFAULT NULL, + `current_version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `current_release_time` datetime NULL DEFAULT NULL, + `status` tinyint(1) NULL DEFAULT NULL COMMENT '1:已完成 ，2:等待重启 ，3:告警 ，4:慢任务 ，5:运行中 ，6:失败任务', + `org_identification` bigint(20) NULL DEFAULT NULL, + `create_by` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `label` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `current_released` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `description` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '作业表' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_job +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_job_alarm_send_history +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_job_alarm_send_history`; +CREATE TABLE `streamis_stream_job_alarm_send_history` ( + `id` bigint(20) NOT NULL, + `job_id` bigint(20) NULL DEFAULT NULL, + `task_id` bigint(20) NULL DEFAULT NULL, + `create_by` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `type` tinyint(1) NULL DEFAULT NULL, + `rule_type` tinyint(1) NULL DEFAULT NULL, + `content` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '报警历史信息' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_job_alarm_send_history +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_job_checkpoints +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_job_checkpoints`; +CREATE TABLE `streamis_stream_job_checkpoints` ( + `id` bigint(20) NOT NULL, + `config_value_id` bigint(20) NULL DEFAULT NULL, + `path` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `size` int(20) NULL DEFAULT NULL, + `status` tinyint(1) NULL DEFAULT NULL, + `trigger_timestamp` datetime NULL DEFAULT NULL, + `latest_ack_timestamp` datetime NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_job_checkpoints +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_job_code_resource +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_job_code_resource`; +CREATE TABLE `streamis_stream_job_code_resource` ( + `id` bigint(20) NOT NULL, + `job_version_id` bigint(20) NULL DEFAULT NULL, + `bml_version_id` bigint(20) NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '其他代码' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_job_code_resource +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_job_role +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_job_role`; +CREATE TABLE `streamis_stream_job_role` ( + `id` bigint(20) NOT NULL, + `job_id` bigint(20) NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `front_name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `update_time` datetime NULL DEFAULT NULL, + `description` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_job_role +-- ---------------------------- +INSERT INTO `streamis_stream_job_role` VALUES (1, -1, '管理员', '管理员', '2021-04-07 20:57:09', NULL); + +-- ---------------------------- +-- Table structure for streamis_stream_job_sql_resource +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_job_sql_resource`; +CREATE TABLE `streamis_stream_job_sql_resource` ( + `id` bigint(20) NOT NULL, + `job_version_id` bigint(20) NULL DEFAULT NULL, + `execute_sql` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_job_sql_resource +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_job_user_role +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_job_user_role`; +CREATE TABLE `streamis_stream_job_user_role` ( + `id` bigint(20) NOT NULL, + `job_id` bigint(20) NULL DEFAULT NULL, + `user_id` bigint(20) NULL DEFAULT NULL, + `role_id` bigint(20) NULL DEFAULT NULL, + `type` tinyint(1) NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '作业角色关系' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_job_user_role +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_job_version +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_job_version`; +CREATE TABLE `streamis_stream_job_version` ( + `id` bigint(20) NOT NULL, + ` job_id` bigint(20) NULL DEFAULT NULL, + `version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `program_arguments` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `bml_version` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `resource_id` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_job_version +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_project +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_project`; +CREATE TABLE `streamis_stream_project` ( + `id` bigint(20) NOT NULL, + `workspace_id` bigint(20) NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `create_by` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '项目表' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_project +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_task +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_task`; +CREATE TABLE `streamis_stream_task` ( + `id` bigint(20) NOT NULL, + `job_version_id` bigint(20) NULL DEFAULT NULL, + `version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `stream_task_identification` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `status` tinyint(10) NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_task +-- ---------------------------- + +-- ---------------------------- +-- Table structure for streamis_stream_user +-- ---------------------------- +DROP TABLE IF EXISTS `streamis_stream_user`; +CREATE TABLE `streamis_stream_user` ( + `id` bigint(20) NOT NULL, + `username` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `name` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '用户表' ROW_FORMAT = Compact; + +-- ---------------------------- +-- Records of streamis_stream_user +-- ---------------------------- +INSERT INTO `streamis_stream_user` VALUES (1, 'hdfs', 'hdfs'); + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/db/streamis_dml.sql b/db/streamis_dml.sql new file mode 100644 index 000000000..f429ea569 --- /dev/null +++ b/db/streamis_dml.sql @@ -0,0 +1,22 @@ +-- ---------------------------- +-- Records of streamis_stream_configuration_config_key +-- ---------------------------- +INSERT INTO `streamis_stream_configuration_config_key` VALUES (1, 'wds.linkis.flink.resource', '资源配置', '资源配置', NULL, 'None', NULL, 0, 0, 1, '资源配置', 1, 0, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (2, 'wds.linkis.flink.taskmanager.num', 'Task Managers数量', 'Task Managers数量', '4', 'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|128)$', 0, 0, 2, '资源配置', 1, 1, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (3, 'wds.linkis.flink.jobmanager.memory', 'JobManager Memory', 'JobManager Memory', '1.5', 'Regex', '^([1-9]\\d{0,2}|1000)(G|g)$', 0, 0, 2, '资源配置', 1, 2, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (4, 'wds.linkis.flink.taskmanager.memory', 'TaskManager Memory', 'TaskManager Memory', '1.5', 'Regex', '^([1-9]\\d{0,2}|1000)(G|g)$', 0, 0, 2, '资源配置', 1, 3, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (5, 'wds.linkis.flink.jobmanager.cpus', 'JobManager CPUs', 'JobManager CPUs', '1', 'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|128)$', 0, 0, 2, '资源配置', 1, 4, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (6, 'wds.linkis.flink.taskManager.cpus', 'TaskManager CPUs', 'TaskManager CPUs', '1', 'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|128)$', 0, 0, 2, '资源配置', 1, 5, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (7, 'wds.linkis.flink.custom', '自定义参数', '自定义参数', NULL, 'None', NULL, 0, 0, 1, '自定义参数', 2, 0, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (8, 'wds.linkis.flink.produce', '生产配置', '生产配置', NULL, 'None', NULL, 0, 0, 1, '生产配置', 3, 0, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (9, 'wds.linkis.flink.checkpoint.interval', 'Checkpoint间隔', 'Checkpoint间隔', NULL, NULL, NULL, 0, 0, 2, '生产配置', 3, 1, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (10, 'wds.linkis.flink.reboot.strategy', '重启策略', '重启策略', '不重启,基于Checkpoint自动重启,无Checkpoint不重启', 'None', NULL, 0, 0, 2, '重启策略', 3, 2, 2); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (11, 'wds.linkis.flink.alert', '告警设置', '告警设置', NULL, 'None', NULL, 0, 0, 1, '告警设置', 4, 0, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (12, 'wds.linkis.flink.alert.rule', '告警规则', '告警规则', '任务日志中出现ERROR/EXCEPTION,任务核心指标出现异常', 'None', NULL, 0, 0, 2, '告警规则', 4, 1, 2); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (13, 'wds.linkis.flink.alert.user', '告警用户', '告警用户', NULL, NULL, NULL, 0, 0, 2, '告警用户', 4, 3, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (14, 'wds.linkis.flink.alert.leve', '告警级别', '告警级别', 'CLEARED,INDETERMINATE,WARNING,MINOR,MAJOR,CRITICAL', 'None', NULL, 0, 0, 2, '告警级别', 4, 2, 2); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (15, 'wds.linkis.flink.alert.failure.level', '失败时告警级别', '失败时告警级别', 'CLEARED,INDETERMINATE,WARNING,MINOR,MAJOR,CRITICAL', 'None', NULL, 0, 0, 2, '失败时告警级别', 4, 4, 2); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (16, 'wds.linkis.flink.alert.failure.user', '失败时告警用户', '失败时告警用户', NULL, 'None', NULL, 0, 0, 2, '失败时告警用户', 4, 5, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (17, 'wds.linkis.flink.authority', '权限设置', '权限设置', NULL, 'None', NULL, 0, 0, 1, '权限设置', 5, 0, 1); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (18, 'wds.linkis.flink.authority.author', '授权模式', '授权模式', '私密,指定全员可见,指定人员可见', 'None', NULL, 0, 0, 2, '授权模式', 5, 1, 2); +INSERT INTO `streamis_stream_configuration_config_key` VALUES (19, 'wds.linkis.flink.authority.visible', '可见人员', '可见人员', NULL, 'None', NULL, 0, 0, 2, '可见人员', 5, 2, 1); From ad4cf4f2757e3075dc4b52ae5463088c6fb23991 Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Tue, 27 Apr 2021 21:30:54 +0800 Subject: [PATCH 016/434] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=B8=8Bdatasource?= =?UTF-8?q?=20appconn=E7=9A=84=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../operation/StreamFlowQueryOperation.java | 2 +- .../StreamisProjectDeleteOperation.java | 2 +- .../streamis-datasource-appconn/pom.xml | 6 + .../StreamisDatasourceExternalException.java | 19 + .../operation/DatasourceImportOperation.java | 77 ++++ .../ref/DatasourceImportRequestRef.java | 23 +- .../ref/DatasourceImportResponseRef.java | 1 + .../StreamisDataSourceDevelopStandard.java | 10 +- .../common/StreamisPluginErrorException.java | 22 ++ .../plugins/common/StreamisPluginUtils.java | 54 +++ .../src/main/assembly/distribution.xml | 356 +++++++++++++++--- 11 files changed, 513 insertions(+), 59 deletions(-) create mode 100644 streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/exception/StreamisDatasourceExternalException.java create mode 100644 streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/operation/DatasourceImportOperation.java create mode 100644 streamis-plugins/streamis-plugins-common/src/main/java/com/webank/wedatasphere/streamis/plugins/common/StreamisPluginErrorException.java diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowQueryOperation.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowQueryOperation.java index f2145cbf2..39b36dd74 100644 --- a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowQueryOperation.java +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowQueryOperation.java @@ -45,6 +45,6 @@ public StreamFlowQueryResponseRef query(StreamFlowQueryRequestRef streamFlowQuer @Override public void setDevelopmentService(DevelopmentService developmentService) { - + this.developmentService = developmentService; } } diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamisProjectDeleteOperation.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamisProjectDeleteOperation.java index d314a37d0..e00bbf658 100644 --- a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamisProjectDeleteOperation.java +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamisProjectDeleteOperation.java @@ -17,7 +17,7 @@ public class StreamisProjectDeleteOperation implements ProjectDeletionOperation private static final Logger LOGGER = LoggerFactory.getLogger(StreamisProjectDeleteOperation.class); - //todo 以后写 + //todo 以后写，先等下写 @Override public ProjectResponseRef deleteProject(ProjectRequestRef projectRequestRef) throws ExternalOperationFailedException { diff --git a/streamis-plugins/streamis-datasource-appconn/pom.xml b/streamis-plugins/streamis-datasource-appconn/pom.xml index 1c95867e6..feb8f6e67 100644 --- a/streamis-plugins/streamis-datasource-appconn/pom.xml +++ b/streamis-plugins/streamis-datasource-appconn/pom.xml @@ -38,6 +38,12 @@ ${dss.version} + + com.webank.wedatasphere.streamis + streamis-plugins-common + ${streamis.version} + + diff --git a/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/exception/StreamisDatasourceExternalException.java b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/exception/StreamisDatasourceExternalException.java new file mode 100644 index 000000000..2aa885d81 --- /dev/null +++ b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/exception/StreamisDatasourceExternalException.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.streamis.datasource.appconn.exception; + +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class StreamisDatasourceExternalException extends ExternalOperationFailedException { + + public StreamisDatasourceExternalException(int errorCode, String message) { + super(errorCode, message); + } + + public StreamisDatasourceExternalException(int errorCode, String message, Throwable cause) { + super(errorCode, message); + this.initCause(cause); + } +} diff --git a/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/operation/DatasourceImportOperation.java b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/operation/DatasourceImportOperation.java new file mode 100644 index 000000000..5b7f6b1c4 --- /dev/null +++ b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/operation/DatasourceImportOperation.java @@ -0,0 +1,77 @@ +package com.webank.wedatasphere.streamis.datasource.appconn.operation; + +import com.webank.wedatasphere.dss.standard.app.development.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.development.publish.RefImportOperation; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.linkis.httpclient.request.HttpAction; +import com.webank.wedatasphere.linkis.httpclient.response.HttpResult; +import com.webank.wedatasphere.streamis.datasource.appconn.action.DatasourcePostAction; +import com.webank.wedatasphere.streamis.datasource.appconn.exception.StreamisDatasourceExternalException; +import com.webank.wedatasphere.streamis.datasource.appconn.ref.DatasourceImportRequestRef; +import com.webank.wedatasphere.streamis.datasource.appconn.ref.DatasourceImportResponseRef; +import com.webank.wedatasphere.streamis.datasource.appconn.utils.DataSourceUtils; +import com.webank.wedatasphere.streamis.plugins.common.StreamisPluginUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class DatasourceImportOperation implements RefImportOperation { + + + + private AppInstance appInstance; + + private static final Logger LOGGER = LoggerFactory.getLogger(DatasourceImportOperation.class); + + private String datasourceImportUrl; + + private SSORequestOperation ssoRequestOperation; + + private static final String DATASOURCE_SUFFIX_URL = "api/res_j/v1/streamis/importDatasource"; + + + public DatasourceImportOperation(AppInstance appInstance){ + this.appInstance = appInstance; + } + + + @Override + public DatasourceImportResponseRef importRef(DatasourceImportRequestRef datasourceImportRequestRef) throws StreamisDatasourceExternalException { + LOGGER.info("begin to export datasource from streamis, request is {}", DataSourceUtils.COMMON_GSON.toJson(datasourceImportRequestRef)); + try{ + SSOUrlBuilderOperation ssoUrlBuilderOperation = datasourceImportRequestRef.getWorkspace().getSSOUrlBuilderOperation().copy(); + ssoUrlBuilderOperation.setAppName(DataSourceUtils.DATASOURCE_APP_NAME.getValue()); + ssoUrlBuilderOperation.setReqUrl(this.datasourceImportUrl); + ssoUrlBuilderOperation.setWorkspace(datasourceImportRequestRef.getWorkspace().getWorkspaceName()); + DatasourcePostAction datasourcePostAction = new DatasourcePostAction(ssoUrlBuilderOperation.getBuiltUrl(), datasourceImportRequestRef.getUser()); + datasourcePostAction.addRequestPayload(StreamisPluginUtils.BML_RESOURCE_ID_STR, datasourceImportRequestRef.getBmlResourceId()); + datasourcePostAction.addRequestPayload(StreamisPluginUtils.BML_RESOURCE_VERSION_STR,datasourceImportRequestRef.getBmlVersion()); + HttpResult httpResult = this.ssoRequestOperation.requestWithSSO(ssoUrlBuilderOperation, datasourcePostAction); + LOGGER.info("end to export datasource from streamis, response is {}", httpResult.getResponseBody()); + Map data = StreamisPluginUtils.getDataFromResponseBody(httpResult.getResponseBody()); + if (data == null){ + LOGGER.error("data from responseBody {} is null", httpResult.getResponseBody()); + throw new StreamisDatasourceExternalException(600401, "failed to import datasource to streamis"); + } + Long streamisTableMetaId = (Long) data.get(StreamisPluginUtils.STREAMIS_RPC_SERVER_NAME); + return new DatasourceImportResponseRef(httpResult.getResponseBody(), 0, streamisTableMetaId); + }catch(final StreamisDatasourceExternalException e){ + throw e; + } + catch(final Exception e){ + throw new StreamisDatasourceExternalException(300601, "failed to export ref ", e); + } + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + + } +} diff --git a/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/ref/DatasourceImportRequestRef.java b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/ref/DatasourceImportRequestRef.java index 5812a4b8a..dffeb7b0b 100644 --- a/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/ref/DatasourceImportRequestRef.java +++ b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/ref/DatasourceImportRequestRef.java @@ -2,6 +2,7 @@ import com.webank.wedatasphere.dss.standard.app.development.publish.ImportRequestRef; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.streamis.plugins.common.StreamisPluginUtils; import java.util.HashMap; import java.util.Map; @@ -21,6 +22,7 @@ public class DatasourceImportRequestRef implements ImportRequestRef { private static final String TYPE = "import"; + private String user; /** * bmlresource includes resourceId and version @@ -28,10 +30,9 @@ public class DatasourceImportRequestRef implements ImportRequestRef { private Map bmlResource; - public DatasourceImportRequestRef(Map bmlResource){ - this.bmlResource = bmlResource; - } + public DatasourceImportRequestRef(){ + } @Override @@ -73,6 +74,22 @@ public Map getBmlResource() { return bmlResource; } + public String getBmlResourceId(){ + return (String) this.parameters.get(StreamisPluginUtils.BML_RESOURCE_ID_STR); + } + + public String getBmlVersion(){ + return (String) this.parameters.get(StreamisPluginUtils.BML_RESOURCE_VERSION_STR); + } + + public String getUser(){ + return (String)this.parameters.get(StreamisPluginUtils.USER_STR); + } + + + + + public void setBmlResource(Map bmlResource) { this.bmlResource = bmlResource; } diff --git a/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/ref/DatasourceImportResponseRef.java b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/ref/DatasourceImportResponseRef.java index ae91d6035..2080a4dc1 100644 --- a/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/ref/DatasourceImportResponseRef.java +++ b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/ref/DatasourceImportResponseRef.java @@ -11,6 +11,7 @@ /** * created by yangzhiyue on 2021/4/13 * Description: + * 将 */ public class DatasourceImportResponseRef extends AbstractResponseRef { diff --git a/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/standard/StreamisDataSourceDevelopStandard.java b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/standard/StreamisDataSourceDevelopStandard.java index db02ab344..4b552a75a 100644 --- a/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/standard/StreamisDataSourceDevelopStandard.java +++ b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/standard/StreamisDataSourceDevelopStandard.java @@ -5,7 +5,6 @@ import com.webank.wedatasphere.dss.standard.app.development.AbstractLabelDevelopmentIntegrationStandard; import com.webank.wedatasphere.dss.standard.app.development.RefOperationService; import com.webank.wedatasphere.dss.standard.common.desc.AppDesc; -import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException; import com.webank.wedatasphere.streamis.datasource.appconn.service.StreamisDatasourceQueryService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,16 +38,11 @@ public class StreamisDataSourceDevelopStandard extends AbstractLabelDevelopmentI private StreamisDataSourceDevelopStandard(AppConn appConn){ this.appConn = appConn; - try { - init(); - } catch (AppStandardErrorException e) { - LOGGER.error("Failed to init in Streamis datasource development standard", e); - } + init(); } public static StreamisDataSourceDevelopStandard getInstance(AppConn appConn){ - if(instance == null){ synchronized (StreamisDataSourceDevelopStandard.class){ if (instance == null){ @@ -88,7 +82,7 @@ public void setAppDesc(AppDesc appDesc) { } @Override - public void init() throws AppStandardErrorException { + public void init() { LOGGER.info("begin to init in streamis dataource develop standard"); //do noting } diff --git a/streamis-plugins/streamis-plugins-common/src/main/java/com/webank/wedatasphere/streamis/plugins/common/StreamisPluginErrorException.java b/streamis-plugins/streamis-plugins-common/src/main/java/com/webank/wedatasphere/streamis/plugins/common/StreamisPluginErrorException.java new file mode 100644 index 000000000..427c86eb2 --- /dev/null +++ b/streamis-plugins/streamis-plugins-common/src/main/java/com/webank/wedatasphere/streamis/plugins/common/StreamisPluginErrorException.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.streamis.plugins.common; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; + +/** + * created by yangzhiyue on 2021/4/27 + * Description: + */ +public class StreamisPluginErrorException extends ErrorException { + + + public StreamisPluginErrorException(int errCode, String desc) { + super(errCode, desc); + } + + public StreamisPluginErrorException(int errCode, String desc, Throwable throwable) { + super(errCode, desc); + this.initCause(throwable); + } + + +} diff --git a/streamis-plugins/streamis-plugins-common/src/main/java/com/webank/wedatasphere/streamis/plugins/common/StreamisPluginUtils.java b/streamis-plugins/streamis-plugins-common/src/main/java/com/webank/wedatasphere/streamis/plugins/common/StreamisPluginUtils.java index a83f4b657..880df7bda 100644 --- a/streamis-plugins/streamis-plugins-common/src/main/java/com/webank/wedatasphere/streamis/plugins/common/StreamisPluginUtils.java +++ b/streamis-plugins/streamis-plugins-common/src/main/java/com/webank/wedatasphere/streamis/plugins/common/StreamisPluginUtils.java @@ -1,8 +1,14 @@ package com.webank.wedatasphere.streamis.plugins.common; +import com.google.gson.*; import com.webank.wedatasphere.dss.common.entity.DSSLabel; import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; import com.webank.wedatasphere.linkis.common.conf.CommonVars; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Type; +import java.util.Map; /** * created by yangzhiyue on 2021/4/21 @@ -11,6 +17,8 @@ public class StreamisPluginUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(StreamisPluginUtils.class); + public static final String DEV_LABEL = "dev"; public static final String PROD_LABEL = "prod"; @@ -18,10 +26,33 @@ public class StreamisPluginUtils { public static final String CS_STR = "contextId"; + public static final String BML_RESOURCE_ID_STR = "bmlResourceId"; + + public static final String BML_RESOURCE_VERSION_STR = "bmlResourceVersion"; + + public static final String USER_STR = "user"; + + public static final String DATA_STR = "data"; + + public static final String STREAMIS_TABLE_META_ID_STR = "streamisTableMetaId"; + public static final String STREAMIS_RPC_SERVER_NAME = CommonVars.apply("wds.streamis.plugin.server.name", "streamis-server").getValue(); + public static final Gson COMMON_GSON = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").serializeNulls() + .registerTypeAdapter(Double.class, new JsonSerializer() { + @Override + public JsonElement serialize(Double src, Type typeOfSrc, JsonSerializationContext context) { + if(src == src.longValue()) { + return new JsonPrimitive(src.longValue()); + } else { + return new JsonPrimitive(src); + } + } + }).create(); + + public static String getLabel(AppInstance appInstance){ for (DSSLabel label : appInstance.getLabels()) { if (label.getLabel().equalsIgnoreCase(PROD_LABEL)) return PROD_LABEL; @@ -30,6 +61,29 @@ public static String getLabel(AppInstance appInstance){ } + /** + * 将 + * @param responseBody json response + * @return linkis response中的data数据，如果有问题 则返回null + */ + @SuppressWarnings("unchecked") + public static Map getDataFromResponseBody(String responseBody){ + try{ + Map responseMap = COMMON_GSON.fromJson(responseBody, Map.class); + Object dataObject = responseMap.get(DATA_STR); + if (dataObject instanceof Map){ + return (Map)dataObject; + }else{ + LOGGER.error("dataObject {} is not Map class, it is a {}", dataObject, dataObject.getClass()); + return null; + } + }catch(final Exception e){ + LOGGER.error("failed to convert a responseBody {} to Map", responseBody, e); + return null; + } + } + + } diff --git a/streamis-server/src/main/assembly/distribution.xml b/streamis-server/src/main/assembly/distribution.xml index 914f21084..d609166b8 100644 --- a/streamis-server/src/main/assembly/distribution.xml +++ b/streamis-server/src/main/assembly/distribution.xml @@ -1,46 +1,310 @@ - - - - streamis-server - - dir - - true - streamis-server - - - - - - lib - true - true - false - true - true - - - - - - - + + + + streamis-server + + zip + + true + streamis-server + + + + + + lib + true + true + false + true + true + + antlr:antlr:jar + aopalliance:aopalliance:jar + asm:asm:jar + cglib:cglib:jar + com.fasterxml.jackson.core:jackson-annotations:jar + com.fasterxml.jackson.core:jackson-core:jar + com.fasterxml.jackson.core:jackson-databind:jar + com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar + com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar + com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar + com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar + com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar + com.fasterxml.jackson.module:jackson-module-parameter-names:jar + com.fasterxml.jackson.module:jackson-module-paranamer:jar + com.google.code.findbugs:annotations:jar + com.google.code.gson:gson:jar + com.google.guava:guava:jar + com.google.inject:guice:jar + com.google.protobuf:protobuf-java:jar + com.netflix.archaius:archaius-core:jar + com.netflix.eureka:eureka-client:jar + com.netflix.eureka:eureka-core:jar + com.netflix.hystrix:hystrix-core:jar + com.netflix.netflix-commons:netflix-commons-util:jar + com.netflix.netflix-commons:netflix-eventbus:jar + com.netflix.netflix-commons:netflix-infix:jar + com.netflix.netflix-commons:netflix-statistics:jar + com.netflix.ribbon:ribbon:jar + com.netflix.ribbon:ribbon-core:jar + com.netflix.ribbon:ribbon-eureka:jar + com.netflix.ribbon:ribbon-httpclient:jar + com.netflix.ribbon:ribbon-loadbalancer:jar + com.netflix.ribbon:ribbon-transport:jar + com.netflix.servo:servo-core:jar + + com.sun.jersey.contribs:jersey-apache-client4:jar + com.sun.jersey:jersey-client:jar + com.sun.jersey:jersey-core:jar + com.sun.jersey:jersey-json:jar + com.sun.jersey:jersey-server:jar + com.sun.jersey:jersey-servlet:jar + com.sun.xml.bind:jaxb-impl:jar + com.thoughtworks.paranamer:paranamer:jar + com.thoughtworks.xstream:xstream:jar + com.webank.wedatasphere.linkis:linkis-common:jar + com.webank.wedatasphere.linkis:linkis-module:jar + commons-beanutils:commons-beanutils:jar + + commons-cli:commons-cli:jar + commons-codec:commons-codec:jar + commons-collections:commons-collections:jar + commons-configuration:commons-configuration:jar + commons-daemon:commons-daemon:jar + commons-dbcp:commons-dbcp:jar + + commons-httpclient:commons-httpclient:jar + commons-io:commons-io:jar + commons-jxpath:commons-jxpath:jar + commons-lang:commons-lang:jar + commons-logging:commons-logging:jar + commons-net:commons-net:jar + commons-pool:commons-pool:jar + io.micrometer:micrometer-core:jar + io.netty:netty:jar + io.netty:netty-all:jar + + io.reactivex:rxjava:jar + io.reactivex:rxnetty:jar + io.reactivex:rxnetty-contexts:jar + io.reactivex:rxnetty-servo:jar + javax.activation:activation:jar + javax.annotation:javax.annotation-api:jar + javax.inject:javax.inject:jar + javax.servlet:javax.servlet-api:jar + javax.servlet.jsp:jsp-api:jar + javax.validation:validation-api:jar + + javax.ws.rs:javax.ws.rs-api:jar + javax.xml.bind:jaxb-api:jar + javax.xml.stream:stax-api:jar + joda-time:joda-time:jar + log4j:log4j:jar + mysql:mysql-connector-java:jar + + + org.antlr:antlr-runtime:jar + org.antlr:stringtemplate:jar + org.apache.commons:commons-compress:jar + + org.apache.commons:commons-math3:jar + org.apache.curator:curator-client:jar + org.apache.curator:curator-framework:jar + org.apache.curator:curator-recipes:jar + org.apache.directory.api:api-asn1-api:jar + org.apache.directory.api:api-util:jar + org.apache.directory.server:apacheds-i18n:jar + org.apache.directory.server:apacheds-kerberos-codec:jar + org.apache.hadoop:hadoop-annotations:jar + org.apache.hadoop:hadoop-auth:jar + org.apache.hadoop:hadoop-common:jar + org.apache.hadoop:hadoop-hdfs:jar + org.apache.htrace:htrace-core:jar + org.apache.httpcomponents:httpclient:jar + org.apache.httpcomponents:httpcore:jar + org.apache.logging.log4j:log4j-api:jar + org.apache.logging.log4j:log4j-core:jar + org.apache.logging.log4j:log4j-jul:jar + org.apache.logging.log4j:log4j-slf4j-impl:jar + org.apache.zookeeper:zookeeper:jar + org.aspectj:aspectjweaver:jar + org.bouncycastle:bcpkix-jdk15on:jar + org.bouncycastle:bcprov-jdk15on:jar + org.codehaus.jackson:jackson-jaxrs:jar + org.codehaus.jackson:jackson-xc:jar + org.codehaus.jettison:jettison:jar + org.codehaus.woodstox:stax2-api:jar + + org.eclipse.jetty:jetty-annotations:jar + org.eclipse.jetty:jetty-client:jar + org.eclipse.jetty:jetty-continuation:jar + org.eclipse.jetty:jetty-http:jar + org.eclipse.jetty:jetty-io:jar + + org.eclipse.jetty:jetty-plus:jar + org.eclipse.jetty:jetty-security:jar + org.eclipse.jetty:jetty-server:jar + org.eclipse.jetty:jetty-servlet:jar + org.eclipse.jetty:jetty-servlets:jar + org.eclipse.jetty:jetty-util:jar + org.eclipse.jetty:jetty-webapp:jar + org.eclipse.jetty:jetty-xml:jar + org.eclipse.jetty.websocket:javax-websocket-client-impl:jar + org.eclipse.jetty.websocket:javax-websocket-server-impl:jar + org.eclipse.jetty.websocket:websocket-api:jar + org.eclipse.jetty.websocket:websocket-client:jar + org.eclipse.jetty.websocket:websocket-common:jar + org.eclipse.jetty.websocket:websocket-server:jar + org.eclipse.jetty.websocket:websocket-servlet:jar + org.fusesource.leveldbjni:leveldbjni-all:jar + org.glassfish.hk2:class-model:jar + org.glassfish.hk2:config-types:jar + org.glassfish.hk2.external:aopalliance-repackaged:jar + org.glassfish.hk2.external:asm-all-repackaged:jar + org.glassfish.hk2.external:bean-validator:jar + org.glassfish.hk2.external:javax.inject:jar + org.glassfish.hk2:hk2:jar + org.glassfish.hk2:hk2-api:jar + org.glassfish.hk2:hk2-config:jar + org.glassfish.hk2:hk2-core:jar + org.glassfish.hk2:hk2-locator:jar + org.glassfish.hk2:hk2-runlevel:jar + org.glassfish.hk2:hk2-utils:jar + org.glassfish.hk2:osgi-resource-locator:jar + org.glassfish.hk2:spring-bridge:jar + org.glassfish.jersey.bundles:jaxrs-ri:jar + org.glassfish.jersey.bundles.repackaged:jersey-guava:jar + org.glassfish.jersey.containers:jersey-container-servlet:jar + org.glassfish.jersey.containers:jersey-container-servlet-core:jar + org.glassfish.jersey.core:jersey-client:jar + org.glassfish.jersey.core:jersey-common:jar + org.glassfish.jersey.core:jersey-server:jar + org.glassfish.jersey.ext:jersey-entity-filtering:jar + org.glassfish.jersey.ext:jersey-spring3:jar + org.glassfish.jersey.media:jersey-media-jaxb:jar + org.glassfish.jersey.media:jersey-media-json-jackson:jar + org.glassfish.jersey.media:jersey-media-multipart:jar + org.hdrhistogram:HdrHistogram:jar + org.javassist:javassist:jar + + + org.jvnet.mimepull:mimepull:jar + org.jvnet:tiger-types:jar + org.latencyutils:LatencyUtils:jar + org.mortbay.jasper:apache-el:jar + org.mortbay.jetty:jetty:jar + org.mortbay.jetty:jetty-util:jar + org.ow2.asm:asm-analysis:jar + org.ow2.asm:asm-commons:jar + org.ow2.asm:asm-tree:jar + org.reflections:reflections:jar + + org.scala-lang:scala-compiler:jar + org.scala-lang:scala-library:jar + org.scala-lang:scala-reflect:jar + org.scala-lang:scalap:jar + org.slf4j:jul-to-slf4j:jar + org.slf4j:slf4j-api:jar + org.springframework.boot:spring-boot:jar + org.springframework.boot:spring-boot-actuator:jar + org.springframework.boot:spring-boot-actuator-autoconfigure:jar + org.springframework.boot:spring-boot-autoconfigure:jar + org.springframework.boot:spring-boot-starter:jar + org.springframework.boot:spring-boot-starter-actuator:jar + org.springframework.boot:spring-boot-starter-aop:jar + org.springframework.boot:spring-boot-starter-jetty:jar + org.springframework.boot:spring-boot-starter-json:jar + org.springframework.boot:spring-boot-starter-log4j2:jar + org.springframework.boot:spring-boot-starter-web:jar + org.springframework.cloud:spring-cloud-commons:jar + org.springframework.cloud:spring-cloud-config-client:jar + org.springframework.cloud:spring-cloud-context:jar + + + org.springframework.cloud:spring-cloud-netflix-eureka-client:jar + + org.springframework.cloud:spring-cloud-starter:jar + org.springframework.cloud:spring-cloud-starter-config:jar + org.springframework.cloud:spring-cloud-starter-eureka:jar + org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar + org.springframework.cloud:spring-cloud-starter-netflix-eureka-client:jar + org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar + org.springframework.security:spring-security-crypto:jar + org.springframework.security:spring-security-rsa:jar + org.springframework:spring-aop:jar + org.springframework:spring-beans:jar + org.springframework:spring-context:jar + org.springframework:spring-core:jar + org.springframework:spring-expression:jar + org.springframework:spring-jcl:jar + org.springframework:spring-web:jar + org.springframework:spring-webmvc:jar + org.tukaani:xz:jar + org.yaml:snakeyaml:jar + + xerces:xercesImpl:jar + xmlenc:xmlenc:jar + xmlpull:xmlpull:jar + xpp3:xpp3_min:jar + + + + + + + ${basedir}/src/main/resources + + * + + 0777 + conf + unix + + + ${basedir}/bin + + * + + 0777 + bin + unix + + + . + + */** + + logs + + + + + From a0a43a3a162817ade0fb183e8463a7a10554c2c3 Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Tue, 27 Apr 2021 21:30:54 +0800 Subject: [PATCH 017/434] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=B8=8Bdatasource?= =?UTF-8?q?=20appconn=E7=9A=84=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../operation/StreamFlowQueryOperation.java | 2 +- .../StreamisProjectDeleteOperation.java | 2 +- .../streamis-datasource-appconn/pom.xml | 6 + .../StreamisDatasourceExternalException.java | 19 + .../operation/DatasourceImportOperation.java | 77 ++++ .../ref/DatasourceImportRequestRef.java | 23 +- .../ref/DatasourceImportResponseRef.java | 1 + .../StreamisDataSourceDevelopStandard.java | 10 +- .../common/StreamisPluginErrorException.java | 22 ++ .../plugins/common/StreamisPluginUtils.java | 54 +++ .../src/main/assembly/distribution.xml | 356 +++++++++++++++--- 11 files changed, 513 insertions(+), 59 deletions(-) create mode 100644 streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/exception/StreamisDatasourceExternalException.java create mode 100644 streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/operation/DatasourceImportOperation.java create mode 100644 streamis-plugins/streamis-plugins-common/src/main/java/com/webank/wedatasphere/streamis/plugins/common/StreamisPluginErrorException.java diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowQueryOperation.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowQueryOperation.java index f2145cbf2..39b36dd74 100644 --- a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowQueryOperation.java +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamFlowQueryOperation.java @@ -45,6 +45,6 @@ public StreamFlowQueryResponseRef query(StreamFlowQueryRequestRef streamFlowQuer @Override public void setDevelopmentService(DevelopmentService developmentService) { - + this.developmentService = developmentService; } } diff --git a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamisProjectDeleteOperation.java b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamisProjectDeleteOperation.java index d314a37d0..e00bbf658 100644 --- a/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamisProjectDeleteOperation.java +++ b/streamis-plugins/streamis-appconn/src/main/java/com/webank/wedatasphere/streamis/appconn/operation/StreamisProjectDeleteOperation.java @@ -17,7 +17,7 @@ public class StreamisProjectDeleteOperation implements ProjectDeletionOperation private static final Logger LOGGER = LoggerFactory.getLogger(StreamisProjectDeleteOperation.class); - //todo 以后写 + //todo 以后写，先等下写 @Override public ProjectResponseRef deleteProject(ProjectRequestRef projectRequestRef) throws ExternalOperationFailedException { diff --git a/streamis-plugins/streamis-datasource-appconn/pom.xml b/streamis-plugins/streamis-datasource-appconn/pom.xml index 1c95867e6..feb8f6e67 100644 --- a/streamis-plugins/streamis-datasource-appconn/pom.xml +++ b/streamis-plugins/streamis-datasource-appconn/pom.xml @@ -38,6 +38,12 @@ ${dss.version} + + com.webank.wedatasphere.streamis + streamis-plugins-common + ${streamis.version} + + diff --git a/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/exception/StreamisDatasourceExternalException.java b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/exception/StreamisDatasourceExternalException.java new file mode 100644 index 000000000..2aa885d81 --- /dev/null +++ b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/exception/StreamisDatasourceExternalException.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.streamis.datasource.appconn.exception; + +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class StreamisDatasourceExternalException extends ExternalOperationFailedException { + + public StreamisDatasourceExternalException(int errorCode, String message) { + super(errorCode, message); + } + + public StreamisDatasourceExternalException(int errorCode, String message, Throwable cause) { + super(errorCode, message); + this.initCause(cause); + } +} diff --git a/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/operation/DatasourceImportOperation.java b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/operation/DatasourceImportOperation.java new file mode 100644 index 000000000..5b7f6b1c4 --- /dev/null +++ b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/operation/DatasourceImportOperation.java @@ -0,0 +1,77 @@ +package com.webank.wedatasphere.streamis.datasource.appconn.operation; + +import com.webank.wedatasphere.dss.standard.app.development.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.development.publish.RefImportOperation; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import com.webank.wedatasphere.linkis.httpclient.request.HttpAction; +import com.webank.wedatasphere.linkis.httpclient.response.HttpResult; +import com.webank.wedatasphere.streamis.datasource.appconn.action.DatasourcePostAction; +import com.webank.wedatasphere.streamis.datasource.appconn.exception.StreamisDatasourceExternalException; +import com.webank.wedatasphere.streamis.datasource.appconn.ref.DatasourceImportRequestRef; +import com.webank.wedatasphere.streamis.datasource.appconn.ref.DatasourceImportResponseRef; +import com.webank.wedatasphere.streamis.datasource.appconn.utils.DataSourceUtils; +import com.webank.wedatasphere.streamis.plugins.common.StreamisPluginUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +/** + * created by yangzhiyue on 2021/4/26 + * Description: + */ +public class DatasourceImportOperation implements RefImportOperation { + + + + private AppInstance appInstance; + + private static final Logger LOGGER = LoggerFactory.getLogger(DatasourceImportOperation.class); + + private String datasourceImportUrl; + + private SSORequestOperation ssoRequestOperation; + + private static final String DATASOURCE_SUFFIX_URL = "api/res_j/v1/streamis/importDatasource"; + + + public DatasourceImportOperation(AppInstance appInstance){ + this.appInstance = appInstance; + } + + + @Override + public DatasourceImportResponseRef importRef(DatasourceImportRequestRef datasourceImportRequestRef) throws StreamisDatasourceExternalException { + LOGGER.info("begin to export datasource from streamis, request is {}", DataSourceUtils.COMMON_GSON.toJson(datasourceImportRequestRef)); + try{ + SSOUrlBuilderOperation ssoUrlBuilderOperation = datasourceImportRequestRef.getWorkspace().getSSOUrlBuilderOperation().copy(); + ssoUrlBuilderOperation.setAppName(DataSourceUtils.DATASOURCE_APP_NAME.getValue()); + ssoUrlBuilderOperation.setReqUrl(this.datasourceImportUrl); + ssoUrlBuilderOperation.setWorkspace(datasourceImportRequestRef.getWorkspace().getWorkspaceName()); + DatasourcePostAction datasourcePostAction = new DatasourcePostAction(ssoUrlBuilderOperation.getBuiltUrl(), datasourceImportRequestRef.getUser()); + datasourcePostAction.addRequestPayload(StreamisPluginUtils.BML_RESOURCE_ID_STR, datasourceImportRequestRef.getBmlResourceId()); + datasourcePostAction.addRequestPayload(StreamisPluginUtils.BML_RESOURCE_VERSION_STR,datasourceImportRequestRef.getBmlVersion()); + HttpResult httpResult = this.ssoRequestOperation.requestWithSSO(ssoUrlBuilderOperation, datasourcePostAction); + LOGGER.info("end to export datasource from streamis, response is {}", httpResult.getResponseBody()); + Map data = StreamisPluginUtils.getDataFromResponseBody(httpResult.getResponseBody()); + if (data == null){ + LOGGER.error("data from responseBody {} is null", httpResult.getResponseBody()); + throw new StreamisDatasourceExternalException(600401, "failed to import datasource to streamis"); + } + Long streamisTableMetaId = (Long) data.get(StreamisPluginUtils.STREAMIS_RPC_SERVER_NAME); + return new DatasourceImportResponseRef(httpResult.getResponseBody(), 0, streamisTableMetaId); + }catch(final StreamisDatasourceExternalException e){ + throw e; + } + catch(final Exception e){ + throw new StreamisDatasourceExternalException(300601, "failed to export ref ", e); + } + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + + } +} diff --git a/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/ref/DatasourceImportRequestRef.java b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/ref/DatasourceImportRequestRef.java index 5812a4b8a..dffeb7b0b 100644 --- a/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/ref/DatasourceImportRequestRef.java +++ b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/ref/DatasourceImportRequestRef.java @@ -2,6 +2,7 @@ import com.webank.wedatasphere.dss.standard.app.development.publish.ImportRequestRef; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.streamis.plugins.common.StreamisPluginUtils; import java.util.HashMap; import java.util.Map; @@ -21,6 +22,7 @@ public class DatasourceImportRequestRef implements ImportRequestRef { private static final String TYPE = "import"; + private String user; /** * bmlresource includes resourceId and version @@ -28,10 +30,9 @@ public class DatasourceImportRequestRef implements ImportRequestRef { private Map bmlResource; - public DatasourceImportRequestRef(Map bmlResource){ - this.bmlResource = bmlResource; - } + public DatasourceImportRequestRef(){ + } @Override @@ -73,6 +74,22 @@ public Map getBmlResource() { return bmlResource; } + public String getBmlResourceId(){ + return (String) this.parameters.get(StreamisPluginUtils.BML_RESOURCE_ID_STR); + } + + public String getBmlVersion(){ + return (String) this.parameters.get(StreamisPluginUtils.BML_RESOURCE_VERSION_STR); + } + + public String getUser(){ + return (String)this.parameters.get(StreamisPluginUtils.USER_STR); + } + + + + + public void setBmlResource(Map bmlResource) { this.bmlResource = bmlResource; } diff --git a/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/ref/DatasourceImportResponseRef.java b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/ref/DatasourceImportResponseRef.java index ae91d6035..2080a4dc1 100644 --- a/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/ref/DatasourceImportResponseRef.java +++ b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/ref/DatasourceImportResponseRef.java @@ -11,6 +11,7 @@ /** * created by yangzhiyue on 2021/4/13 * Description: + * 将 */ public class DatasourceImportResponseRef extends AbstractResponseRef { diff --git a/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/standard/StreamisDataSourceDevelopStandard.java b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/standard/StreamisDataSourceDevelopStandard.java index db02ab344..4b552a75a 100644 --- a/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/standard/StreamisDataSourceDevelopStandard.java +++ b/streamis-plugins/streamis-datasource-appconn/src/main/java/com/webank/wedatasphere/streamis/datasource/appconn/standard/StreamisDataSourceDevelopStandard.java @@ -5,7 +5,6 @@ import com.webank.wedatasphere.dss.standard.app.development.AbstractLabelDevelopmentIntegrationStandard; import com.webank.wedatasphere.dss.standard.app.development.RefOperationService; import com.webank.wedatasphere.dss.standard.common.desc.AppDesc; -import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException; import com.webank.wedatasphere.streamis.datasource.appconn.service.StreamisDatasourceQueryService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,16 +38,11 @@ public class StreamisDataSourceDevelopStandard extends AbstractLabelDevelopmentI private StreamisDataSourceDevelopStandard(AppConn appConn){ this.appConn = appConn; - try { - init(); - } catch (AppStandardErrorException e) { - LOGGER.error("Failed to init in Streamis datasource development standard", e); - } + init(); } public static StreamisDataSourceDevelopStandard getInstance(AppConn appConn){ - if(instance == null){ synchronized (StreamisDataSourceDevelopStandard.class){ if (instance == null){ @@ -88,7 +82,7 @@ public void setAppDesc(AppDesc appDesc) { } @Override - public void init() throws AppStandardErrorException { + public void init() { LOGGER.info("begin to init in streamis dataource develop standard"); //do noting } diff --git a/streamis-plugins/streamis-plugins-common/src/main/java/com/webank/wedatasphere/streamis/plugins/common/StreamisPluginErrorException.java b/streamis-plugins/streamis-plugins-common/src/main/java/com/webank/wedatasphere/streamis/plugins/common/StreamisPluginErrorException.java new file mode 100644 index 000000000..427c86eb2 --- /dev/null +++ b/streamis-plugins/streamis-plugins-common/src/main/java/com/webank/wedatasphere/streamis/plugins/common/StreamisPluginErrorException.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.streamis.plugins.common; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; + +/** + * created by yangzhiyue on 2021/4/27 + * Description: + */ +public class StreamisPluginErrorException extends ErrorException { + + + public StreamisPluginErrorException(int errCode, String desc) { + super(errCode, desc); + } + + public StreamisPluginErrorException(int errCode, String desc, Throwable throwable) { + super(errCode, desc); + this.initCause(throwable); + } + + +} diff --git a/streamis-plugins/streamis-plugins-common/src/main/java/com/webank/wedatasphere/streamis/plugins/common/StreamisPluginUtils.java b/streamis-plugins/streamis-plugins-common/src/main/java/com/webank/wedatasphere/streamis/plugins/common/StreamisPluginUtils.java index a83f4b657..880df7bda 100644 --- a/streamis-plugins/streamis-plugins-common/src/main/java/com/webank/wedatasphere/streamis/plugins/common/StreamisPluginUtils.java +++ b/streamis-plugins/streamis-plugins-common/src/main/java/com/webank/wedatasphere/streamis/plugins/common/StreamisPluginUtils.java @@ -1,8 +1,14 @@ package com.webank.wedatasphere.streamis.plugins.common; +import com.google.gson.*; import com.webank.wedatasphere.dss.common.entity.DSSLabel; import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; import com.webank.wedatasphere.linkis.common.conf.CommonVars; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Type; +import java.util.Map; /** * created by yangzhiyue on 2021/4/21 @@ -11,6 +17,8 @@ public class StreamisPluginUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(StreamisPluginUtils.class); + public static final String DEV_LABEL = "dev"; public static final String PROD_LABEL = "prod"; @@ -18,10 +26,33 @@ public class StreamisPluginUtils { public static final String CS_STR = "contextId"; + public static final String BML_RESOURCE_ID_STR = "bmlResourceId"; + + public static final String BML_RESOURCE_VERSION_STR = "bmlResourceVersion"; + + public static final String USER_STR = "user"; + + public static final String DATA_STR = "data"; + + public static final String STREAMIS_TABLE_META_ID_STR = "streamisTableMetaId"; + public static final String STREAMIS_RPC_SERVER_NAME = CommonVars.apply("wds.streamis.plugin.server.name", "streamis-server").getValue(); + public static final Gson COMMON_GSON = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").serializeNulls() + .registerTypeAdapter(Double.class, new JsonSerializer() { + @Override + public JsonElement serialize(Double src, Type typeOfSrc, JsonSerializationContext context) { + if(src == src.longValue()) { + return new JsonPrimitive(src.longValue()); + } else { + return new JsonPrimitive(src); + } + } + }).create(); + + public static String getLabel(AppInstance appInstance){ for (DSSLabel label : appInstance.getLabels()) { if (label.getLabel().equalsIgnoreCase(PROD_LABEL)) return PROD_LABEL; @@ -30,6 +61,29 @@ public static String getLabel(AppInstance appInstance){ } + /** + * 将 + * @param responseBody json response + * @return linkis response中的data数据，如果有问题 则返回null + */ + @SuppressWarnings("unchecked") + public static Map getDataFromResponseBody(String responseBody){ + try{ + Map responseMap = COMMON_GSON.fromJson(responseBody, Map.class); + Object dataObject = responseMap.get(DATA_STR); + if (dataObject instanceof Map){ + return (Map)dataObject; + }else{ + LOGGER.error("dataObject {} is not Map class, it is a {}", dataObject, dataObject.getClass()); + return null; + } + }catch(final Exception e){ + LOGGER.error("failed to convert a responseBody {} to Map", responseBody, e); + return null; + } + } + + } diff --git a/streamis-server/src/main/assembly/distribution.xml b/streamis-server/src/main/assembly/distribution.xml index 914f21084..d609166b8 100644 --- a/streamis-server/src/main/assembly/distribution.xml +++ b/streamis-server/src/main/assembly/distribution.xml @@ -1,46 +1,310 @@ - - - - streamis-server - - dir - - true - streamis-server - - - - - - lib - true - true - false - true - true - - - - - - - + + + + streamis-server + + zip + + true + streamis-server + + + + + + lib + true + true + false + true + true + + antlr:antlr:jar + aopalliance:aopalliance:jar + asm:asm:jar + cglib:cglib:jar + com.fasterxml.jackson.core:jackson-annotations:jar + com.fasterxml.jackson.core:jackson-core:jar + com.fasterxml.jackson.core:jackson-databind:jar + com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar + com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar + com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar + com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar + com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar + com.fasterxml.jackson.module:jackson-module-parameter-names:jar + com.fasterxml.jackson.module:jackson-module-paranamer:jar + com.google.code.findbugs:annotations:jar + com.google.code.gson:gson:jar + com.google.guava:guava:jar + com.google.inject:guice:jar + com.google.protobuf:protobuf-java:jar + com.netflix.archaius:archaius-core:jar + com.netflix.eureka:eureka-client:jar + com.netflix.eureka:eureka-core:jar + com.netflix.hystrix:hystrix-core:jar + com.netflix.netflix-commons:netflix-commons-util:jar + com.netflix.netflix-commons:netflix-eventbus:jar + com.netflix.netflix-commons:netflix-infix:jar + com.netflix.netflix-commons:netflix-statistics:jar + com.netflix.ribbon:ribbon:jar + com.netflix.ribbon:ribbon-core:jar + com.netflix.ribbon:ribbon-eureka:jar + com.netflix.ribbon:ribbon-httpclient:jar + com.netflix.ribbon:ribbon-loadbalancer:jar + com.netflix.ribbon:ribbon-transport:jar + com.netflix.servo:servo-core:jar + + com.sun.jersey.contribs:jersey-apache-client4:jar + com.sun.jersey:jersey-client:jar + com.sun.jersey:jersey-core:jar + com.sun.jersey:jersey-json:jar + com.sun.jersey:jersey-server:jar + com.sun.jersey:jersey-servlet:jar + com.sun.xml.bind:jaxb-impl:jar + com.thoughtworks.paranamer:paranamer:jar + com.thoughtworks.xstream:xstream:jar + com.webank.wedatasphere.linkis:linkis-common:jar + com.webank.wedatasphere.linkis:linkis-module:jar + commons-beanutils:commons-beanutils:jar + + commons-cli:commons-cli:jar + commons-codec:commons-codec:jar + commons-collections:commons-collections:jar + commons-configuration:commons-configuration:jar + commons-daemon:commons-daemon:jar + commons-dbcp:commons-dbcp:jar + + commons-httpclient:commons-httpclient:jar + commons-io:commons-io:jar + commons-jxpath:commons-jxpath:jar + commons-lang:commons-lang:jar + commons-logging:commons-logging:jar + commons-net:commons-net:jar + commons-pool:commons-pool:jar + io.micrometer:micrometer-core:jar + io.netty:netty:jar + io.netty:netty-all:jar + + io.reactivex:rxjava:jar + io.reactivex:rxnetty:jar + io.reactivex:rxnetty-contexts:jar + io.reactivex:rxnetty-servo:jar + javax.activation:activation:jar + javax.annotation:javax.annotation-api:jar + javax.inject:javax.inject:jar + javax.servlet:javax.servlet-api:jar + javax.servlet.jsp:jsp-api:jar + javax.validation:validation-api:jar + + javax.ws.rs:javax.ws.rs-api:jar + javax.xml.bind:jaxb-api:jar + javax.xml.stream:stax-api:jar + joda-time:joda-time:jar + log4j:log4j:jar + mysql:mysql-connector-java:jar + + + org.antlr:antlr-runtime:jar + org.antlr:stringtemplate:jar + org.apache.commons:commons-compress:jar + + org.apache.commons:commons-math3:jar + org.apache.curator:curator-client:jar + org.apache.curator:curator-framework:jar + org.apache.curator:curator-recipes:jar + org.apache.directory.api:api-asn1-api:jar + org.apache.directory.api:api-util:jar + org.apache.directory.server:apacheds-i18n:jar + org.apache.directory.server:apacheds-kerberos-codec:jar + org.apache.hadoop:hadoop-annotations:jar + org.apache.hadoop:hadoop-auth:jar + org.apache.hadoop:hadoop-common:jar + org.apache.hadoop:hadoop-hdfs:jar + org.apache.htrace:htrace-core:jar + org.apache.httpcomponents:httpclient:jar + org.apache.httpcomponents:httpcore:jar + org.apache.logging.log4j:log4j-api:jar + org.apache.logging.log4j:log4j-core:jar + org.apache.logging.log4j:log4j-jul:jar + org.apache.logging.log4j:log4j-slf4j-impl:jar + org.apache.zookeeper:zookeeper:jar + org.aspectj:aspectjweaver:jar + org.bouncycastle:bcpkix-jdk15on:jar + org.bouncycastle:bcprov-jdk15on:jar + org.codehaus.jackson:jackson-jaxrs:jar + org.codehaus.jackson:jackson-xc:jar + org.codehaus.jettison:jettison:jar + org.codehaus.woodstox:stax2-api:jar + + org.eclipse.jetty:jetty-annotations:jar + org.eclipse.jetty:jetty-client:jar + org.eclipse.jetty:jetty-continuation:jar + org.eclipse.jetty:jetty-http:jar + org.eclipse.jetty:jetty-io:jar + + org.eclipse.jetty:jetty-plus:jar + org.eclipse.jetty:jetty-security:jar + org.eclipse.jetty:jetty-server:jar + org.eclipse.jetty:jetty-servlet:jar + org.eclipse.jetty:jetty-servlets:jar + org.eclipse.jetty:jetty-util:jar + org.eclipse.jetty:jetty-webapp:jar + org.eclipse.jetty:jetty-xml:jar + org.eclipse.jetty.websocket:javax-websocket-client-impl:jar + org.eclipse.jetty.websocket:javax-websocket-server-impl:jar + org.eclipse.jetty.websocket:websocket-api:jar + org.eclipse.jetty.websocket:websocket-client:jar + org.eclipse.jetty.websocket:websocket-common:jar + org.eclipse.jetty.websocket:websocket-server:jar + org.eclipse.jetty.websocket:websocket-servlet:jar + org.fusesource.leveldbjni:leveldbjni-all:jar + org.glassfish.hk2:class-model:jar + org.glassfish.hk2:config-types:jar + org.glassfish.hk2.external:aopalliance-repackaged:jar + org.glassfish.hk2.external:asm-all-repackaged:jar + org.glassfish.hk2.external:bean-validator:jar + org.glassfish.hk2.external:javax.inject:jar + org.glassfish.hk2:hk2:jar + org.glassfish.hk2:hk2-api:jar + org.glassfish.hk2:hk2-config:jar + org.glassfish.hk2:hk2-core:jar + org.glassfish.hk2:hk2-locator:jar + org.glassfish.hk2:hk2-runlevel:jar + org.glassfish.hk2:hk2-utils:jar + org.glassfish.hk2:osgi-resource-locator:jar + org.glassfish.hk2:spring-bridge:jar + org.glassfish.jersey.bundles:jaxrs-ri:jar + org.glassfish.jersey.bundles.repackaged:jersey-guava:jar + org.glassfish.jersey.containers:jersey-container-servlet:jar + org.glassfish.jersey.containers:jersey-container-servlet-core:jar + org.glassfish.jersey.core:jersey-client:jar + org.glassfish.jersey.core:jersey-common:jar + org.glassfish.jersey.core:jersey-server:jar + org.glassfish.jersey.ext:jersey-entity-filtering:jar + org.glassfish.jersey.ext:jersey-spring3:jar + org.glassfish.jersey.media:jersey-media-jaxb:jar + org.glassfish.jersey.media:jersey-media-json-jackson:jar + org.glassfish.jersey.media:jersey-media-multipart:jar + org.hdrhistogram:HdrHistogram:jar + org.javassist:javassist:jar + + + org.jvnet.mimepull:mimepull:jar + org.jvnet:tiger-types:jar + org.latencyutils:LatencyUtils:jar + org.mortbay.jasper:apache-el:jar + org.mortbay.jetty:jetty:jar + org.mortbay.jetty:jetty-util:jar + org.ow2.asm:asm-analysis:jar + org.ow2.asm:asm-commons:jar + org.ow2.asm:asm-tree:jar + org.reflections:reflections:jar + + org.scala-lang:scala-compiler:jar + org.scala-lang:scala-library:jar + org.scala-lang:scala-reflect:jar + org.scala-lang:scalap:jar + org.slf4j:jul-to-slf4j:jar + org.slf4j:slf4j-api:jar + org.springframework.boot:spring-boot:jar + org.springframework.boot:spring-boot-actuator:jar + org.springframework.boot:spring-boot-actuator-autoconfigure:jar + org.springframework.boot:spring-boot-autoconfigure:jar + org.springframework.boot:spring-boot-starter:jar + org.springframework.boot:spring-boot-starter-actuator:jar + org.springframework.boot:spring-boot-starter-aop:jar + org.springframework.boot:spring-boot-starter-jetty:jar + org.springframework.boot:spring-boot-starter-json:jar + org.springframework.boot:spring-boot-starter-log4j2:jar + org.springframework.boot:spring-boot-starter-web:jar + org.springframework.cloud:spring-cloud-commons:jar + org.springframework.cloud:spring-cloud-config-client:jar + org.springframework.cloud:spring-cloud-context:jar + + + org.springframework.cloud:spring-cloud-netflix-eureka-client:jar + + org.springframework.cloud:spring-cloud-starter:jar + org.springframework.cloud:spring-cloud-starter-config:jar + org.springframework.cloud:spring-cloud-starter-eureka:jar + org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar + org.springframework.cloud:spring-cloud-starter-netflix-eureka-client:jar + org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar + org.springframework.security:spring-security-crypto:jar + org.springframework.security:spring-security-rsa:jar + org.springframework:spring-aop:jar + org.springframework:spring-beans:jar + org.springframework:spring-context:jar + org.springframework:spring-core:jar + org.springframework:spring-expression:jar + org.springframework:spring-jcl:jar + org.springframework:spring-web:jar + org.springframework:spring-webmvc:jar + org.tukaani:xz:jar + org.yaml:snakeyaml:jar + + xerces:xercesImpl:jar + xmlenc:xmlenc:jar + xmlpull:xmlpull:jar + xpp3:xpp3_min:jar + + + + + + + ${basedir}/src/main/resources + + * + + 0777 + conf + unix + + + ${basedir}/bin + + * + + 0777 + bin + unix + + + . + + */** + + logs + + + + + From 844a86a737f02f6e3c4ee7027dda66f8d26635ec Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Tue, 27 Apr 2021 21:59:03 +0800 Subject: [PATCH 018/434] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=B8=8Bpom?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 6 +- streamis-jobmanager/pom.xml | 1 + .../streamis-jobmanager-common/pom.xml | 3 +- streamis-plugins/pom.xml | 1 + .../streamis-plugins-common/pom.xml | 3 +- streamis-project/pom.xml | 8 +- .../streamis-project-common/pom.xml | 3 +- .../streamis-project-server/pom.xml | 3 +- .../src/main/assembly/distribution.xml | 241 +----------------- .../src/main/resources/application.yml | 22 ++ .../src/main/resources/linkis.properties | 48 ++++ .../src/main/resources/log4j.properties | 37 +++ streamis-server/src/main/resources/log4j2.xml | 39 +++ .../receiver/StreamisWorkflowReceiver.scala | 5 +- 14 files changed, 165 insertions(+), 255 deletions(-) create mode 100644 streamis-server/src/main/resources/application.yml create mode 100644 streamis-server/src/main/resources/linkis.properties create mode 100644 streamis-server/src/main/resources/log4j.properties create mode 100644 streamis-server/src/main/resources/log4j2.xml diff --git a/pom.xml b/pom.xml index a8685fb22..78f1136d9 100644 --- a/pom.xml +++ b/pom.xml @@ -25,13 +25,9 @@ streamis-datasource streamis-jobmanager streamis-workflow + streamis-project streamis-plugins streamis-server - streamis-plugins/streamis-plugins-common - streamis-project - streamis-project/streamis-project-server - streamis-project/streamis-project-common - streamis-jobmanager/streamis-jobmanager-common diff --git a/streamis-jobmanager/pom.xml b/streamis-jobmanager/pom.xml index 666749376..9383415ab 100644 --- a/streamis-jobmanager/pom.xml +++ b/streamis-jobmanager/pom.xml @@ -12,6 +12,7 @@ streamis-jobmanager pom + streamis-jobmanager-common streamis-job-manager streamis-job-launcher streamis-jobmanager-server diff --git a/streamis-jobmanager/streamis-jobmanager-common/pom.xml b/streamis-jobmanager/streamis-jobmanager-common/pom.xml index e4ef14f2e..9bea248ba 100644 --- a/streamis-jobmanager/streamis-jobmanager-common/pom.xml +++ b/streamis-jobmanager/streamis-jobmanager-common/pom.xml @@ -3,10 +3,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - streamis + streamis-jobmanager com.webank.wedatasphere.streamis 0.1.0-SNAPSHOT - ../../pom.xml 4.0.0 diff --git a/streamis-plugins/pom.xml b/streamis-plugins/pom.xml index 436aa431a..3920a3621 100644 --- a/streamis-plugins/pom.xml +++ b/streamis-plugins/pom.xml @@ -12,6 +12,7 @@ streamis-plugins pom + streamis-plugins-common streamis-appconn streamis-jobmanager-appconn streamis-datasource-appconn diff --git a/streamis-plugins/streamis-plugins-common/pom.xml b/streamis-plugins/streamis-plugins-common/pom.xml index fad9ecbb9..f264d0ff9 100644 --- a/streamis-plugins/streamis-plugins-common/pom.xml +++ b/streamis-plugins/streamis-plugins-common/pom.xml @@ -3,10 +3,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - streamis + streamis-plugins com.webank.wedatasphere.streamis 0.1.0-SNAPSHOT - ../../pom.xml 4.0.0 diff --git a/streamis-project/pom.xml b/streamis-project/pom.xml index f69348e19..0f7c495ba 100644 --- a/streamis-project/pom.xml +++ b/streamis-project/pom.xml @@ -10,10 +10,16 @@ 4.0.0 streamis-project - + pom 8 8 + + + streamis-project-common + streamis-project-server + + \ No newline at end of file diff --git a/streamis-project/streamis-project-common/pom.xml b/streamis-project/streamis-project-common/pom.xml index 8618f0a52..7d1b74efb 100644 --- a/streamis-project/streamis-project-common/pom.xml +++ b/streamis-project/streamis-project-common/pom.xml @@ -3,10 +3,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - streamis + streamis-project com.webank.wedatasphere.streamis 0.1.0-SNAPSHOT - ../../pom.xml 4.0.0 diff --git a/streamis-project/streamis-project-server/pom.xml b/streamis-project/streamis-project-server/pom.xml index f8a872af5..274c57c3a 100644 --- a/streamis-project/streamis-project-server/pom.xml +++ b/streamis-project/streamis-project-server/pom.xml @@ -3,10 +3,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - streamis + streamis-project com.webank.wedatasphere.streamis 0.1.0-SNAPSHOT - ../../pom.xml 4.0.0 diff --git a/streamis-server/src/main/assembly/distribution.xml b/streamis-server/src/main/assembly/distribution.xml index d609166b8..14f716ab7 100644 --- a/streamis-server/src/main/assembly/distribution.xml +++ b/streamis-server/src/main/assembly/distribution.xml @@ -34,247 +34,8 @@ true true false - true + false true - - antlr:antlr:jar - aopalliance:aopalliance:jar - asm:asm:jar - cglib:cglib:jar - com.fasterxml.jackson.core:jackson-annotations:jar - com.fasterxml.jackson.core:jackson-core:jar - com.fasterxml.jackson.core:jackson-databind:jar - com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar - com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar - com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar - com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar - com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar - com.fasterxml.jackson.module:jackson-module-parameter-names:jar - com.fasterxml.jackson.module:jackson-module-paranamer:jar - com.google.code.findbugs:annotations:jar - com.google.code.gson:gson:jar - com.google.guava:guava:jar - com.google.inject:guice:jar - com.google.protobuf:protobuf-java:jar - com.netflix.archaius:archaius-core:jar - com.netflix.eureka:eureka-client:jar - com.netflix.eureka:eureka-core:jar - com.netflix.hystrix:hystrix-core:jar - com.netflix.netflix-commons:netflix-commons-util:jar - com.netflix.netflix-commons:netflix-eventbus:jar - com.netflix.netflix-commons:netflix-infix:jar - com.netflix.netflix-commons:netflix-statistics:jar - com.netflix.ribbon:ribbon:jar - com.netflix.ribbon:ribbon-core:jar - com.netflix.ribbon:ribbon-eureka:jar - com.netflix.ribbon:ribbon-httpclient:jar - com.netflix.ribbon:ribbon-loadbalancer:jar - com.netflix.ribbon:ribbon-transport:jar - com.netflix.servo:servo-core:jar - - com.sun.jersey.contribs:jersey-apache-client4:jar - com.sun.jersey:jersey-client:jar - com.sun.jersey:jersey-core:jar - com.sun.jersey:jersey-json:jar - com.sun.jersey:jersey-server:jar - com.sun.jersey:jersey-servlet:jar - com.sun.xml.bind:jaxb-impl:jar - com.thoughtworks.paranamer:paranamer:jar - com.thoughtworks.xstream:xstream:jar - com.webank.wedatasphere.linkis:linkis-common:jar - com.webank.wedatasphere.linkis:linkis-module:jar - commons-beanutils:commons-beanutils:jar - - commons-cli:commons-cli:jar - commons-codec:commons-codec:jar - commons-collections:commons-collections:jar - commons-configuration:commons-configuration:jar - commons-daemon:commons-daemon:jar - commons-dbcp:commons-dbcp:jar - - commons-httpclient:commons-httpclient:jar - commons-io:commons-io:jar - commons-jxpath:commons-jxpath:jar - commons-lang:commons-lang:jar - commons-logging:commons-logging:jar - commons-net:commons-net:jar - commons-pool:commons-pool:jar - io.micrometer:micrometer-core:jar - io.netty:netty:jar - io.netty:netty-all:jar - - io.reactivex:rxjava:jar - io.reactivex:rxnetty:jar - io.reactivex:rxnetty-contexts:jar - io.reactivex:rxnetty-servo:jar - javax.activation:activation:jar - javax.annotation:javax.annotation-api:jar - javax.inject:javax.inject:jar - javax.servlet:javax.servlet-api:jar - javax.servlet.jsp:jsp-api:jar - javax.validation:validation-api:jar - - javax.ws.rs:javax.ws.rs-api:jar - javax.xml.bind:jaxb-api:jar - javax.xml.stream:stax-api:jar - joda-time:joda-time:jar - log4j:log4j:jar - mysql:mysql-connector-java:jar - - - org.antlr:antlr-runtime:jar - org.antlr:stringtemplate:jar - org.apache.commons:commons-compress:jar - - org.apache.commons:commons-math3:jar - org.apache.curator:curator-client:jar - org.apache.curator:curator-framework:jar - org.apache.curator:curator-recipes:jar - org.apache.directory.api:api-asn1-api:jar - org.apache.directory.api:api-util:jar - org.apache.directory.server:apacheds-i18n:jar - org.apache.directory.server:apacheds-kerberos-codec:jar - org.apache.hadoop:hadoop-annotations:jar - org.apache.hadoop:hadoop-auth:jar - org.apache.hadoop:hadoop-common:jar - org.apache.hadoop:hadoop-hdfs:jar - org.apache.htrace:htrace-core:jar - org.apache.httpcomponents:httpclient:jar - org.apache.httpcomponents:httpcore:jar - org.apache.logging.log4j:log4j-api:jar - org.apache.logging.log4j:log4j-core:jar - org.apache.logging.log4j:log4j-jul:jar - org.apache.logging.log4j:log4j-slf4j-impl:jar - org.apache.zookeeper:zookeeper:jar - org.aspectj:aspectjweaver:jar - org.bouncycastle:bcpkix-jdk15on:jar - org.bouncycastle:bcprov-jdk15on:jar - org.codehaus.jackson:jackson-jaxrs:jar - org.codehaus.jackson:jackson-xc:jar - org.codehaus.jettison:jettison:jar - org.codehaus.woodstox:stax2-api:jar - - org.eclipse.jetty:jetty-annotations:jar - org.eclipse.jetty:jetty-client:jar - org.eclipse.jetty:jetty-continuation:jar - org.eclipse.jetty:jetty-http:jar - org.eclipse.jetty:jetty-io:jar - - org.eclipse.jetty:jetty-plus:jar - org.eclipse.jetty:jetty-security:jar - org.eclipse.jetty:jetty-server:jar - org.eclipse.jetty:jetty-servlet:jar - org.eclipse.jetty:jetty-servlets:jar - org.eclipse.jetty:jetty-util:jar - org.eclipse.jetty:jetty-webapp:jar - org.eclipse.jetty:jetty-xml:jar - org.eclipse.jetty.websocket:javax-websocket-client-impl:jar - org.eclipse.jetty.websocket:javax-websocket-server-impl:jar - org.eclipse.jetty.websocket:websocket-api:jar - org.eclipse.jetty.websocket:websocket-client:jar - org.eclipse.jetty.websocket:websocket-common:jar - org.eclipse.jetty.websocket:websocket-server:jar - org.eclipse.jetty.websocket:websocket-servlet:jar - org.fusesource.leveldbjni:leveldbjni-all:jar - org.glassfish.hk2:class-model:jar - org.glassfish.hk2:config-types:jar - org.glassfish.hk2.external:aopalliance-repackaged:jar - org.glassfish.hk2.external:asm-all-repackaged:jar - org.glassfish.hk2.external:bean-validator:jar - org.glassfish.hk2.external:javax.inject:jar - org.glassfish.hk2:hk2:jar - org.glassfish.hk2:hk2-api:jar - org.glassfish.hk2:hk2-config:jar - org.glassfish.hk2:hk2-core:jar - org.glassfish.hk2:hk2-locator:jar - org.glassfish.hk2:hk2-runlevel:jar - org.glassfish.hk2:hk2-utils:jar - org.glassfish.hk2:osgi-resource-locator:jar - org.glassfish.hk2:spring-bridge:jar - org.glassfish.jersey.bundles:jaxrs-ri:jar - org.glassfish.jersey.bundles.repackaged:jersey-guava:jar - org.glassfish.jersey.containers:jersey-container-servlet:jar - org.glassfish.jersey.containers:jersey-container-servlet-core:jar - org.glassfish.jersey.core:jersey-client:jar - org.glassfish.jersey.core:jersey-common:jar - org.glassfish.jersey.core:jersey-server:jar - org.glassfish.jersey.ext:jersey-entity-filtering:jar - org.glassfish.jersey.ext:jersey-spring3:jar - org.glassfish.jersey.media:jersey-media-jaxb:jar - org.glassfish.jersey.media:jersey-media-json-jackson:jar - org.glassfish.jersey.media:jersey-media-multipart:jar - org.hdrhistogram:HdrHistogram:jar - org.javassist:javassist:jar - - - org.jvnet.mimepull:mimepull:jar - org.jvnet:tiger-types:jar - org.latencyutils:LatencyUtils:jar - org.mortbay.jasper:apache-el:jar - org.mortbay.jetty:jetty:jar - org.mortbay.jetty:jetty-util:jar - org.ow2.asm:asm-analysis:jar - org.ow2.asm:asm-commons:jar - org.ow2.asm:asm-tree:jar - org.reflections:reflections:jar - - org.scala-lang:scala-compiler:jar - org.scala-lang:scala-library:jar - org.scala-lang:scala-reflect:jar - org.scala-lang:scalap:jar - org.slf4j:jul-to-slf4j:jar - org.slf4j:slf4j-api:jar - org.springframework.boot:spring-boot:jar - org.springframework.boot:spring-boot-actuator:jar - org.springframework.boot:spring-boot-actuator-autoconfigure:jar - org.springframework.boot:spring-boot-autoconfigure:jar - org.springframework.boot:spring-boot-starter:jar - org.springframework.boot:spring-boot-starter-actuator:jar - org.springframework.boot:spring-boot-starter-aop:jar - org.springframework.boot:spring-boot-starter-jetty:jar - org.springframework.boot:spring-boot-starter-json:jar - org.springframework.boot:spring-boot-starter-log4j2:jar - org.springframework.boot:spring-boot-starter-web:jar - org.springframework.cloud:spring-cloud-commons:jar - org.springframework.cloud:spring-cloud-config-client:jar - org.springframework.cloud:spring-cloud-context:jar - - - org.springframework.cloud:spring-cloud-netflix-eureka-client:jar - - org.springframework.cloud:spring-cloud-starter:jar - org.springframework.cloud:spring-cloud-starter-config:jar - org.springframework.cloud:spring-cloud-starter-eureka:jar - org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar - org.springframework.cloud:spring-cloud-starter-netflix-eureka-client:jar - org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar - org.springframework.security:spring-security-crypto:jar - org.springframework.security:spring-security-rsa:jar - org.springframework:spring-aop:jar - org.springframework:spring-beans:jar - org.springframework:spring-context:jar - org.springframework:spring-core:jar - org.springframework:spring-expression:jar - org.springframework:spring-jcl:jar - org.springframework:spring-web:jar - org.springframework:spring-webmvc:jar - org.tukaani:xz:jar - org.yaml:snakeyaml:jar - - xerces:xercesImpl:jar - xmlenc:xmlenc:jar - xmlpull:xmlpull:jar - xpp3:xpp3_min:jar - diff --git a/streamis-server/src/main/resources/application.yml b/streamis-server/src/main/resources/application.yml new file mode 100644 index 000000000..7d7a5dcb2 --- /dev/null +++ b/streamis-server/src/main/resources/application.yml @@ -0,0 +1,22 @@ +server: + port: 9321 +spring: + application: + name: streamis-server + + +eureka: + client: + serviceUrl: + defaultZone: http://127.0.0.1:20303/eureka/ + instance: + metadata-map: + test: wedatasphere + +management: + endpoints: + web: + exposure: + include: refresh,info +logging: + config: classpath:log4j2.xml diff --git a/streamis-server/src/main/resources/linkis.properties b/streamis-server/src/main/resources/linkis.properties new file mode 100644 index 000000000..1befd5885 --- /dev/null +++ b/streamis-server/src/main/resources/linkis.properties @@ -0,0 +1,48 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +wds.linkis.test.mode=true + + +wds.linkis.server.mybatis.datasource.url=jdbc:mysql://127.0.0.1:3306/db?characterEncoding=UTF-8 + +wds.linkis.server.mybatis.datasource.username=username + +wds.linkis.server.mybatis.datasource.password=* + + +wds.linkis.log.clear=true + +wds.linkis.server.version=v1 + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.server.restful,com.webank.wedatasphere.dss.application.restful,com.webank.wedatasphere.dss.workspace.server.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com\\webank\\wedatasphere\\dss\\server\\dao\\impl\\*.xml,classpath*:com\\webank\\wedatasphere\\dss\\application\\dao\\impl\\*.xml,classpath*:com\\webank\\wedatasphere\\dss\\workspace\\mapper\\impl\\*.xml,classpath*:com\\webank\\wedatasphere\\dss\\workspace\\common\\dao\\impl\\*.xml + +wds.linkis.server.mybatis.typeAliasesPackage=com + +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.server.dao,com.webank.wedatasphere.dss.application.dao,com.webank.wedatasphere.dss.workspace.mapper,com.webank.wedatasphere.dss.workspace.common.dao + + + +wds.linkis.gateway.ip=127.0.0.1 +wds.linkis.gateway.port=9001 + + + diff --git a/streamis-server/src/main/resources/log4j.properties b/streamis-server/src/main/resources/log4j.properties new file mode 100644 index 000000000..6f1673f17 --- /dev/null +++ b/streamis-server/src/main/resources/log4j.properties @@ -0,0 +1,37 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO diff --git a/streamis-server/src/main/resources/log4j2.xml b/streamis-server/src/main/resources/log4j2.xml new file mode 100644 index 000000000..7076c8dd9 --- /dev/null +++ b/streamis-server/src/main/resources/log4j2.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala b/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala index 06fabc9b5..63b91dfc2 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala +++ b/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala @@ -10,6 +10,7 @@ import com.webank.wedatasphere.streamis.workflow.server.service.StreamFlowServic import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component +import java.util import scala.concurrent.duration.Duration import scala.collection.JavaConversions._ import scala.collection.JavaConverters._ @@ -51,7 +52,9 @@ class StreamisWorkflowReceiver extends Receiver with Logging { } case streamFlowImportRequest: StreamFlowImportRequest => Utils.tryCatch{ val dssFlows = streamFlowService.importStreamFlow(streamFlowImportRequest) - StreamFlowImportResponse(0, Lists.newArrayList(dssFlows.map(_.getId).toList.asJavaCollection), "") + val idList = new util.ArrayList[Long]() + dssFlows.foreach(flow => idList.add(flow.getId)) + StreamFlowImportResponse(0, idList, "") }{ t => logger.error(s"failed to import resourceId: ${streamFlowImportRequest.bmlResourceId} version is ${streamFlowImportRequest.bmlVersion}", t) StreamFlowImportResponse(-1, null, t.getCause.getMessage) From 3058d5e76f6f1af33020a51d52732901802700f8 Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Tue, 27 Apr 2021 21:59:03 +0800 Subject: [PATCH 019/434] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=B8=8Bpom?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 6 +- streamis-jobmanager/pom.xml | 1 + .../streamis-jobmanager-common/pom.xml | 3 +- streamis-plugins/pom.xml | 1 + .../streamis-plugins-common/pom.xml | 3 +- streamis-project/pom.xml | 8 +- .../streamis-project-common/pom.xml | 3 +- .../streamis-project-server/pom.xml | 3 +- .../src/main/assembly/distribution.xml | 241 +----------------- .../src/main/resources/application.yml | 22 ++ .../src/main/resources/linkis.properties | 48 ++++ .../src/main/resources/log4j.properties | 37 +++ streamis-server/src/main/resources/log4j2.xml | 39 +++ .../receiver/StreamisWorkflowReceiver.scala | 5 +- 14 files changed, 165 insertions(+), 255 deletions(-) create mode 100644 streamis-server/src/main/resources/application.yml create mode 100644 streamis-server/src/main/resources/linkis.properties create mode 100644 streamis-server/src/main/resources/log4j.properties create mode 100644 streamis-server/src/main/resources/log4j2.xml diff --git a/pom.xml b/pom.xml index a8685fb22..78f1136d9 100644 --- a/pom.xml +++ b/pom.xml @@ -25,13 +25,9 @@ streamis-datasource streamis-jobmanager streamis-workflow + streamis-project streamis-plugins streamis-server - streamis-plugins/streamis-plugins-common - streamis-project - streamis-project/streamis-project-server - streamis-project/streamis-project-common - streamis-jobmanager/streamis-jobmanager-common diff --git a/streamis-jobmanager/pom.xml b/streamis-jobmanager/pom.xml index 666749376..9383415ab 100644 --- a/streamis-jobmanager/pom.xml +++ b/streamis-jobmanager/pom.xml @@ -12,6 +12,7 @@ streamis-jobmanager pom + streamis-jobmanager-common streamis-job-manager streamis-job-launcher streamis-jobmanager-server diff --git a/streamis-jobmanager/streamis-jobmanager-common/pom.xml b/streamis-jobmanager/streamis-jobmanager-common/pom.xml index e4ef14f2e..9bea248ba 100644 --- a/streamis-jobmanager/streamis-jobmanager-common/pom.xml +++ b/streamis-jobmanager/streamis-jobmanager-common/pom.xml @@ -3,10 +3,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - streamis + streamis-jobmanager com.webank.wedatasphere.streamis 0.1.0-SNAPSHOT - ../../pom.xml 4.0.0 diff --git a/streamis-plugins/pom.xml b/streamis-plugins/pom.xml index 436aa431a..3920a3621 100644 --- a/streamis-plugins/pom.xml +++ b/streamis-plugins/pom.xml @@ -12,6 +12,7 @@ streamis-plugins pom + streamis-plugins-common streamis-appconn streamis-jobmanager-appconn streamis-datasource-appconn diff --git a/streamis-plugins/streamis-plugins-common/pom.xml b/streamis-plugins/streamis-plugins-common/pom.xml index fad9ecbb9..f264d0ff9 100644 --- a/streamis-plugins/streamis-plugins-common/pom.xml +++ b/streamis-plugins/streamis-plugins-common/pom.xml @@ -3,10 +3,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - streamis + streamis-plugins com.webank.wedatasphere.streamis 0.1.0-SNAPSHOT - ../../pom.xml 4.0.0 diff --git a/streamis-project/pom.xml b/streamis-project/pom.xml index f69348e19..0f7c495ba 100644 --- a/streamis-project/pom.xml +++ b/streamis-project/pom.xml @@ -10,10 +10,16 @@ 4.0.0 streamis-project - + pom 8 8 + + + streamis-project-common + streamis-project-server + + \ No newline at end of file diff --git a/streamis-project/streamis-project-common/pom.xml b/streamis-project/streamis-project-common/pom.xml index 8618f0a52..7d1b74efb 100644 --- a/streamis-project/streamis-project-common/pom.xml +++ b/streamis-project/streamis-project-common/pom.xml @@ -3,10 +3,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - streamis + streamis-project com.webank.wedatasphere.streamis 0.1.0-SNAPSHOT - ../../pom.xml 4.0.0 diff --git a/streamis-project/streamis-project-server/pom.xml b/streamis-project/streamis-project-server/pom.xml index f8a872af5..274c57c3a 100644 --- a/streamis-project/streamis-project-server/pom.xml +++ b/streamis-project/streamis-project-server/pom.xml @@ -3,10 +3,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - streamis + streamis-project com.webank.wedatasphere.streamis 0.1.0-SNAPSHOT - ../../pom.xml 4.0.0 diff --git a/streamis-server/src/main/assembly/distribution.xml b/streamis-server/src/main/assembly/distribution.xml index d609166b8..14f716ab7 100644 --- a/streamis-server/src/main/assembly/distribution.xml +++ b/streamis-server/src/main/assembly/distribution.xml @@ -34,247 +34,8 @@ true true false - true + false true - - antlr:antlr:jar - aopalliance:aopalliance:jar - asm:asm:jar - cglib:cglib:jar - com.fasterxml.jackson.core:jackson-annotations:jar - com.fasterxml.jackson.core:jackson-core:jar - com.fasterxml.jackson.core:jackson-databind:jar - com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar - com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar - com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar - com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar - com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar - com.fasterxml.jackson.module:jackson-module-parameter-names:jar - com.fasterxml.jackson.module:jackson-module-paranamer:jar - com.google.code.findbugs:annotations:jar - com.google.code.gson:gson:jar - com.google.guava:guava:jar - com.google.inject:guice:jar - com.google.protobuf:protobuf-java:jar - com.netflix.archaius:archaius-core:jar - com.netflix.eureka:eureka-client:jar - com.netflix.eureka:eureka-core:jar - com.netflix.hystrix:hystrix-core:jar - com.netflix.netflix-commons:netflix-commons-util:jar - com.netflix.netflix-commons:netflix-eventbus:jar - com.netflix.netflix-commons:netflix-infix:jar - com.netflix.netflix-commons:netflix-statistics:jar - com.netflix.ribbon:ribbon:jar - com.netflix.ribbon:ribbon-core:jar - com.netflix.ribbon:ribbon-eureka:jar - com.netflix.ribbon:ribbon-httpclient:jar - com.netflix.ribbon:ribbon-loadbalancer:jar - com.netflix.ribbon:ribbon-transport:jar - com.netflix.servo:servo-core:jar - - com.sun.jersey.contribs:jersey-apache-client4:jar - com.sun.jersey:jersey-client:jar - com.sun.jersey:jersey-core:jar - com.sun.jersey:jersey-json:jar - com.sun.jersey:jersey-server:jar - com.sun.jersey:jersey-servlet:jar - com.sun.xml.bind:jaxb-impl:jar - com.thoughtworks.paranamer:paranamer:jar - com.thoughtworks.xstream:xstream:jar - com.webank.wedatasphere.linkis:linkis-common:jar - com.webank.wedatasphere.linkis:linkis-module:jar - commons-beanutils:commons-beanutils:jar - - commons-cli:commons-cli:jar - commons-codec:commons-codec:jar - commons-collections:commons-collections:jar - commons-configuration:commons-configuration:jar - commons-daemon:commons-daemon:jar - commons-dbcp:commons-dbcp:jar - - commons-httpclient:commons-httpclient:jar - commons-io:commons-io:jar - commons-jxpath:commons-jxpath:jar - commons-lang:commons-lang:jar - commons-logging:commons-logging:jar - commons-net:commons-net:jar - commons-pool:commons-pool:jar - io.micrometer:micrometer-core:jar - io.netty:netty:jar - io.netty:netty-all:jar - - io.reactivex:rxjava:jar - io.reactivex:rxnetty:jar - io.reactivex:rxnetty-contexts:jar - io.reactivex:rxnetty-servo:jar - javax.activation:activation:jar - javax.annotation:javax.annotation-api:jar - javax.inject:javax.inject:jar - javax.servlet:javax.servlet-api:jar - javax.servlet.jsp:jsp-api:jar - javax.validation:validation-api:jar - - javax.ws.rs:javax.ws.rs-api:jar - javax.xml.bind:jaxb-api:jar - javax.xml.stream:stax-api:jar - joda-time:joda-time:jar - log4j:log4j:jar - mysql:mysql-connector-java:jar - - - org.antlr:antlr-runtime:jar - org.antlr:stringtemplate:jar - org.apache.commons:commons-compress:jar - - org.apache.commons:commons-math3:jar - org.apache.curator:curator-client:jar - org.apache.curator:curator-framework:jar - org.apache.curator:curator-recipes:jar - org.apache.directory.api:api-asn1-api:jar - org.apache.directory.api:api-util:jar - org.apache.directory.server:apacheds-i18n:jar - org.apache.directory.server:apacheds-kerberos-codec:jar - org.apache.hadoop:hadoop-annotations:jar - org.apache.hadoop:hadoop-auth:jar - org.apache.hadoop:hadoop-common:jar - org.apache.hadoop:hadoop-hdfs:jar - org.apache.htrace:htrace-core:jar - org.apache.httpcomponents:httpclient:jar - org.apache.httpcomponents:httpcore:jar - org.apache.logging.log4j:log4j-api:jar - org.apache.logging.log4j:log4j-core:jar - org.apache.logging.log4j:log4j-jul:jar - org.apache.logging.log4j:log4j-slf4j-impl:jar - org.apache.zookeeper:zookeeper:jar - org.aspectj:aspectjweaver:jar - org.bouncycastle:bcpkix-jdk15on:jar - org.bouncycastle:bcprov-jdk15on:jar - org.codehaus.jackson:jackson-jaxrs:jar - org.codehaus.jackson:jackson-xc:jar - org.codehaus.jettison:jettison:jar - org.codehaus.woodstox:stax2-api:jar - - org.eclipse.jetty:jetty-annotations:jar - org.eclipse.jetty:jetty-client:jar - org.eclipse.jetty:jetty-continuation:jar - org.eclipse.jetty:jetty-http:jar - org.eclipse.jetty:jetty-io:jar - - org.eclipse.jetty:jetty-plus:jar - org.eclipse.jetty:jetty-security:jar - org.eclipse.jetty:jetty-server:jar - org.eclipse.jetty:jetty-servlet:jar - org.eclipse.jetty:jetty-servlets:jar - org.eclipse.jetty:jetty-util:jar - org.eclipse.jetty:jetty-webapp:jar - org.eclipse.jetty:jetty-xml:jar - org.eclipse.jetty.websocket:javax-websocket-client-impl:jar - org.eclipse.jetty.websocket:javax-websocket-server-impl:jar - org.eclipse.jetty.websocket:websocket-api:jar - org.eclipse.jetty.websocket:websocket-client:jar - org.eclipse.jetty.websocket:websocket-common:jar - org.eclipse.jetty.websocket:websocket-server:jar - org.eclipse.jetty.websocket:websocket-servlet:jar - org.fusesource.leveldbjni:leveldbjni-all:jar - org.glassfish.hk2:class-model:jar - org.glassfish.hk2:config-types:jar - org.glassfish.hk2.external:aopalliance-repackaged:jar - org.glassfish.hk2.external:asm-all-repackaged:jar - org.glassfish.hk2.external:bean-validator:jar - org.glassfish.hk2.external:javax.inject:jar - org.glassfish.hk2:hk2:jar - org.glassfish.hk2:hk2-api:jar - org.glassfish.hk2:hk2-config:jar - org.glassfish.hk2:hk2-core:jar - org.glassfish.hk2:hk2-locator:jar - org.glassfish.hk2:hk2-runlevel:jar - org.glassfish.hk2:hk2-utils:jar - org.glassfish.hk2:osgi-resource-locator:jar - org.glassfish.hk2:spring-bridge:jar - org.glassfish.jersey.bundles:jaxrs-ri:jar - org.glassfish.jersey.bundles.repackaged:jersey-guava:jar - org.glassfish.jersey.containers:jersey-container-servlet:jar - org.glassfish.jersey.containers:jersey-container-servlet-core:jar - org.glassfish.jersey.core:jersey-client:jar - org.glassfish.jersey.core:jersey-common:jar - org.glassfish.jersey.core:jersey-server:jar - org.glassfish.jersey.ext:jersey-entity-filtering:jar - org.glassfish.jersey.ext:jersey-spring3:jar - org.glassfish.jersey.media:jersey-media-jaxb:jar - org.glassfish.jersey.media:jersey-media-json-jackson:jar - org.glassfish.jersey.media:jersey-media-multipart:jar - org.hdrhistogram:HdrHistogram:jar - org.javassist:javassist:jar - - - org.jvnet.mimepull:mimepull:jar - org.jvnet:tiger-types:jar - org.latencyutils:LatencyUtils:jar - org.mortbay.jasper:apache-el:jar - org.mortbay.jetty:jetty:jar - org.mortbay.jetty:jetty-util:jar - org.ow2.asm:asm-analysis:jar - org.ow2.asm:asm-commons:jar - org.ow2.asm:asm-tree:jar - org.reflections:reflections:jar - - org.scala-lang:scala-compiler:jar - org.scala-lang:scala-library:jar - org.scala-lang:scala-reflect:jar - org.scala-lang:scalap:jar - org.slf4j:jul-to-slf4j:jar - org.slf4j:slf4j-api:jar - org.springframework.boot:spring-boot:jar - org.springframework.boot:spring-boot-actuator:jar - org.springframework.boot:spring-boot-actuator-autoconfigure:jar - org.springframework.boot:spring-boot-autoconfigure:jar - org.springframework.boot:spring-boot-starter:jar - org.springframework.boot:spring-boot-starter-actuator:jar - org.springframework.boot:spring-boot-starter-aop:jar - org.springframework.boot:spring-boot-starter-jetty:jar - org.springframework.boot:spring-boot-starter-json:jar - org.springframework.boot:spring-boot-starter-log4j2:jar - org.springframework.boot:spring-boot-starter-web:jar - org.springframework.cloud:spring-cloud-commons:jar - org.springframework.cloud:spring-cloud-config-client:jar - org.springframework.cloud:spring-cloud-context:jar - - - org.springframework.cloud:spring-cloud-netflix-eureka-client:jar - - org.springframework.cloud:spring-cloud-starter:jar - org.springframework.cloud:spring-cloud-starter-config:jar - org.springframework.cloud:spring-cloud-starter-eureka:jar - org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar - org.springframework.cloud:spring-cloud-starter-netflix-eureka-client:jar - org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar - org.springframework.security:spring-security-crypto:jar - org.springframework.security:spring-security-rsa:jar - org.springframework:spring-aop:jar - org.springframework:spring-beans:jar - org.springframework:spring-context:jar - org.springframework:spring-core:jar - org.springframework:spring-expression:jar - org.springframework:spring-jcl:jar - org.springframework:spring-web:jar - org.springframework:spring-webmvc:jar - org.tukaani:xz:jar - org.yaml:snakeyaml:jar - - xerces:xercesImpl:jar - xmlenc:xmlenc:jar - xmlpull:xmlpull:jar - xpp3:xpp3_min:jar - diff --git a/streamis-server/src/main/resources/application.yml b/streamis-server/src/main/resources/application.yml new file mode 100644 index 000000000..7d7a5dcb2 --- /dev/null +++ b/streamis-server/src/main/resources/application.yml @@ -0,0 +1,22 @@ +server: + port: 9321 +spring: + application: + name: streamis-server + + +eureka: + client: + serviceUrl: + defaultZone: http://127.0.0.1:20303/eureka/ + instance: + metadata-map: + test: wedatasphere + +management: + endpoints: + web: + exposure: + include: refresh,info +logging: + config: classpath:log4j2.xml diff --git a/streamis-server/src/main/resources/linkis.properties b/streamis-server/src/main/resources/linkis.properties new file mode 100644 index 000000000..1befd5885 --- /dev/null +++ b/streamis-server/src/main/resources/linkis.properties @@ -0,0 +1,48 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +wds.linkis.test.mode=true + + +wds.linkis.server.mybatis.datasource.url=jdbc:mysql://127.0.0.1:3306/db?characterEncoding=UTF-8 + +wds.linkis.server.mybatis.datasource.username=username + +wds.linkis.server.mybatis.datasource.password=* + + +wds.linkis.log.clear=true + +wds.linkis.server.version=v1 + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.server.restful,com.webank.wedatasphere.dss.application.restful,com.webank.wedatasphere.dss.workspace.server.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com\\webank\\wedatasphere\\dss\\server\\dao\\impl\\*.xml,classpath*:com\\webank\\wedatasphere\\dss\\application\\dao\\impl\\*.xml,classpath*:com\\webank\\wedatasphere\\dss\\workspace\\mapper\\impl\\*.xml,classpath*:com\\webank\\wedatasphere\\dss\\workspace\\common\\dao\\impl\\*.xml + +wds.linkis.server.mybatis.typeAliasesPackage=com + +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.server.dao,com.webank.wedatasphere.dss.application.dao,com.webank.wedatasphere.dss.workspace.mapper,com.webank.wedatasphere.dss.workspace.common.dao + + + +wds.linkis.gateway.ip=127.0.0.1 +wds.linkis.gateway.port=9001 + + + diff --git a/streamis-server/src/main/resources/log4j.properties b/streamis-server/src/main/resources/log4j.properties new file mode 100644 index 000000000..6f1673f17 --- /dev/null +++ b/streamis-server/src/main/resources/log4j.properties @@ -0,0 +1,37 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO diff --git a/streamis-server/src/main/resources/log4j2.xml b/streamis-server/src/main/resources/log4j2.xml new file mode 100644 index 000000000..7076c8dd9 --- /dev/null +++ b/streamis-server/src/main/resources/log4j2.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala b/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala index 06fabc9b5..63b91dfc2 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala +++ b/streamis-workflow/streamis-workflow-server/src/main/scala/com/webank/wedatasphere/streamis/worflow/server/receiver/StreamisWorkflowReceiver.scala @@ -10,6 +10,7 @@ import com.webank.wedatasphere.streamis.workflow.server.service.StreamFlowServic import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component +import java.util import scala.concurrent.duration.Duration import scala.collection.JavaConversions._ import scala.collection.JavaConverters._ @@ -51,7 +52,9 @@ class StreamisWorkflowReceiver extends Receiver with Logging { } case streamFlowImportRequest: StreamFlowImportRequest => Utils.tryCatch{ val dssFlows = streamFlowService.importStreamFlow(streamFlowImportRequest) - StreamFlowImportResponse(0, Lists.newArrayList(dssFlows.map(_.getId).toList.asJavaCollection), "") + val idList = new util.ArrayList[Long]() + dssFlows.foreach(flow => idList.add(flow.getId)) + StreamFlowImportResponse(0, idList, "") }{ t => logger.error(s"failed to import resourceId: ${streamFlowImportRequest.bmlResourceId} version is ${streamFlowImportRequest.bmlVersion}", t) StreamFlowImportResponse(-1, null, t.getCause.getMessage) From 7f62db8600c114e217d1f26146f5abdc6d04498e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8F=B8=E6=B5=B7=E6=B4=8B?= Date: Wed, 28 Apr 2021 18:31:33 +0800 Subject: [PATCH 020/434] remove mock api --- web/src/apps/streamis/common/common.js | 13 ++ web/src/apps/streamis/i18n/en.json | 2 +- web/src/apps/streamis/i18n/zh.json | 2 +- .../apps/streamis/module/coreIndex/index.vue | 63 +++++---- .../apps/streamis/module/jobList/index.vue | 122 ++++++++++++++---- web/vue.config.js | 6 +- 6 files changed, 150 insertions(+), 58 deletions(-) create mode 100644 web/src/apps/streamis/common/common.js diff --git a/web/src/apps/streamis/common/common.js b/web/src/apps/streamis/common/common.js new file mode 100644 index 000000000..fcd37540d --- /dev/null +++ b/web/src/apps/streamis/common/common.js @@ -0,0 +1,13 @@ +export const jobStatuses = [ + { name: "failure", code: 6, icon: "md-close-circle", color: "#990033" }, + { name: "running", code: 5, color: "#008000" }, + { name: "slowTask", code: 4, icon: "md-help-circle", color: "#6666FF" }, + { name: "alert", code: 3, icon: "md-warning", color: "#FF99CC" }, + { name: "waitRestart", code: 2, icon: "md-alert", color: "#FF00CC" }, + { + name: "success", + code: 1, + icon: "md-checkmark-circle", + color: "#008000" + } +]; diff --git a/web/src/apps/streamis/i18n/en.json b/web/src/apps/streamis/i18n/en.json index 90180a5fc..949aa3bde 100644 --- a/web/src/apps/streamis/i18n/en.json +++ b/web/src/apps/streamis/i18n/en.json @@ -29,7 +29,7 @@ "formItems": { "queryBtn": "Query", "jobStatus": "Job status", - "submitter": "Submitter", + "jobCreator": "Job Creator", "jobName": "Job name", "stopBtn": "Stop", "startBtn": "Start", diff --git a/web/src/apps/streamis/i18n/zh.json b/web/src/apps/streamis/i18n/zh.json index 8be736a0a..3a75ebf02 100644 --- a/web/src/apps/streamis/i18n/zh.json +++ b/web/src/apps/streamis/i18n/zh.json @@ -28,7 +28,7 @@ "formItems": { "queryBtn": "查询", "jobStatus": "作业状态", - "submitter": "提交人", + "jobCreator": "提交人", "jobName": "作业名称", "stopBtn": "停止", "startBtn": "启动", diff --git a/web/src/apps/streamis/module/coreIndex/index.vue b/web/src/apps/streamis/module/coreIndex/index.vue index 786c107b3..2b1c9d646 100644 --- a/web/src/apps/streamis/module/coreIndex/index.vue +++ b/web/src/apps/streamis/module/coreIndex/index.vue @@ -8,9 +8,21 @@ style="margin-left: 50px; margin-top: 10px;" >
- + +
+ +
+

- {{ item.num }} + {{ item.num || 0 }}

{{ $t(`message.streamis.jobStatus.${item.name}`) }}

@@ -21,24 +33,13 @@ + diff --git a/web/src/apps/streamis/module/jobConfigure/index.vue b/web/src/apps/streamis/module/jobConfig/indexOld.vue similarity index 78% rename from web/src/apps/streamis/module/jobConfigure/index.vue rename to web/src/apps/streamis/module/jobConfig/indexOld.vue index fc4ed2ee1..63662f7d1 100644 --- a/web/src/apps/streamis/module/jobConfigure/index.vue +++ b/web/src/apps/streamis/module/jobConfig/indexOld.vue @@ -3,7 +3,7 @@
-

{{ $t("message.streamis.jobConfigure.resourceConfigure") }}

+

{{ $t("message.streamis.jobConfig.resourceConfig") }}

@@ -11,13 +11,13 @@ @@ -25,7 +25,7 @@ @@ -39,7 +39,7 @@ >
G
@@ -52,7 +52,7 @@ :label-width="labelWidth2" > @@ -66,7 +66,7 @@ >
G
@@ -78,7 +78,7 @@
-

{{ $t("message.streamis.jobConfigure.productionConfigure") }}

+

{{ $t("message.streamis.jobConfig.productionConfig") }}

@@ -86,13 +86,13 @@ @@ -101,13 +101,13 @@ - INSERT INTO linkis_stream_job(`project_id`,`name`,`current_task_id`,`current_version`,`current_release_time`,`org_identification`,`create_by`,`label`,`description`,`job_type`) VALUES(##{projectId},#{name},#{currentTaskId},#{currentVersion},#{currentReleaseTime},#{orgIdentification},#{createBy},#{label},#{description},#{jobType}) + INSERT INTO linkis_stream_job(`project_id`,`name`,`current_task_id`,`current_version`,`current_release_time`,`org_identification`,`create_by`,`label`,`description`,`job_type`) VALUES(#{projectId},#{name},#{currentTaskId},#{currentVersion},#{currentReleaseTime},#{orgIdentification},#{createBy},#{label},#{description},#{jobType}) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamProjectMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamProjectMapper.xml index 5bc274a3a..dfdc54449 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamProjectMapper.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamProjectMapper.xml @@ -50,7 +50,26 @@ name=#{name}, + + create_by=#{createBy}, + WHERE id=#{id} + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PublishRequestVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PublishRequestVO.java new file mode 100644 index 000000000..1509bb002 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PublishRequestVO.java @@ -0,0 +1,139 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +import java.util.List; + +public class PublishRequestVO { + /** + * sql或者scala等 + */ + private String type = "sql"; + + /** + * 提交到jobmanager的执行代码 + */ + private String executionCode; + + /** + * job的名字 + */ + private String streamisJobName; + + /** + * 创建人 + */ + private String createBy; + + /** + * 修改人 + */ + private String updateBy; + + /** + * 任务的描述信息 + */ + private String description; + + + /** + * 标签 + */ + private List tags; + + + /** + * 发布用户 + */ + private String publishUser; + + /** + * 是用来进行判断是新建一个jobmanager 的任务 + * 传v0001 v0002 + */ + private String version; + + + /** + * 工程名字，必须是要传到的streamis jobmanager + */ + private String projectName; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getExecutionCode() { + return executionCode; + } + + public void setExecutionCode(String executionCode) { + this.executionCode = executionCode; + } + + public String getStreamisJobName() { + return streamisJobName; + } + + public void setStreamisJobName(String streamisJobName) { + this.streamisJobName = streamisJobName; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public String getUpdateBy() { + return updateBy; + } + + public void setUpdateBy(String updateBy) { + this.updateBy = updateBy; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public List getTags() { + return tags; + } + + public void setTags(List tags) { + this.tags = tags; + } + + public String getPublishUser() { + return publishUser; + } + + public void setPublishUser(String publishUser) { + this.publishUser = publishUser; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala index 1755652a5..ab5480030 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala @@ -3,10 +3,12 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.service import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf -import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamBmlMapper, StreamJobMapper} -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{JobFlowVO, JobProgressVO, QueryJobListVO, TaskCoreNumVO} +import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamBmlMapper, StreamJobMapper, StreamProjectMapper} +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamProject +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{JobFlowVO, JobProgressVO, PublishRequestVO, QueryJobListVO, TaskCoreNumVO} import com.webank.wedatasphere.streamis.jobmanager.manager.util.DateUtils -import org.slf4j.Logger +import org.apache.commons.lang.StringUtils +import org.springframework.beans.BeanUtils import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -20,6 +22,8 @@ class JobService extends Logging{ @Autowired private var streamJobMapper:StreamJobMapper=_ @Autowired private var streamBmlMapper:StreamBmlMapper=_ + @Autowired private var streamProjectMapper:StreamProjectMapper=_ + def getByProList(projectId:Long,jobName:String,jobStatus:Integer,jobCreator:String):java.util.List[QueryJobListVO] = { val jobLists = streamJobMapper.getJobLists(projectId, jobName, jobStatus, jobCreator).asScala @@ -96,4 +100,52 @@ class JobService extends Logging{ } + /** + * 发布 + * @param publishRequestVO + */ + def publishToJobManager(publishRequestVO:PublishRequestVO): Unit ={ + //处理项目 + val projectName = publishRequestVO.getProjectName + if(StringUtils.isEmpty(projectName)){ + error("projectName is null") + return + } + if(StringUtils.isEmpty(publishRequestVO.getStreamisJobName)){ + error("jobName is null") + return + } + + addPublishProject(publishRequestVO) + //作业 + + + } + + /** + * 存储项目 + * @param publishRequestVO + */ + def addPublishProject(publishRequestVO:PublishRequestVO): Unit ={ + val projectName = publishRequestVO.getProjectName + val projectList = streamProjectMapper.getByProjects(null, null, projectName) + if(projectList!=null && projectList.size() > 0){ + val project = new StreamProject() + BeanUtils.copyProperties(projectList.asScala.head,project) + streamProjectMapper.updateProject(project) + }else{ + val project = new StreamProject() + project.setName(projectName) + project.setCreateBy(publishRequestVO.getCreateBy) + streamProjectMapper.instertProject(project) + } + } + + def addPublishJob(publishRequestVO:PublishRequestVO): Unit ={ + val jobName = publishRequestVO.getStreamisJobName + + + } + + } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/ProjectService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/ProjectService.scala index e927f4f7d..9e245baa7 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/ProjectService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/ProjectService.scala @@ -1,6 +1,8 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.service import com.webank.wedatasphere.linkis.common.utils.Logging +import com.webank.wedatasphere.streamis.jobmanager.manager.dao.StreamProjectMapper +import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service /** @@ -9,4 +11,6 @@ import org.springframework.stereotype.Service @Service class ProjectService extends Logging{ + + } diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobExceptionManager.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobExceptionManager.java index 4f52b9a9e..4e3b1d471 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobExceptionManager.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobExceptionManager.java @@ -9,6 +9,7 @@ public class JobExceptionManager { { put("30300", "上传失败"); put("30301","%s cannot be empty!"); + } }; diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java index 6b0e9602c..57ea3bfbe 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java @@ -36,7 +36,6 @@ public class ConfigurationRestfulApi { @GET @Path("/view") public Response getView(@Context HttpServletRequest req,@QueryParam("jobId") Long jobId) throws IOException, ConfigurationException { - String username = SecurityFilter.getLoginUsername(req); if(jobId==null){ throw new ConfigurationException("params cannot be empty!"); } diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 448789452..99e51b377 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -2,20 +2,20 @@ import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; - import com.webank.wedatasphere.linkis.server.Message; import com.webank.wedatasphere.linkis.server.security.SecurityFilter; import com.webank.wedatasphere.streamis.jobmanager.exception.JobException; import com.webank.wedatasphere.streamis.jobmanager.exception.JobExceptionManager; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.*; import com.webank.wedatasphere.streamis.jobmanager.manager.service.JobService; +import com.webank.wedatasphere.streamis.jobmanager.manager.service.ProjectService; import com.webank.wedatasphere.streamis.jobmanager.manager.service.TaskService; +import org.codehaus.jackson.map.ObjectMapper; import org.glassfish.jersey.media.multipart.FormDataMultiPart; import org.glassfish.jersey.media.multipart.FormDataParam; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; - import javax.servlet.http.HttpServletRequest; import javax.ws.rs.*; import javax.ws.rs.core.Context; @@ -35,7 +35,10 @@ public class JobRestfulApi { JobService jobService; @Autowired TaskService taskService; + @Autowired + ProjectService projectService; + ObjectMapper mapper = new ObjectMapper(); @GET @Path("/list") @@ -81,7 +84,7 @@ public Response uploadJar(@Context HttpServletRequest req, @FormDataParam("projectId") Long projectId, @FormDataParam("path") String path, FormDataMultiPart form) throws IOException, JobException { - String userName = SecurityFilter.getLoginUsername(req); + if (StringUtils.isEmpty(path)) { JobExceptionManager.createException(30300,path); } @@ -172,6 +175,22 @@ public Response releaseJob(@Context HttpServletRequest req, Map return Message.messageToResponse(Message.ok()); } + @POST + @Path("/publishToJobManager") + public Response publishToJobManager(@Context HttpServletRequest req, Map json) throws IOException, JobException{ + if(json == null){ + JobExceptionManager.createException(30301,"version"); + } + PublishRequestVO publishRequestVO = mapper.readValue(mapper.writeValueAsString(json), PublishRequestVO.class); + if(publishRequestVO == null){ + JobExceptionManager.createException(30301,"version"); + } + + + //todo 发布 + return Message.messageToResponse(Message.ok()); + } + @GET @Path("/flow") public Response flowJob(@Context HttpServletRequest req,@QueryParam("jobId") Long jobId,@QueryParam("version") String version) throws IOException, JobException{ diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java index 6de09b319..c2abebe1a 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java @@ -28,7 +28,7 @@ public class ProjectRestfulApi { @GET @Path("/core/target") public Response getView(@Context HttpServletRequest req, @QueryParam("projectId") Long projectId) throws IOException, ProjectException { - String username = SecurityFilter.getLoginUsername(req); + if(projectId==null){ throw new ProjectException("params cannot be empty!"); } diff --git a/streamis-plugins/streamis-jobmanager-appconn/src/main/java/com/webank/wedatasphere/streamis/jobmanager/appconn/operation/PublishOperation.java b/streamis-plugins/streamis-jobmanager-appconn/src/main/java/com/webank/wedatasphere/streamis/jobmanager/appconn/operation/PublishOperation.java index 3f06efa55..90cf20896 100644 --- a/streamis-plugins/streamis-jobmanager-appconn/src/main/java/com/webank/wedatasphere/streamis/jobmanager/appconn/operation/PublishOperation.java +++ b/streamis-plugins/streamis-jobmanager-appconn/src/main/java/com/webank/wedatasphere/streamis/jobmanager/appconn/operation/PublishOperation.java @@ -30,7 +30,7 @@ public class PublishOperation implements Operation ssoRequestOperation; From 294e2e07da50f0109965006d694fb8c82784df60 Mon Sep 17 00:00:00 2001 From: "77954309@qq.com" Date: Wed, 12 May 2021 22:06:17 +0800 Subject: [PATCH 037/434] publish --- .../manager/dao/StreamJobMapper.java | 2 + .../manager/dao/StreamProjectMapper.java | 2 +- .../manager/dao/impl/StreamJobMapper.xml | 2 +- .../manager/dao/impl/StreamProjectMapper.xml | 19 +++ .../manager/entity/vo/PublishRequestVO.java | 139 ++++++++++++++++++ .../manager/service/JobService.scala | 58 +++++++- .../manager/service/ProjectService.scala | 4 + .../exception/JobExceptionManager.java | 1 + .../restful/api/ConfigurationRestfulApi.java | 1 - .../jobmanager/restful/api/JobRestfulApi.java | 25 +++- .../restful/api/ProjectRestfulApi.java | 2 +- .../appconn/operation/PublishOperation.java | 2 +- 12 files changed, 246 insertions(+), 11 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PublishRequestVO.java diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java index 07d885720..4f230a4ce 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java @@ -15,6 +15,8 @@ public interface StreamJobMapper { List getJobLists(@Param("projectId") Long projectId,@Param("name") String name,@Param("status") Integer status,@Param("createBy") String createBy); + List getJobByName(@Param("name") String name); + StreamJob getJobById(@Param("jobId") Long jobId); List getByJobCount(@Param("projectId") Long projectId); diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamProjectMapper.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamProjectMapper.java index 553251cf0..abc30aaf5 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamProjectMapper.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamProjectMapper.java @@ -14,6 +14,6 @@ public interface StreamProjectMapper { void updateProject(StreamProject streamProject); - List getByProjects(@Param("id") Long id,@Param("workspaceId") Long workspaceId); + List getByProjects(@Param("id") Long id,@Param("workspaceId") Long workspaceId,@Param("name") String name); } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml index 96a89e105..5bf2b6db3 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml @@ -104,7 +104,7 @@ - INSERT INTO linkis_stream_job(`project_id`,`name`,`current_task_id`,`current_version`,`current_release_time`,`org_identification`,`create_by`,`label`,`description`,`job_type`) VALUES(##{projectId},#{name},#{currentTaskId},#{currentVersion},#{currentReleaseTime},#{orgIdentification},#{createBy},#{label},#{description},#{jobType}) + INSERT INTO linkis_stream_job(`project_id`,`name`,`current_task_id`,`current_version`,`current_release_time`,`org_identification`,`create_by`,`label`,`description`,`job_type`) VALUES(#{projectId},#{name},#{currentTaskId},#{currentVersion},#{currentReleaseTime},#{orgIdentification},#{createBy},#{label},#{description},#{jobType}) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamProjectMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamProjectMapper.xml index 5bc274a3a..dfdc54449 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamProjectMapper.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamProjectMapper.xml @@ -50,7 +50,26 @@ name=#{name}, + + create_by=#{createBy}, + WHERE id=#{id} + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PublishRequestVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PublishRequestVO.java new file mode 100644 index 000000000..1509bb002 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PublishRequestVO.java @@ -0,0 +1,139 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo; + +import java.util.List; + +public class PublishRequestVO { + /** + * sql或者scala等 + */ + private String type = "sql"; + + /** + * 提交到jobmanager的执行代码 + */ + private String executionCode; + + /** + * job的名字 + */ + private String streamisJobName; + + /** + * 创建人 + */ + private String createBy; + + /** + * 修改人 + */ + private String updateBy; + + /** + * 任务的描述信息 + */ + private String description; + + + /** + * 标签 + */ + private List tags; + + + /** + * 发布用户 + */ + private String publishUser; + + /** + * 是用来进行判断是新建一个jobmanager 的任务 + * 传v0001 v0002 + */ + private String version; + + + /** + * 工程名字，必须是要传到的streamis jobmanager + */ + private String projectName; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getExecutionCode() { + return executionCode; + } + + public void setExecutionCode(String executionCode) { + this.executionCode = executionCode; + } + + public String getStreamisJobName() { + return streamisJobName; + } + + public void setStreamisJobName(String streamisJobName) { + this.streamisJobName = streamisJobName; + } + + public String getCreateBy() { + return createBy; + } + + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public String getUpdateBy() { + return updateBy; + } + + public void setUpdateBy(String updateBy) { + this.updateBy = updateBy; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public List getTags() { + return tags; + } + + public void setTags(List tags) { + this.tags = tags; + } + + public String getPublishUser() { + return publishUser; + } + + public void setPublishUser(String publishUser) { + this.publishUser = publishUser; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala index 1755652a5..ab5480030 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala @@ -3,10 +3,12 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.service import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf -import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamBmlMapper, StreamJobMapper} -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{JobFlowVO, JobProgressVO, QueryJobListVO, TaskCoreNumVO} +import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamBmlMapper, StreamJobMapper, StreamProjectMapper} +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamProject +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{JobFlowVO, JobProgressVO, PublishRequestVO, QueryJobListVO, TaskCoreNumVO} import com.webank.wedatasphere.streamis.jobmanager.manager.util.DateUtils -import org.slf4j.Logger +import org.apache.commons.lang.StringUtils +import org.springframework.beans.BeanUtils import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -20,6 +22,8 @@ class JobService extends Logging{ @Autowired private var streamJobMapper:StreamJobMapper=_ @Autowired private var streamBmlMapper:StreamBmlMapper=_ + @Autowired private var streamProjectMapper:StreamProjectMapper=_ + def getByProList(projectId:Long,jobName:String,jobStatus:Integer,jobCreator:String):java.util.List[QueryJobListVO] = { val jobLists = streamJobMapper.getJobLists(projectId, jobName, jobStatus, jobCreator).asScala @@ -96,4 +100,52 @@ class JobService extends Logging{ } + /** + * 发布 + * @param publishRequestVO + */ + def publishToJobManager(publishRequestVO:PublishRequestVO): Unit ={ + //处理项目 + val projectName = publishRequestVO.getProjectName + if(StringUtils.isEmpty(projectName)){ + error("projectName is null") + return + } + if(StringUtils.isEmpty(publishRequestVO.getStreamisJobName)){ + error("jobName is null") + return + } + + addPublishProject(publishRequestVO) + //作业 + + + } + + /** + * 存储项目 + * @param publishRequestVO + */ + def addPublishProject(publishRequestVO:PublishRequestVO): Unit ={ + val projectName = publishRequestVO.getProjectName + val projectList = streamProjectMapper.getByProjects(null, null, projectName) + if(projectList!=null && projectList.size() > 0){ + val project = new StreamProject() + BeanUtils.copyProperties(projectList.asScala.head,project) + streamProjectMapper.updateProject(project) + }else{ + val project = new StreamProject() + project.setName(projectName) + project.setCreateBy(publishRequestVO.getCreateBy) + streamProjectMapper.instertProject(project) + } + } + + def addPublishJob(publishRequestVO:PublishRequestVO): Unit ={ + val jobName = publishRequestVO.getStreamisJobName + + + } + + } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/ProjectService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/ProjectService.scala index e927f4f7d..9e245baa7 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/ProjectService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/ProjectService.scala @@ -1,6 +1,8 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.service import com.webank.wedatasphere.linkis.common.utils.Logging +import com.webank.wedatasphere.streamis.jobmanager.manager.dao.StreamProjectMapper +import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service /** @@ -9,4 +11,6 @@ import org.springframework.stereotype.Service @Service class ProjectService extends Logging{ + + } diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobExceptionManager.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobExceptionManager.java index 4f52b9a9e..4e3b1d471 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobExceptionManager.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/exception/JobExceptionManager.java @@ -9,6 +9,7 @@ public class JobExceptionManager { { put("30300", "上传失败"); put("30301","%s cannot be empty!"); + } }; diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java index 6b0e9602c..57ea3bfbe 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java @@ -36,7 +36,6 @@ public class ConfigurationRestfulApi { @GET @Path("/view") public Response getView(@Context HttpServletRequest req,@QueryParam("jobId") Long jobId) throws IOException, ConfigurationException { - String username = SecurityFilter.getLoginUsername(req); if(jobId==null){ throw new ConfigurationException("params cannot be empty!"); } diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 448789452..99e51b377 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -2,20 +2,20 @@ import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; - import com.webank.wedatasphere.linkis.server.Message; import com.webank.wedatasphere.linkis.server.security.SecurityFilter; import com.webank.wedatasphere.streamis.jobmanager.exception.JobException; import com.webank.wedatasphere.streamis.jobmanager.exception.JobExceptionManager; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.*; import com.webank.wedatasphere.streamis.jobmanager.manager.service.JobService; +import com.webank.wedatasphere.streamis.jobmanager.manager.service.ProjectService; import com.webank.wedatasphere.streamis.jobmanager.manager.service.TaskService; +import org.codehaus.jackson.map.ObjectMapper; import org.glassfish.jersey.media.multipart.FormDataMultiPart; import org.glassfish.jersey.media.multipart.FormDataParam; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; - import javax.servlet.http.HttpServletRequest; import javax.ws.rs.*; import javax.ws.rs.core.Context; @@ -35,7 +35,10 @@ public class JobRestfulApi { JobService jobService; @Autowired TaskService taskService; + @Autowired + ProjectService projectService; + ObjectMapper mapper = new ObjectMapper(); @GET @Path("/list") @@ -81,7 +84,7 @@ public Response uploadJar(@Context HttpServletRequest req, @FormDataParam("projectId") Long projectId, @FormDataParam("path") String path, FormDataMultiPart form) throws IOException, JobException { - String userName = SecurityFilter.getLoginUsername(req); + if (StringUtils.isEmpty(path)) { JobExceptionManager.createException(30300,path); } @@ -172,6 +175,22 @@ public Response releaseJob(@Context HttpServletRequest req, Map return Message.messageToResponse(Message.ok()); } + @POST + @Path("/publishToJobManager") + public Response publishToJobManager(@Context HttpServletRequest req, Map json) throws IOException, JobException{ + if(json == null){ + JobExceptionManager.createException(30301,"version"); + } + PublishRequestVO publishRequestVO = mapper.readValue(mapper.writeValueAsString(json), PublishRequestVO.class); + if(publishRequestVO == null){ + JobExceptionManager.createException(30301,"version"); + } + + + //todo 发布 + return Message.messageToResponse(Message.ok()); + } + @GET @Path("/flow") public Response flowJob(@Context HttpServletRequest req,@QueryParam("jobId") Long jobId,@QueryParam("version") String version) throws IOException, JobException{ diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java index 6de09b319..c2abebe1a 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ProjectRestfulApi.java @@ -28,7 +28,7 @@ public class ProjectRestfulApi { @GET @Path("/core/target") public Response getView(@Context HttpServletRequest req, @QueryParam("projectId") Long projectId) throws IOException, ProjectException { - String username = SecurityFilter.getLoginUsername(req); + if(projectId==null){ throw new ProjectException("params cannot be empty!"); } diff --git a/streamis-plugins/streamis-jobmanager-appconn/src/main/java/com/webank/wedatasphere/streamis/jobmanager/appconn/operation/PublishOperation.java b/streamis-plugins/streamis-jobmanager-appconn/src/main/java/com/webank/wedatasphere/streamis/jobmanager/appconn/operation/PublishOperation.java index 3f06efa55..90cf20896 100644 --- a/streamis-plugins/streamis-jobmanager-appconn/src/main/java/com/webank/wedatasphere/streamis/jobmanager/appconn/operation/PublishOperation.java +++ b/streamis-plugins/streamis-jobmanager-appconn/src/main/java/com/webank/wedatasphere/streamis/jobmanager/appconn/operation/PublishOperation.java @@ -30,7 +30,7 @@ public class PublishOperation implements Operation ssoRequestOperation; From 41fa8ca95aaa33d30c8971bc00027080ba89a9de Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Thu, 13 May 2021 20:51:53 +0800 Subject: [PATCH 038/434] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=80=E4=BA=9Bwor?= =?UTF-8?q?kflow=20server=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DataSourceServer2managerRestfulApi.java | 2 +- streamis-plugins/streamis-appconn/pom.xml | 13 +++- .../src/main/assembly/distribution.xml | 71 +++++++++++++++++++ .../standard/StreamisJMDevStandard.java | 10 +-- .../server/restful/StreamisFlowRestful.java | 46 ++++++++++-- .../server/service/StreamFlowService.java | 1 + .../service/impl/StreamFlowServiceImpl.java | 16 +++++ 7 files changed, 145 insertions(+), 14 deletions(-) create mode 100644 streamis-plugins/streamis-appconn/src/main/assembly/distribution.xml diff --git a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java index aa54a3026..c070d2ec0 100644 --- a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java +++ b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java @@ -63,7 +63,7 @@ public Response updateStreamisTableMeta(@Context HttpServletRequest req, JsonNo } - @GET + @POST @Path("streamisTableMetaInfo/{streamis_table_meta_Id}") public Response deleteStreamisTableMeta(@Context HttpServletRequest req, @PathParam("streamis_table_meta_Id")Long id) throws ErrorException { return streamisTableMetaApi.deleteStreamisTableMeta(req,id); diff --git a/streamis-plugins/streamis-appconn/pom.xml b/streamis-plugins/streamis-appconn/pom.xml index c99218d3e..d4fcd781d 100644 --- a/streamis-plugins/streamis-appconn/pom.xml +++ b/streamis-plugins/streamis-appconn/pom.xml @@ -17,29 +17,34 @@ com.webank.wedatasphere.dss dss-appconn-core ${dss.version} + provided com.webank.wedatasphere.dss spring-origin-dss-project-plugin ${dss.version} + provided com.webank.wedatasphere.dss dss-structure-integration-standard ${dss.version} + provided com.webank.wedatasphere.dss dss-development-process-standard ${dss.version} + provided com.webank.wedatasphere.dss dss-development-process-standard-execution ${dss.version} + provided @@ -83,6 +88,7 @@ org.json4s + provided @@ -91,6 +97,7 @@ com.webank.wedatasphere.dss dss-orchestrator-core ${dss.version} + provided @@ -139,7 +146,7 @@ org.apache.maven.plugins maven-jar-plugin - + org.apache.maven.plugins maven-gpg-plugin diff --git a/streamis-plugins/streamis-appconn/src/main/assembly/distribution.xml b/streamis-plugins/streamis-appconn/src/main/assembly/distribution.xml new file mode 100644 index 000000000..5fbadae5b --- /dev/null +++ b/streamis-plugins/streamis-appconn/src/main/assembly/distribution.xml @@ -0,0 +1,71 @@ + + + + streamis-appconn + + zip + + true + streamis-appconn + + + + + + lib + true + true + false + false + true + + + + + + ${basedir}/src/main/resources + + * + + 0777 + conf + unix + + + ${basedir}/bin + + * + + 0777 + bin + unix + + + . + + */** + + logs + + + + + diff --git a/streamis-plugins/streamis-jobmanager-appconn/src/main/java/com/webank/wedatasphere/streamis/jobmanager/appconn/standard/StreamisJMDevStandard.java b/streamis-plugins/streamis-jobmanager-appconn/src/main/java/com/webank/wedatasphere/streamis/jobmanager/appconn/standard/StreamisJMDevStandard.java index 49f0d9705..1f7cb47e6 100644 --- a/streamis-plugins/streamis-jobmanager-appconn/src/main/java/com/webank/wedatasphere/streamis/jobmanager/appconn/standard/StreamisJMDevStandard.java +++ b/streamis-plugins/streamis-jobmanager-appconn/src/main/java/com/webank/wedatasphere/streamis/jobmanager/appconn/standard/StreamisJMDevStandard.java @@ -4,7 +4,6 @@ import com.webank.wedatasphere.dss.standard.app.development.AbstractLabelDevelopmentIntegrationStandard; import com.webank.wedatasphere.dss.standard.app.development.RefOperationService; import com.webank.wedatasphere.dss.standard.common.desc.AppDesc; -import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException; import java.io.IOException; import java.util.List; @@ -27,9 +26,11 @@ public class StreamisJMDevStandard extends AbstractLabelDevelopmentIntegrationSt + private StreamisJMDevStandard(AppConn appConn){ this.appConn = appConn; this.appDesc = appConn.getAppDesc(); + init(); } @@ -44,6 +45,10 @@ public static StreamisJMDevStandard getInstance(AppConn appConn){ return instance; } + @Override + public void init() { + + } @@ -63,10 +68,7 @@ public void setAppDesc(AppDesc appDesc) { } - @Override - public void init() throws AppStandardErrorException { - } @Override public void close() throws IOException { diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/restful/StreamisFlowRestful.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/restful/StreamisFlowRestful.java index 7a817fb6d..54bc65473 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/restful/StreamisFlowRestful.java +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/restful/StreamisFlowRestful.java @@ -1,5 +1,10 @@ package com.webank.wedatasphere.streamis.workflow.server.restful; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.linkis.server.Message; +import com.webank.wedatasphere.linkis.server.security.SecurityFilter; +import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowCreateRequest; +import com.webank.wedatasphere.streamis.workflow.server.exception.StreamisFlowErrorException; import com.webank.wedatasphere.streamis.workflow.server.service.StreamFlowService; import org.codehaus.jackson.JsonNode; import org.slf4j.Logger; @@ -8,13 +13,11 @@ import org.springframework.stereotype.Component; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.Consumes; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.util.ArrayList; /** * created by yangzhiyue on 2021/4/19 @@ -40,10 +43,41 @@ public class StreamisFlowRestful { @POST @Path("createStreamFlow") public Response createStreamFlow(@Context HttpServletRequest request, JsonNode jsonNode){ - //todo 先不实现，先通过rpc方式 - return null; + String createBy = SecurityFilter.getLoginUsername(request); + String flowName = jsonNode.get("flowName").getTextValue(); + String description = jsonNode.get("description").getTextValue(); + String workspaceName = jsonNode.get("workspaceName").getTextValue(); + Long parentFlowId = jsonNode.get("parentFlowId").getLongValue(); + String uses = jsonNode.get("uses").getTextValue(); + String contextId = jsonNode.get("contextId").getTextValue(); + Long projectId = jsonNode.get("projectId").getLongValue(); + String projectName = jsonNode.get("projectName").getTextValue(); + StreamFlowCreateRequest streamFlowCreateRequest = new StreamFlowCreateRequest(flowName, + createBy, description, workspaceName, parentFlowId, uses, new ArrayList<>(), contextId, projectId, projectName); + try { + DSSFlow dssFlow = streamingFlowService.createStreamFlow(streamFlowCreateRequest); + return Message.messageToResponse(Message.ok().data("streamFlow", dssFlow.getId())); + } catch (StreamisFlowErrorException e) { + LOGGER.error("failed to create stream flow ", e); + return Message.messageToResponse(Message.error("Failed to create stream flow")); + } } + @GET + @Path("getStreamFlow") + public Response getStreamFlow(@Context HttpServletRequest request, @QueryParam("flowId") Long flowId){ + String username = SecurityFilter.getLoginUsername(request); + try{ + DSSFlow dssFlow = streamingFlowService.getStreamFlow(flowId); + return Message.messageToResponse(Message.ok().data("flow", dssFlow)); + }catch(final Exception e){ + LOGGER.error("failed to get stream flow", e); + return Message.messageToResponse(Message.error("failed to get stream flow")); + } + + } + + @POST @Path("updateStreamFlow") diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java index 4beaf56eb..1d6af25f4 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java @@ -30,4 +30,5 @@ public interface StreamFlowService { void deleteStreamFlow() throws StreamisFlowErrorException; + DSSFlow getStreamFlow(Long flowId) throws StreamisFlowErrorException; } diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java index 93b9edf1c..58a8a03c7 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java @@ -3,11 +3,13 @@ import com.google.common.collect.Lists; import com.webank.wedatasphere.dss.common.entity.DSSLabel; import com.webank.wedatasphere.dss.common.entity.IOEnv; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.workflow.WorkFlowManager; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.entity.DSSFlowImportParam; +import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowCopyRequest; import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowCreateRequest; import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowExportRequest; @@ -38,6 +40,10 @@ public class StreamFlowServiceImpl implements StreamFlowService { @Autowired private WorkFlowManager workFlowManager; + + @Autowired + DSSFlowService dssFlowService; + @Override public DSSFlow createStreamFlow(StreamFlowCreateRequest streamFlowCreateRequest) throws StreamisFlowErrorException { try{ @@ -128,4 +134,14 @@ public DSSFlow copyStreamFlow(StreamFlowCopyRequest streamFlowCopyRequest) throw public void deleteStreamFlow() throws StreamisFlowErrorException { } + + @Override + public DSSFlow getStreamFlow(Long flowId) throws StreamisFlowErrorException { + try { + return dssFlowService.getLatestVersionFlow(flowId); + } catch (DSSErrorException e) { + LOGGER.error("failed to get dssFlow for {}", flowId); + throw new StreamisFlowErrorException(600407, "failed to get flow", e); + } + } } From ceb29deb5a931b9155721d7cbcdaad1518193518 Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Thu, 13 May 2021 20:51:53 +0800 Subject: [PATCH 039/434] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=80=E4=BA=9Bwor?= =?UTF-8?q?kflow=20server=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DataSourceServer2managerRestfulApi.java | 2 +- streamis-plugins/streamis-appconn/pom.xml | 13 +++- .../src/main/assembly/distribution.xml | 71 +++++++++++++++++++ .../standard/StreamisJMDevStandard.java | 10 +-- .../server/restful/StreamisFlowRestful.java | 46 ++++++++++-- .../server/service/StreamFlowService.java | 1 + .../service/impl/StreamFlowServiceImpl.java | 16 +++++ 7 files changed, 145 insertions(+), 14 deletions(-) create mode 100644 streamis-plugins/streamis-appconn/src/main/assembly/distribution.xml diff --git a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java index aa54a3026..c070d2ec0 100644 --- a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java +++ b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java @@ -63,7 +63,7 @@ public Response updateStreamisTableMeta(@Context HttpServletRequest req, JsonNo } - @GET + @POST @Path("streamisTableMetaInfo/{streamis_table_meta_Id}") public Response deleteStreamisTableMeta(@Context HttpServletRequest req, @PathParam("streamis_table_meta_Id")Long id) throws ErrorException { return streamisTableMetaApi.deleteStreamisTableMeta(req,id); diff --git a/streamis-plugins/streamis-appconn/pom.xml b/streamis-plugins/streamis-appconn/pom.xml index c99218d3e..d4fcd781d 100644 --- a/streamis-plugins/streamis-appconn/pom.xml +++ b/streamis-plugins/streamis-appconn/pom.xml @@ -17,29 +17,34 @@ com.webank.wedatasphere.dss dss-appconn-core ${dss.version} + provided com.webank.wedatasphere.dss spring-origin-dss-project-plugin ${dss.version} + provided com.webank.wedatasphere.dss dss-structure-integration-standard ${dss.version} + provided com.webank.wedatasphere.dss dss-development-process-standard ${dss.version} + provided com.webank.wedatasphere.dss dss-development-process-standard-execution ${dss.version} + provided @@ -83,6 +88,7 @@ org.json4s + provided @@ -91,6 +97,7 @@ com.webank.wedatasphere.dss dss-orchestrator-core ${dss.version} + provided @@ -139,7 +146,7 @@ org.apache.maven.plugins maven-jar-plugin - + org.apache.maven.plugins maven-gpg-plugin diff --git a/streamis-plugins/streamis-appconn/src/main/assembly/distribution.xml b/streamis-plugins/streamis-appconn/src/main/assembly/distribution.xml new file mode 100644 index 000000000..5fbadae5b --- /dev/null +++ b/streamis-plugins/streamis-appconn/src/main/assembly/distribution.xml @@ -0,0 +1,71 @@ + + + + streamis-appconn + + zip + + true + streamis-appconn + + + + + + lib + true + true + false + false + true + + + + + + ${basedir}/src/main/resources + + * + + 0777 + conf + unix + + + ${basedir}/bin + + * + + 0777 + bin + unix + + + . + + */** + + logs + + + + + diff --git a/streamis-plugins/streamis-jobmanager-appconn/src/main/java/com/webank/wedatasphere/streamis/jobmanager/appconn/standard/StreamisJMDevStandard.java b/streamis-plugins/streamis-jobmanager-appconn/src/main/java/com/webank/wedatasphere/streamis/jobmanager/appconn/standard/StreamisJMDevStandard.java index 49f0d9705..1f7cb47e6 100644 --- a/streamis-plugins/streamis-jobmanager-appconn/src/main/java/com/webank/wedatasphere/streamis/jobmanager/appconn/standard/StreamisJMDevStandard.java +++ b/streamis-plugins/streamis-jobmanager-appconn/src/main/java/com/webank/wedatasphere/streamis/jobmanager/appconn/standard/StreamisJMDevStandard.java @@ -4,7 +4,6 @@ import com.webank.wedatasphere.dss.standard.app.development.AbstractLabelDevelopmentIntegrationStandard; import com.webank.wedatasphere.dss.standard.app.development.RefOperationService; import com.webank.wedatasphere.dss.standard.common.desc.AppDesc; -import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException; import java.io.IOException; import java.util.List; @@ -27,9 +26,11 @@ public class StreamisJMDevStandard extends AbstractLabelDevelopmentIntegrationSt + private StreamisJMDevStandard(AppConn appConn){ this.appConn = appConn; this.appDesc = appConn.getAppDesc(); + init(); } @@ -44,6 +45,10 @@ public static StreamisJMDevStandard getInstance(AppConn appConn){ return instance; } + @Override + public void init() { + + } @@ -63,10 +68,7 @@ public void setAppDesc(AppDesc appDesc) { } - @Override - public void init() throws AppStandardErrorException { - } @Override public void close() throws IOException { diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/restful/StreamisFlowRestful.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/restful/StreamisFlowRestful.java index 7a817fb6d..54bc65473 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/restful/StreamisFlowRestful.java +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/restful/StreamisFlowRestful.java @@ -1,5 +1,10 @@ package com.webank.wedatasphere.streamis.workflow.server.restful; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.linkis.server.Message; +import com.webank.wedatasphere.linkis.server.security.SecurityFilter; +import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowCreateRequest; +import com.webank.wedatasphere.streamis.workflow.server.exception.StreamisFlowErrorException; import com.webank.wedatasphere.streamis.workflow.server.service.StreamFlowService; import org.codehaus.jackson.JsonNode; import org.slf4j.Logger; @@ -8,13 +13,11 @@ import org.springframework.stereotype.Component; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.Consumes; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.util.ArrayList; /** * created by yangzhiyue on 2021/4/19 @@ -40,10 +43,41 @@ public class StreamisFlowRestful { @POST @Path("createStreamFlow") public Response createStreamFlow(@Context HttpServletRequest request, JsonNode jsonNode){ - //todo 先不实现，先通过rpc方式 - return null; + String createBy = SecurityFilter.getLoginUsername(request); + String flowName = jsonNode.get("flowName").getTextValue(); + String description = jsonNode.get("description").getTextValue(); + String workspaceName = jsonNode.get("workspaceName").getTextValue(); + Long parentFlowId = jsonNode.get("parentFlowId").getLongValue(); + String uses = jsonNode.get("uses").getTextValue(); + String contextId = jsonNode.get("contextId").getTextValue(); + Long projectId = jsonNode.get("projectId").getLongValue(); + String projectName = jsonNode.get("projectName").getTextValue(); + StreamFlowCreateRequest streamFlowCreateRequest = new StreamFlowCreateRequest(flowName, + createBy, description, workspaceName, parentFlowId, uses, new ArrayList<>(), contextId, projectId, projectName); + try { + DSSFlow dssFlow = streamingFlowService.createStreamFlow(streamFlowCreateRequest); + return Message.messageToResponse(Message.ok().data("streamFlow", dssFlow.getId())); + } catch (StreamisFlowErrorException e) { + LOGGER.error("failed to create stream flow ", e); + return Message.messageToResponse(Message.error("Failed to create stream flow")); + } } + @GET + @Path("getStreamFlow") + public Response getStreamFlow(@Context HttpServletRequest request, @QueryParam("flowId") Long flowId){ + String username = SecurityFilter.getLoginUsername(request); + try{ + DSSFlow dssFlow = streamingFlowService.getStreamFlow(flowId); + return Message.messageToResponse(Message.ok().data("flow", dssFlow)); + }catch(final Exception e){ + LOGGER.error("failed to get stream flow", e); + return Message.messageToResponse(Message.error("failed to get stream flow")); + } + + } + + @POST @Path("updateStreamFlow") diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java index 4beaf56eb..1d6af25f4 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/StreamFlowService.java @@ -30,4 +30,5 @@ public interface StreamFlowService { void deleteStreamFlow() throws StreamisFlowErrorException; + DSSFlow getStreamFlow(Long flowId) throws StreamisFlowErrorException; } diff --git a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java index 93b9edf1c..58a8a03c7 100644 --- a/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java +++ b/streamis-workflow/streamis-workflow-server/src/main/java/com/webank/wedatasphere/streamis/workflow/server/service/impl/StreamFlowServiceImpl.java @@ -3,11 +3,13 @@ import com.google.common.collect.Lists; import com.webank.wedatasphere.dss.common.entity.DSSLabel; import com.webank.wedatasphere.dss.common.entity.IOEnv; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.workflow.WorkFlowManager; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.entity.DSSFlowImportParam; +import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowCopyRequest; import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowCreateRequest; import com.webank.wedatasphere.streamis.workflow.common.protocol.StreamFlowExportRequest; @@ -38,6 +40,10 @@ public class StreamFlowServiceImpl implements StreamFlowService { @Autowired private WorkFlowManager workFlowManager; + + @Autowired + DSSFlowService dssFlowService; + @Override public DSSFlow createStreamFlow(StreamFlowCreateRequest streamFlowCreateRequest) throws StreamisFlowErrorException { try{ @@ -128,4 +134,14 @@ public DSSFlow copyStreamFlow(StreamFlowCopyRequest streamFlowCopyRequest) throw public void deleteStreamFlow() throws StreamisFlowErrorException { } + + @Override + public DSSFlow getStreamFlow(Long flowId) throws StreamisFlowErrorException { + try { + return dssFlowService.getLatestVersionFlow(flowId); + } catch (DSSErrorException e) { + LOGGER.error("failed to get dssFlow for {}", flowId); + throw new StreamisFlowErrorException(600407, "failed to get flow", e); + } + } } From 785cc6add563de2451d5298aa58128a480315fa3 Mon Sep 17 00:00:00 2001 From: Dlimeng <77954309@qq.com> Date: Fri, 14 May 2021 11:20:38 +0800 Subject: [PATCH 040/434] publish --- .../manager/dao/StreamJobMapper.java | 10 +- .../manager/dao/impl/StreamJobMapper.xml | 1 + .../manager/entity/StreamJobRunRelation.java | 61 +++++++++++ .../manager/entity/vo/PublishRequestVO.java | 10 ++ .../jobmanager/manager/conf/JobConf.scala | 1 + .../manager/service/JobService.scala | 100 +++++++++++++++++- .../jobmanager/restful/api/JobRestfulApi.java | 3 +- streamis-server/pom.xml | 2 +- 8 files changed, 178 insertions(+), 10 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobRunRelation.java diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java index 4f230a4ce..c4a33aaa7 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java @@ -1,6 +1,7 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.dao; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob; +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJobRunRelation; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJobSqlResource; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJobVersion; import org.apache.ibatis.annotations.Param; @@ -15,7 +16,7 @@ public interface StreamJobMapper { List getJobLists(@Param("projectId") Long projectId,@Param("name") String name,@Param("status") Integer status,@Param("createBy") String createBy); - List getJobByName(@Param("name") String name); + //List getJobByName(@Param("name") String name); StreamJob getJobById(@Param("jobId") Long jobId); @@ -25,18 +26,23 @@ public interface StreamJobMapper { List getJobVersionsById(@Param("jobId") Long jobId,@Param("version") String version); + List getJobRunRelationList(@Param("jobId") Long jobId); + void insertJob(StreamJob streamJob); void insertJobVersion(StreamJobVersion streamJobVersion); void insertJobSqlResource(StreamJobSqlResource streamJobSqlResource); + void insertJobRunRelation(StreamJobRunRelation streamJobRunRelation); + void updateJob(StreamJob streamJob); void updateJobVersion(StreamJobVersion streamJobVersion); void updateJobSqlResource(StreamJobSqlResource streamJobSqlResource); + void updateJobRunRelation(StreamJobRunRelation streamJobRunRelation); - + void deleteJobRunRelation(@Param("id") Long id); } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml index 5bf2b6db3..c7699d4c6 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml @@ -103,6 +103,7 @@ + INSERT INTO linkis_stream_job(`project_id`,`name`,`current_task_id`,`current_version`,`current_release_time`,`org_identification`,`create_by`,`label`,`description`,`job_type`) VALUES(#{projectId},#{name},#{currentTaskId},#{currentVersion},#{currentReleaseTime},#{orgIdentification},#{createBy},#{label},#{description},#{jobType}) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobRunRelation.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobRunRelation.java new file mode 100644 index 000000000..e4f450015 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobRunRelation.java @@ -0,0 +1,61 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity; + +import java.util.List; + +public class StreamJobRunRelation { + private Long id; + private Long jobId; + private Long jobVersionId; + private Long parentId; + private Long level; + + private List childs; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getJobId() { + return jobId; + } + + public void setJobId(Long jobId) { + this.jobId = jobId; + } + + public Long getJobVersionId() { + return jobVersionId; + } + + public void setJobVersionId(Long jobVersionId) { + this.jobVersionId = jobVersionId; + } + + public Long getParentId() { + return parentId; + } + + public void setParentId(Long parentId) { + this.parentId = parentId; + } + + public Long getLevel() { + return level; + } + + public void setLevel(Long level) { + this.level = level; + } + + public List getChilds() { + return childs; + } + + public void setChilds(List childs) { + this.childs = childs; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PublishRequestVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PublishRequestVO.java index 1509bb002..b6b097927 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PublishRequestVO.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PublishRequestVO.java @@ -3,6 +3,8 @@ import java.util.List; public class PublishRequestVO { + + private Long projectId; /** * sql或者scala等 */ @@ -136,4 +138,12 @@ public String getProjectName() { public void setProjectName(String projectName) { this.projectName = projectName; } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala index c7d243579..e4fff2839 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala @@ -20,4 +20,5 @@ object JobConf { val JOBMANAGER_FLINK_JOB_STATUS_SEVEN = CommonVars("wds.linkis.flink.job.status.seven", 7,"Stop task") + val JOBMANAGER_FLINK_SQL = CommonVars("wds.linkis.flink.job.sql", 1,"flink type sql") } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala index ab5480030..208a602f7 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala @@ -4,7 +4,7 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.service import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamBmlMapper, StreamJobMapper, StreamProjectMapper} -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamProject +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{StreamJob, StreamJobRunRelation, StreamJobSqlResource, StreamJobVersion, StreamProject} import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{JobFlowVO, JobProgressVO, PublishRequestVO, QueryJobListVO, TaskCoreNumVO} import com.webank.wedatasphere.streamis.jobmanager.manager.util.DateUtils import org.apache.commons.lang.StringUtils @@ -116,9 +116,9 @@ class JobService extends Logging{ return } - addPublishProject(publishRequestVO) - //作业 - + val projectId = addPublishProject(publishRequestVO) + publishRequestVO.setProjectId(projectId) + addPublishJob(publishRequestVO) } @@ -126,26 +126,116 @@ class JobService extends Logging{ * 存储项目 * @param publishRequestVO */ - def addPublishProject(publishRequestVO:PublishRequestVO): Unit ={ + def addPublishProject(publishRequestVO:PublishRequestVO): Long ={ val projectName = publishRequestVO.getProjectName val projectList = streamProjectMapper.getByProjects(null, null, projectName) if(projectList!=null && projectList.size() > 0){ val project = new StreamProject() BeanUtils.copyProperties(projectList.asScala.head,project) streamProjectMapper.updateProject(project) + project.getId }else{ val project = new StreamProject() project.setName(projectName) project.setCreateBy(publishRequestVO.getCreateBy) streamProjectMapper.instertProject(project) + project.getId } } def addPublishJob(publishRequestVO:PublishRequestVO): Unit ={ val jobName = publishRequestVO.getStreamisJobName + val jobLists = streamJobMapper.getJobLists(null, jobName, null, null) + val job = new StreamJob() + val jobVersion = new StreamJobVersion() + val jobSql = new StreamJobSqlResource() + + if(jobLists != null && jobLists.size() > 0) { + val jobHead = jobLists.asScala.head + BeanUtils.copyProperties(jobHead,job) + val tags = publishRequestVO.getTags + if(tags != null && tags.size() > 0){ + job.setLabel(tags.asScala.mkString(",")) + } + + job.setCreateBy(publishRequestVO.getCreateBy) + publishRequestVO.getType match { + case "sql"=> job.setJobType(JobConf.JOBMANAGER_FLINK_SQL.getValue) + } + val preVersion = job.getCurrentVersion + + job.setCurrentVersion(publishRequestVO.getVersion) + streamJobMapper.updateJob(job) + + jobVersion.setJobId(job.getId) + jobVersion.setVersion(publishRequestVO.getVersion) + streamJobMapper.insertJobVersion(jobVersion) + + publishRequestVO.getType match { + case "sql"=> { + jobSql.setJobVersionId(jobVersion.getId) + jobSql.setExecuteSql(publishRequestVO.getExecutionCode) + streamJobMapper.updateJobSqlResource(jobSql) + } + } + + val preVersionList = streamJobMapper.getJobVersionsById(job.getId,preVersion) + val jobRunRelation = new StreamJobRunRelation() + jobRunRelation.setJobId(job.getId) + jobRunRelation.setParentId(preVersionList.asScala.head.getId) + jobRunRelation.setJobVersionId(jobVersion.getId) + + addJobRunRelation(job.getStatus.toInt,jobRunRelation) + }else{ + val tags = publishRequestVO.getTags + if(tags != null && tags.size() > 0){ + job.setLabel(tags.asScala.mkString(",")) + } + + publishRequestVO.getType match { + case "sql"=> job.setJobType(JobConf.JOBMANAGER_FLINK_SQL.getValue) + } + job.setProjectId(publishRequestVO.getProjectId) + job.setName(publishRequestVO.getStreamisJobName) + job.setCurrentVersion(publishRequestVO.getVersion) + job.setCreateBy(publishRequestVO.getCreateBy) + streamJobMapper.insertJob(job) + + jobVersion.setJobId(job.getId) + jobVersion.setVersion(publishRequestVO.getVersion) + streamJobMapper.insertJobVersion(jobVersion) + + publishRequestVO.getType match { + case "sql"=> { + jobSql.setJobVersionId(jobVersion.getId) + jobSql.setExecuteSql(publishRequestVO.getExecutionCode) + streamJobMapper.insertJobSqlResource(jobSql) + } + } + + } } + def addJobRunRelation(status:Int,jobRunRelation:StreamJobRunRelation): Unit ={ + val relations = streamJobMapper.getJobRunRelationList(jobRunRelation.getJobId) + if(relations == null || relations.isEmpty){ + jobRunRelation.setParentId(null) + } + status match { + case JobConf.JOBMANAGER_FLINK_JOB_STATUS_TWO.getValue => { + streamJobMapper.insertJobRunRelation(jobRunRelation) + } + case JobConf.JOBMANAGER_FLINK_JOB_STATUS_THREE.getValue => { + streamJobMapper.insertJobRunRelation(jobRunRelation) + } + case JobConf.JOBMANAGER_FLINK_JOB_STATUS_FOUR.getValue => { + streamJobMapper.insertJobRunRelation(jobRunRelation) + } + } + } + + } diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 99e51b377..3d0b4f93c 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -185,8 +185,7 @@ public Response publishToJobManager(@Context HttpServletRequest req, Maporg.apache.maven.plugins maven-assembly-plugin 2.3 - falseFLo + false make-assembly From 6be4571021d06a967491806f7b21591cf288fe20 Mon Sep 17 00:00:00 2001 From: "77954309@qq.com" Date: Fri, 14 May 2021 11:20:38 +0800 Subject: [PATCH 041/434] publish --- .../manager/dao/StreamJobMapper.java | 10 +- .../manager/dao/impl/StreamJobMapper.xml | 1 + .../manager/entity/StreamJobRunRelation.java | 61 +++++++++++ .../manager/entity/vo/PublishRequestVO.java | 10 ++ .../jobmanager/manager/conf/JobConf.scala | 1 + .../manager/service/JobService.scala | 100 +++++++++++++++++- .../jobmanager/restful/api/JobRestfulApi.java | 3 +- streamis-server/pom.xml | 2 +- 8 files changed, 178 insertions(+), 10 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobRunRelation.java diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java index 4f230a4ce..c4a33aaa7 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java @@ -1,6 +1,7 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.dao; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob; +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJobRunRelation; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJobSqlResource; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJobVersion; import org.apache.ibatis.annotations.Param; @@ -15,7 +16,7 @@ public interface StreamJobMapper { List getJobLists(@Param("projectId") Long projectId,@Param("name") String name,@Param("status") Integer status,@Param("createBy") String createBy); - List getJobByName(@Param("name") String name); + //List getJobByName(@Param("name") String name); StreamJob getJobById(@Param("jobId") Long jobId); @@ -25,18 +26,23 @@ public interface StreamJobMapper { List getJobVersionsById(@Param("jobId") Long jobId,@Param("version") String version); + List getJobRunRelationList(@Param("jobId") Long jobId); + void insertJob(StreamJob streamJob); void insertJobVersion(StreamJobVersion streamJobVersion); void insertJobSqlResource(StreamJobSqlResource streamJobSqlResource); + void insertJobRunRelation(StreamJobRunRelation streamJobRunRelation); + void updateJob(StreamJob streamJob); void updateJobVersion(StreamJobVersion streamJobVersion); void updateJobSqlResource(StreamJobSqlResource streamJobSqlResource); + void updateJobRunRelation(StreamJobRunRelation streamJobRunRelation); - + void deleteJobRunRelation(@Param("id") Long id); } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml index 5bf2b6db3..c7699d4c6 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml @@ -103,6 +103,7 @@ + INSERT INTO linkis_stream_job(`project_id`,`name`,`current_task_id`,`current_version`,`current_release_time`,`org_identification`,`create_by`,`label`,`description`,`job_type`) VALUES(#{projectId},#{name},#{currentTaskId},#{currentVersion},#{currentReleaseTime},#{orgIdentification},#{createBy},#{label},#{description},#{jobType}) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobRunRelation.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobRunRelation.java new file mode 100644 index 000000000..e4f450015 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobRunRelation.java @@ -0,0 +1,61 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.entity; + +import java.util.List; + +public class StreamJobRunRelation { + private Long id; + private Long jobId; + private Long jobVersionId; + private Long parentId; + private Long level; + + private List childs; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getJobId() { + return jobId; + } + + public void setJobId(Long jobId) { + this.jobId = jobId; + } + + public Long getJobVersionId() { + return jobVersionId; + } + + public void setJobVersionId(Long jobVersionId) { + this.jobVersionId = jobVersionId; + } + + public Long getParentId() { + return parentId; + } + + public void setParentId(Long parentId) { + this.parentId = parentId; + } + + public Long getLevel() { + return level; + } + + public void setLevel(Long level) { + this.level = level; + } + + public List getChilds() { + return childs; + } + + public void setChilds(List childs) { + this.childs = childs; + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PublishRequestVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PublishRequestVO.java index 1509bb002..b6b097927 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PublishRequestVO.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/PublishRequestVO.java @@ -3,6 +3,8 @@ import java.util.List; public class PublishRequestVO { + + private Long projectId; /** * sql或者scala等 */ @@ -136,4 +138,12 @@ public String getProjectName() { public void setProjectName(String projectName) { this.projectName = projectName; } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala index c7d243579..e4fff2839 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala @@ -20,4 +20,5 @@ object JobConf { val JOBMANAGER_FLINK_JOB_STATUS_SEVEN = CommonVars("wds.linkis.flink.job.status.seven", 7,"Stop task") + val JOBMANAGER_FLINK_SQL = CommonVars("wds.linkis.flink.job.sql", 1,"flink type sql") } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala index ab5480030..208a602f7 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala @@ -4,7 +4,7 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.service import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamBmlMapper, StreamJobMapper, StreamProjectMapper} -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamProject +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{StreamJob, StreamJobRunRelation, StreamJobSqlResource, StreamJobVersion, StreamProject} import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{JobFlowVO, JobProgressVO, PublishRequestVO, QueryJobListVO, TaskCoreNumVO} import com.webank.wedatasphere.streamis.jobmanager.manager.util.DateUtils import org.apache.commons.lang.StringUtils @@ -116,9 +116,9 @@ class JobService extends Logging{ return } - addPublishProject(publishRequestVO) - //作业 - + val projectId = addPublishProject(publishRequestVO) + publishRequestVO.setProjectId(projectId) + addPublishJob(publishRequestVO) } @@ -126,26 +126,116 @@ class JobService extends Logging{ * 存储项目 * @param publishRequestVO */ - def addPublishProject(publishRequestVO:PublishRequestVO): Unit ={ + def addPublishProject(publishRequestVO:PublishRequestVO): Long ={ val projectName = publishRequestVO.getProjectName val projectList = streamProjectMapper.getByProjects(null, null, projectName) if(projectList!=null && projectList.size() > 0){ val project = new StreamProject() BeanUtils.copyProperties(projectList.asScala.head,project) streamProjectMapper.updateProject(project) + project.getId }else{ val project = new StreamProject() project.setName(projectName) project.setCreateBy(publishRequestVO.getCreateBy) streamProjectMapper.instertProject(project) + project.getId } } def addPublishJob(publishRequestVO:PublishRequestVO): Unit ={ val jobName = publishRequestVO.getStreamisJobName + val jobLists = streamJobMapper.getJobLists(null, jobName, null, null) + val job = new StreamJob() + val jobVersion = new StreamJobVersion() + val jobSql = new StreamJobSqlResource() + + if(jobLists != null && jobLists.size() > 0) { + val jobHead = jobLists.asScala.head + BeanUtils.copyProperties(jobHead,job) + val tags = publishRequestVO.getTags + if(tags != null && tags.size() > 0){ + job.setLabel(tags.asScala.mkString(",")) + } + + job.setCreateBy(publishRequestVO.getCreateBy) + publishRequestVO.getType match { + case "sql"=> job.setJobType(JobConf.JOBMANAGER_FLINK_SQL.getValue) + } + val preVersion = job.getCurrentVersion + + job.setCurrentVersion(publishRequestVO.getVersion) + streamJobMapper.updateJob(job) + + jobVersion.setJobId(job.getId) + jobVersion.setVersion(publishRequestVO.getVersion) + streamJobMapper.insertJobVersion(jobVersion) + + publishRequestVO.getType match { + case "sql"=> { + jobSql.setJobVersionId(jobVersion.getId) + jobSql.setExecuteSql(publishRequestVO.getExecutionCode) + streamJobMapper.updateJobSqlResource(jobSql) + } + } + + val preVersionList = streamJobMapper.getJobVersionsById(job.getId,preVersion) + val jobRunRelation = new StreamJobRunRelation() + jobRunRelation.setJobId(job.getId) + jobRunRelation.setParentId(preVersionList.asScala.head.getId) + jobRunRelation.setJobVersionId(jobVersion.getId) + + addJobRunRelation(job.getStatus.toInt,jobRunRelation) + }else{ + val tags = publishRequestVO.getTags + if(tags != null && tags.size() > 0){ + job.setLabel(tags.asScala.mkString(",")) + } + + publishRequestVO.getType match { + case "sql"=> job.setJobType(JobConf.JOBMANAGER_FLINK_SQL.getValue) + } + job.setProjectId(publishRequestVO.getProjectId) + job.setName(publishRequestVO.getStreamisJobName) + job.setCurrentVersion(publishRequestVO.getVersion) + job.setCreateBy(publishRequestVO.getCreateBy) + streamJobMapper.insertJob(job) + + jobVersion.setJobId(job.getId) + jobVersion.setVersion(publishRequestVO.getVersion) + streamJobMapper.insertJobVersion(jobVersion) + + publishRequestVO.getType match { + case "sql"=> { + jobSql.setJobVersionId(jobVersion.getId) + jobSql.setExecuteSql(publishRequestVO.getExecutionCode) + streamJobMapper.insertJobSqlResource(jobSql) + } + } + + } } + def addJobRunRelation(status:Int,jobRunRelation:StreamJobRunRelation): Unit ={ + val relations = streamJobMapper.getJobRunRelationList(jobRunRelation.getJobId) + if(relations == null || relations.isEmpty){ + jobRunRelation.setParentId(null) + } + status match { + case JobConf.JOBMANAGER_FLINK_JOB_STATUS_TWO.getValue => { + streamJobMapper.insertJobRunRelation(jobRunRelation) + } + case JobConf.JOBMANAGER_FLINK_JOB_STATUS_THREE.getValue => { + streamJobMapper.insertJobRunRelation(jobRunRelation) + } + case JobConf.JOBMANAGER_FLINK_JOB_STATUS_FOUR.getValue => { + streamJobMapper.insertJobRunRelation(jobRunRelation) + } + } + } + + } diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 99e51b377..3d0b4f93c 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -185,8 +185,7 @@ public Response publishToJobManager(@Context HttpServletRequest req, Maporg.apache.maven.plugins maven-assembly-plugin 2.3 - falseFLo + false make-assembly From 313339d6abed7cf2f2a68df78ad578ae134cbdd7 Mon Sep 17 00:00:00 2001 From: Dlimeng <77954309@qq.com> Date: Fri, 14 May 2021 16:55:37 +0800 Subject: [PATCH 042/434] publish --- .../jobmanager/restful/api/ConfigurationRestfulApi.java | 1 - .../streamis/jobmanager/restful/api/JobRestfulApi.java | 8 +------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java index 57ea3bfbe..02e6f46cb 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java @@ -1,7 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.restful.api; import com.webank.wedatasphere.linkis.server.Message; -import com.webank.wedatasphere.linkis.server.security.SecurityFilter; import com.webank.wedatasphere.streamis.jobmanager.exception.ConfigurationException; import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.vo.ConfigKeyVO; import com.webank.wedatasphere.streamis.jobmanager.launcher.service.ConfigurationService; diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 3d0b4f93c..172f2213d 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -169,12 +169,6 @@ public Response executeHistoryJob(@Context HttpServletRequest req,@QueryParam("j return Message.messageToResponse(Message.ok().data("details",streamTaskListVOS)); } - @POST - @Path("/release") - public Response releaseJob(@Context HttpServletRequest req, Map json) throws IOException, JobException{ - - return Message.messageToResponse(Message.ok()); - } @POST @Path("/publishToJobManager") public Response publishToJobManager(@Context HttpServletRequest req, Map json) throws IOException, JobException{ @@ -183,7 +177,7 @@ public Response publishToJobManager(@Context HttpServletRequest req, Map Date: Fri, 14 May 2021 16:55:37 +0800 Subject: [PATCH 043/434] publish --- .../jobmanager/restful/api/ConfigurationRestfulApi.java | 1 - .../streamis/jobmanager/restful/api/JobRestfulApi.java | 8 +------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java index 57ea3bfbe..02e6f46cb 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/ConfigurationRestfulApi.java @@ -1,7 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.restful.api; import com.webank.wedatasphere.linkis.server.Message; -import com.webank.wedatasphere.linkis.server.security.SecurityFilter; import com.webank.wedatasphere.streamis.jobmanager.exception.ConfigurationException; import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.vo.ConfigKeyVO; import com.webank.wedatasphere.streamis.jobmanager.launcher.service.ConfigurationService; diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 3d0b4f93c..172f2213d 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -169,12 +169,6 @@ public Response executeHistoryJob(@Context HttpServletRequest req,@QueryParam("j return Message.messageToResponse(Message.ok().data("details",streamTaskListVOS)); } - @POST - @Path("/release") - public Response releaseJob(@Context HttpServletRequest req, Map json) throws IOException, JobException{ - - return Message.messageToResponse(Message.ok()); - } @POST @Path("/publishToJobManager") public Response publishToJobManager(@Context HttpServletRequest req, Map json) throws IOException, JobException{ @@ -183,7 +177,7 @@ public Response publishToJobManager(@Context HttpServletRequest req, Map Date: Mon, 24 May 2021 11:04:35 +0800 Subject: [PATCH 044/434] datasource management --- web/src/apps/streamis/assets/images/u2616.png | Bin 0 -> 869 bytes web/src/apps/streamis/assets/images/u2618.png | Bin 0 -> 598 bytes web/src/apps/streamis/assets/images/u2662.png | Bin 0 -> 359 bytes web/src/apps/streamis/assets/images/u2665.png | Bin 0 -> 548 bytes web/src/apps/streamis/assets/images/u2680.png | Bin 0 -> 3361 bytes web/src/apps/streamis/assets/images/u458.png | Bin 0 -> 274 bytes .../streamis/assets/streamisIconFont/demo.css | 539 ++++++++++++++++++ .../assets/streamisIconFont/demo_index.html | 349 ++++++++++++ .../assets/streamisIconFont/iconfont.css | 43 ++ .../assets/streamisIconFont/iconfont.js | 1 + .../assets/streamisIconFont/iconfont.json | 58 ++ .../assets/streamisIconFont/iconfont.ttf | Bin 0 -> 4212 bytes .../assets/streamisIconFont/iconfont.woff | Bin 0 -> 2708 bytes .../assets/streamisIconFont/iconfont.woff2 | Bin 0 -> 2156 bytes .../streamis/module/dataSourceInit/index.vue | 41 ++ .../module/datasourceToolbar/index.vue | 84 +++ .../module/sourceTablePanel/index.vue | 126 ++++ .../streamis/module/tableFieldsList/index.js | 23 + .../module/tableFieldsList/index.scss | 19 + .../streamis/module/tableFieldsList/index.vue | 354 ++++++++++++ .../apps/streamis/module/tableInfo/index.vue | 91 +++ .../apps/streamis/module/treeSource/index.vue | 233 ++++++++ web/src/apps/streamis/router.js | 11 + .../apps/streamis/view/dataSource/index.vue | 71 +++ .../view/realTimeDataSource/index.vue | 11 + web/src/main.js | 1 + 26 files changed, 2055 insertions(+) create mode 100644 web/src/apps/streamis/assets/images/u2616.png create mode 100644 web/src/apps/streamis/assets/images/u2618.png create mode 100644 web/src/apps/streamis/assets/images/u2662.png create mode 100644 web/src/apps/streamis/assets/images/u2665.png create mode 100644 web/src/apps/streamis/assets/images/u2680.png create mode 100644 web/src/apps/streamis/assets/images/u458.png create mode 100644 web/src/apps/streamis/assets/streamisIconFont/demo.css create mode 100644 web/src/apps/streamis/assets/streamisIconFont/demo_index.html create mode 100644 web/src/apps/streamis/assets/streamisIconFont/iconfont.css create mode 100644 web/src/apps/streamis/assets/streamisIconFont/iconfont.js create mode 100644 web/src/apps/streamis/assets/streamisIconFont/iconfont.json create mode 100644 web/src/apps/streamis/assets/streamisIconFont/iconfont.ttf create mode 100644 web/src/apps/streamis/assets/streamisIconFont/iconfont.woff create mode 100644 web/src/apps/streamis/assets/streamisIconFont/iconfont.woff2 create mode 100644 web/src/apps/streamis/module/dataSourceInit/index.vue create mode 100644 web/src/apps/streamis/module/datasourceToolbar/index.vue create mode 100644 web/src/apps/streamis/module/sourceTablePanel/index.vue create mode 100644 web/src/apps/streamis/module/tableFieldsList/index.js create mode 100644 web/src/apps/streamis/module/tableFieldsList/index.scss create mode 100644 web/src/apps/streamis/module/tableFieldsList/index.vue create mode 100644 web/src/apps/streamis/module/tableInfo/index.vue create mode 100644 web/src/apps/streamis/module/treeSource/index.vue create mode 100644 web/src/apps/streamis/view/dataSource/index.vue create mode 100644 web/src/apps/streamis/view/realTimeDataSource/index.vue diff --git a/web/src/apps/streamis/assets/images/u2616.png b/web/src/apps/streamis/assets/images/u2616.png new file mode 100644 index 0000000000000000000000000000000000000000..a2e6d40f02f88eef3b51b06318f131fc015d28d9 GIT binary patch literal 869 zcmV-r1DgDaP)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D0{TfrK~#8N?VCSn z6LA#BfA6@Y9UQtgI5;{KEU3_egANXXx>*oJY9R?}5wzf-Q^=6Xp+lwEG(n0P1PfgR z9UL4AItW&9=+METLxv6xo!aJ}?_J)V)%3!pU~~ETK11(*-}X}Ir|rF3{KntHJ7kmS$0YihG?f8`g&!Qiq1@{ zozJri1@8^Tx235JEWG;^CaYBlC*NUZ2he}QEOqDKrKt=c5$D2x>G5* zNtc}JtplaMkmx1=WJ&a~nAksX2NC90SK)&MvFJ&50B><2 v;Y)P+b%~TSfM?otsAMwmBJP`;zcJ<)FdzQ(^P000>X1^@s6#OZ}&00001b5ch_0Itp) z=>Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D0qaRbK~zXft(GBA z!$1_r*Jfg30x>a(n3&iHKumx`Op;&_2;>9;ffx$Ohd`jn3=V-nAV-3P1QJLPTY?w~ z5;FL`yVqt{uI$?JOI}{@y?gJUYwy}dl?BXZvkq<>YpFG`ckBs!!Or72zQq@Z<~{8C zI{@a5upKR33^g?hPHlNHsJq0iXe&QVXfpB6RFs7JXIxubx{#0+EG3XYpI4kbI~NkA zZc!feyTVnqy?w|w^}o=!uM$QiXcy%{{Uh!WKeiEz#1nyz#!5l`J+3Le@NJLKridKt zxG?hq35?fGQenKBc34855UEfB z*{SyZ#l^0PddLb93K9^QH_$Q*0r}u~QK18D4+|BLU0c)k76L0I%9vM{kqK_3gceastU11{ZXpoIK)^>FEb7jNAL k&7A)(Vi(vK*B&aBA66)k8B)zzHUIzs07*qoM6N<$f&-)WTmS$7 literal 0 HcmV?d00001 diff --git a/web/src/apps/streamis/assets/images/u2662.png b/web/src/apps/streamis/assets/images/u2662.png new file mode 100644 index 0000000000000000000000000000000000000000..9eee4dd33f35b07f274e3c0823f4bb339456976f GIT binary patch literal 359 zcmV-t0hs=YP)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D0Q^ZrK~yMHb&;=b z0znYQzu6!WCl60jk~X0!^bPs|6$mV@mZY`18tcH|4INheFW8W#cm0eb z^^gsiG^fZ3>fw0fH-uTd^af$Ugi$=HMV~C#tm2gdq_|iWJv%#lR=slL4+6b(^OV4m zF7jRNqb4>py}fCK|5AY$UvjPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D0l7&;K~y+Tl~W-| zgHaHE_qMTOSTJm0_=CYPQG;OAXc!n=5QHKA+h7>123ar~*2egUNuxzD2nq@=4qQ+$ z7%YwjqXS{%Kfil>|Mt({;s=j+ckk}q_wMeI`UThJWvzK1VwPDsMCuK4M=84UJZgzg zwaW3250xDUS1Ozy4EU6YcC9G_X1>6MM_Q{IF55Q22%H%X`4)yX;|0L+6RmrA@ap~?>k+zbZ_97la& zOOLb(G%}S(B7eu@w1Yr#oIPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D47N!`K~#8N?VW9C zRM{EF&14b-6&k2eKls5?%cdyQWi54Kp^?nQd|1Jy78QhUSeLDAiAo;=y2 z9Q50ijfbQK75KXc{9l^lBd+x?TQ^Ugj#E87J*nNhcQ2~G(FmPjh}@V?r?>OK1Kr)- zPbnvD_XuqiL^$_Mk!Dv`R{lah7mY?|Ab#HdNhk=EU`Zup4k9^2JkT!ERvu)dJ>8O^ zjf*0ibdU1K2tgu9X|J2$iHV6FH0l7wK^nM|hDI}I+8Q&YB+qj*9*^J6RMm!>2)3kt zi{e4jR#Uk~0m;(u*U;tHYazR0f_eS>=#&m~vsXFXk*V!C%t7=;I?eY`BkE+!qAbA> z+mBxT0mWB%{WoQ5I~&s52-;=R&&qX@MG1!3C=z=FQcsgwx}$0qAX-xVHrB{_h<=o* zU5$bSTOIp8#ZFZP)nX2^kD{ZK_Si?n=01|ibGI@AdGyN1d*gv!v!@-qoWa?dz&>j(YSevE=E_Ao>6;{(Q1W_&!Z&Q4L%=|a2c?6zXAT{L z1_Td?U~f{}Az(;N1JR&jk9vb}$tSko3*)QK&~-2#ulplJH1Y=YIOM>OgZsqET=MuJ8cTxiPU+65;IWYSm&F*wptSWNAjJSLx&(C!Eup2Awt|nc>O-G#}2{c12Z3tnwTbY zE}>vChlTQ03N{3VeuCa+m4D#xHlLjg1q%HTer0tr))25)>g;l{PRfS3A z0$%zwUHNy)qu9ZDo2u0cF09yqoitRF*V`bzsR1Xspe@+}C>msaN*^;5q+{1^?svtX zaOXxB`1MZmtA#e26Y<`}H#h$>`3_5bC>9FW_VxyzMSuDMC}DcYc}7L`XoI>_+%KBz z(U6`w!ARsr=CHA#af;Kw!-=`_+zu0xPNfevvCoR=VI%x^}XoL}ti?KP^d z-dDR;D9(>Io}=i;*=$H+$uu_D zm#N7|If4!J8gh}#=*{tFy?(6%)Yv9;?2GX69S+NSperIc2Y1kSzr^5{RCmP88>DIi zOp{*N_8@-}Rl3KtdS;Be)o7tUeH;@G42WB7J6?5$PkT z#Qc-}Exq{+Nzpt?p>E6Chi>1HHFae;EF029qtJtBXI&R_K=KvqBhrvXZozcCjd&kqj|UsBn3+f#8KGs#JE z{djk1Xy~{t=lkTQrl#cbx#HQ84wV7H)JXPdEEZd}gI8YQiCjl0=coCHrcAoGDKr80 zY%aM%Nw2bMZgzESr1S}ZhJ<1`^k=xIxQBW7_?x1OeSu*IhCs7BIGGWeH`O; zIEZ9-)V4CV^OvNKDH7FGR)CrUPo}4*H>|9z?Bk{iI?)95nj~AK@@&$tzv}7fxlzm1 zYfS`a!;<|x4N=gEd{luhpeG3!r|)MQYLBPDoT@0^(r2AX$$StGj4>0rUl`j_oz);ZipQm*i- zMoGIIwI+%OA{Br#i%{uv?4fvDCMPF%@vQo$+?G`n?%Y_>J=(jcAq!$4UiLRi1p&7e@rUq+bc@2 z2T3%q-c>F%K0G`u=Viwukw^^ihHKZ8!?_Zhi?RggKv9Z6N2;~0Oh+w{QAdrCDMacO z(k@e30jec98`(RZ{2s#B+%zZFfVPX&K&s}QPwFAq2RBzQtpM&n(1?a^whQ4FV61;9}rmGr~8i#6us?|y?ZOq{hrV9Gxu#L6w<9+_2rS0QZ0qRvt zc8|wmvCFo~M_y!)pcS6zkgrgD$5|M~0XOnGUlLb^$Q8eA!3sDRr zmh2XsyjAY24HdiX=>)YW#H(EVNT;d}se(u(a+x%So~>xbYt?ZP5G}{v2scyEiMHHs z>ejengXffrOz@o7>A)0xQk4U>YJ#(oMfL#>6)3nSBwLb7mgISE)LNH7@>TKr zXrfx*Gm;0OEeMBDO`+=JP#>dL6G46rCs2ZOp}~s{I5FQc;sG?VtzeU8g;z;$YQN+Z zME;e0h7Mc-D4~TXgi3HRK)k=K)snuRrs*dQjEnx?oJHr&dIz4a#+$GeH|GX zHuiJ>Nn{1`ISV`@iy0XB4ude`@%$AjK*8mnE{-7;ac?hey1`iH869**BMY*}7BhN@o06Lk$)78&q Iol`;+0Bp, +.markdown>blockquote, +.markdown>.highlight, +.markdown>ol, +.markdown>ul { + width: 80%; +} + +.markdown ul>li { + list-style: circle; +} + +.markdown>ul li, +.markdown blockquote ul>li { + margin-left: 20px; + padding-left: 4px; +} + +.markdown>ul li p, +.markdown>ol li p { + margin: 0.6em 0; +} + +.markdown ol>li { + list-style: decimal; +} + +.markdown>ol li, +.markdown blockquote ol>li { + margin-left: 20px; + padding-left: 4px; +} + +.markdown code { + margin: 0 3px; + padding: 0 5px; + background: #eee; + border-radius: 3px; +} + +.markdown strong, +.markdown b { + font-weight: 600; +} + +.markdown>table { + border-collapse: collapse; + border-spacing: 0px; + empty-cells: show; + border: 1px solid #e9e9e9; + width: 95%; + margin-bottom: 24px; +} + +.markdown>table th { + white-space: nowrap; + color: #333; + font-weight: 600; +} + +.markdown>table th, +.markdown>table td { + border: 1px solid #e9e9e9; + padding: 8px 16px; + text-align: left; +} + +.markdown>table th { + background: #F7F7F7; +} + +.markdown blockquote { + font-size: 90%; + color: #999; + border-left: 4px solid #e9e9e9; + padding-left: 0.8em; + margin: 1em 0; +} + +.markdown blockquote p { + margin: 0; +} + +.markdown .anchor { + opacity: 0; + transition: opacity 0.3s ease; + margin-left: 8px; +} + +.markdown .waiting { + color: #ccc; +} + +.markdown h1:hover .anchor, +.markdown h2:hover .anchor, +.markdown h3:hover .anchor, +.markdown h4:hover .anchor, +.markdown h5:hover .anchor, +.markdown h6:hover .anchor { + opacity: 1; + display: inline-block; +} + +.markdown>br, +.markdown>p>br { + clear: both; +} + + +.hljs { + display: block; + background: white; + padding: 0.5em; + color: #333333; + overflow-x: auto; +} + +.hljs-comment, +.hljs-meta { + color: #969896; +} + +.hljs-string, +.hljs-variable, +.hljs-template-variable, +.hljs-strong, +.hljs-emphasis, +.hljs-quote { + color: #df5000; +} + +.hljs-keyword, +.hljs-selector-tag, +.hljs-type { + color: #a71d5d; +} + +.hljs-literal, +.hljs-symbol, +.hljs-bullet, +.hljs-attribute { + color: #0086b3; +} + +.hljs-section, +.hljs-name { + color: #63a35c; +} + +.hljs-tag { + color: #333333; +} + +.hljs-title, +.hljs-attr, +.hljs-selector-id, +.hljs-selector-class, +.hljs-selector-attr, +.hljs-selector-pseudo { + color: #795da3; +} + +.hljs-addition { + color: #55a532; + background-color: #eaffea; +} + +.hljs-deletion { + color: #bd2c00; + background-color: #ffecec; +} + +.hljs-link { + text-decoration: underline; +} + +/* 代码高亮 */ +/* PrismJS 1.15.0 +https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript */ +/** + * prism.js default theme for JavaScript, CSS and HTML + * Based on dabblet (http://dabblet.com) + * @author Lea Verou + */ +code[class*="language-"], +pre[class*="language-"] { + color: black; + background: none; + text-shadow: 0 1px white; + font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace; + text-align: left; + white-space: pre; + word-spacing: normal; + word-break: normal; + word-wrap: normal; + line-height: 1.5; + + -moz-tab-size: 4; + -o-tab-size: 4; + tab-size: 4; + + -webkit-hyphens: none; + -moz-hyphens: none; + -ms-hyphens: none; + hyphens: none; +} + +pre[class*="language-"]::-moz-selection, +pre[class*="language-"] ::-moz-selection, +code[class*="language-"]::-moz-selection, +code[class*="language-"] ::-moz-selection { + text-shadow: none; + background: #b3d4fc; +} + +pre[class*="language-"]::selection, +pre[class*="language-"] ::selection, +code[class*="language-"]::selection, +code[class*="language-"] ::selection { + text-shadow: none; + background: #b3d4fc; +} + +@media print { + + code[class*="language-"], + pre[class*="language-"] { + text-shadow: none; + } +} + +/* Code blocks */ +pre[class*="language-"] { + padding: 1em; + margin: .5em 0; + overflow: auto; +} + +:not(pre)>code[class*="language-"], +pre[class*="language-"] { + background: #f5f2f0; +} + +/* Inline code */ +:not(pre)>code[class*="language-"] { + padding: .1em; + border-radius: .3em; + white-space: normal; +} + +.token.comment, +.token.prolog, +.token.doctype, +.token.cdata { + color: slategray; +} + +.token.punctuation { + color: #999; +} + +.namespace { + opacity: .7; +} + +.token.property, +.token.tag, +.token.boolean, +.token.number, +.token.constant, +.token.symbol, +.token.deleted { + color: #905; +} + +.token.selector, +.token.attr-name, +.token.string, +.token.char, +.token.builtin, +.token.inserted { + color: #690; +} + +.token.operator, +.token.entity, +.token.url, +.language-css .token.string, +.style .token.string { + color: #9a6e3a; + background: hsla(0, 0%, 100%, .5); +} + +.token.atrule, +.token.attr-value, +.token.keyword { + color: #07a; +} + +.token.function, +.token.class-name { + color: #DD4A68; +} + +.token.regex, +.token.important, +.token.variable { + color: #e90; +} + +.token.important, +.token.bold { + font-weight: bold; +} + +.token.italic { + font-style: italic; +} + +.token.entity { + cursor: help; +} diff --git a/web/src/apps/streamis/assets/streamisIconFont/demo_index.html b/web/src/apps/streamis/assets/streamisIconFont/demo_index.html new file mode 100644 index 000000000..5e079b8ff --- /dev/null +++ b/web/src/apps/streamis/assets/streamisIconFont/demo_index.html @@ -0,0 +1,349 @@ + + + + + iconfont Demo + + + + + + + + + + + + + +
+

+ + +

+
+
    +
  • Unicode
    • +
  • Font class
    • +
  • Symbol
    • +
+ + 查看项目 + +
+
+
+
    + +
  • + +
    数据
    +
    &#xe6ab;
    +
    • + +
  • + +
    数据
    +
    &#xe62d;
    +
    • + +
  • + +
    集群
    +
    &#xe610;
    +
    • + +
  • + +
    mysql
    +
    &#xe613;
    +
    • + +
  • + +
    apachekafka
    +
    &#xeb3f;
    +
    • + +
  • + +
    表格
    +
    &#xe617;
    +
    • + +
  • + +
    数据
    +
    &#xe832;
    +
    • + +
+
+

Unicode 引用

+
+ +

Unicode 是字体在网页端最原始的应用方式，特点是：

+
    +
  • 支持按字体的方式去动态调整图标大小，颜色等等。
    • +
  • 默认情况下不支持多色，直接添加多色图标会自动去色。
    • +
+
+

注意：新版 iconfont 支持两种方式引用多色图标：SVG symbol 引用方式和彩色字体图标模式。（使用彩色字体图标需要在「编辑项目」中开启「彩色」选项后并重新生成。）

+
+

Unicode 使用步骤如下：

+

第一步：拷贝项目下面生成的 @font-face

+
@font-face {
+  font-family: 'iconfont';
+  src: url('iconfont.woff2?t=1621579006110') format('woff2'),
+       url('iconfont.woff?t=1621579006110') format('woff'),
+       url('iconfont.ttf?t=1621579006110') format('truetype');
+}
+
+

第二步：定义使用 iconfont 的样式

+
.iconfont {
+  font-family: "iconfont" !important;
+  font-size: 16px;
+  font-style: normal;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+

第三步：挑选相应图标并获取字体编码，应用于页面

+
+<span class="iconfont">&#x33;</span>
+
+
+

"iconfont" 是你项目下的 font-family。可以通过编辑项目查看，默认是 "iconfont"。

+
+
+
+
+
    + +
  • + +
    + 数据 +
    +
    .icon-shuju +
    +
    • + +
  • + +
    + 数据 +
    +
    .icon-fl-shuju +
    +
    • + +
  • + +
    + 集群 +
    +
    .icon-jiqun +
    +
    • + +
  • + +
    + mysql +
    +
    .icon-mysql +
    +
    • + +
  • + +
    + apachekafka +
    +
    .icon-apachekafka +
    +
    • + +
  • + +
    + 表格 +
    +
    .icon-table +
    +
    • + +
  • + +
    + 数据 +
    +
    .icon-shuju1 +
    +
    • + +
+
+

font-class 引用

+
+ +

font-class 是 Unicode 使用方式的一种变种，主要是解决 Unicode 书写不直观，语意不明确的问题。

+

与 Unicode 使用方式相比，具有如下特点：

+
    +
  • 相比于 Unicode 语意明确，书写更直观。可以很容易分辨这个 icon 是什么。
    • +
  • 因为使用 class 来定义图标，所以当要替换图标时，只需要修改 class 里面的 Unicode 引用。
    • +
+

使用步骤如下：

+

第一步：引入项目下面生成的 fontclass 代码：

+
<link rel="stylesheet" href="./iconfont.css">
+
+

第二步：挑选相应图标并获取类名，应用于页面：

+
<span class="iconfont icon-xxx"></span>
+
+
+

" + iconfont" 是你项目下的 font-family。可以通过编辑项目查看，默认是 "iconfont"。

+
+
+
+
+
    + +
  • + +
    数据
    +
    #icon-shuju
    +
    • + +
  • + +
    数据
    +
    #icon-fl-shuju
    +
    • + +
  • + +
    集群
    +
    #icon-jiqun
    +
    • + +
  • + +
    mysql
    +
    #icon-mysql
    +
    • + +
  • + +
    apachekafka
    +
    #icon-apachekafka
    +
    • + +
  • + +
    表格
    +
    #icon-table
    +
    • + +
  • + +
    数据
    +
    #icon-shuju1
    +
    • + +
+
+

Symbol 引用

+
+ +

这是一种全新的使用方式，应该说这才是未来的主流，也是平台目前推荐的用法。相关介绍可以参考这篇文章 + 这种用法其实是做了一个 SVG 的集合，与另外两种相比具有如下特点：

+
    +
  • 支持多色图标了，不再受单色限制。
    • +
  • 通过一些技巧，支持像字体那样，通过 font-size, color 来调整样式。
    • +
  • 兼容性较差，支持 IE9+，及现代浏览器。
    • +
  • 浏览器渲染 SVG 的性能一般，还不如 png。
    • +
+

使用步骤如下：

+

第一步：引入项目下面生成的 symbol 代码：

+
<script src="./iconfont.js"></script>
+
+

第二步：加入通用 CSS 代码（引入一次就行）：

+
<style>
+.icon {
+  width: 1em;
+  height: 1em;
+  vertical-align: -0.15em;
+  fill: currentColor;
+  overflow: hidden;
+}
+</style>
+
+

第三步：挑选相应图标并获取类名，应用于页面：

+
<svg class="icon" aria-hidden="true">
+  <use xlink:href="#icon-xxx"></use>
+</svg>
+
+
+
+ +
+
+ + + diff --git a/web/src/apps/streamis/assets/streamisIconFont/iconfont.css b/web/src/apps/streamis/assets/streamisIconFont/iconfont.css new file mode 100644 index 000000000..a24c7c15d --- /dev/null +++ b/web/src/apps/streamis/assets/streamisIconFont/iconfont.css @@ -0,0 +1,43 @@ +@font-face { + font-family: "iconfont"; /* Project id 2562871 */ + src: url('iconfont.woff2?t=1621579006110') format('woff2'), + url('iconfont.woff?t=1621579006110') format('woff'), + url('iconfont.ttf?t=1621579006110') format('truetype'); +} + +.iconfont { + font-family: "iconfont" !important; + font-size: 16px; + font-style: normal; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +.icon-shuju:before { + content: "\e6ab"; +} + +.icon-fl-shuju:before { + content: "\e62d"; +} + +.icon-jiqun:before { + content: "\e610"; +} + +.icon-mysql:before { + content: "\e613"; +} + +.icon-apachekafka:before { + content: "\eb3f"; +} + +.icon-table:before { + content: "\e617"; +} + +.icon-shuju1:before { + content: "\e832"; +} + diff --git a/web/src/apps/streamis/assets/streamisIconFont/iconfont.js b/web/src/apps/streamis/assets/streamisIconFont/iconfont.js new file mode 100644 index 000000000..fe9dcc11d --- /dev/null +++ b/web/src/apps/streamis/assets/streamisIconFont/iconfont.js @@ -0,0 +1 @@ +!function(a){var t,e,c,o,h,n,l='',d=(d=document.getElementsByTagName("script"))[d.length-1].getAttribute("data-injectcss");if(d&&!a.__iconfont__svg__cssinject__){a.__iconfont__svg__cssinject__=!0;try{document.write("")}catch(a){console&&console.log(a)}}function s(){h||(h=!0,c())}t=function(){var a,t,e;(e=document.createElement("div")).innerHTML=l,l=null,(t=e.getElementsByTagName("svg")[0])&&(t.setAttribute("aria-hidden","true"),t.style.position="absolute",t.style.width=0,t.style.height=0,t.style.overflow="hidden",a=t,(e=document.body).firstChild?(t=e.firstChild).parentNode.insertBefore(a,t):e.appendChild(a))},document.addEventListener?~["complete","loaded","interactive"].indexOf(document.readyState)?setTimeout(t,0):(e=function(){document.removeEventListener("DOMContentLoaded",e,!1),t()},document.addEventListener("DOMContentLoaded",e,!1)):document.attachEvent&&(c=t,o=a.document,h=!1,(n=function(){try{o.documentElement.doScroll("left")}catch(a){return void setTimeout(n,50)}s()})(),o.onreadystatechange=function(){"complete"==o.readyState&&(o.onreadystatechange=null,s())})}(window); \ No newline at end of file diff --git a/web/src/apps/streamis/assets/streamisIconFont/iconfont.json b/web/src/apps/streamis/assets/streamisIconFont/iconfont.json new file mode 100644 index 000000000..2a70b25e8 --- /dev/null +++ b/web/src/apps/streamis/assets/streamisIconFont/iconfont.json @@ -0,0 +1,58 @@ +{ + "id": "2562871", + "name": "streamis", + "font_family": "iconfont", + "css_prefix_text": "icon-", + "description": "", + "glyphs": [ + { + "icon_id": "1789126", + "name": "数据", + "font_class": "shuju", + "unicode": "e6ab", + "unicode_decimal": 59051 + }, + { + "icon_id": "2525864", + "name": "数据", + "font_class": "fl-shuju", + "unicode": "e62d", + "unicode_decimal": 58925 + }, + { + "icon_id": "7159338", + "name": "集群", + "font_class": "jiqun", + "unicode": "e610", + "unicode_decimal": 58896 + }, + { + "icon_id": "11493372", + "name": "mysql", + "font_class": "mysql", + "unicode": "e613", + "unicode_decimal": 58899 + }, + { + "icon_id": "15378137", + "name": "apachekafka", + "font_class": "apachekafka", + "unicode": "eb3f", + "unicode_decimal": 60223 + }, + { + "icon_id": "18828425", + "name": "表格", + "font_class": "table", + "unicode": "e617", + "unicode_decimal": 58903 + }, + { + "icon_id": "20016505", + "name": "数据", + "font_class": "shuju1", + "unicode": "e832", + "unicode_decimal": 59442 + } + ] +} diff --git a/web/src/apps/streamis/assets/streamisIconFont/iconfont.ttf b/web/src/apps/streamis/assets/streamisIconFont/iconfont.ttf new file mode 100644 index 0000000000000000000000000000000000000000..4a33f0047f581ebc3c75ca514f5b8f42ec959ea7 GIT binary patch literal 4212 zcmd^Cdu&@*8UN0G*}it|T;F?ro#tgb_H~-pcH-F1(l$=t=}Rq5N?WR}mFqaKI*%sK zXvfOZH3Gq=q`;=FlRzL^0^1)RJDHaAN9M=6=n8zTBeg z6Wmtz49a$3D2s(gvcq3NH6+T4cmZJcMxlBUnjj0zmTkNDgS(h34Z}(>!<+v1Vp1IF z%V9)QL(qg!9H;X0N~hCQ*zGo{W7vN${K9(>5=E)2+9?Bq5XWtygaYX z$L3#pKY6Qz&g}gRTHh_i@TA`6MSJD&`5ElRMzbJvEdJk%%NT$jYZz>FRLyFPxr`Vq zou_%ttaJs<$IMDs@cc_=rE~Xw(yTbXaI3?t#(V?3OgN^WUCVyJZQ_235uLcapJ88M zi=aRR(y$ElI%zlhQ+i_5j4d=;d@X_LM@+HE=CRiimbesAIAl@G5Z z7iXx}fJo=>6@FalR^9lkT-LJ1ctc~$!gR8+Awdx{*3#GzU!}+$+cwAwHk}=^vcT1M z$SMlqv$0Gjw*C$}TaQN9P^_P#im4EaPl>c2_uEdR`J?FZC~mj8!+;GkuYrW)+lu4q zCaXzEkJ{8)vW^aEnuL%LY$^^ZB|1ntPUQQn*t+0zN4!U#3zE9?N4-bS*Dd~;VOL;F z?((nRg@y@vHIhg~n8SKglYW@k@y3*i#qAbKB=p0$4)C3~^aT4Mz7bjQLp3zxTG$DD zpbPqNB}~E;oQJQ&x8M@|2z~>CoU|4?*XN9efWP{r#GQFOfHI6r-EnuxOK zM2+foim0oeA@$eEb@WGLB6@v}KcJKJ`t-~tr(;gULj7q>$4&e5Gcz;jPW{lGGBxKG zADJW9jjo6&`KSx=zd1d9i5iK-=pGRozvFm*`rHh5LD}mLpV49hUps`~%*D6)DXu~! z;Pi_8Brl6gPOtOzu8aHf0!pfvKf~kCt9pe=;k#X3-HQV1mqZzDMFdW2K%|S^7Z%^N zqVZmF;htdPYxf7E-m_i%_dOsOM=CJb#2F(q>=^r!MBy|AP@%oieW5$gpB6A5eu~E* zP3$y3+udy>hL3@nou(<4TooP;j|MD(a}*>w8W!uOEoqnD~^ zWvKaD@%`0`n@{s)=}M;daonY7ZMfq4rCd5?Xs8+5wq}etny-%3hunk;wYX-fLrXGC zQpliXYt{^>vwx9ywr`9-`SdG_syz~@al05+)tdatfZJiy)JnE={y+ zZp|Opctv#+&f{@=1e+M)#A>I#ns6H`ly$C3zfj@W^@Z%zD&Dm@tK>zGoe`Rnf+V?k zPFU$5wtE~4U9!meIL0P2jA-}Ey@8g-;Nw5PRu&uQ9%Hmx~wueo+auPsAJ!PI4Y^kOJYz>fq^$6(N6xxMm;E?bnG{^i>Kxqilcp z%4c0daJ5^MytcNCAaE{85~`{l9*KdV+g@vD_$rU&a@Vam(hlVS?#MP&Kb~P;J4w3VC`(5VQZdeIO8v1=ttyzHpXEi+@}E0kJQB zV%DuhBD>21ZP`t!@B)dnq{7TnIJNlf?Cjpz*^lw|@hmNSsc4*PmR@9k#vTA4#9#}2 z2KQ|UcM{Gf-K(;r-=W7Wr;WNvq z&Ql|k%P49!M(yu7Qjkz9n`){ut0Icmt412zs;Zk;YQOOLo2skY8Y60%j8__@ucmF4 zuE?^YuWGBRXu44&<(EX*PwvM`6Z z(ZW2Q?vfT3kZ-rJh%2D{zsZRaNAa6aI0yEj!&wW{Q~UQU%zzue`lbzw{8bBch~Kg> zk8}5)g$3jnEG)tSvcken&`EytSRq#`^aPHb3Jl~&hI>bbOXaq!&^J-cjg`f6d!R5j zJ}@#IXpT3Psm{W1VXPcx{L#K-sni?j9UB=6Y{PhkVsRueIyN#`$d}^%rPAoyM1uB1 z`FMV02sXnQ6!4H+!g~({@UVLd?*ovB5g3ME<6gQ??n3K6JQ5etbL>9ta_<9ZJ%%r~ z0V7%f4^(md>$u#i6Rm~~%jMaO!=u<+5_2qJMggp27-jZW>Q2PzzL+fL5!0} z>p1iqUZb!UH+tgE^OWu5#&ZrC-%4X$(f_y)Dt5vUmT-h80A}VPK}=^I&-7Be1GA1uJ?G3FXfIE3xZ+OY()MSzOdO2 literal 0 HcmV?d00001 diff --git a/web/src/apps/streamis/assets/streamisIconFont/iconfont.woff b/web/src/apps/streamis/assets/streamisIconFont/iconfont.woff new file mode 100644 index 0000000000000000000000000000000000000000..807b60f86df4a7239b285c318f2e52c4b6c08ee8 GIT binary patch literal 2708 zcmY+Gc{J2tAIHCAvScY{N>fszUuwiJdzQi= zSxUAOktU3(@XHc1DMUo2&}5!(J zk+#VZb^`P%w68usXa|y#m?8%3KP-+0eyB$KRC2Ge_9#ITQ?|?Mp zI%5-pI3%c6S5~rUdwbJ)>iU~b`R?c|B@i8+R3;RF8_F@jJ(k$c(ur=ecGv~a5<2n_ z4IchhV#3j(w!Cb)J4XDm-Dq@@$2=^Lruyzt`%ViRIo#7p9i+jwYZV+=WRKa`uI>f? z;O2v&bCN3IljSi-Gw^+<&;`QV9cKcYM=^^X*OTKWNPhgP@KZNd+Og3XE6rJ?)&5zn z*8NA@LY#y}F__9XGDvL%cp;2}Z11zdd;d||dd}m2&%)8Zyy+yloq2kv;N z0SHDV4gy1413MX$BHlPg*@IK~I_}5Trvl5ctuP7y_^bFY245;{B@xz#00+SzLiyNj z71O2#5Tv*fN;!?XmRa>?<;b0LmmQna$}&3wS@=wYP}BkYd7KO&)){orqzV}CjT%6mK zr>hJ+(xT|^ei5BvJT>~`GV0|tgy&X`k7D3#iq7<{g78ktSY-0U! zueL%o?Q91BUAZ^s5Vt;Ef}e>0j9)!p&)tshmg2L5)*-wuENwrk3+9`2YTAM(9=kIx zv7@_VIt(Z|j=JlnuCJ@RrpTeJ{-C6&YH0QA`Pdd`e43Gj`27X1_cwEjP8dj{B0C%7 zDbqH=LuQWYKh(yuL{BLi`JZnECzH}%`%f%IC5JZj1}IrXmfy|vI{u+b6YgV{ll9N{ z@ufHJL2_OnGdiR2uS8EjPi=M$c_|H7J_eRgxNZWj2pf@kFH4zh+nMSGa?$ZyN!@ z|6-14dM?=ybyzhu*N-ysr={22#2TDfcD7}Rh65dkqnMO)C;Q!4jbi7jO48EbUTka( zQ&E^f;7*|I>?lB~Z&BNDOKb1`O8U?J#+&@_y{EtReVjCTs4l}+)jS+#K7AaOy!(K@ zp?GA?_gVdc0U4Qn-y{*F5F%GGDZ8bmn7poxZt6VUvG~p9=hB20uw!nuU${qT%h+2a zURUTrOIB&RAZQ@xel(?Dbbi2-sKv{nj`j!F=kiwcy>&|yaRst2j5TBzAthuzzt?B* zREmNLt?5v5rPtZpn4A!^Gk#WAL@xPMOf&YFsGNHG`%&^?15%u*YsLSk7VKJ$K26hU-+3EpQm1_ z9Ck?)tH4f0EY(-$sHM5v$z9tl{Uhx_T8@+RC01fIY7ikv!uh7!ZxAeZts)M7ju4y) z(;U5s=_*^nMO5f+ta+5(+(w>nD0yFF7z@8=ryMwgMzjVg8DY-9??p|xoq1Ay81e*W zp_1~xx?Eqepfz;lXKhvZQdMsROJDk|^mj_AE8^z+OLvQO&C?n0mudb%zM=k;MjpYg z=IIfC|IiMO!p1RcMhk*>b0=!66kVQRVu#hi`wK5`56(TrtZ(NeJz~IVH8V~ zzo!Du^vvQKb2v>6ZJCw=!@{c3zk2?xc-Jjp_qK2pE+u2y^7%nl8vZu|TxBAYQ{V02 zK2&+m6k{IY`b4y5o@_s%dhTYfz!IR7$uq1Mv9ShEY@sq9*I7q(9?Rp3lV=JI>aI@K zysr*E8od%v0fls^P+8^v)q@rJvIP_hz4N7+YlMWo zXD+*N`{x5t;a+xL6Q>doNhIeEH~y|CziTT%>RDUfGqVx$$e7k>7~fggSQBozDH4g+ zP_?|PHu{oKIC>Z0&@B7wGx6Vg{Hs=WcE0_rCUx3ZDSn?$JQOJHmdN`(P9zWrkCXkj zM3OIookn70a1|JJkZodtP%G&2w&UCS{0Auucr z%^W=ewQCL@V5^ra=uD zaB2hj>5ihfo*GsJap%Kz`PylTTU}lY_^zWHXPc8Hg;FCNk+g;3YAq{2kSXX~;0%SC zz2zC&ETwHh}-(C3w?ln0-x^3Qa59e9lNjYZq%xEPq@4*hg zI=cSi!`JM2+d!Bsp~Q$qE;vC5I|% zz8bBAT}&(2jDz?^_i(x%wLdS(H0#WOBdOu-PV0%goTOnWD!ocJ=6*{lJkE`GT?iSy z5qj#^JCWi6eTk3piR@H%E}O;9W?y9&vJ=@i5*{2XGp)|jIOwp4M;Pcsp$5nUcTxI0 i0S1M3;iCsIF8vt)gN`nJjsehWBYy(;-NVAx0pQ=8BiM!j literal 0 HcmV?d00001 diff --git a/web/src/apps/streamis/assets/streamisIconFont/iconfont.woff2 b/web/src/apps/streamis/assets/streamisIconFont/iconfont.woff2 new file mode 100644 index 0000000000000000000000000000000000000000..2effb7dcaa011b2d837e7ab4804debd0c23f546c GIT binary patch literal 2156 zcmV-y2$T1BPew8T0RR9100?XV3jhEB01$Kl003X9sB;`3(_OtlJ>_abU-g547PX6P`wk zXj?9HdhZY1^j}EYLQ+ElHGzt+-Mk`3m?-U}oeRkoCQy|bH74S;Gesum|2st1YCV_i z-d`w;#|SVMgODsv$2-PEm|Z!`^VfGJZFgRpax_k%6hh*#7{&PewJIjbbzN)tcJj{D zaE(wQvPV_gM+n#>9L0FekAYr{Z6-Y9=cN-3Oq z?f4-Mr7g6v=tH1Srvw1-Jpcs|aywk4c2KMav;ZL)`51D>;0pkb>L;S$wc~64Ytie_ z*W2Gleu%SJI>SLm4jO`g@c>Y0n6N-#g#i$2kYrnmEhX8!P|TpOo5Dvuz`$#f0;~Zr z)B_fHy}ceF;B6#w3eJZ(pr4uyzZn@m02mf|gaTl0AgR%GDFOis-+J39FgXTMTkC(T&^}`Uj>ZLxn|Uu^KKwkOzw44u|{e%(uQJ#nlhOf zv&>vpV>cp4?s~M@*SIKa?Nz=kKs~e@og_fT%;JDfpD}bFwg3y@5ktu4F8A5;7q_n9GfNt%kq=-g(>$Ec+B3Vl@-6|+ zdTjKRwFq~)G$RbvEHruaTq3syHZ_UNMk#%66KTwm;OfTKHu7viQ;)8tqg}Wyh|sjO zF!OejO{=a~^E==+?YKE`OZS(0stNz9u#)(f36=Xhw^x~O@P>tx5mo>qdWb!qv!vja*g zegUr}Hnk)!mB{!P8pJdW!9?*5|M=!ynV`=>T;bL~=(||WZO(|&4yVF*Z?632V5+Y& zMd{;qRPBsEB9+eRt}{{Ayzh)ZCLasdfdStRW6fdXdE;$!3>p2ZHGBeZg8Qq{LK_xp zSImG+&2Wet6&4ZZ3_RHb1^jQ;_rn_>R8;Kdz^Qd6%VLGH6Sr*aa_VGNK~GiwemSms z*bOAL%15-4Yx~hVjf~rFJ3lFLzEO@7q<4QVf2@6rj3wCwvsV?mH z8UFAb(GPcgSfSM7|KWAcLghUtlXLRK*46n3*hqOqQ=Yjh$W9IGWel}{84T6(Js zs*9@g|9120m5-Whl~3sTZVo*)MV$*C&`N(?`S74~5f)A@N2q$RqOp#E$D+PdRfXL; z_<$T&K3buw!|o%Z0+^ti5zvQVFQKDDafOyXtkJe3^mA%@Yx3^2!*}pW@ zDLyLm%7L<$#btM9uLxZ*BYkMnO&l$M^YqD!;__PYin#TAbqYMhH6M$LEBY0oJ*rwF z{mEqVRcSF}D*6K!WgSfPx*t=5znjCQ{v@SV=<)anY5Ve7?6=k2offYRL&go^mbQC1 zb}XOg0|(PR{3RcEjM)A`GT2?LkIsnJi@jD9?5zGUboR^ad&AT1CDC`%_0f{BVdktAm&XF}}R?b?B`CB)57+~%bhTO{YavLLa3MY)cv%R`OqUu+#akN8`& zbSW*I%e2YG`M?sUEEg zUu@#fQ}^(i)v65UM=JUd~~s>guN}2-=7U09CoipF)*9jhr0pWM1{h*L6;_0Pz#6gt3LF)?^AY?alL!Ul1=0s(vZs&paxzu*$a?%_j4HiOa_wj;BX^E3*QPk@J0002hbMMap literal 0 HcmV?d00001 diff --git a/web/src/apps/streamis/module/dataSourceInit/index.vue b/web/src/apps/streamis/module/dataSourceInit/index.vue new file mode 100644 index 000000000..64c53821e --- /dev/null +++ b/web/src/apps/streamis/module/dataSourceInit/index.vue @@ -0,0 +1,41 @@ + + + + + \ No newline at end of file diff --git a/web/src/apps/streamis/module/datasourceToolbar/index.vue b/web/src/apps/streamis/module/datasourceToolbar/index.vue new file mode 100644 index 000000000..f0a426dde --- /dev/null +++ b/web/src/apps/streamis/module/datasourceToolbar/index.vue @@ -0,0 +1,84 @@ + + + + + \ No newline at end of file diff --git a/web/src/apps/streamis/module/sourceTablePanel/index.vue b/web/src/apps/streamis/module/sourceTablePanel/index.vue new file mode 100644 index 000000000..cda94a954 --- /dev/null +++ b/web/src/apps/streamis/module/sourceTablePanel/index.vue @@ -0,0 +1,126 @@ + + + + + \ No newline at end of file diff --git a/web/src/apps/streamis/module/tableFieldsList/index.js b/web/src/apps/streamis/module/tableFieldsList/index.js new file mode 100644 index 000000000..7a7281ab0 --- /dev/null +++ b/web/src/apps/streamis/module/tableFieldsList/index.js @@ -0,0 +1,23 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export default { + name: 'tableFieldsList', + dispatchs: { + }, + component: () => import('./index.vue'), +}; diff --git a/web/src/apps/streamis/module/tableFieldsList/index.scss b/web/src/apps/streamis/module/tableFieldsList/index.scss new file mode 100644 index 000000000..1cc5bce90 --- /dev/null +++ b/web/src/apps/streamis/module/tableFieldsList/index.scss @@ -0,0 +1,19 @@ +.contianer { + padding: 20px 30px 0; +} + +.cardWrap { + display: flex; +} + +.cardInner { + display: flex; + width: 100px; + flex-direction: column; + justify-content: center; + align-content: center; + + & p { + text-align: center; + } +} diff --git a/web/src/apps/streamis/module/tableFieldsList/index.vue b/web/src/apps/streamis/module/tableFieldsList/index.vue new file mode 100644 index 000000000..aa3d4562a --- /dev/null +++ b/web/src/apps/streamis/module/tableFieldsList/index.vue @@ -0,0 +1,354 @@ + + + diff --git a/web/src/apps/streamis/module/tableInfo/index.vue b/web/src/apps/streamis/module/tableInfo/index.vue new file mode 100644 index 000000000..ae4e3b44f --- /dev/null +++ b/web/src/apps/streamis/module/tableInfo/index.vue @@ -0,0 +1,91 @@ + + + + + \ No newline at end of file diff --git a/web/src/apps/streamis/module/treeSource/index.vue b/web/src/apps/streamis/module/treeSource/index.vue new file mode 100644 index 000000000..c9c4d553f --- /dev/null +++ b/web/src/apps/streamis/module/treeSource/index.vue @@ -0,0 +1,233 @@ + + + + + \ No newline at end of file diff --git a/web/src/apps/streamis/router.js b/web/src/apps/streamis/router.js index 7814b1225..0f8e6016e 100644 --- a/web/src/apps/streamis/router.js +++ b/web/src/apps/streamis/router.js @@ -20,5 +20,16 @@ export default [ }, component: () => import('./view/jobDetail/index.vue'), + }, + { + path: '/dataSource', + name: 'DataSource', + meta: { + title: 'Streamis', + keepAlive: false, // 缓存导致页面有多个编辑器，广播事件会触发报错 + publicPage: true, // 权限公开 + }, + component: () => + import('./view/dataSource/index.vue'), } ] diff --git a/web/src/apps/streamis/view/dataSource/index.vue b/web/src/apps/streamis/view/dataSource/index.vue new file mode 100644 index 000000000..8b74272c9 --- /dev/null +++ b/web/src/apps/streamis/view/dataSource/index.vue @@ -0,0 +1,71 @@ + + + diff --git a/web/src/apps/streamis/view/realTimeDataSource/index.vue b/web/src/apps/streamis/view/realTimeDataSource/index.vue new file mode 100644 index 000000000..b98d9d525 --- /dev/null +++ b/web/src/apps/streamis/view/realTimeDataSource/index.vue @@ -0,0 +1,11 @@ + + + + \ No newline at end of file diff --git a/web/src/main.js b/web/src/main.js index e774911bf..6ab47f994 100644 --- a/web/src/main.js +++ b/web/src/main.js @@ -32,6 +32,7 @@ import './components/svgIcon/index.js' import './dss/module/index.js' import './apps/scriptis/module/index.js' import './apps/workflows/module/index.js' +import './apps/streamis/assets/streamisIconFont/iconfont.css' Vue.use(VueRouter) Vue.use(component) From 91ab58e5e739100dcded19be0fdcef1e6d817942 Mon Sep 17 00:00:00 2001 From: pengmengsi 90080 Date: Mon, 24 May 2021 11:04:35 +0800 Subject: [PATCH 045/434] datasource management --- web/src/apps/streamis/assets/images/u2616.png | Bin 0 -> 869 bytes web/src/apps/streamis/assets/images/u2618.png | Bin 0 -> 598 bytes web/src/apps/streamis/assets/images/u2662.png | Bin 0 -> 359 bytes web/src/apps/streamis/assets/images/u2665.png | Bin 0 -> 548 bytes web/src/apps/streamis/assets/images/u2680.png | Bin 0 -> 3361 bytes web/src/apps/streamis/assets/images/u458.png | Bin 0 -> 274 bytes .../streamis/assets/streamisIconFont/demo.css | 539 ++++++++++++++++++ .../assets/streamisIconFont/demo_index.html | 349 ++++++++++++ .../assets/streamisIconFont/iconfont.css | 43 ++ .../assets/streamisIconFont/iconfont.js | 1 + .../assets/streamisIconFont/iconfont.json | 58 ++ .../assets/streamisIconFont/iconfont.ttf | Bin 0 -> 4212 bytes .../assets/streamisIconFont/iconfont.woff | Bin 0 -> 2708 bytes .../assets/streamisIconFont/iconfont.woff2 | Bin 0 -> 2156 bytes .../streamis/module/dataSourceInit/index.vue | 41 ++ .../module/datasourceToolbar/index.vue | 84 +++ .../module/sourceTablePanel/index.vue | 126 ++++ .../streamis/module/tableFieldsList/index.js | 23 + .../module/tableFieldsList/index.scss | 19 + .../streamis/module/tableFieldsList/index.vue | 354 ++++++++++++ .../apps/streamis/module/tableInfo/index.vue | 91 +++ .../apps/streamis/module/treeSource/index.vue | 233 ++++++++ web/src/apps/streamis/router.js | 11 + .../apps/streamis/view/dataSource/index.vue | 71 +++ .../view/realTimeDataSource/index.vue | 11 + web/src/main.js | 1 + 26 files changed, 2055 insertions(+) create mode 100644 web/src/apps/streamis/assets/images/u2616.png create mode 100644 web/src/apps/streamis/assets/images/u2618.png create mode 100644 web/src/apps/streamis/assets/images/u2662.png create mode 100644 web/src/apps/streamis/assets/images/u2665.png create mode 100644 web/src/apps/streamis/assets/images/u2680.png create mode 100644 web/src/apps/streamis/assets/images/u458.png create mode 100644 web/src/apps/streamis/assets/streamisIconFont/demo.css create mode 100644 web/src/apps/streamis/assets/streamisIconFont/demo_index.html create mode 100644 web/src/apps/streamis/assets/streamisIconFont/iconfont.css create mode 100644 web/src/apps/streamis/assets/streamisIconFont/iconfont.js create mode 100644 web/src/apps/streamis/assets/streamisIconFont/iconfont.json create mode 100644 web/src/apps/streamis/assets/streamisIconFont/iconfont.ttf create mode 100644 web/src/apps/streamis/assets/streamisIconFont/iconfont.woff create mode 100644 web/src/apps/streamis/assets/streamisIconFont/iconfont.woff2 create mode 100644 web/src/apps/streamis/module/dataSourceInit/index.vue create mode 100644 web/src/apps/streamis/module/datasourceToolbar/index.vue create mode 100644 web/src/apps/streamis/module/sourceTablePanel/index.vue create mode 100644 web/src/apps/streamis/module/tableFieldsList/index.js create mode 100644 web/src/apps/streamis/module/tableFieldsList/index.scss create mode 100644 web/src/apps/streamis/module/tableFieldsList/index.vue create mode 100644 web/src/apps/streamis/module/tableInfo/index.vue create mode 100644 web/src/apps/streamis/module/treeSource/index.vue create mode 100644 web/src/apps/streamis/view/dataSource/index.vue create mode 100644 web/src/apps/streamis/view/realTimeDataSource/index.vue diff --git a/web/src/apps/streamis/assets/images/u2616.png b/web/src/apps/streamis/assets/images/u2616.png new file mode 100644 index 0000000000000000000000000000000000000000..a2e6d40f02f88eef3b51b06318f131fc015d28d9 GIT binary patch literal 869 zcmV-r1DgDaP)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D0{TfrK~#8N?VCSn z6LA#BfA6@Y9UQtgI5;{KEU3_egANXXx>*oJY9R?}5wzf-Q^=6Xp+lwEG(n0P1PfgR z9UL4AItW&9=+METLxv6xo!aJ}?_J)V)%3!pU~~ETK11(*-}X}Ir|rF3{KntHJ7kmS$0YihG?f8`g&!Qiq1@{ zozJri1@8^Tx235JEWG;^CaYBlC*NUZ2he}QEOqDKrKt=c5$D2x>G5* zNtc}JtplaMkmx1=WJ&a~nAksX2NC90SK)&MvFJ&50B><2 v;Y)P+b%~TSfM?otsAMwmBJP`;zcJ<)FdzQ(^P000>X1^@s6#OZ}&00001b5ch_0Itp) z=>Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D0qaRbK~zXft(GBA z!$1_r*Jfg30x>a(n3&iHKumx`Op;&_2;>9;ffx$Ohd`jn3=V-nAV-3P1QJLPTY?w~ z5;FL`yVqt{uI$?JOI}{@y?gJUYwy}dl?BXZvkq<>YpFG`ckBs!!Or72zQq@Z<~{8C zI{@a5upKR33^g?hPHlNHsJq0iXe&QVXfpB6RFs7JXIxubx{#0+EG3XYpI4kbI~NkA zZc!feyTVnqy?w|w^}o=!uM$QiXcy%{{Uh!WKeiEz#1nyz#!5l`J+3Le@NJLKridKt zxG?hq35?fGQenKBc34855UEfB z*{SyZ#l^0PddLb93K9^QH_$Q*0r}u~QK18D4+|BLU0c)k76L0I%9vM{kqK_3gceastU11{ZXpoIK)^>FEb7jNAL k&7A)(Vi(vK*B&aBA66)k8B)zzHUIzs07*qoM6N<$f&-)WTmS$7 literal 0 HcmV?d00001 diff --git a/web/src/apps/streamis/assets/images/u2662.png b/web/src/apps/streamis/assets/images/u2662.png new file mode 100644 index 0000000000000000000000000000000000000000..9eee4dd33f35b07f274e3c0823f4bb339456976f GIT binary patch literal 359 zcmV-t0hs=YP)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D0Q^ZrK~yMHb&;=b z0znYQzu6!WCl60jk~X0!^bPs|6$mV@mZY`18tcH|4INheFW8W#cm0eb z^^gsiG^fZ3>fw0fH-uTd^af$Ugi$=HMV~C#tm2gdq_|iWJv%#lR=slL4+6b(^OV4m zF7jRNqb4>py}fCK|5AY$UvjPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D0l7&;K~y+Tl~W-| zgHaHE_qMTOSTJm0_=CYPQG;OAXc!n=5QHKA+h7>123ar~*2egUNuxzD2nq@=4qQ+$ z7%YwjqXS{%Kfil>|Mt({;s=j+ckk}q_wMeI`UThJWvzK1VwPDsMCuK4M=84UJZgzg zwaW3250xDUS1Ozy4EU6YcC9G_X1>6MM_Q{IF55Q22%H%X`4)yX;|0L+6RmrA@ap~?>k+zbZ_97la& zOOLb(G%}S(B7eu@w1Yr#oIPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D47N!`K~#8N?VW9C zRM{EF&14b-6&k2eKls5?%cdyQWi54Kp^?nQd|1Jy78QhUSeLDAiAo;=y2 z9Q50ijfbQK75KXc{9l^lBd+x?TQ^Ugj#E87J*nNhcQ2~G(FmPjh}@V?r?>OK1Kr)- zPbnvD_XuqiL^$_Mk!Dv`R{lah7mY?|Ab#HdNhk=EU`Zup4k9^2JkT!ERvu)dJ>8O^ zjf*0ibdU1K2tgu9X|J2$iHV6FH0l7wK^nM|hDI}I+8Q&YB+qj*9*^J6RMm!>2)3kt zi{e4jR#Uk~0m;(u*U;tHYazR0f_eS>=#&m~vsXFXk*V!C%t7=;I?eY`BkE+!qAbA> z+mBxT0mWB%{WoQ5I~&s52-;=R&&qX@MG1!3C=z=FQcsgwx}$0qAX-xVHrB{_h<=o* zU5$bSTOIp8#ZFZP)nX2^kD{ZK_Si?n=01|ibGI@AdGyN1d*gv!v!@-qoWa?dz&>j(YSevE=E_Ao>6;{(Q1W_&!Z&Q4L%=|a2c?6zXAT{L z1_Td?U~f{}Az(;N1JR&jk9vb}$tSko3*)QK&~-2#ulplJH1Y=YIOM>OgZsqET=MuJ8cTxiPU+65;IWYSm&F*wptSWNAjJSLx&(C!Eup2Awt|nc>O-G#}2{c12Z3tnwTbY zE}>vChlTQ03N{3VeuCa+m4D#xHlLjg1q%HTer0tr))25)>g;l{PRfS3A z0$%zwUHNy)qu9ZDo2u0cF09yqoitRF*V`bzsR1Xspe@+}C>msaN*^;5q+{1^?svtX zaOXxB`1MZmtA#e26Y<`}H#h$>`3_5bC>9FW_VxyzMSuDMC}DcYc}7L`XoI>_+%KBz z(U6`w!ARsr=CHA#af;Kw!-=`_+zu0xPNfevvCoR=VI%x^}XoL}ti?KP^d z-dDR;D9(>Io}=i;*=$H+$uu_D zm#N7|If4!J8gh}#=*{tFy?(6%)Yv9;?2GX69S+NSperIc2Y1kSzr^5{RCmP88>DIi zOp{*N_8@-}Rl3KtdS;Be)o7tUeH;@G42WB7J6?5$PkT z#Qc-}Exq{+Nzpt?p>E6Chi>1HHFae;EF029qtJtBXI&R_K=KvqBhrvXZozcCjd&kqj|UsBn3+f#8KGs#JE z{djk1Xy~{t=lkTQrl#cbx#HQ84wV7H)JXPdEEZd}gI8YQiCjl0=coCHrcAoGDKr80 zY%aM%Nw2bMZgzESr1S}ZhJ<1`^k=xIxQBW7_?x1OeSu*IhCs7BIGGWeH`O; zIEZ9-)V4CV^OvNKDH7FGR)CrUPo}4*H>|9z?Bk{iI?)95nj~AK@@&$tzv}7fxlzm1 zYfS`a!;<|x4N=gEd{luhpeG3!r|)MQYLBPDoT@0^(r2AX$$StGj4>0rUl`j_oz);ZipQm*i- zMoGIIwI+%OA{Br#i%{uv?4fvDCMPF%@vQo$+?G`n?%Y_>J=(jcAq!$4UiLRi1p&7e@rUq+bc@2 z2T3%q-c>F%K0G`u=Viwukw^^ihHKZ8!?_Zhi?RggKv9Z6N2;~0Oh+w{QAdrCDMacO z(k@e30jec98`(RZ{2s#B+%zZFfVPX&K&s}QPwFAq2RBzQtpM&n(1?a^whQ4FV61;9}rmGr~8i#6us?|y?ZOq{hrV9Gxu#L6w<9+_2rS0QZ0qRvt zc8|wmvCFo~M_y!)pcS6zkgrgD$5|M~0XOnGUlLb^$Q8eA!3sDRr zmh2XsyjAY24HdiX=>)YW#H(EVNT;d}se(u(a+x%So~>xbYt?ZP5G}{v2scyEiMHHs z>ejengXffrOz@o7>A)0xQk4U>YJ#(oMfL#>6)3nSBwLb7mgISE)LNH7@>TKr zXrfx*Gm;0OEeMBDO`+=JP#>dL6G46rCs2ZOp}~s{I5FQc;sG?VtzeU8g;z;$YQN+Z zME;e0h7Mc-D4~TXgi3HRK)k=K)snuRrs*dQjEnx?oJHr&dIz4a#+$GeH|GX zHuiJ>Nn{1`ISV`@iy0XB4ude`@%$AjK*8mnE{-7;ac?hey1`iH869**BMY*}7BhN@o06Lk$)78&q Iol`;+0Bp, +.markdown>blockquote, +.markdown>.highlight, +.markdown>ol, +.markdown>ul { + width: 80%; +} + +.markdown ul>li { + list-style: circle; +} + +.markdown>ul li, +.markdown blockquote ul>li { + margin-left: 20px; + padding-left: 4px; +} + +.markdown>ul li p, +.markdown>ol li p { + margin: 0.6em 0; +} + +.markdown ol>li { + list-style: decimal; +} + +.markdown>ol li, +.markdown blockquote ol>li { + margin-left: 20px; + padding-left: 4px; +} + +.markdown code { + margin: 0 3px; + padding: 0 5px; + background: #eee; + border-radius: 3px; +} + +.markdown strong, +.markdown b { + font-weight: 600; +} + +.markdown>table { + border-collapse: collapse; + border-spacing: 0px; + empty-cells: show; + border: 1px solid #e9e9e9; + width: 95%; + margin-bottom: 24px; +} + +.markdown>table th { + white-space: nowrap; + color: #333; + font-weight: 600; +} + +.markdown>table th, +.markdown>table td { + border: 1px solid #e9e9e9; + padding: 8px 16px; + text-align: left; +} + +.markdown>table th { + background: #F7F7F7; +} + +.markdown blockquote { + font-size: 90%; + color: #999; + border-left: 4px solid #e9e9e9; + padding-left: 0.8em; + margin: 1em 0; +} + +.markdown blockquote p { + margin: 0; +} + +.markdown .anchor { + opacity: 0; + transition: opacity 0.3s ease; + margin-left: 8px; +} + +.markdown .waiting { + color: #ccc; +} + +.markdown h1:hover .anchor, +.markdown h2:hover .anchor, +.markdown h3:hover .anchor, +.markdown h4:hover .anchor, +.markdown h5:hover .anchor, +.markdown h6:hover .anchor { + opacity: 1; + display: inline-block; +} + +.markdown>br, +.markdown>p>br { + clear: both; +} + + +.hljs { + display: block; + background: white; + padding: 0.5em; + color: #333333; + overflow-x: auto; +} + +.hljs-comment, +.hljs-meta { + color: #969896; +} + +.hljs-string, +.hljs-variable, +.hljs-template-variable, +.hljs-strong, +.hljs-emphasis, +.hljs-quote { + color: #df5000; +} + +.hljs-keyword, +.hljs-selector-tag, +.hljs-type { + color: #a71d5d; +} + +.hljs-literal, +.hljs-symbol, +.hljs-bullet, +.hljs-attribute { + color: #0086b3; +} + +.hljs-section, +.hljs-name { + color: #63a35c; +} + +.hljs-tag { + color: #333333; +} + +.hljs-title, +.hljs-attr, +.hljs-selector-id, +.hljs-selector-class, +.hljs-selector-attr, +.hljs-selector-pseudo { + color: #795da3; +} + +.hljs-addition { + color: #55a532; + background-color: #eaffea; +} + +.hljs-deletion { + color: #bd2c00; + background-color: #ffecec; +} + +.hljs-link { + text-decoration: underline; +} + +/* 代码高亮 */ +/* PrismJS 1.15.0 +https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript */ +/** + * prism.js default theme for JavaScript, CSS and HTML + * Based on dabblet (http://dabblet.com) + * @author Lea Verou + */ +code[class*="language-"], +pre[class*="language-"] { + color: black; + background: none; + text-shadow: 0 1px white; + font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace; + text-align: left; + white-space: pre; + word-spacing: normal; + word-break: normal; + word-wrap: normal; + line-height: 1.5; + + -moz-tab-size: 4; + -o-tab-size: 4; + tab-size: 4; + + -webkit-hyphens: none; + -moz-hyphens: none; + -ms-hyphens: none; + hyphens: none; +} + +pre[class*="language-"]::-moz-selection, +pre[class*="language-"] ::-moz-selection, +code[class*="language-"]::-moz-selection, +code[class*="language-"] ::-moz-selection { + text-shadow: none; + background: #b3d4fc; +} + +pre[class*="language-"]::selection, +pre[class*="language-"] ::selection, +code[class*="language-"]::selection, +code[class*="language-"] ::selection { + text-shadow: none; + background: #b3d4fc; +} + +@media print { + + code[class*="language-"], + pre[class*="language-"] { + text-shadow: none; + } +} + +/* Code blocks */ +pre[class*="language-"] { + padding: 1em; + margin: .5em 0; + overflow: auto; +} + +:not(pre)>code[class*="language-"], +pre[class*="language-"] { + background: #f5f2f0; +} + +/* Inline code */ +:not(pre)>code[class*="language-"] { + padding: .1em; + border-radius: .3em; + white-space: normal; +} + +.token.comment, +.token.prolog, +.token.doctype, +.token.cdata { + color: slategray; +} + +.token.punctuation { + color: #999; +} + +.namespace { + opacity: .7; +} + +.token.property, +.token.tag, +.token.boolean, +.token.number, +.token.constant, +.token.symbol, +.token.deleted { + color: #905; +} + +.token.selector, +.token.attr-name, +.token.string, +.token.char, +.token.builtin, +.token.inserted { + color: #690; +} + +.token.operator, +.token.entity, +.token.url, +.language-css .token.string, +.style .token.string { + color: #9a6e3a; + background: hsla(0, 0%, 100%, .5); +} + +.token.atrule, +.token.attr-value, +.token.keyword { + color: #07a; +} + +.token.function, +.token.class-name { + color: #DD4A68; +} + +.token.regex, +.token.important, +.token.variable { + color: #e90; +} + +.token.important, +.token.bold { + font-weight: bold; +} + +.token.italic { + font-style: italic; +} + +.token.entity { + cursor: help; +} diff --git a/web/src/apps/streamis/assets/streamisIconFont/demo_index.html b/web/src/apps/streamis/assets/streamisIconFont/demo_index.html new file mode 100644 index 000000000..5e079b8ff --- /dev/null +++ b/web/src/apps/streamis/assets/streamisIconFont/demo_index.html @@ -0,0 +1,349 @@ + + + + + iconfont Demo + + + + + + + + + + + + + +
+

+ + +

+
+
    +
  • Unicode
    • +
  • Font class
    • +
  • Symbol
    • +
+ + 查看项目 + +
+
+
+
    + +
  • + +
    数据
    +
    &#xe6ab;
    +
    • + +
  • + +
    数据
    +
    &#xe62d;
    +
    • + +
  • + +
    集群
    +
    &#xe610;
    +
    • + +
  • + +
    mysql
    +
    &#xe613;
    +
    • + +
  • + +
    apachekafka
    +
    &#xeb3f;
    +
    • + +
  • + +
    表格
    +
    &#xe617;
    +
    • + +
  • + +
    数据
    +
    &#xe832;
    +
    • + +
+
+

Unicode 引用

+
+ +

Unicode 是字体在网页端最原始的应用方式，特点是：

+
    +
  • 支持按字体的方式去动态调整图标大小，颜色等等。
    • +
  • 默认情况下不支持多色，直接添加多色图标会自动去色。
    • +
+
+

注意：新版 iconfont 支持两种方式引用多色图标：SVG symbol 引用方式和彩色字体图标模式。（使用彩色字体图标需要在「编辑项目」中开启「彩色」选项后并重新生成。）

+
+

Unicode 使用步骤如下：

+

第一步：拷贝项目下面生成的 @font-face

+
@font-face {
+  font-family: 'iconfont';
+  src: url('iconfont.woff2?t=1621579006110') format('woff2'),
+       url('iconfont.woff?t=1621579006110') format('woff'),
+       url('iconfont.ttf?t=1621579006110') format('truetype');
+}
+
+

第二步：定义使用 iconfont 的样式

+
.iconfont {
+  font-family: "iconfont" !important;
+  font-size: 16px;
+  font-style: normal;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+

第三步：挑选相应图标并获取字体编码，应用于页面

+
+<span class="iconfont">&#x33;</span>
+
+
+

"iconfont" 是你项目下的 font-family。可以通过编辑项目查看，默认是 "iconfont"。

+
+
+
+
+
    + +
  • + +
    + 数据 +
    +
    .icon-shuju +
    +
    • + +
  • + +
    + 数据 +
    +
    .icon-fl-shuju +
    +
    • + +
  • + +
    + 集群 +
    +
    .icon-jiqun +
    +
    • + +
  • + +
    + mysql +
    +
    .icon-mysql +
    +
    • + +
  • + +
    + apachekafka +
    +
    .icon-apachekafka +
    +
    • + +
  • + +
    + 表格 +
    +
    .icon-table +
    +
    • + +
  • + +
    + 数据 +
    +
    .icon-shuju1 +
    +
    • + +
+
+

font-class 引用

+
+ +

font-class 是 Unicode 使用方式的一种变种，主要是解决 Unicode 书写不直观，语意不明确的问题。

+

与 Unicode 使用方式相比，具有如下特点：

+
    +
  • 相比于 Unicode 语意明确，书写更直观。可以很容易分辨这个 icon 是什么。
    • +
  • 因为使用 class 来定义图标，所以当要替换图标时，只需要修改 class 里面的 Unicode 引用。
    • +
+

使用步骤如下：

+

第一步：引入项目下面生成的 fontclass 代码：

+
<link rel="stylesheet" href="./iconfont.css">
+
+

第二步：挑选相应图标并获取类名，应用于页面：

+
<span class="iconfont icon-xxx"></span>
+
+
+

" + iconfont" 是你项目下的 font-family。可以通过编辑项目查看，默认是 "iconfont"。

+
+
+
+
+
    + +
  • + +
    数据
    +
    #icon-shuju
    +
    • + +
  • + +
    数据
    +
    #icon-fl-shuju
    +
    • + +
  • + +
    集群
    +
    #icon-jiqun
    +
    • + +
  • + +
    mysql
    +
    #icon-mysql
    +
    • + +
  • + +
    apachekafka
    +
    #icon-apachekafka
    +
    • + +
  • + +
    表格
    +
    #icon-table
    +
    • + +
  • + +
    数据
    +
    #icon-shuju1
    +
    • + +
+
+

Symbol 引用

+
+ +

这是一种全新的使用方式，应该说这才是未来的主流，也是平台目前推荐的用法。相关介绍可以参考这篇文章 + 这种用法其实是做了一个 SVG 的集合，与另外两种相比具有如下特点：

+
    +
  • 支持多色图标了，不再受单色限制。
    • +
  • 通过一些技巧，支持像字体那样，通过 font-size, color 来调整样式。
    • +
  • 兼容性较差，支持 IE9+，及现代浏览器。
    • +
  • 浏览器渲染 SVG 的性能一般，还不如 png。
    • +
+

使用步骤如下：

+

第一步：引入项目下面生成的 symbol 代码：

+
<script src="./iconfont.js"></script>
+
+

第二步：加入通用 CSS 代码（引入一次就行）：

+
<style>
+.icon {
+  width: 1em;
+  height: 1em;
+  vertical-align: -0.15em;
+  fill: currentColor;
+  overflow: hidden;
+}
+</style>
+
+

第三步：挑选相应图标并获取类名，应用于页面：

+
<svg class="icon" aria-hidden="true">
+  <use xlink:href="#icon-xxx"></use>
+</svg>
+
+
+
+ +
+
+ + + diff --git a/web/src/apps/streamis/assets/streamisIconFont/iconfont.css b/web/src/apps/streamis/assets/streamisIconFont/iconfont.css new file mode 100644 index 000000000..a24c7c15d --- /dev/null +++ b/web/src/apps/streamis/assets/streamisIconFont/iconfont.css @@ -0,0 +1,43 @@ +@font-face { + font-family: "iconfont"; /* Project id 2562871 */ + src: url('iconfont.woff2?t=1621579006110') format('woff2'), + url('iconfont.woff?t=1621579006110') format('woff'), + url('iconfont.ttf?t=1621579006110') format('truetype'); +} + +.iconfont { + font-family: "iconfont" !important; + font-size: 16px; + font-style: normal; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +.icon-shuju:before { + content: "\e6ab"; +} + +.icon-fl-shuju:before { + content: "\e62d"; +} + +.icon-jiqun:before { + content: "\e610"; +} + +.icon-mysql:before { + content: "\e613"; +} + +.icon-apachekafka:before { + content: "\eb3f"; +} + +.icon-table:before { + content: "\e617"; +} + +.icon-shuju1:before { + content: "\e832"; +} + diff --git a/web/src/apps/streamis/assets/streamisIconFont/iconfont.js b/web/src/apps/streamis/assets/streamisIconFont/iconfont.js new file mode 100644 index 000000000..fe9dcc11d --- /dev/null +++ b/web/src/apps/streamis/assets/streamisIconFont/iconfont.js @@ -0,0 +1 @@ +!function(a){var t,e,c,o,h,n,l='',d=(d=document.getElementsByTagName("script"))[d.length-1].getAttribute("data-injectcss");if(d&&!a.__iconfont__svg__cssinject__){a.__iconfont__svg__cssinject__=!0;try{document.write("")}catch(a){console&&console.log(a)}}function s(){h||(h=!0,c())}t=function(){var a,t,e;(e=document.createElement("div")).innerHTML=l,l=null,(t=e.getElementsByTagName("svg")[0])&&(t.setAttribute("aria-hidden","true"),t.style.position="absolute",t.style.width=0,t.style.height=0,t.style.overflow="hidden",a=t,(e=document.body).firstChild?(t=e.firstChild).parentNode.insertBefore(a,t):e.appendChild(a))},document.addEventListener?~["complete","loaded","interactive"].indexOf(document.readyState)?setTimeout(t,0):(e=function(){document.removeEventListener("DOMContentLoaded",e,!1),t()},document.addEventListener("DOMContentLoaded",e,!1)):document.attachEvent&&(c=t,o=a.document,h=!1,(n=function(){try{o.documentElement.doScroll("left")}catch(a){return void setTimeout(n,50)}s()})(),o.onreadystatechange=function(){"complete"==o.readyState&&(o.onreadystatechange=null,s())})}(window); \ No newline at end of file diff --git a/web/src/apps/streamis/assets/streamisIconFont/iconfont.json b/web/src/apps/streamis/assets/streamisIconFont/iconfont.json new file mode 100644 index 000000000..2a70b25e8 --- /dev/null +++ b/web/src/apps/streamis/assets/streamisIconFont/iconfont.json @@ -0,0 +1,58 @@ +{ + "id": "2562871", + "name": "streamis", + "font_family": "iconfont", + "css_prefix_text": "icon-", + "description": "", + "glyphs": [ + { + "icon_id": "1789126", + "name": "数据", + "font_class": "shuju", + "unicode": "e6ab", + "unicode_decimal": 59051 + }, + { + "icon_id": "2525864", + "name": "数据", + "font_class": "fl-shuju", + "unicode": "e62d", + "unicode_decimal": 58925 + }, + { + "icon_id": "7159338", + "name": "集群", + "font_class": "jiqun", + "unicode": "e610", + "unicode_decimal": 58896 + }, + { + "icon_id": "11493372", + "name": "mysql", + "font_class": "mysql", + "unicode": "e613", + "unicode_decimal": 58899 + }, + { + "icon_id": "15378137", + "name": "apachekafka", + "font_class": "apachekafka", + "unicode": "eb3f", + "unicode_decimal": 60223 + }, + { + "icon_id": "18828425", + "name": "表格", + "font_class": "table", + "unicode": "e617", + "unicode_decimal": 58903 + }, + { + "icon_id": "20016505", + "name": "数据", + "font_class": "shuju1", + "unicode": "e832", + "unicode_decimal": 59442 + } + ] +} diff --git a/web/src/apps/streamis/assets/streamisIconFont/iconfont.ttf b/web/src/apps/streamis/assets/streamisIconFont/iconfont.ttf new file mode 100644 index 0000000000000000000000000000000000000000..4a33f0047f581ebc3c75ca514f5b8f42ec959ea7 GIT binary patch literal 4212 zcmd^Cdu&@*8UN0G*}it|T;F?ro#tgb_H~-pcH-F1(l$=t=}Rq5N?WR}mFqaKI*%sK zXvfOZH3Gq=q`;=FlRzL^0^1)RJDHaAN9M=6=n8zTBeg z6Wmtz49a$3D2s(gvcq3NH6+T4cmZJcMxlBUnjj0zmTkNDgS(h34Z}(>!<+v1Vp1IF z%V9)QL(qg!9H;X0N~hCQ*zGo{W7vN${K9(>5=E)2+9?Bq5XWtygaYX z$L3#pKY6Qz&g}gRTHh_i@TA`6MSJD&`5ElRMzbJvEdJk%%NT$jYZz>FRLyFPxr`Vq zou_%ttaJs<$IMDs@cc_=rE~Xw(yTbXaI3?t#(V?3OgN^WUCVyJZQ_235uLcapJ88M zi=aRR(y$ElI%zlhQ+i_5j4d=;d@X_LM@+HE=CRiimbesAIAl@G5Z z7iXx}fJo=>6@FalR^9lkT-LJ1ctc~$!gR8+Awdx{*3#GzU!}+$+cwAwHk}=^vcT1M z$SMlqv$0Gjw*C$}TaQN9P^_P#im4EaPl>c2_uEdR`J?FZC~mj8!+;GkuYrW)+lu4q zCaXzEkJ{8)vW^aEnuL%LY$^^ZB|1ntPUQQn*t+0zN4!U#3zE9?N4-bS*Dd~;VOL;F z?((nRg@y@vHIhg~n8SKglYW@k@y3*i#qAbKB=p0$4)C3~^aT4Mz7bjQLp3zxTG$DD zpbPqNB}~E;oQJQ&x8M@|2z~>CoU|4?*XN9efWP{r#GQFOfHI6r-EnuxOK zM2+foim0oeA@$eEb@WGLB6@v}KcJKJ`t-~tr(;gULj7q>$4&e5Gcz;jPW{lGGBxKG zADJW9jjo6&`KSx=zd1d9i5iK-=pGRozvFm*`rHh5LD}mLpV49hUps`~%*D6)DXu~! z;Pi_8Brl6gPOtOzu8aHf0!pfvKf~kCt9pe=;k#X3-HQV1mqZzDMFdW2K%|S^7Z%^N zqVZmF;htdPYxf7E-m_i%_dOsOM=CJb#2F(q>=^r!MBy|AP@%oieW5$gpB6A5eu~E* zP3$y3+udy>hL3@nou(<4TooP;j|MD(a}*>w8W!uOEoqnD~^ zWvKaD@%`0`n@{s)=}M;daonY7ZMfq4rCd5?Xs8+5wq}etny-%3hunk;wYX-fLrXGC zQpliXYt{^>vwx9ywr`9-`SdG_syz~@al05+)tdatfZJiy)JnE={y+ zZp|Opctv#+&f{@=1e+M)#A>I#ns6H`ly$C3zfj@W^@Z%zD&Dm@tK>zGoe`Rnf+V?k zPFU$5wtE~4U9!meIL0P2jA-}Ey@8g-;Nw5PRu&uQ9%Hmx~wueo+auPsAJ!PI4Y^kOJYz>fq^$6(N6xxMm;E?bnG{^i>Kxqilcp z%4c0daJ5^MytcNCAaE{85~`{l9*KdV+g@vD_$rU&a@Vam(hlVS?#MP&Kb~P;J4w3VC`(5VQZdeIO8v1=ttyzHpXEi+@}E0kJQB zV%DuhBD>21ZP`t!@B)dnq{7TnIJNlf?Cjpz*^lw|@hmNSsc4*PmR@9k#vTA4#9#}2 z2KQ|UcM{Gf-K(;r-=W7Wr;WNvq z&Ql|k%P49!M(yu7Qjkz9n`){ut0Icmt412zs;Zk;YQOOLo2skY8Y60%j8__@ucmF4 zuE?^YuWGBRXu44&<(EX*PwvM`6Z z(ZW2Q?vfT3kZ-rJh%2D{zsZRaNAa6aI0yEj!&wW{Q~UQU%zzue`lbzw{8bBch~Kg> zk8}5)g$3jnEG)tSvcken&`EytSRq#`^aPHb3Jl~&hI>bbOXaq!&^J-cjg`f6d!R5j zJ}@#IXpT3Psm{W1VXPcx{L#K-sni?j9UB=6Y{PhkVsRueIyN#`$d}^%rPAoyM1uB1 z`FMV02sXnQ6!4H+!g~({@UVLd?*ovB5g3ME<6gQ??n3K6JQ5etbL>9ta_<9ZJ%%r~ z0V7%f4^(md>$u#i6Rm~~%jMaO!=u<+5_2qJMggp27-jZW>Q2PzzL+fL5!0} z>p1iqUZb!UH+tgE^OWu5#&ZrC-%4X$(f_y)Dt5vUmT-h80A}VPK}=^I&-7Be1GA1uJ?G3FXfIE3xZ+OY()MSzOdO2 literal 0 HcmV?d00001 diff --git a/web/src/apps/streamis/assets/streamisIconFont/iconfont.woff b/web/src/apps/streamis/assets/streamisIconFont/iconfont.woff new file mode 100644 index 0000000000000000000000000000000000000000..807b60f86df4a7239b285c318f2e52c4b6c08ee8 GIT binary patch literal 2708 zcmY+Gc{J2tAIHCAvScY{N>fszUuwiJdzQi= zSxUAOktU3(@XHc1DMUo2&}5!(J zk+#VZb^`P%w68usXa|y#m?8%3KP-+0eyB$KRC2Ge_9#ITQ?|?Mp zI%5-pI3%c6S5~rUdwbJ)>iU~b`R?c|B@i8+R3;RF8_F@jJ(k$c(ur=ecGv~a5<2n_ z4IchhV#3j(w!Cb)J4XDm-Dq@@$2=^Lruyzt`%ViRIo#7p9i+jwYZV+=WRKa`uI>f? z;O2v&bCN3IljSi-Gw^+<&;`QV9cKcYM=^^X*OTKWNPhgP@KZNd+Og3XE6rJ?)&5zn z*8NA@LY#y}F__9XGDvL%cp;2}Z11zdd;d||dd}m2&%)8Zyy+yloq2kv;N z0SHDV4gy1413MX$BHlPg*@IK~I_}5Trvl5ctuP7y_^bFY245;{B@xz#00+SzLiyNj z71O2#5Tv*fN;!?XmRa>?<;b0LmmQna$}&3wS@=wYP}BkYd7KO&)){orqzV}CjT%6mK zr>hJ+(xT|^ei5BvJT>~`GV0|tgy&X`k7D3#iq7<{g78ktSY-0U! zueL%o?Q91BUAZ^s5Vt;Ef}e>0j9)!p&)tshmg2L5)*-wuENwrk3+9`2YTAM(9=kIx zv7@_VIt(Z|j=JlnuCJ@RrpTeJ{-C6&YH0QA`Pdd`e43Gj`27X1_cwEjP8dj{B0C%7 zDbqH=LuQWYKh(yuL{BLi`JZnECzH}%`%f%IC5JZj1}IrXmfy|vI{u+b6YgV{ll9N{ z@ufHJL2_OnGdiR2uS8EjPi=M$c_|H7J_eRgxNZWj2pf@kFH4zh+nMSGa?$ZyN!@ z|6-14dM?=ybyzhu*N-ysr={22#2TDfcD7}Rh65dkqnMO)C;Q!4jbi7jO48EbUTka( zQ&E^f;7*|I>?lB~Z&BNDOKb1`O8U?J#+&@_y{EtReVjCTs4l}+)jS+#K7AaOy!(K@ zp?GA?_gVdc0U4Qn-y{*F5F%GGDZ8bmn7poxZt6VUvG~p9=hB20uw!nuU${qT%h+2a zURUTrOIB&RAZQ@xel(?Dbbi2-sKv{nj`j!F=kiwcy>&|yaRst2j5TBzAthuzzt?B* zREmNLt?5v5rPtZpn4A!^Gk#WAL@xPMOf&YFsGNHG`%&^?15%u*YsLSk7VKJ$K26hU-+3EpQm1_ z9Ck?)tH4f0EY(-$sHM5v$z9tl{Uhx_T8@+RC01fIY7ikv!uh7!ZxAeZts)M7ju4y) z(;U5s=_*^nMO5f+ta+5(+(w>nD0yFF7z@8=ryMwgMzjVg8DY-9??p|xoq1Ay81e*W zp_1~xx?Eqepfz;lXKhvZQdMsROJDk|^mj_AE8^z+OLvQO&C?n0mudb%zM=k;MjpYg z=IIfC|IiMO!p1RcMhk*>b0=!66kVQRVu#hi`wK5`56(TrtZ(NeJz~IVH8V~ zzo!Du^vvQKb2v>6ZJCw=!@{c3zk2?xc-Jjp_qK2pE+u2y^7%nl8vZu|TxBAYQ{V02 zK2&+m6k{IY`b4y5o@_s%dhTYfz!IR7$uq1Mv9ShEY@sq9*I7q(9?Rp3lV=JI>aI@K zysr*E8od%v0fls^P+8^v)q@rJvIP_hz4N7+YlMWo zXD+*N`{x5t;a+xL6Q>doNhIeEH~y|CziTT%>RDUfGqVx$$e7k>7~fggSQBozDH4g+ zP_?|PHu{oKIC>Z0&@B7wGx6Vg{Hs=WcE0_rCUx3ZDSn?$JQOJHmdN`(P9zWrkCXkj zM3OIookn70a1|JJkZodtP%G&2w&UCS{0Auucr z%^W=ewQCL@V5^ra=uD zaB2hj>5ihfo*GsJap%Kz`PylTTU}lY_^zWHXPc8Hg;FCNk+g;3YAq{2kSXX~;0%SC zz2zC&ETwHh}-(C3w?ln0-x^3Qa59e9lNjYZq%xEPq@4*hg zI=cSi!`JM2+d!Bsp~Q$qE;vC5I|% zz8bBAT}&(2jDz?^_i(x%wLdS(H0#WOBdOu-PV0%goTOnWD!ocJ=6*{lJkE`GT?iSy z5qj#^JCWi6eTk3piR@H%E}O;9W?y9&vJ=@i5*{2XGp)|jIOwp4M;Pcsp$5nUcTxI0 i0S1M3;iCsIF8vt)gN`nJjsehWBYy(;-NVAx0pQ=8BiM!j literal 0 HcmV?d00001 diff --git a/web/src/apps/streamis/assets/streamisIconFont/iconfont.woff2 b/web/src/apps/streamis/assets/streamisIconFont/iconfont.woff2 new file mode 100644 index 0000000000000000000000000000000000000000..2effb7dcaa011b2d837e7ab4804debd0c23f546c GIT binary patch literal 2156 zcmV-y2$T1BPew8T0RR9100?XV3jhEB01$Kl003X9sB;`3(_OtlJ>_abU-g547PX6P`wk zXj?9HdhZY1^j}EYLQ+ElHGzt+-Mk`3m?-U}oeRkoCQy|bH74S;Gesum|2st1YCV_i z-d`w;#|SVMgODsv$2-PEm|Z!`^VfGJZFgRpax_k%6hh*#7{&PewJIjbbzN)tcJj{D zaE(wQvPV_gM+n#>9L0FekAYr{Z6-Y9=cN-3Oq z?f4-Mr7g6v=tH1Srvw1-Jpcs|aywk4c2KMav;ZL)`51D>;0pkb>L;S$wc~64Ytie_ z*W2Gleu%SJI>SLm4jO`g@c>Y0n6N-#g#i$2kYrnmEhX8!P|TpOo5Dvuz`$#f0;~Zr z)B_fHy}ceF;B6#w3eJZ(pr4uyzZn@m02mf|gaTl0AgR%GDFOis-+J39FgXTMTkC(T&^}`Uj>ZLxn|Uu^KKwkOzw44u|{e%(uQJ#nlhOf zv&>vpV>cp4?s~M@*SIKa?Nz=kKs~e@og_fT%;JDfpD}bFwg3y@5ktu4F8A5;7q_n9GfNt%kq=-g(>$Ec+B3Vl@-6|+ zdTjKRwFq~)G$RbvEHruaTq3syHZ_UNMk#%66KTwm;OfTKHu7viQ;)8tqg}Wyh|sjO zF!OejO{=a~^E==+?YKE`OZS(0stNz9u#)(f36=Xhw^x~O@P>tx5mo>qdWb!qv!vja*g zegUr}Hnk)!mB{!P8pJdW!9?*5|M=!ynV`=>T;bL~=(||WZO(|&4yVF*Z?632V5+Y& zMd{;qRPBsEB9+eRt}{{Ayzh)ZCLasdfdStRW6fdXdE;$!3>p2ZHGBeZg8Qq{LK_xp zSImG+&2Wet6&4ZZ3_RHb1^jQ;_rn_>R8;Kdz^Qd6%VLGH6Sr*aa_VGNK~GiwemSms z*bOAL%15-4Yx~hVjf~rFJ3lFLzEO@7q<4QVf2@6rj3wCwvsV?mH z8UFAb(GPcgSfSM7|KWAcLghUtlXLRK*46n3*hqOqQ=Yjh$W9IGWel}{84T6(Js zs*9@g|9120m5-Whl~3sTZVo*)MV$*C&`N(?`S74~5f)A@N2q$RqOp#E$D+PdRfXL; z_<$T&K3buw!|o%Z0+^ti5zvQVFQKDDafOyXtkJe3^mA%@Yx3^2!*}pW@ zDLyLm%7L<$#btM9uLxZ*BYkMnO&l$M^YqD!;__PYin#TAbqYMhH6M$LEBY0oJ*rwF z{mEqVRcSF}D*6K!WgSfPx*t=5znjCQ{v@SV=<)anY5Ve7?6=k2offYRL&go^mbQC1 zb}XOg0|(PR{3RcEjM)A`GT2?LkIsnJi@jD9?5zGUboR^ad&AT1CDC`%_0f{BVdktAm&XF}}R?b?B`CB)57+~%bhTO{YavLLa3MY)cv%R`OqUu+#akN8`& zbSW*I%e2YG`M?sUEEg zUu@#fQ}^(i)v65UM=JUd~~s>guN}2-=7U09CoipF)*9jhr0pWM1{h*L6;_0Pz#6gt3LF)?^AY?alL!Ul1=0s(vZs&paxzu*$a?%_j4HiOa_wj;BX^E3*QPk@J0002hbMMap literal 0 HcmV?d00001 diff --git a/web/src/apps/streamis/module/dataSourceInit/index.vue b/web/src/apps/streamis/module/dataSourceInit/index.vue new file mode 100644 index 000000000..64c53821e --- /dev/null +++ b/web/src/apps/streamis/module/dataSourceInit/index.vue @@ -0,0 +1,41 @@ + + + + + \ No newline at end of file diff --git a/web/src/apps/streamis/module/datasourceToolbar/index.vue b/web/src/apps/streamis/module/datasourceToolbar/index.vue new file mode 100644 index 000000000..f0a426dde --- /dev/null +++ b/web/src/apps/streamis/module/datasourceToolbar/index.vue @@ -0,0 +1,84 @@ + + + + + \ No newline at end of file diff --git a/web/src/apps/streamis/module/sourceTablePanel/index.vue b/web/src/apps/streamis/module/sourceTablePanel/index.vue new file mode 100644 index 000000000..cda94a954 --- /dev/null +++ b/web/src/apps/streamis/module/sourceTablePanel/index.vue @@ -0,0 +1,126 @@ + + + + + \ No newline at end of file diff --git a/web/src/apps/streamis/module/tableFieldsList/index.js b/web/src/apps/streamis/module/tableFieldsList/index.js new file mode 100644 index 000000000..7a7281ab0 --- /dev/null +++ b/web/src/apps/streamis/module/tableFieldsList/index.js @@ -0,0 +1,23 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export default { + name: 'tableFieldsList', + dispatchs: { + }, + component: () => import('./index.vue'), +}; diff --git a/web/src/apps/streamis/module/tableFieldsList/index.scss b/web/src/apps/streamis/module/tableFieldsList/index.scss new file mode 100644 index 000000000..1cc5bce90 --- /dev/null +++ b/web/src/apps/streamis/module/tableFieldsList/index.scss @@ -0,0 +1,19 @@ +.contianer { + padding: 20px 30px 0; +} + +.cardWrap { + display: flex; +} + +.cardInner { + display: flex; + width: 100px; + flex-direction: column; + justify-content: center; + align-content: center; + + & p { + text-align: center; + } +} diff --git a/web/src/apps/streamis/module/tableFieldsList/index.vue b/web/src/apps/streamis/module/tableFieldsList/index.vue new file mode 100644 index 000000000..aa3d4562a --- /dev/null +++ b/web/src/apps/streamis/module/tableFieldsList/index.vue @@ -0,0 +1,354 @@ + + + diff --git a/web/src/apps/streamis/module/tableInfo/index.vue b/web/src/apps/streamis/module/tableInfo/index.vue new file mode 100644 index 000000000..ae4e3b44f --- /dev/null +++ b/web/src/apps/streamis/module/tableInfo/index.vue @@ -0,0 +1,91 @@ + + + + + \ No newline at end of file diff --git a/web/src/apps/streamis/module/treeSource/index.vue b/web/src/apps/streamis/module/treeSource/index.vue new file mode 100644 index 000000000..c9c4d553f --- /dev/null +++ b/web/src/apps/streamis/module/treeSource/index.vue @@ -0,0 +1,233 @@ + + + + + \ No newline at end of file diff --git a/web/src/apps/streamis/router.js b/web/src/apps/streamis/router.js index 7814b1225..0f8e6016e 100644 --- a/web/src/apps/streamis/router.js +++ b/web/src/apps/streamis/router.js @@ -20,5 +20,16 @@ export default [ }, component: () => import('./view/jobDetail/index.vue'), + }, + { + path: '/dataSource', + name: 'DataSource', + meta: { + title: 'Streamis', + keepAlive: false, // 缓存导致页面有多个编辑器，广播事件会触发报错 + publicPage: true, // 权限公开 + }, + component: () => + import('./view/dataSource/index.vue'), } ] diff --git a/web/src/apps/streamis/view/dataSource/index.vue b/web/src/apps/streamis/view/dataSource/index.vue new file mode 100644 index 000000000..8b74272c9 --- /dev/null +++ b/web/src/apps/streamis/view/dataSource/index.vue @@ -0,0 +1,71 @@ + + + diff --git a/web/src/apps/streamis/view/realTimeDataSource/index.vue b/web/src/apps/streamis/view/realTimeDataSource/index.vue new file mode 100644 index 000000000..b98d9d525 --- /dev/null +++ b/web/src/apps/streamis/view/realTimeDataSource/index.vue @@ -0,0 +1,11 @@ + + + + \ No newline at end of file diff --git a/web/src/main.js b/web/src/main.js index e774911bf..6ab47f994 100644 --- a/web/src/main.js +++ b/web/src/main.js @@ -32,6 +32,7 @@ import './components/svgIcon/index.js' import './dss/module/index.js' import './apps/scriptis/module/index.js' import './apps/workflows/module/index.js' +import './apps/streamis/assets/streamisIconFont/iconfont.css' Vue.use(VueRouter) Vue.use(component) From f4587083dccba1f04efae830c8beb5b56e7d907f Mon Sep 17 00:00:00 2001 From: pengmengsi 90080 Date: Mon, 31 May 2021 11:16:15 +0800 Subject: [PATCH 046/434] datasource update --- .../module/sourceTablePanel/index.vue | 16 ++- .../streamis/module/tableFieldsList/index.vue | 107 +++++++++++++----- .../apps/streamis/module/treeSource/index.vue | 39 +++++-- .../apps/streamis/view/dataSource/index.vue | 9 +- 4 files changed, 126 insertions(+), 45 deletions(-) diff --git a/web/src/apps/streamis/module/sourceTablePanel/index.vue b/web/src/apps/streamis/module/sourceTablePanel/index.vue index cda94a954..0e3e46400 100644 --- a/web/src/apps/streamis/module/sourceTablePanel/index.vue +++ b/web/src/apps/streamis/module/sourceTablePanel/index.vue @@ -30,7 +30,7 @@
- +
@@ -49,6 +49,20 @@ import tableFieldsList from '@/apps/streamis/module/tableFieldsList'; import tableInfo from '@/apps/streamis/module/tableInfo'; export default { + props: ["fieldsListInfo"], + data(){ + return { + nodeNameValue: this.fieldsListInfo + } + }, + watch: { + fieldsListInfo(){ + this.nodeNameValue = this.fieldsListInfo; + } + }, + mounted(){ + // this.nodeNameValue = this.fieldsListInfo; + }, components: { tableFieldsList: tableFieldsList.component, tableInfo: tableInfo diff --git a/web/src/apps/streamis/module/tableFieldsList/index.vue b/web/src/apps/streamis/module/tableFieldsList/index.vue index aa3d4562a..63175ef03 100644 --- a/web/src/apps/streamis/module/tableFieldsList/index.vue +++ b/web/src/apps/streamis/module/tableFieldsList/index.vue @@ -2,11 +2,11 @@
- diff --git a/web/src/apps/streamis/module/treeSource/index.vue b/web/src/apps/streamis/module/treeSource/index.vue index 240b23305..4fc4ad4ff 100644 --- a/web/src/apps/streamis/module/treeSource/index.vue +++ b/web/src/apps/streamis/module/treeSource/index.vue @@ -161,9 +161,6 @@ export default { diff --git a/web/src/apps/streamis/view/realTimeDataSource/index.vue b/web/src/apps/streamis/view/realTimeDataSource/index.vue deleted file mode 100644 index b98d9d525..000000000 --- a/web/src/apps/streamis/view/realTimeDataSource/index.vue +++ /dev/null @@ -1,11 +0,0 @@ - - - - \ No newline at end of file From 0361ec9bdd64a2c5ccf0d68e7a75358b782a0851 Mon Sep 17 00:00:00 2001 From: pengmengsi 90080 Date: Mon, 7 Jun 2021 15:13:48 +0800 Subject: [PATCH 065/434] datasource update --- .../module/datasourceToolbar/index.vue | 2 - .../module/sourceTablePanel/index.vue | 140 ---------------- .../streamis/module/tableFieldsList/index.vue | 54 ++----- .../apps/streamis/module/treeSource/index.vue | 21 ++- .../apps/streamis/view/dataSource/index.vue | 153 +++++++++++++++--- .../view/realTimeDataSource/index.vue | 11 -- 6 files changed, 152 insertions(+), 229 deletions(-) delete mode 100644 web/src/apps/streamis/module/sourceTablePanel/index.vue delete mode 100644 web/src/apps/streamis/view/realTimeDataSource/index.vue diff --git a/web/src/apps/streamis/module/datasourceToolbar/index.vue b/web/src/apps/streamis/module/datasourceToolbar/index.vue index f0a426dde..ec2f1f90c 100644 --- a/web/src/apps/streamis/module/datasourceToolbar/index.vue +++ b/web/src/apps/streamis/module/datasourceToolbar/index.vue @@ -41,8 +41,6 @@ export default { \ No newline at end of file diff --git a/web/src/apps/streamis/module/tableFieldsList/index.vue b/web/src/apps/streamis/module/tableFieldsList/index.vue index 63175ef03..eb139b72d 100644 --- a/web/src/apps/streamis/module/tableFieldsList/index.vue +++ b/web/src/apps/streamis/module/tableFieldsList/index.vue @@ -98,8 +98,7 @@ diff --git a/web/src/apps/streamis/module/treeSource/index.vue b/web/src/apps/streamis/module/treeSource/index.vue index 240b23305..4fc4ad4ff 100644 --- a/web/src/apps/streamis/module/treeSource/index.vue +++ b/web/src/apps/streamis/module/treeSource/index.vue @@ -161,9 +161,6 @@ export default { diff --git a/web/src/apps/streamis/view/realTimeDataSource/index.vue b/web/src/apps/streamis/view/realTimeDataSource/index.vue deleted file mode 100644 index b98d9d525..000000000 --- a/web/src/apps/streamis/view/realTimeDataSource/index.vue +++ /dev/null @@ -1,11 +0,0 @@ - - - - \ No newline at end of file From a2eca3aab757759bab9134b02d83ebefcdda37ff Mon Sep 17 00:00:00 2001 From: kongslove <42604208+kongslove@users.noreply.github.com> Date: Wed, 9 Jun 2021 15:42:43 +0800 Subject: [PATCH 066/434] add transfer and manager --- streamis-datasource/pom.xml | 4 + .../streamis-datasource-manager/pom.xml | 6 + .../dao/StreamisDatasourceExtraInfoDao.java | 17 ++ .../dao/StreamisDatasourceExtraUiDao.java | 17 ++ .../impl/StreamisDatasourceExtraInfoDao.xml | 18 ++ .../dao/impl/StreamisDatasourceExtraUiDao.xml | 29 +++ .../domain/StreamisDataSourceTableVO.java | 32 +++ .../domain/StreamisDatasourceExtraInfo.java | 78 ++++++ .../domain/StreamisDatasourceExtraUi.java | 194 +++++++++++++++ .../restful/api/StreamisTableMetaApi.java | 234 +++++++++++++++--- .../StreamisDatasourceExtraInfoService.java | 17 ++ .../StreamisDatasourceExtraUiService.java | 17 ++ ...treamisDatasourceExtraInfoServiceImpl.java | 21 ++ .../StreamisDatasourceExtraUiServiceImpl.java | 21 ++ .../streamis-datasource-server/pom.xml | 15 ++ .../DataSourceServer2managerRestfulApi.java | 30 ++- .../api/DataSourceTransferRestfulApi.java | 188 ++++++++++++-- .../transfer/entity/StreamisTableEntity.java | 5 + .../transfer/service/DataSourceTransfer.java | 2 +- .../service/impl/DataSourceTransferImpl.java | 42 +++- .../client/LinkisDataSourceClient.scala | 84 ++++++- 21 files changed, 1002 insertions(+), 69 deletions(-) create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceExtraInfoDao.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceExtraUiDao.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraInfoDao.xml create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraUiDao.xml create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDataSourceTableVO.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceExtraInfo.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceExtraUi.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceExtraInfoService.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceExtraUiService.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceExtraInfoServiceImpl.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceExtraUiServiceImpl.java diff --git a/streamis-datasource/pom.xml b/streamis-datasource/pom.xml index c941ed67d..0df441d56 100644 --- a/streamis-datasource/pom.xml +++ b/streamis-datasource/pom.xml @@ -9,6 +9,10 @@ 4.0.0 + + 1.0.0-RC1 + + streamis-datasource pom diff --git a/streamis-datasource/streamis-datasource-manager/pom.xml b/streamis-datasource/streamis-datasource-manager/pom.xml index 5d3cbe7f0..6ddfed869 100644 --- a/streamis-datasource/streamis-datasource-manager/pom.xml +++ b/streamis-datasource/streamis-datasource-manager/pom.xml @@ -54,6 +54,12 @@ linkis-mybatis ${linkis.version} + + com.webank.wedatasphere.streamis + streamis-datasource-transfer + 0.1.0-SNAPSHOT + compile + diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceExtraInfoDao.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceExtraInfoDao.java new file mode 100644 index 000000000..f1afc19a5 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceExtraInfoDao.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.streamis.datasource.manager.dao; + + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraInfo; + +/** + *

+ * Mapper 接口 + *

+ * + * @author c01013 + * @since 2021-06-04 + */ +public interface StreamisDatasourceExtraInfoDao extends BaseMapper { + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceExtraUiDao.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceExtraUiDao.java new file mode 100644 index 000000000..4aaa96c92 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceExtraUiDao.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.streamis.datasource.manager.dao; + + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraUi; + +/** + *

+ * Mapper 接口 + *

+ * + * @author c01013 + * @since 2021-06-04 + */ +public interface StreamisDatasourceExtraUiDao extends BaseMapper { + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraInfoDao.xml b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraInfoDao.xml new file mode 100644 index 000000000..0d8b1e773 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraInfoDao.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + id, key, value, streamis_table_meta_id + + + diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraUiDao.xml b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraUiDao.xml new file mode 100644 index 000000000..5f15d5831 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraUiDao.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + id, datasource_type, key, description, description_en, lable_name, lable_name_en, ui_type, required, value, default_value, is_hidden, condition, is_advanced, order + + + diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDataSourceTableVO.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDataSourceTableVO.java new file mode 100644 index 000000000..14f1cdc08 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDataSourceTableVO.java @@ -0,0 +1,32 @@ +package com.webank.wedatasphere.streamis.datasource.manager.domain; + + +public class StreamisDataSourceTableVO { + private String tableName; + private Boolean isStreamisDataSource; + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public Boolean getStreamisDataSource() { + return isStreamisDataSource; + } + + public void setStreamisDataSource(Boolean streamisDataSource) { + isStreamisDataSource = streamisDataSource; + } + + + @Override + public String toString() { + return "StreamisDataSourceTableVO{" + + "tableName='" + tableName + '\'' + + ", isStreamisDataSource=" + isStreamisDataSource + + '}'; + } +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceExtraInfo.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceExtraInfo.java new file mode 100644 index 000000000..6ac566e90 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceExtraInfo.java @@ -0,0 +1,78 @@ +package com.webank.wedatasphere.streamis.datasource.manager.domain; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.activerecord.Model; + +import java.io.Serializable; + +/** + *

+ * + *

+ * + * @author c01013 + * @since 2021-06-04 + */ +@TableName("streamis_datasource_extra_info") +public class StreamisDatasourceExtraInfo extends Model { + + private static final long serialVersionUID = 1L; + + @TableId(value = "id", type = IdType.AUTO) + private Integer id; + private String key; + private String value; + @TableField("streamis_table_meta_id") + private Long streamisTableMetaId; + + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public Long getStreamisTableMetaId() { + return streamisTableMetaId; + } + + public void setStreamisTableMetaId(Long streamisTableMetaId) { + this.streamisTableMetaId = streamisTableMetaId; + } + + @Override + protected Serializable pkVal() { + return this.id; + } + + @Override + public String toString() { + return "StreamisDatasourceExtraInfo{" + + ", id=" + id + + ", key=" + key + + ", value=" + value + + ", streamisTableMetaId=" + streamisTableMetaId + + "}"; + } +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceExtraUi.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceExtraUi.java new file mode 100644 index 000000000..5b099ee13 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceExtraUi.java @@ -0,0 +1,194 @@ +package com.webank.wedatasphere.streamis.datasource.manager.domain; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.activerecord.Model; +import java.io.Serializable; + +/** + *

+ * + *

+ * + * @author c01013 + * @since 2021-06-04 + */ +@TableName("streamis_datasource_extra_ui") +public class StreamisDatasourceExtraUi extends Model { + + private static final long serialVersionUID = 1L; + + @TableId(value = "id", type = IdType.AUTO) + private Integer id; + @TableField("datasource_type") + private String datasourceType; + private String key; + private String description; + @TableField("description_en") + private String descriptionEn; + @TableField("lable_name") + private String lableName; + @TableField("lable_name_en") + private String lableNameEn; + @TableField("ui_type") + private String uiType; + private Integer required; + private String value; + @TableField("default_value") + private String defaultValue; + @TableField("is_hidden") + private Integer isHidden; + private String condition; + @TableField("is_advanced") + private Integer isAdvanced; + private Integer order; + + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getDatasourceType() { + return datasourceType; + } + + public void setDatasourceType(String datasourceType) { + this.datasourceType = datasourceType; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getDescriptionEn() { + return descriptionEn; + } + + public void setDescriptionEn(String descriptionEn) { + this.descriptionEn = descriptionEn; + } + + public String getLableName() { + return lableName; + } + + public void setLableName(String lableName) { + this.lableName = lableName; + } + + public String getLableNameEn() { + return lableNameEn; + } + + public void setLableNameEn(String lableNameEn) { + this.lableNameEn = lableNameEn; + } + + public String getUiType() { + return uiType; + } + + public void setUiType(String uiType) { + this.uiType = uiType; + } + + public Integer getRequired() { + return required; + } + + public void setRequired(Integer required) { + this.required = required; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(String defaultValue) { + this.defaultValue = defaultValue; + } + + public Integer getIsHidden() { + return isHidden; + } + + public void setIsHidden(Integer isHidden) { + this.isHidden = isHidden; + } + + public String getCondition() { + return condition; + } + + public void setCondition(String condition) { + this.condition = condition; + } + + public Integer getIsAdvanced() { + return isAdvanced; + } + + public void setIsAdvanced(Integer isAdvanced) { + this.isAdvanced = isAdvanced; + } + + public Integer getOrder() { + return order; + } + + public void setOrder(Integer order) { + this.order = order; + } + + @Override + protected Serializable pkVal() { + return this.id; + } + + @Override + public String toString() { + return "StreamisDatasourceExtraUi{" + + ", id=" + id + + ", datasourceType=" + datasourceType + + ", key=" + key + + ", description=" + description + + ", descriptionEn=" + descriptionEn + + ", lableName=" + lableName + + ", lableNameEn=" + lableNameEn + + ", uiType=" + uiType + + ", required=" + required + + ", value=" + value + + ", defaultValue=" + defaultValue + + ", isHidden=" + isHidden + + ", condition=" + condition + + ", isAdvanced=" + isAdvanced + + ", order=" + order + + "}"; + } +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java index 81409604c..d8088e54c 100644 --- a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java @@ -18,13 +18,21 @@ import com.baomidou.mybatisplus.core.conditions.Wrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; +import com.baomidou.mybatisplus.extension.conditions.update.UpdateChainWrapper; import com.webank.wedatasphere.linkis.common.exception.ErrorException; -import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceFields; -import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisTableMeta; +import com.webank.wedatasphere.linkis.datasource.client.response.MetadataGetDatabasesResult; +import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSource; +import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceType; +import com.webank.wedatasphere.streamis.datasource.manager.domain.*; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceAuthorityService; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceExtraInfoService; import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceFieldsService; import com.webank.wedatasphere.linkis.server.Message; import com.webank.wedatasphere.linkis.server.security.SecurityFilter; import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisTableMetaService; +import com.webank.wedatasphere.streamis.datasource.transfer.client.LinkisDataSourceClient; +import org.apache.commons.collections.CollectionUtils; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; @@ -38,7 +46,6 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; import java.util.*; - @Service public class StreamisTableMetaApi { @@ -49,40 +56,99 @@ public class StreamisTableMetaApi { @Autowired private StreamisDatasourceFieldsService streamisDatasourceFieldsService; + + @Autowired + private StreamisDatasourceAuthorityService streamisDatasourceAuthorityService; + + @Autowired + StreamisDatasourceExtraInfoService streamisDatasourceExtraInfoService; ObjectMapper mapper = new ObjectMapper(); - /** - * 获取左侧数据源树，需要调用API获取linkis数据源，还需获取streamis数据源 - */ - public Response getStreamisDataSourceList( HttpServletRequest req, Map json + + + + public Response getDataSourceCluster( HttpServletRequest req, Map json ) throws ErrorException { Message message = null; try{ String userName = SecurityFilter.getLoginUsername(req); - String workSpaceName = (String) json.get("workSpaceName"); - String projectName = (String) json.get("projectName"); - String task = (String) json.get("task"); - String dataSourceType = (String) json.get("dataSourceType"); - //TODO: invoke rpc interface to get linkisDataSource Information - - - StreamisTableMeta streamisTableMeta = streamisTableMetaService.getById(1); - logger.warn(streamisTableMeta.toString()); - QueryWrapper wrapper = new QueryWrapper<>(); - wrapper.eq("id",1); - StreamisTableMeta one = streamisTableMetaService.getOne(wrapper); + String system = json.get("system"); + String name = json.get("name"); + Long dataSourceTypeId = Long.valueOf(json.get("dataSourceTypeId")); + List dataSourceClusters = getDataSourceClusters(system, name, dataSourceTypeId, userName); + message = Message.ok().data("dataSourceCluster",dataSourceClusters); }catch (Throwable e){ - logger.error("",e); - throw new ErrorException(30001, "抱歉，后台获取数据源列表失败"); + logger.error("failed to get dataSourceType list",e); + throw new ErrorException(30016, "sorry,failed to get dataSource list"); } return Message.messageToResponse(message); + } + public Response getDataSourceTypes( HttpServletRequest req, Map json + ) throws ErrorException { + Message message = null; + try{ + String userName = SecurityFilter.getLoginUsername(req); + List dataSourceTypes = getDataSourceTypes(userName); + message = Message.ok().data("dataSourceTypes",dataSourceTypes); + }catch (Throwable e){ + logger.error("failed to get dataSourceType list",e); + throw new ErrorException(30015, "sorry,failed to get dataSourceType list"); + } + return Message.messageToResponse(message); } + public Response getDataBasesByCuster( HttpServletRequest req, Map json + ) throws ErrorException { + Message message = null; + try{ + String userName = SecurityFilter.getLoginUsername(req); + String system = json.get("system"); + Long dataSourceTypeId = Long.valueOf(json.get("dataSourceId")); + List dataBases = queryDataBasesByCuster(system, dataSourceTypeId, userName); + message = Message.ok().data("dataBases",dataBases); + }catch (Throwable e){ + logger.error("failed to get dataBases list",e); + throw new ErrorException(30017, "sorry,failed to get dataBases list"); + } + return Message.messageToResponse(message); + } + + public Response getTablesByDataBase( HttpServletRequest req, Map json + ) throws ErrorException { + Message message = null; + try{ + String userName = SecurityFilter.getLoginUsername(req); + String system = json.get("system"); + Long dataSourceId = Long.valueOf(json.get("dataSourceId")); + String dataBase = json.get("dataBase"); + QueryWrapper wrapper = new QueryWrapper<>();; + List tables = queryTablesByDataBase(system, dataSourceId, dataBase,userName); + List tableList = new ArrayList<>(); + StreamisDataSourceTableVO tableVO = null; + wrapper.eq("node_name",dataBase); + List list = streamisTableMetaService.list(wrapper); + + for (String table : tables) { + for (StreamisTableMeta streamisTableMeta : list) { + tableVO = new StreamisDataSourceTableVO(); + tableVO.setTableName(table); + tableVO.setStreamisDataSource(table.equals(streamisTableMeta.getTableName())); + tableList.add(tableVO); + } + + } + message = Message.ok().data("tables",tableList); + }catch (Throwable e){ + logger.error("failed to get tables list",e); + throw new ErrorException(30018, "sorry,failed to get tables list"); + } + return Message.messageToResponse(message); + } public Response getStreamisTableMeta( HttpServletRequest req, Long streamisTableMetaId - ) throws ErrorException { + ) throws ErrorException { Message message = null; try{ StreamisTableMeta streamisTableMeta = streamisTableMetaService.getById(streamisTableMetaId); @@ -92,7 +158,7 @@ public Response getStreamisTableMeta( HttpServletRequest req, Long streamisTable message=Message.ok().data("streamisTableMeta",streamisTableMeta).data("streamisDatasourceFields",streamisDatasourceFields); }catch (Throwable e){ logger.error("Failed to get streamisTableMetaInfo",e); - throw new ErrorException(30002, "抱歉，后台获取数据源详细信息失败"); + throw new ErrorException(30002, "Failed to get streamisTableMetaInfo(后台获取数据源详细信息失败)"); } return Message.messageToResponse(message); @@ -107,6 +173,10 @@ public Response getStreamisTableMetaByNodeNames( HttpServletRequest req,JsonNode try{ List nodeNames = mapper.readValue(json.get("nodeNames"), new TypeReference>() {}); + if(CollectionUtils.isEmpty(nodeNames)){ + logger.error("Failed to get streamisDataSource Tree,node_name is null"); + throw new ErrorException(30012, "Failed to get streamisDataSource Tree,node_name is null"); + } for (String nodeName : nodeNames) { wrapper = new QueryWrapper<>(); wrapper.eq("node_name",nodeName); @@ -116,8 +186,9 @@ public Response getStreamisTableMetaByNodeNames( HttpServletRequest req,JsonNode message = Message.ok().data("nodeTables", nodeTablesMap); }catch (Throwable e){ + logger.error("Failed to get streamisDataSource Tree",e); - throw new ErrorException(30011, "抱歉，后台获取左侧数据源树失败"); + throw new ErrorException(30011, e.getMessage()); } return Message.messageToResponse(message); @@ -128,24 +199,51 @@ public Response addStreamisTableMeta( HttpServletRequest req, JsonNode json) th Message message = null; try { String userName = SecurityFilter.getLoginUsername(req); + String authorityId = json.get("authorityId").asText(); + StreamisTableMeta streamisTableMeta = mapper.readValue(json.get("streamisTableMeta"), StreamisTableMeta.class); //第一次保存linkisDataSource数据源 streamisTableMeta.setCreateBy(userName); boolean result = streamisTableMetaService.save(streamisTableMeta); if(!result){ - throw new ErrorException(30003, "抱歉，添加streamis数据源失败"); + throw new ErrorException(30003, "Sorry, failed to add streamis data source(抱歉，添加streamis数据源失败_"); } Long streamisTableMetaId = streamisTableMeta.getId(); + //add StreamisDatasourceFields List fieldsList = mapper.readValue(json.get("streamisTableFields"), new TypeReference>() { }); fieldsList.forEach(obj ->obj.setStreamisTableMetaId(streamisTableMetaId)); - boolean b = streamisDatasourceFieldsService.saveBatch(fieldsList); + result = streamisDatasourceFieldsService.saveBatch(fieldsList); + if(!result){ + throw new ErrorException(30013, "Sorry, failed to add StreamisTableFields table field information(抱歉，添加StreamisTableFields表字段信息失败)"); + } +// String scope = streamisTableMeta.getScope(); + StreamisDatasourceAuthority streamisDatasourceAuthority = new StreamisDatasourceAuthority(); + + streamisDatasourceAuthority.setAuthorityScope(streamisTableMeta.getScope()); + streamisDatasourceAuthority.setStreamisTableMetaId(streamisTableMetaId); + streamisDatasourceAuthority.setAuthorityId(authorityId); + streamisDatasourceAuthority.setGrantUser("*"); + result = streamisDatasourceAuthorityService.save(streamisDatasourceAuthority); + if(!result){ + throw new ErrorException(30014, " sorry, add streamisDatasourceAuthority information failure (抱歉，添加streamisDatasourceAuthority信息失败)"); + } + List extraInfoList = mapper.readValue(json.get("streamisExtraInfo"), new TypeReference>() { + }); + + extraInfoList.forEach(obj-> obj.setStreamisTableMetaId(streamisTableMetaId)); + + result = streamisDatasourceExtraInfoService.saveBatch(extraInfoList); + if(!result){ + throw new ErrorException(30019, " sorry, add extraInfo information failure"); + } + //add to StreamisDatasourceAuthority message = Message.ok().data("streamisTableMetaId",streamisTableMetaId); } catch (Exception e) { logger.error("Failed to add StreamisTableMeta: ", e); - throw new ErrorException(30003, "抱歉，添加streamis数据源失败"); + throw new ErrorException(30003, e.getMessage()); } return Message.messageToResponse(message); } @@ -158,16 +256,31 @@ public Response updateStreamisTableMeta( HttpServletRequest req, JsonNode json) StreamisTableMeta streamisTableMeta = mapper.readValue(json.get("streamisTableMeta"), StreamisTableMeta.class); Long streamisTableMetaId = streamisTableMeta.getId(); if(null == streamisTableMetaId ){ - throw new ErrorException(30004, "抱歉，更新streamis数据源信息失败，streamisTableMetaId is null"); + throw new ErrorException(30004, "sorry，Failed to update streamis data source information(更新streamis数据源信息失败)，streamisTableMetaId is null"); } streamisTableMeta.setUpdateBy(userName); // update StreamisTableMeta streamisTableMetaService.updateById(streamisTableMeta); + + List extraInfoList = mapper.readValue(json.get("streamisExtraInfo"), new TypeReference>() { + }); + + extraInfoList.forEach(obj-> obj.setStreamisTableMetaId(streamisTableMetaId)); + UpdateWrapper wrapper = null; + boolean result ; + for (StreamisDatasourceExtraInfo extraInfo : extraInfoList) { + wrapper.eq("key",extraInfo.getKey()).eq("streamis_table_meta_id",streamisTableMetaId).set("value",extraInfo.getValue()); + result = streamisDatasourceExtraInfoService.update(null, wrapper); + if(!result){ + throw new ErrorException(30020, String.format("sorry，Failed to update the %s information(更新%s失败)",extraInfo.getKey(),extraInfo.getKey())); + } + } + message = Message.ok(); } catch (Exception e) { logger.error("Failed to update StreamisTableMeta: ", e); - throw new ErrorException(30004, "抱歉，更新streamis数据源信息失败"); + throw new ErrorException(30004,e.getMessage()); } return Message.messageToResponse(message); } @@ -180,10 +293,16 @@ public Response deleteStreamisTableMeta(HttpServletRequest req, Long id) throws streamisDatasourceFieldsQueryWrapper.eq("streamis_table_meta_id",id); boolean remove = streamisDatasourceFieldsService.remove(streamisDatasourceFieldsQueryWrapper); if (!remove){ - throw new ErrorException(30008, String.format("抱歉，删除streamis数据源 表[%s]字段信息失败",id)); + throw new ErrorException(30008, String.format("Sorry, failed to delete the streamis data source table [%s] field information(抱歉，删除streamis数据源 表[%s]字段信息失败)",id,id)); + } + QueryWrapper datasourceAuthorityQueryWrapper = new QueryWrapper<>(); + datasourceAuthorityQueryWrapper.eq("streamis_table_meta_id",id); + remove = streamisDatasourceAuthorityService.remove(datasourceAuthorityQueryWrapper); + if (!remove){ + throw new ErrorException(30015, String.format("抱歉，删除streamisDatasourceAuthority数据源 表[%s]字段信息失败",id)); } // delete StreamisTableMeta - boolean result = streamisDatasourceFieldsService.removeById(id); + boolean result = streamisTableMetaService.removeById(id); if (!result){ throw new ErrorException(30009, String.format("抱歉，删除streamis数据源 表[%s]失败",id)); } @@ -191,7 +310,7 @@ public Response deleteStreamisTableMeta(HttpServletRequest req, Long id) throws } catch (Exception e) { logger.error("Failed to delete StreamisTableMeta: ", e); - throw new ErrorException(30010, "抱歉，删除streamis数据源信息失败"); + throw new ErrorException(30010, e.getMessage()); } return Message.messageToResponse(message); } @@ -208,7 +327,7 @@ public Response addStreamisTableFields( HttpServletRequest req, JsonNode json) message = Message.ok(); } catch (Exception e) { logger.error("Failed to add streamisDatasourceFields: ", e); - throw new ErrorException(30005, "抱歉，添加StreamisTableFields表字段信息失败"); + throw new ErrorException(30005, e.getMessage()); } return Message.messageToResponse(message); } @@ -241,4 +360,51 @@ public Response deleteStreamisTableFields( HttpServletRequest req, Long id) thro } -} + /** + * 获取数据源类型 + * @return 数据源类型列表 + */ + public List getDataSourceTypes(String userName){ + return LinkisDataSourceClient.queryDataSourceTypes(userName); + } + + /** + * + * @param system system + * @param name username + * @param typeId typeId + * @param user hadoop + * @return 获取数据源集群 + */ + public List getDataSourceClusters(String system,String name,Long typeId ,String user){ + return LinkisDataSourceClient.queryClusterByDataSourceType(system,name,typeId,user); + } + + /** + * 获取数据库列表 + * @param system system + * @param dataSourceId dataSourceId + * @param user hadoop + * @return list + */ + public List queryDataBasesByCuster(String system, Long dataSourceId , String user){ + return LinkisDataSourceClient.queryDataBasesByCuster(system,dataSourceId,user).dbs(); + } + + /** + * 获取table列表 + * @param system system + * @param dataSourceId dataSourceId + * @param dataBase dataBase + * @param user user + * @return * @return + */ + public List queryTablesByDataBase(String system, Long dataSourceId , String dataBase,String user){ + return LinkisDataSourceClient.queryTablesByDataBase(system,dataSourceId,dataBase ,user).tables(); + } + + + + + +} \ No newline at end of file diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceExtraInfoService.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceExtraInfoService.java new file mode 100644 index 000000000..35056d4a6 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceExtraInfoService.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.streamis.datasource.manager.service; + + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraInfo; + +/** + *

+ * 服务类 + *

+ * + * @author c01013 + * @since 2021-06-04 + */ +public interface StreamisDatasourceExtraInfoService extends IService { + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceExtraUiService.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceExtraUiService.java new file mode 100644 index 000000000..205e6ecfa --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceExtraUiService.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.streamis.datasource.manager.service; + + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraUi; + +/** + *

+ * 服务类 + *

+ * + * @author c01013 + * @since 2021-06-04 + */ +public interface StreamisDatasourceExtraUiService extends IService { + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceExtraInfoServiceImpl.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceExtraInfoServiceImpl.java new file mode 100644 index 000000000..6b4fd8ff2 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceExtraInfoServiceImpl.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.streamis.datasource.manager.service.impl; + + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.streamis.datasource.manager.dao.StreamisDatasourceExtraInfoDao; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraInfo; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceExtraInfoService; +import org.springframework.stereotype.Service; + +/** + *

+ * 服务实现类 + *

+ * + * @author c01013 + * @since 2021-06-04 + */ +@Service +public class StreamisDatasourceExtraInfoServiceImpl extends ServiceImpl implements StreamisDatasourceExtraInfoService { + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceExtraUiServiceImpl.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceExtraUiServiceImpl.java new file mode 100644 index 000000000..10b57bcc3 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceExtraUiServiceImpl.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.streamis.datasource.manager.service.impl; + + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.streamis.datasource.manager.dao.StreamisDatasourceExtraUiDao; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraUi; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceExtraUiService; +import org.springframework.stereotype.Service; + +/** + *

+ * 服务实现类 + *

+ * + * @author c01013 + * @since 2021-06-04 + */ +@Service +public class StreamisDatasourceExtraUiServiceImpl extends ServiceImpl implements StreamisDatasourceExtraUiService { + +} diff --git a/streamis-datasource/streamis-datasource-server/pom.xml b/streamis-datasource/streamis-datasource-server/pom.xml index e3bda5bfd..f95280fb9 100644 --- a/streamis-datasource/streamis-datasource-server/pom.xml +++ b/streamis-datasource/streamis-datasource-server/pom.xml @@ -27,6 +27,21 @@ linkis-mybatis ${linkis.version} + + com.webank.wedatasphere.linkis + linkis-bmlserver + ${linkis.version} + + + com.webank.wedatasphere.linkis + linkis-storage + ${linkis.version} + + + com.webank.wedatasphere.dss + dss-server + ${dss.version} + com.webank.wedatasphere.streamis streamis-datasource-transfer diff --git a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java index c070d2ec0..fcf32ceb6 100644 --- a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java +++ b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java @@ -23,14 +23,32 @@ public class DataSourceServer2managerRestfulApi { private StreamisTableMetaApi streamisTableMetaApi; - /** - * 获取左侧数据源树，需要调用API获取linkis数据源，还需获取streamis数据源 - */ @POST - @Path("streamisDataSourceList") - public Response getStreamisDataSourceList(@Context HttpServletRequest req, Map json + @Path("dataSourceType") + public Response getDataSourceType(@Context HttpServletRequest req, Map json + ) throws ErrorException { + return streamisTableMetaApi.getDataSourceTypes(req, json); + } + + @POST + @Path("dataSourceCluster") + public Response getDataSourceCluster(@Context HttpServletRequest req, Map json + ) throws ErrorException { + return streamisTableMetaApi.getDataSourceCluster(req,json); + } + + @POST + @Path("dataBases") + public Response getDataBasesByCuster(@Context HttpServletRequest req, Map json + ) throws ErrorException { + return streamisTableMetaApi.getDataBasesByCuster(req,json); + } + + @POST + @Path("tables") + public Response getTablesByDataBase(@Context HttpServletRequest req, Map json ) throws ErrorException { - return streamisTableMetaApi.getStreamisDataSourceList(req, json); + return streamisTableMetaApi.getTablesByDataBase(req,json); } /** diff --git a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java index 1972a0749..46f12286e 100644 --- a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java +++ b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java @@ -1,59 +1,219 @@ package com.webank.wedatasphere.streamis.datasource.server.restful.api; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.google.gson.Gson; +import com.webank.wedatasphere.dss.server.service.BMLService; +import com.webank.wedatasphere.linkis.bml.Entity.Resource; +import com.webank.wedatasphere.linkis.bml.Entity.ResourceVersion; +import com.webank.wedatasphere.linkis.bml.common.*; +import com.webank.wedatasphere.linkis.bml.dao.ResourceDao; +import com.webank.wedatasphere.linkis.bml.dao.VersionDao; +import com.webank.wedatasphere.linkis.bml.restful.RestfulUtils; +import com.webank.wedatasphere.linkis.bml.service.ResourceService; +import com.webank.wedatasphere.linkis.bml.service.VersionService; +import com.webank.wedatasphere.linkis.bml.util.HttpRequestHelper; import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.common.io.Fs; +import com.webank.wedatasphere.linkis.common.io.FsPath; import com.webank.wedatasphere.linkis.server.Message; +import com.webank.wedatasphere.linkis.server.security.SecurityFilter; +import com.webank.wedatasphere.linkis.storage.FSFactory; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceFields; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisTableMeta; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceFieldsService; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisTableMetaService; import com.webank.wedatasphere.streamis.datasource.transfer.entity.StreamisDataSourceCode; import com.webank.wedatasphere.streamis.datasource.transfer.entity.StreamisTableEntity; import com.webank.wedatasphere.streamis.datasource.transfer.service.DataSourceTransfer; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; +import org.codehaus.jackson.JsonNode; import org.glassfish.jersey.media.multipart.FormDataParam; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import scala.tools.cmd.Opt; +import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.QueryParam; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; -import java.util.HashMap; -import java.util.Map; +import java.io.*; +import java.util.*; +@Component +@Path("streamis") public class DataSourceTransferRestfulApi { private static final Logger logger = LoggerFactory.getLogger(DataSourceTransferRestfulApi.class); @Autowired private DataSourceTransfer dataSourceTransfer; + @Autowired + private StreamisTableMetaService streamisTableMetaService; + @Autowired + private StreamisDatasourceFieldsService streamisDatasourceFieldsService; + + @Autowired + private ResourceDao resourceDao; + @Autowired + private VersionDao versionDao; + @Autowired + private ResourceService resourceService; + @Autowired + private VersionService versionService; + @GET @Path("/transfer") - public Response transfer(@Context HttpServletRequest request, @QueryParam("streamisTableMetaId") String streamisTableMetaId, @FormDataParam("labels") Map labels) throws ErrorException { + public Response transfer(@Context HttpServletRequest request, @QueryParam("streamisTableMetaId") String streamisTableMetaId, @QueryParam("dataSourceId") String dataSource,@QueryParam("nodeName") String nodeName, @FormDataParam("labels") Map labels) throws ErrorException { Message message; if(StringUtils.isBlank(streamisTableMetaId)){ throw new ErrorException(30201,"streamisTableMetaId is null."); } + String userName = SecurityFilter.getLoginUsername(request); + Map params = new HashMap<>(); try { + + //TODO: 如何获取runType engineType version + params.put("nodeName",nodeName); + params.put("runType","sql"); + params.put("engineType","flink"); + params.put("version","1.12.2"); + Long dataSourceId = Long.parseLong(dataSource); //通过streamisTableMetaId拿到StreamisTableMeta实体 - StreamisTableEntity streamisTableEntity = dataSourceTransfer.getStreamisTableMetaById(streamisTableMetaId); + StreamisTableEntity streamisTableEntity = dataSourceTransfer.getStreamisTableMetaById(streamisTableMetaId,userName,dataSourceId); + //转换为FlinkSql - StreamisDataSourceCode streamisDataSourceCode = dataSourceTransfer.transfer(streamisTableEntity, labels, new HashMap()); + StreamisDataSourceCode streamisDataSourceCode = dataSourceTransfer.transfer(streamisTableEntity, labels, params); message = Message.ok().data("streamisDataSourceCode",streamisDataSourceCode); } catch (Exception e) { - logger.error("传入streamisTableMetaId {} 数据源转换失败：", streamisTableMetaId, e); - throw new ErrorException(30202, "抱歉，后台数据源转换失败"); + logger.error("传入streamisTableMetaId {} data source conversion failed：", streamisTableMetaId, e); + throw new ErrorException(30202, "Sorry, background data source conversion failed(后台数据源转换失败)"); } return Message.messageToResponse(message); } + //上传数据源到hdfs @POST @Path("/exportStreamisTableMeta") - public Response exportStreamisTableMeta(@Context HttpServletRequest request, @FormDataParam("streamisTableMetaId") String streamisTableMetaId){ - Message message; + public Response exportStreamisTableMeta(@Context HttpServletRequest request, JsonNode jsonNode) throws ErrorException{ + Message message = new Message(); + String user = RestfulUtils.getUserName(request); + String clientIp = HttpRequestHelper.getIp(request); + Map streamisEntity = new HashMap<>(); + ResourceHelper resourceHelper = ResourceHelperFactory.getResourceHelper(); + //通过streamisTableMetaId拿到StreamisTableMeta实体 + String streamisTableMetaId = jsonNode.get("streamisTableMetaId").asText(); + StreamisTableMeta streamisTableMeta = streamisTableMetaService.getById(streamisTableMetaId); + + QueryWrapper streamisDatasourceFieldsWrapper = new QueryWrapper<>(); + streamisDatasourceFieldsWrapper.eq("streamis_table_meta_id",streamisTableMetaId); + List streamisDatasourceFields = streamisDatasourceFieldsService.list(streamisDatasourceFieldsWrapper); + + streamisEntity.put("streamisTableMeta",streamisTableMeta); + streamisEntity.put("streamisDatasourceFields",streamisDatasourceFields); + Gson gson = new Gson(); + String json = gson.toJson(streamisEntity); + + if(json != null) { + String resourceId = UUID.randomUUID().toString(); + String fileName = UUID.randomUUID().toString() + ".json"; + String path = resourceHelper.generatePath(user, fileName, new HashMap()); + InputStream in_nocode = new ByteArrayInputStream(json.getBytes()); + StringBuilder sb = new StringBuilder(); + //上传资源到HDFS + long size = resourceHelper.upload(path, user, in_nocode, sb, true); + String md5String = sb.toString(); + boolean isSuccess = false; + if (StringUtils.isNotEmpty(md5String) && size >= 0) { + isSuccess = true; + } + Resource resource = Resource.createNewResource(resourceId, user, fileName, new HashMap()); + //插入一条记录到resource表 + long id = resourceDao.uploadResource(resource); + logger.info("{} uploaded a resource and resourceId is {}", user, resource.getResourceId()); + //插入一条记录到resource version表 + //TODO 版本是否固定为v000001？ + ResourceVersion resourceVersion = ResourceVersion.createNewResourceVersion(resourceId, path, md5String, + clientIp, size, Constant.FIRST_VERSION, 1); + versionDao.insertNewVersion(resourceVersion); + + message = Message.ok().data("resourceId",resourceId).data("version",resourceVersion.getVersion()); + } + + return Message.messageToResponse(message); + } - return Message.messageToResponse(Message.ok()); + //从hdfs下载数据源，插入数据库 + @POST + @Path("/importStreamisTableMeta") + public Response importStreamisTableMeta(@Context HttpServletRequest request, @Context HttpServletResponse resp,JsonNode jsonNode ) + throws ErrorException, IOException { + Message message = new Message(); + String user = RestfulUtils.getUserName(request); + String resourceId = jsonNode.get("resourceId").asText(); + String version = jsonNode.get("version").asText(); + if (StringUtils.isBlank(resourceId) || !resourceService.checkResourceId(resourceId)) { + message = Message.error("resourceId:"+resourceId+"为空,非法或者已被删除!"); + message.setStatus(1); + return Message.messageToResponse(message); + } + if (!resourceService.checkAuthority(user, resourceId)){ + throw new BmlPermissionDeniedException("您没有权限下载此资源"); + } + //判version空,返回最新版本 + if (StringUtils.isBlank(version)){ + version = versionService.getNewestVersion(resourceId); + } + //判version不存在或者非法 + if (!versionService.checkVersion(resourceId, version)) { + message = Message.error("version:"+version+"不存在,非法或者已被删除"); + message.setStatus(1); + return Message.messageToResponse(message); + } + //判resourceId和version是否过期 + if (!resourceService.checkExpire(resourceId, version)){ + throw new BmlResourceExpiredException(resourceId); + } + + ResourceVersion resourceVersion = versionDao.findResourceVersion(resourceId, version); + long startByte = resourceVersion.getStartByte(); + long endByte = resourceVersion.getEndByte(); + String path = resourceVersion.getResource(); + Fs fileSystem = FSFactory.getFsByProxyUser(new FsPath(path), user); + fileSystem.init(new HashMap()); + InputStream inputStream = fileSystem.read(new FsPath(path)); + inputStream.skip(startByte - 1L); + logger.info("{} 下载资源 {} inputStream skipped {} bytes", new Object[]{user, resourceId, startByte - 1L}); + + try { + //将流转换为json + Gson gson = new Gson(); + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); + StringBuilder sb = new StringBuilder(); + String s = null; + while ((s = bufferedReader.readLine()) != null) { + sb.append(s); + sb.append("\n"); + } + String json = sb.toString().trim(); + //将存放入数组的数据转为map格式 + HashMap map = gson.fromJson(sb.toString(), HashMap.class); + message.data("streamisEntity",map); + } catch (IOException e) { + logger.error("用户 {} 下载资源 resourceId: {}, version:{} 出现IO异常", user, resourceId, version, e); + throw new ErrorException(73562, "抱歉,后台IO错误造成您本次下载资源失败"); + } catch (final Throwable t){ + logger.error("用户 {} 下载资源 resourceId: {}, version:{} 出现异常", user, resourceId, version, t); + throw new ErrorException(73561, "抱歉，后台服务出错导致您本次下载资源失败"); + }finally { + IOUtils.closeQuietly(inputStream); + } + return Message.messageToResponse(message); } } diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisTableEntity.java b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisTableEntity.java index 831556180..058c453e3 100644 --- a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisTableEntity.java +++ b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisTableEntity.java @@ -42,4 +42,9 @@ public StreamisTableMeta getTableInfo() { public void setTableInfo(StreamisTableMeta tableInfo) { this.tableInfo = tableInfo; } + + public String toString(){ + + return null; + } } diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/DataSourceTransfer.java b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/DataSourceTransfer.java index da8906832..35a513049 100644 --- a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/DataSourceTransfer.java +++ b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/DataSourceTransfer.java @@ -9,6 +9,6 @@ public interface DataSourceTransfer { StreamisDataSourceCode transfer(StreamisTableEntity streamisTableEntity, Map labels, Map params); - StreamisTableEntity getStreamisTableMetaById(String streamisTableMetaId) throws ErrorException; + StreamisTableEntity getStreamisTableMetaById(String tableMetaId, String userName, Long dataSourceId) throws ErrorException; } diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java index bb7ffb3f4..9ac33bec4 100644 --- a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java +++ b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java @@ -2,12 +2,13 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.webank.wedatasphere.linkis.common.exception.ErrorException; -import com.webank.wedatasphere.linkis.datasource.client.response.QueryDataSourceResult; -import com.webank.wedatasphere.linkis.manager.label.entity.Label; +import com.webank.wedatasphere.linkis.datasource.client.response.GetConnectParamsByDataSourceIdResult; import com.webank.wedatasphere.linkis.manager.label.entity.engine.EngineType; import com.webank.wedatasphere.linkis.manager.label.entity.engine.RunType; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraInfo; import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceFields; import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisTableMeta; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceExtraInfoService; import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceFieldsService; import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisTableMetaService; import com.webank.wedatasphere.streamis.datasource.transfer.client.LinkisDataSourceClient; @@ -34,12 +35,16 @@ public class DataSourceTransferImpl implements DataSourceTransfer { @Autowired StreamisDatasourceFieldsService streamisDatasourceFieldsService; + @Autowired + StreamisDatasourceExtraInfoService streamisDatasourceExtraInfoService; + @Override public StreamisDataSourceCode transfer(StreamisTableEntity streamisTableEntity, Map labels, Map params) { String runType = (String)params.get("runType"); - String engineType = (String)params.get("engingType"); + String engineType = (String)params.get("engineType"); String version = (String)params.get("version"); + //获取表名，列名 StreamisDataSourceCode streamisDataSourceCode = new StreamisDataSourceCode(); String tableName = streamisTableEntity.getTableInfo().getTableName(); @@ -51,10 +56,15 @@ public StreamisDataSourceCode transfer(StreamisTableEntity streamisTableEntity, if(EngineType.FLINK().equals(engineType) && version.equals("1.12.2") && RunType.SQL().equals(runType)){ Map linkisDataSourceContent = streamisTableEntity.getLinkisDatasource().getLinkisDataSourceContent(); + String dataSourceType = linkisDataSourceContent.get("dataSourceType").toString(); if(dataSourceType.equals("kafka")){ + String groupId = getKaflaGroupIdInfo(streamisTableEntity.getTableInfo().getId()); + linkisDataSourceContent.put("groupId",groupId); + linkisDataSourceContent.put("topicName",tableName); streamisDataSourceCode = transferKafkaSql(tableName, cols, labels, linkisDataSourceContent); }else if(dataSourceType.equals("mysql")){ + linkisDataSourceContent.put("dbName",params.get("nodeName")); streamisDataSourceCode = transferMysql(tableName, cols, labels, linkisDataSourceContent); } } @@ -62,18 +72,18 @@ public StreamisDataSourceCode transfer(StreamisTableEntity streamisTableEntity, } public StreamisDataSourceCode transferKafkaSql(String tableName, String cols, Map labels, Map linkisDataSourceContent) { - String servers = (String)linkisDataSourceContent.get("servers"); + String servers = (String)linkisDataSourceContent.get("brokers"); String topicName = (String)linkisDataSourceContent.get("topicName"); String groupId = (String)linkisDataSourceContent.get("groupId"); String offsetMode = (String)linkisDataSourceContent.getOrDefault("offsetMode","earliest-offset"); - String formatMode = (String)linkisDataSourceContent.getOrDefault("formatMode","formatMode"); + String formatMode = (String)linkisDataSourceContent.getOrDefault("formatMode","json"); String executionCode = DataSourceTransferFlinksqlUtils.kafkaDataSourceTransfer(tableName , cols , topicName , servers , groupId , offsetMode , formatMode); StreamisDataSourceCode streamisDataSourceCode = new StreamisDataSourceCode(executionCode, labels, new HashMap()); return streamisDataSourceCode; } public StreamisDataSourceCode transferMysql(String tableName, String cols, Map labels, Map linkisDataSourceContent){ - String serverIp = (String)linkisDataSourceContent.get("serverIp"); + String serverIp = (String)linkisDataSourceContent.get("host");//serverIp String dbName = (String)linkisDataSourceContent.get("dbName"); String executionCode = DataSourceTransferFlinksqlUtils.mysqlDataSourceTransfer(tableName,cols,serverIp,dbName); StreamisDataSourceCode streamisDataSourceCode = new StreamisDataSourceCode(executionCode, labels, new HashMap()); @@ -82,7 +92,7 @@ public StreamisDataSourceCode transferMysql(String tableName, String cols, Map linkisDataSourceContent = LinkisDataSourceClient.queryConnectParams(dataSourceId, "system", userName).getConnectParams(); //拿到linkis数据源里的连接信息 - Map linkisDataSourceContent = queryDataSourceResult.getQuery_list().get(0).getConnectParams(); +// Map linkisDataSourceContent = queryDataSourceResult.getQuery_list().get(0); LinkisDataSource linkisDatasource = new LinkisDataSource(linkisDataSourceUniqueId, linkisDataSourceContent); //返回实体 @@ -106,9 +117,18 @@ public StreamisTableEntity getStreamisTableMetaById(String streamisTableMetaId) streamisTableEntity.setFields(fields); streamisTableEntity.setLinkisDatasource(linkisDatasource); } catch (Exception e) { - logger.error("传入streamisTableMetaId {} 查询元数据失败：", streamisTableMetaId, e); - throw new ErrorException(30203, "抱歉，后台查询streamis元数据失败"); + logger.error("the param streamisTableMetaId {} Failed to query metadata(查询元数据失败)：", streamisTableMetaId, e); + throw new ErrorException(30203, "Sorry, the background query for streamis metadata failed(抱歉，后台查询streamis元数据失败)"); } return streamisTableEntity; } + + + public String getKaflaGroupIdInfo(Long streamisTableMetaId){ + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.eq("streamis_table_meta_id",streamisTableMetaId); + wrapper.eq("key","kafka.group.id"); + StreamisDatasourceExtraInfo extraInfo = streamisDatasourceExtraInfoService.getOne(wrapper); + return extraInfo.getValue(); + } } diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala b/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala index 9f6e6d063..c00803557 100644 --- a/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala +++ b/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala @@ -2,9 +2,10 @@ package com.webank.wedatasphere.streamis.datasource.transfer.client import java.util.concurrent.TimeUnit -import com.webank.wedatasphere.linkis.datasource.client.impl.LinkisDataSourceRemoteClient -import com.webank.wedatasphere.linkis.datasource.client.request.QueryDataSourceAction -import com.webank.wedatasphere.linkis.datasource.client.response.QueryDataSourceResult +import com.webank.wedatasphere.linkis.datasource.client.impl.{LinkisDataSourceRemoteClient, LinkisMetaDataRemoteClient} +import com.webank.wedatasphere.linkis.datasource.client.request.{GetAllDataSourceTypesAction, GetConnectParamsByDataSourceIdAction, MetadataGetDatabasesAction, MetadataGetTablesAction, QueryDataSourceAction, QueryDataSourceEnvAction} +import com.webank.wedatasphere.linkis.datasource.client.response.{GetConnectParamsByDataSourceIdResult, MetadataGetDatabasesResult, MetadataGetTablesResult, QueryDataSourceResult} +import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.{DataSource, DataSourceEnv, DataSourceType} import com.webank.wedatasphere.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfigBuilder import com.webank.wedatasphere.streamis.datasource.transfer.conf.TransferConfiguration @@ -41,6 +42,8 @@ object LinkisDataSourceClient { val dataSourceClient = new LinkisDataSourceRemoteClient(clientConfig) + val metaDataClient = new LinkisMetaDataRemoteClient(clientConfig) + def queryDataSource(linkisDatasourceName:String) : QueryDataSourceResult = { dataSourceClient.queryDataSource(QueryDataSourceAction.builder() .setSystem("") @@ -48,8 +51,83 @@ object LinkisDataSourceClient { .setTypeId(1) .setIdentifies("") .setCurrentPage(1) + .setUser("hadoop") .setPageSize(1).build() ) } + /** + * get datasourceConnect information + * @param dataSourceId id + * @param system dssSystem + * @param user username + * @return + */ + def queryConnectParams(dataSourceId:Long,system: String,user:String) : GetConnectParamsByDataSourceIdResult = { + dataSourceClient.getConnectParams(GetConnectParamsByDataSourceIdAction.builder() + .setDataSourceId(dataSourceId) + .setSystem(system) + .setUser(user) + .build() + ) + } + /** + * get all DataSourceTypes + * @param user user + * @return + */ + def queryDataSourceTypes(user:String):java.util.List[DataSourceType] = { + dataSourceClient.getAllDataSourceTypes(GetAllDataSourceTypesAction.builder() + .setUser(user) + .build() + ).getAllDataSourceType + } + + + + def queryClusterByDataSourceType(system: String,name:String,typeId:Long,user:String):java.util.List[DataSource] = { + dataSourceClient.queryDataSource(QueryDataSourceAction.builder() + .setSystem(system) + .setName(name) + .setTypeId(typeId) + .setIdentifies("") + .setCurrentPage(1) + .setPageSize(10) + .setUser(user) + .build() + ).getAllDataSource + } + + + /** + * get DataBases list + * @param system + * @param dataSourceId + * @param user + * @return list + */ + def queryDataBasesByCuster(system: String,dataSourceId:Long,user:String):MetadataGetDatabasesResult = { + metaDataClient.getDatabases(MetadataGetDatabasesAction.builder() + .setSystem(system) + .setDataSourceId(dataSourceId) + .setUser(user) + .build() + ) + } + + def queryTablesByDataBase(system: String,dataSourceId:Long,dataBase:String , user:String):MetadataGetTablesResult = { + metaDataClient.getTables(MetadataGetTablesAction.builder() + .setSystem(system) + .setDataSourceId(dataSourceId) + .setDatabase(dataBase) + .setUser(user) + .build() + ) + } + + + + + + } From 6bb5843e9364b8bdb142889e46f58126e4f75964 Mon Sep 17 00:00:00 2001 From: kongslove <42604208+kongslove@users.noreply.github.com> Date: Wed, 9 Jun 2021 15:42:43 +0800 Subject: [PATCH 067/434] add transfer and manager --- streamis-datasource/pom.xml | 4 + .../streamis-datasource-manager/pom.xml | 6 + .../dao/StreamisDatasourceExtraInfoDao.java | 17 ++ .../dao/StreamisDatasourceExtraUiDao.java | 17 ++ .../impl/StreamisDatasourceExtraInfoDao.xml | 18 ++ .../dao/impl/StreamisDatasourceExtraUiDao.xml | 29 +++ .../domain/StreamisDataSourceTableVO.java | 32 +++ .../domain/StreamisDatasourceExtraInfo.java | 78 ++++++ .../domain/StreamisDatasourceExtraUi.java | 194 +++++++++++++++ .../restful/api/StreamisTableMetaApi.java | 234 +++++++++++++++--- .../StreamisDatasourceExtraInfoService.java | 17 ++ .../StreamisDatasourceExtraUiService.java | 17 ++ ...treamisDatasourceExtraInfoServiceImpl.java | 21 ++ .../StreamisDatasourceExtraUiServiceImpl.java | 21 ++ .../streamis-datasource-server/pom.xml | 15 ++ .../DataSourceServer2managerRestfulApi.java | 30 ++- .../api/DataSourceTransferRestfulApi.java | 188 ++++++++++++-- .../transfer/entity/StreamisTableEntity.java | 5 + .../transfer/service/DataSourceTransfer.java | 2 +- .../service/impl/DataSourceTransferImpl.java | 42 +++- .../client/LinkisDataSourceClient.scala | 84 ++++++- 21 files changed, 1002 insertions(+), 69 deletions(-) create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceExtraInfoDao.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceExtraUiDao.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraInfoDao.xml create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraUiDao.xml create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDataSourceTableVO.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceExtraInfo.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceExtraUi.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceExtraInfoService.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceExtraUiService.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceExtraInfoServiceImpl.java create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceExtraUiServiceImpl.java diff --git a/streamis-datasource/pom.xml b/streamis-datasource/pom.xml index c941ed67d..0df441d56 100644 --- a/streamis-datasource/pom.xml +++ b/streamis-datasource/pom.xml @@ -9,6 +9,10 @@ 4.0.0 + + 1.0.0-RC1 + + streamis-datasource pom diff --git a/streamis-datasource/streamis-datasource-manager/pom.xml b/streamis-datasource/streamis-datasource-manager/pom.xml index 5d3cbe7f0..6ddfed869 100644 --- a/streamis-datasource/streamis-datasource-manager/pom.xml +++ b/streamis-datasource/streamis-datasource-manager/pom.xml @@ -54,6 +54,12 @@ linkis-mybatis ${linkis.version} + + com.webank.wedatasphere.streamis + streamis-datasource-transfer + 0.1.0-SNAPSHOT + compile + diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceExtraInfoDao.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceExtraInfoDao.java new file mode 100644 index 000000000..f1afc19a5 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceExtraInfoDao.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.streamis.datasource.manager.dao; + + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraInfo; + +/** + *

+ * Mapper 接口 + *

+ * + * @author c01013 + * @since 2021-06-04 + */ +public interface StreamisDatasourceExtraInfoDao extends BaseMapper { + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceExtraUiDao.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceExtraUiDao.java new file mode 100644 index 000000000..4aaa96c92 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/StreamisDatasourceExtraUiDao.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.streamis.datasource.manager.dao; + + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraUi; + +/** + *

+ * Mapper 接口 + *

+ * + * @author c01013 + * @since 2021-06-04 + */ +public interface StreamisDatasourceExtraUiDao extends BaseMapper { + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraInfoDao.xml b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraInfoDao.xml new file mode 100644 index 000000000..0d8b1e773 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraInfoDao.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + id, key, value, streamis_table_meta_id + + + diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraUiDao.xml b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraUiDao.xml new file mode 100644 index 000000000..5f15d5831 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraUiDao.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + id, datasource_type, key, description, description_en, lable_name, lable_name_en, ui_type, required, value, default_value, is_hidden, condition, is_advanced, order + + + diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDataSourceTableVO.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDataSourceTableVO.java new file mode 100644 index 000000000..14f1cdc08 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDataSourceTableVO.java @@ -0,0 +1,32 @@ +package com.webank.wedatasphere.streamis.datasource.manager.domain; + + +public class StreamisDataSourceTableVO { + private String tableName; + private Boolean isStreamisDataSource; + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public Boolean getStreamisDataSource() { + return isStreamisDataSource; + } + + public void setStreamisDataSource(Boolean streamisDataSource) { + isStreamisDataSource = streamisDataSource; + } + + + @Override + public String toString() { + return "StreamisDataSourceTableVO{" + + "tableName='" + tableName + '\'' + + ", isStreamisDataSource=" + isStreamisDataSource + + '}'; + } +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceExtraInfo.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceExtraInfo.java new file mode 100644 index 000000000..6ac566e90 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceExtraInfo.java @@ -0,0 +1,78 @@ +package com.webank.wedatasphere.streamis.datasource.manager.domain; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.activerecord.Model; + +import java.io.Serializable; + +/** + *

+ * + *

+ * + * @author c01013 + * @since 2021-06-04 + */ +@TableName("streamis_datasource_extra_info") +public class StreamisDatasourceExtraInfo extends Model { + + private static final long serialVersionUID = 1L; + + @TableId(value = "id", type = IdType.AUTO) + private Integer id; + private String key; + private String value; + @TableField("streamis_table_meta_id") + private Long streamisTableMetaId; + + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public Long getStreamisTableMetaId() { + return streamisTableMetaId; + } + + public void setStreamisTableMetaId(Long streamisTableMetaId) { + this.streamisTableMetaId = streamisTableMetaId; + } + + @Override + protected Serializable pkVal() { + return this.id; + } + + @Override + public String toString() { + return "StreamisDatasourceExtraInfo{" + + ", id=" + id + + ", key=" + key + + ", value=" + value + + ", streamisTableMetaId=" + streamisTableMetaId + + "}"; + } +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceExtraUi.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceExtraUi.java new file mode 100644 index 000000000..5b099ee13 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDatasourceExtraUi.java @@ -0,0 +1,194 @@ +package com.webank.wedatasphere.streamis.datasource.manager.domain; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.activerecord.Model; +import java.io.Serializable; + +/** + *

+ * + *

+ * + * @author c01013 + * @since 2021-06-04 + */ +@TableName("streamis_datasource_extra_ui") +public class StreamisDatasourceExtraUi extends Model { + + private static final long serialVersionUID = 1L; + + @TableId(value = "id", type = IdType.AUTO) + private Integer id; + @TableField("datasource_type") + private String datasourceType; + private String key; + private String description; + @TableField("description_en") + private String descriptionEn; + @TableField("lable_name") + private String lableName; + @TableField("lable_name_en") + private String lableNameEn; + @TableField("ui_type") + private String uiType; + private Integer required; + private String value; + @TableField("default_value") + private String defaultValue; + @TableField("is_hidden") + private Integer isHidden; + private String condition; + @TableField("is_advanced") + private Integer isAdvanced; + private Integer order; + + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getDatasourceType() { + return datasourceType; + } + + public void setDatasourceType(String datasourceType) { + this.datasourceType = datasourceType; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getDescriptionEn() { + return descriptionEn; + } + + public void setDescriptionEn(String descriptionEn) { + this.descriptionEn = descriptionEn; + } + + public String getLableName() { + return lableName; + } + + public void setLableName(String lableName) { + this.lableName = lableName; + } + + public String getLableNameEn() { + return lableNameEn; + } + + public void setLableNameEn(String lableNameEn) { + this.lableNameEn = lableNameEn; + } + + public String getUiType() { + return uiType; + } + + public void setUiType(String uiType) { + this.uiType = uiType; + } + + public Integer getRequired() { + return required; + } + + public void setRequired(Integer required) { + this.required = required; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(String defaultValue) { + this.defaultValue = defaultValue; + } + + public Integer getIsHidden() { + return isHidden; + } + + public void setIsHidden(Integer isHidden) { + this.isHidden = isHidden; + } + + public String getCondition() { + return condition; + } + + public void setCondition(String condition) { + this.condition = condition; + } + + public Integer getIsAdvanced() { + return isAdvanced; + } + + public void setIsAdvanced(Integer isAdvanced) { + this.isAdvanced = isAdvanced; + } + + public Integer getOrder() { + return order; + } + + public void setOrder(Integer order) { + this.order = order; + } + + @Override + protected Serializable pkVal() { + return this.id; + } + + @Override + public String toString() { + return "StreamisDatasourceExtraUi{" + + ", id=" + id + + ", datasourceType=" + datasourceType + + ", key=" + key + + ", description=" + description + + ", descriptionEn=" + descriptionEn + + ", lableName=" + lableName + + ", lableNameEn=" + lableNameEn + + ", uiType=" + uiType + + ", required=" + required + + ", value=" + value + + ", defaultValue=" + defaultValue + + ", isHidden=" + isHidden + + ", condition=" + condition + + ", isAdvanced=" + isAdvanced + + ", order=" + order + + "}"; + } +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java index 81409604c..d8088e54c 100644 --- a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java @@ -18,13 +18,21 @@ import com.baomidou.mybatisplus.core.conditions.Wrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; +import com.baomidou.mybatisplus.extension.conditions.update.UpdateChainWrapper; import com.webank.wedatasphere.linkis.common.exception.ErrorException; -import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceFields; -import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisTableMeta; +import com.webank.wedatasphere.linkis.datasource.client.response.MetadataGetDatabasesResult; +import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSource; +import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceType; +import com.webank.wedatasphere.streamis.datasource.manager.domain.*; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceAuthorityService; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceExtraInfoService; import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceFieldsService; import com.webank.wedatasphere.linkis.server.Message; import com.webank.wedatasphere.linkis.server.security.SecurityFilter; import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisTableMetaService; +import com.webank.wedatasphere.streamis.datasource.transfer.client.LinkisDataSourceClient; +import org.apache.commons.collections.CollectionUtils; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; @@ -38,7 +46,6 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; import java.util.*; - @Service public class StreamisTableMetaApi { @@ -49,40 +56,99 @@ public class StreamisTableMetaApi { @Autowired private StreamisDatasourceFieldsService streamisDatasourceFieldsService; + + @Autowired + private StreamisDatasourceAuthorityService streamisDatasourceAuthorityService; + + @Autowired + StreamisDatasourceExtraInfoService streamisDatasourceExtraInfoService; ObjectMapper mapper = new ObjectMapper(); - /** - * 获取左侧数据源树，需要调用API获取linkis数据源，还需获取streamis数据源 - */ - public Response getStreamisDataSourceList( HttpServletRequest req, Map json + + + + public Response getDataSourceCluster( HttpServletRequest req, Map json ) throws ErrorException { Message message = null; try{ String userName = SecurityFilter.getLoginUsername(req); - String workSpaceName = (String) json.get("workSpaceName"); - String projectName = (String) json.get("projectName"); - String task = (String) json.get("task"); - String dataSourceType = (String) json.get("dataSourceType"); - //TODO: invoke rpc interface to get linkisDataSource Information - - - StreamisTableMeta streamisTableMeta = streamisTableMetaService.getById(1); - logger.warn(streamisTableMeta.toString()); - QueryWrapper wrapper = new QueryWrapper<>(); - wrapper.eq("id",1); - StreamisTableMeta one = streamisTableMetaService.getOne(wrapper); + String system = json.get("system"); + String name = json.get("name"); + Long dataSourceTypeId = Long.valueOf(json.get("dataSourceTypeId")); + List dataSourceClusters = getDataSourceClusters(system, name, dataSourceTypeId, userName); + message = Message.ok().data("dataSourceCluster",dataSourceClusters); }catch (Throwable e){ - logger.error("",e); - throw new ErrorException(30001, "抱歉，后台获取数据源列表失败"); + logger.error("failed to get dataSourceType list",e); + throw new ErrorException(30016, "sorry,failed to get dataSource list"); } return Message.messageToResponse(message); + } + public Response getDataSourceTypes( HttpServletRequest req, Map json + ) throws ErrorException { + Message message = null; + try{ + String userName = SecurityFilter.getLoginUsername(req); + List dataSourceTypes = getDataSourceTypes(userName); + message = Message.ok().data("dataSourceTypes",dataSourceTypes); + }catch (Throwable e){ + logger.error("failed to get dataSourceType list",e); + throw new ErrorException(30015, "sorry,failed to get dataSourceType list"); + } + return Message.messageToResponse(message); } + public Response getDataBasesByCuster( HttpServletRequest req, Map json + ) throws ErrorException { + Message message = null; + try{ + String userName = SecurityFilter.getLoginUsername(req); + String system = json.get("system"); + Long dataSourceTypeId = Long.valueOf(json.get("dataSourceId")); + List dataBases = queryDataBasesByCuster(system, dataSourceTypeId, userName); + message = Message.ok().data("dataBases",dataBases); + }catch (Throwable e){ + logger.error("failed to get dataBases list",e); + throw new ErrorException(30017, "sorry,failed to get dataBases list"); + } + return Message.messageToResponse(message); + } + + public Response getTablesByDataBase( HttpServletRequest req, Map json + ) throws ErrorException { + Message message = null; + try{ + String userName = SecurityFilter.getLoginUsername(req); + String system = json.get("system"); + Long dataSourceId = Long.valueOf(json.get("dataSourceId")); + String dataBase = json.get("dataBase"); + QueryWrapper wrapper = new QueryWrapper<>();; + List tables = queryTablesByDataBase(system, dataSourceId, dataBase,userName); + List tableList = new ArrayList<>(); + StreamisDataSourceTableVO tableVO = null; + wrapper.eq("node_name",dataBase); + List list = streamisTableMetaService.list(wrapper); + + for (String table : tables) { + for (StreamisTableMeta streamisTableMeta : list) { + tableVO = new StreamisDataSourceTableVO(); + tableVO.setTableName(table); + tableVO.setStreamisDataSource(table.equals(streamisTableMeta.getTableName())); + tableList.add(tableVO); + } + + } + message = Message.ok().data("tables",tableList); + }catch (Throwable e){ + logger.error("failed to get tables list",e); + throw new ErrorException(30018, "sorry,failed to get tables list"); + } + return Message.messageToResponse(message); + } public Response getStreamisTableMeta( HttpServletRequest req, Long streamisTableMetaId - ) throws ErrorException { + ) throws ErrorException { Message message = null; try{ StreamisTableMeta streamisTableMeta = streamisTableMetaService.getById(streamisTableMetaId); @@ -92,7 +158,7 @@ public Response getStreamisTableMeta( HttpServletRequest req, Long streamisTable message=Message.ok().data("streamisTableMeta",streamisTableMeta).data("streamisDatasourceFields",streamisDatasourceFields); }catch (Throwable e){ logger.error("Failed to get streamisTableMetaInfo",e); - throw new ErrorException(30002, "抱歉，后台获取数据源详细信息失败"); + throw new ErrorException(30002, "Failed to get streamisTableMetaInfo(后台获取数据源详细信息失败)"); } return Message.messageToResponse(message); @@ -107,6 +173,10 @@ public Response getStreamisTableMetaByNodeNames( HttpServletRequest req,JsonNode try{ List nodeNames = mapper.readValue(json.get("nodeNames"), new TypeReference>() {}); + if(CollectionUtils.isEmpty(nodeNames)){ + logger.error("Failed to get streamisDataSource Tree,node_name is null"); + throw new ErrorException(30012, "Failed to get streamisDataSource Tree,node_name is null"); + } for (String nodeName : nodeNames) { wrapper = new QueryWrapper<>(); wrapper.eq("node_name",nodeName); @@ -116,8 +186,9 @@ public Response getStreamisTableMetaByNodeNames( HttpServletRequest req,JsonNode message = Message.ok().data("nodeTables", nodeTablesMap); }catch (Throwable e){ + logger.error("Failed to get streamisDataSource Tree",e); - throw new ErrorException(30011, "抱歉，后台获取左侧数据源树失败"); + throw new ErrorException(30011, e.getMessage()); } return Message.messageToResponse(message); @@ -128,24 +199,51 @@ public Response addStreamisTableMeta( HttpServletRequest req, JsonNode json) th Message message = null; try { String userName = SecurityFilter.getLoginUsername(req); + String authorityId = json.get("authorityId").asText(); + StreamisTableMeta streamisTableMeta = mapper.readValue(json.get("streamisTableMeta"), StreamisTableMeta.class); //第一次保存linkisDataSource数据源 streamisTableMeta.setCreateBy(userName); boolean result = streamisTableMetaService.save(streamisTableMeta); if(!result){ - throw new ErrorException(30003, "抱歉，添加streamis数据源失败"); + throw new ErrorException(30003, "Sorry, failed to add streamis data source(抱歉，添加streamis数据源失败_"); } Long streamisTableMetaId = streamisTableMeta.getId(); + //add StreamisDatasourceFields List fieldsList = mapper.readValue(json.get("streamisTableFields"), new TypeReference>() { }); fieldsList.forEach(obj ->obj.setStreamisTableMetaId(streamisTableMetaId)); - boolean b = streamisDatasourceFieldsService.saveBatch(fieldsList); + result = streamisDatasourceFieldsService.saveBatch(fieldsList); + if(!result){ + throw new ErrorException(30013, "Sorry, failed to add StreamisTableFields table field information(抱歉，添加StreamisTableFields表字段信息失败)"); + } +// String scope = streamisTableMeta.getScope(); + StreamisDatasourceAuthority streamisDatasourceAuthority = new StreamisDatasourceAuthority(); + + streamisDatasourceAuthority.setAuthorityScope(streamisTableMeta.getScope()); + streamisDatasourceAuthority.setStreamisTableMetaId(streamisTableMetaId); + streamisDatasourceAuthority.setAuthorityId(authorityId); + streamisDatasourceAuthority.setGrantUser("*"); + result = streamisDatasourceAuthorityService.save(streamisDatasourceAuthority); + if(!result){ + throw new ErrorException(30014, " sorry, add streamisDatasourceAuthority information failure (抱歉，添加streamisDatasourceAuthority信息失败)"); + } + List extraInfoList = mapper.readValue(json.get("streamisExtraInfo"), new TypeReference>() { + }); + + extraInfoList.forEach(obj-> obj.setStreamisTableMetaId(streamisTableMetaId)); + + result = streamisDatasourceExtraInfoService.saveBatch(extraInfoList); + if(!result){ + throw new ErrorException(30019, " sorry, add extraInfo information failure"); + } + //add to StreamisDatasourceAuthority message = Message.ok().data("streamisTableMetaId",streamisTableMetaId); } catch (Exception e) { logger.error("Failed to add StreamisTableMeta: ", e); - throw new ErrorException(30003, "抱歉，添加streamis数据源失败"); + throw new ErrorException(30003, e.getMessage()); } return Message.messageToResponse(message); } @@ -158,16 +256,31 @@ public Response updateStreamisTableMeta( HttpServletRequest req, JsonNode json) StreamisTableMeta streamisTableMeta = mapper.readValue(json.get("streamisTableMeta"), StreamisTableMeta.class); Long streamisTableMetaId = streamisTableMeta.getId(); if(null == streamisTableMetaId ){ - throw new ErrorException(30004, "抱歉，更新streamis数据源信息失败，streamisTableMetaId is null"); + throw new ErrorException(30004, "sorry，Failed to update streamis data source information(更新streamis数据源信息失败)，streamisTableMetaId is null"); } streamisTableMeta.setUpdateBy(userName); // update StreamisTableMeta streamisTableMetaService.updateById(streamisTableMeta); + + List extraInfoList = mapper.readValue(json.get("streamisExtraInfo"), new TypeReference>() { + }); + + extraInfoList.forEach(obj-> obj.setStreamisTableMetaId(streamisTableMetaId)); + UpdateWrapper wrapper = null; + boolean result ; + for (StreamisDatasourceExtraInfo extraInfo : extraInfoList) { + wrapper.eq("key",extraInfo.getKey()).eq("streamis_table_meta_id",streamisTableMetaId).set("value",extraInfo.getValue()); + result = streamisDatasourceExtraInfoService.update(null, wrapper); + if(!result){ + throw new ErrorException(30020, String.format("sorry，Failed to update the %s information(更新%s失败)",extraInfo.getKey(),extraInfo.getKey())); + } + } + message = Message.ok(); } catch (Exception e) { logger.error("Failed to update StreamisTableMeta: ", e); - throw new ErrorException(30004, "抱歉，更新streamis数据源信息失败"); + throw new ErrorException(30004,e.getMessage()); } return Message.messageToResponse(message); } @@ -180,10 +293,16 @@ public Response deleteStreamisTableMeta(HttpServletRequest req, Long id) throws streamisDatasourceFieldsQueryWrapper.eq("streamis_table_meta_id",id); boolean remove = streamisDatasourceFieldsService.remove(streamisDatasourceFieldsQueryWrapper); if (!remove){ - throw new ErrorException(30008, String.format("抱歉，删除streamis数据源 表[%s]字段信息失败",id)); + throw new ErrorException(30008, String.format("Sorry, failed to delete the streamis data source table [%s] field information(抱歉，删除streamis数据源 表[%s]字段信息失败)",id,id)); + } + QueryWrapper datasourceAuthorityQueryWrapper = new QueryWrapper<>(); + datasourceAuthorityQueryWrapper.eq("streamis_table_meta_id",id); + remove = streamisDatasourceAuthorityService.remove(datasourceAuthorityQueryWrapper); + if (!remove){ + throw new ErrorException(30015, String.format("抱歉，删除streamisDatasourceAuthority数据源 表[%s]字段信息失败",id)); } // delete StreamisTableMeta - boolean result = streamisDatasourceFieldsService.removeById(id); + boolean result = streamisTableMetaService.removeById(id); if (!result){ throw new ErrorException(30009, String.format("抱歉，删除streamis数据源 表[%s]失败",id)); } @@ -191,7 +310,7 @@ public Response deleteStreamisTableMeta(HttpServletRequest req, Long id) throws } catch (Exception e) { logger.error("Failed to delete StreamisTableMeta: ", e); - throw new ErrorException(30010, "抱歉，删除streamis数据源信息失败"); + throw new ErrorException(30010, e.getMessage()); } return Message.messageToResponse(message); } @@ -208,7 +327,7 @@ public Response addStreamisTableFields( HttpServletRequest req, JsonNode json) message = Message.ok(); } catch (Exception e) { logger.error("Failed to add streamisDatasourceFields: ", e); - throw new ErrorException(30005, "抱歉，添加StreamisTableFields表字段信息失败"); + throw new ErrorException(30005, e.getMessage()); } return Message.messageToResponse(message); } @@ -241,4 +360,51 @@ public Response deleteStreamisTableFields( HttpServletRequest req, Long id) thro } -} + /** + * 获取数据源类型 + * @return 数据源类型列表 + */ + public List getDataSourceTypes(String userName){ + return LinkisDataSourceClient.queryDataSourceTypes(userName); + } + + /** + * + * @param system system + * @param name username + * @param typeId typeId + * @param user hadoop + * @return 获取数据源集群 + */ + public List getDataSourceClusters(String system,String name,Long typeId ,String user){ + return LinkisDataSourceClient.queryClusterByDataSourceType(system,name,typeId,user); + } + + /** + * 获取数据库列表 + * @param system system + * @param dataSourceId dataSourceId + * @param user hadoop + * @return list + */ + public List queryDataBasesByCuster(String system, Long dataSourceId , String user){ + return LinkisDataSourceClient.queryDataBasesByCuster(system,dataSourceId,user).dbs(); + } + + /** + * 获取table列表 + * @param system system + * @param dataSourceId dataSourceId + * @param dataBase dataBase + * @param user user + * @return * @return + */ + public List queryTablesByDataBase(String system, Long dataSourceId , String dataBase,String user){ + return LinkisDataSourceClient.queryTablesByDataBase(system,dataSourceId,dataBase ,user).tables(); + } + + + + + +} \ No newline at end of file diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceExtraInfoService.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceExtraInfoService.java new file mode 100644 index 000000000..35056d4a6 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceExtraInfoService.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.streamis.datasource.manager.service; + + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraInfo; + +/** + *

+ * 服务类 + *

+ * + * @author c01013 + * @since 2021-06-04 + */ +public interface StreamisDatasourceExtraInfoService extends IService { + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceExtraUiService.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceExtraUiService.java new file mode 100644 index 000000000..205e6ecfa --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/StreamisDatasourceExtraUiService.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.streamis.datasource.manager.service; + + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraUi; + +/** + *

+ * 服务类 + *

+ * + * @author c01013 + * @since 2021-06-04 + */ +public interface StreamisDatasourceExtraUiService extends IService { + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceExtraInfoServiceImpl.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceExtraInfoServiceImpl.java new file mode 100644 index 000000000..6b4fd8ff2 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceExtraInfoServiceImpl.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.streamis.datasource.manager.service.impl; + + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.streamis.datasource.manager.dao.StreamisDatasourceExtraInfoDao; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraInfo; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceExtraInfoService; +import org.springframework.stereotype.Service; + +/** + *

+ * 服务实现类 + *

+ * + * @author c01013 + * @since 2021-06-04 + */ +@Service +public class StreamisDatasourceExtraInfoServiceImpl extends ServiceImpl implements StreamisDatasourceExtraInfoService { + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceExtraUiServiceImpl.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceExtraUiServiceImpl.java new file mode 100644 index 000000000..10b57bcc3 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/service/impl/StreamisDatasourceExtraUiServiceImpl.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.streamis.datasource.manager.service.impl; + + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.streamis.datasource.manager.dao.StreamisDatasourceExtraUiDao; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraUi; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceExtraUiService; +import org.springframework.stereotype.Service; + +/** + *

+ * 服务实现类 + *

+ * + * @author c01013 + * @since 2021-06-04 + */ +@Service +public class StreamisDatasourceExtraUiServiceImpl extends ServiceImpl implements StreamisDatasourceExtraUiService { + +} diff --git a/streamis-datasource/streamis-datasource-server/pom.xml b/streamis-datasource/streamis-datasource-server/pom.xml index e3bda5bfd..f95280fb9 100644 --- a/streamis-datasource/streamis-datasource-server/pom.xml +++ b/streamis-datasource/streamis-datasource-server/pom.xml @@ -27,6 +27,21 @@ linkis-mybatis ${linkis.version} + + com.webank.wedatasphere.linkis + linkis-bmlserver + ${linkis.version} + + + com.webank.wedatasphere.linkis + linkis-storage + ${linkis.version} + + + com.webank.wedatasphere.dss + dss-server + ${dss.version} + com.webank.wedatasphere.streamis streamis-datasource-transfer diff --git a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java index c070d2ec0..fcf32ceb6 100644 --- a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java +++ b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java @@ -23,14 +23,32 @@ public class DataSourceServer2managerRestfulApi { private StreamisTableMetaApi streamisTableMetaApi; - /** - * 获取左侧数据源树，需要调用API获取linkis数据源，还需获取streamis数据源 - */ @POST - @Path("streamisDataSourceList") - public Response getStreamisDataSourceList(@Context HttpServletRequest req, Map json + @Path("dataSourceType") + public Response getDataSourceType(@Context HttpServletRequest req, Map json + ) throws ErrorException { + return streamisTableMetaApi.getDataSourceTypes(req, json); + } + + @POST + @Path("dataSourceCluster") + public Response getDataSourceCluster(@Context HttpServletRequest req, Map json + ) throws ErrorException { + return streamisTableMetaApi.getDataSourceCluster(req,json); + } + + @POST + @Path("dataBases") + public Response getDataBasesByCuster(@Context HttpServletRequest req, Map json + ) throws ErrorException { + return streamisTableMetaApi.getDataBasesByCuster(req,json); + } + + @POST + @Path("tables") + public Response getTablesByDataBase(@Context HttpServletRequest req, Map json ) throws ErrorException { - return streamisTableMetaApi.getStreamisDataSourceList(req, json); + return streamisTableMetaApi.getTablesByDataBase(req,json); } /** diff --git a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java index 1972a0749..46f12286e 100644 --- a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java +++ b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java @@ -1,59 +1,219 @@ package com.webank.wedatasphere.streamis.datasource.server.restful.api; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.google.gson.Gson; +import com.webank.wedatasphere.dss.server.service.BMLService; +import com.webank.wedatasphere.linkis.bml.Entity.Resource; +import com.webank.wedatasphere.linkis.bml.Entity.ResourceVersion; +import com.webank.wedatasphere.linkis.bml.common.*; +import com.webank.wedatasphere.linkis.bml.dao.ResourceDao; +import com.webank.wedatasphere.linkis.bml.dao.VersionDao; +import com.webank.wedatasphere.linkis.bml.restful.RestfulUtils; +import com.webank.wedatasphere.linkis.bml.service.ResourceService; +import com.webank.wedatasphere.linkis.bml.service.VersionService; +import com.webank.wedatasphere.linkis.bml.util.HttpRequestHelper; import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.common.io.Fs; +import com.webank.wedatasphere.linkis.common.io.FsPath; import com.webank.wedatasphere.linkis.server.Message; +import com.webank.wedatasphere.linkis.server.security.SecurityFilter; +import com.webank.wedatasphere.linkis.storage.FSFactory; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceFields; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisTableMeta; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceFieldsService; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisTableMetaService; import com.webank.wedatasphere.streamis.datasource.transfer.entity.StreamisDataSourceCode; import com.webank.wedatasphere.streamis.datasource.transfer.entity.StreamisTableEntity; import com.webank.wedatasphere.streamis.datasource.transfer.service.DataSourceTransfer; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; +import org.codehaus.jackson.JsonNode; import org.glassfish.jersey.media.multipart.FormDataParam; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import scala.tools.cmd.Opt; +import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.QueryParam; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; -import java.util.HashMap; -import java.util.Map; +import java.io.*; +import java.util.*; +@Component +@Path("streamis") public class DataSourceTransferRestfulApi { private static final Logger logger = LoggerFactory.getLogger(DataSourceTransferRestfulApi.class); @Autowired private DataSourceTransfer dataSourceTransfer; + @Autowired + private StreamisTableMetaService streamisTableMetaService; + @Autowired + private StreamisDatasourceFieldsService streamisDatasourceFieldsService; + + @Autowired + private ResourceDao resourceDao; + @Autowired + private VersionDao versionDao; + @Autowired + private ResourceService resourceService; + @Autowired + private VersionService versionService; + @GET @Path("/transfer") - public Response transfer(@Context HttpServletRequest request, @QueryParam("streamisTableMetaId") String streamisTableMetaId, @FormDataParam("labels") Map labels) throws ErrorException { + public Response transfer(@Context HttpServletRequest request, @QueryParam("streamisTableMetaId") String streamisTableMetaId, @QueryParam("dataSourceId") String dataSource,@QueryParam("nodeName") String nodeName, @FormDataParam("labels") Map labels) throws ErrorException { Message message; if(StringUtils.isBlank(streamisTableMetaId)){ throw new ErrorException(30201,"streamisTableMetaId is null."); } + String userName = SecurityFilter.getLoginUsername(request); + Map params = new HashMap<>(); try { + + //TODO: 如何获取runType engineType version + params.put("nodeName",nodeName); + params.put("runType","sql"); + params.put("engineType","flink"); + params.put("version","1.12.2"); + Long dataSourceId = Long.parseLong(dataSource); //通过streamisTableMetaId拿到StreamisTableMeta实体 - StreamisTableEntity streamisTableEntity = dataSourceTransfer.getStreamisTableMetaById(streamisTableMetaId); + StreamisTableEntity streamisTableEntity = dataSourceTransfer.getStreamisTableMetaById(streamisTableMetaId,userName,dataSourceId); + //转换为FlinkSql - StreamisDataSourceCode streamisDataSourceCode = dataSourceTransfer.transfer(streamisTableEntity, labels, new HashMap()); + StreamisDataSourceCode streamisDataSourceCode = dataSourceTransfer.transfer(streamisTableEntity, labels, params); message = Message.ok().data("streamisDataSourceCode",streamisDataSourceCode); } catch (Exception e) { - logger.error("传入streamisTableMetaId {} 数据源转换失败：", streamisTableMetaId, e); - throw new ErrorException(30202, "抱歉，后台数据源转换失败"); + logger.error("传入streamisTableMetaId {} data source conversion failed：", streamisTableMetaId, e); + throw new ErrorException(30202, "Sorry, background data source conversion failed(后台数据源转换失败)"); } return Message.messageToResponse(message); } + //上传数据源到hdfs @POST @Path("/exportStreamisTableMeta") - public Response exportStreamisTableMeta(@Context HttpServletRequest request, @FormDataParam("streamisTableMetaId") String streamisTableMetaId){ - Message message; + public Response exportStreamisTableMeta(@Context HttpServletRequest request, JsonNode jsonNode) throws ErrorException{ + Message message = new Message(); + String user = RestfulUtils.getUserName(request); + String clientIp = HttpRequestHelper.getIp(request); + Map streamisEntity = new HashMap<>(); + ResourceHelper resourceHelper = ResourceHelperFactory.getResourceHelper(); + //通过streamisTableMetaId拿到StreamisTableMeta实体 + String streamisTableMetaId = jsonNode.get("streamisTableMetaId").asText(); + StreamisTableMeta streamisTableMeta = streamisTableMetaService.getById(streamisTableMetaId); + + QueryWrapper streamisDatasourceFieldsWrapper = new QueryWrapper<>(); + streamisDatasourceFieldsWrapper.eq("streamis_table_meta_id",streamisTableMetaId); + List streamisDatasourceFields = streamisDatasourceFieldsService.list(streamisDatasourceFieldsWrapper); + + streamisEntity.put("streamisTableMeta",streamisTableMeta); + streamisEntity.put("streamisDatasourceFields",streamisDatasourceFields); + Gson gson = new Gson(); + String json = gson.toJson(streamisEntity); + + if(json != null) { + String resourceId = UUID.randomUUID().toString(); + String fileName = UUID.randomUUID().toString() + ".json"; + String path = resourceHelper.generatePath(user, fileName, new HashMap()); + InputStream in_nocode = new ByteArrayInputStream(json.getBytes()); + StringBuilder sb = new StringBuilder(); + //上传资源到HDFS + long size = resourceHelper.upload(path, user, in_nocode, sb, true); + String md5String = sb.toString(); + boolean isSuccess = false; + if (StringUtils.isNotEmpty(md5String) && size >= 0) { + isSuccess = true; + } + Resource resource = Resource.createNewResource(resourceId, user, fileName, new HashMap()); + //插入一条记录到resource表 + long id = resourceDao.uploadResource(resource); + logger.info("{} uploaded a resource and resourceId is {}", user, resource.getResourceId()); + //插入一条记录到resource version表 + //TODO 版本是否固定为v000001？ + ResourceVersion resourceVersion = ResourceVersion.createNewResourceVersion(resourceId, path, md5String, + clientIp, size, Constant.FIRST_VERSION, 1); + versionDao.insertNewVersion(resourceVersion); + + message = Message.ok().data("resourceId",resourceId).data("version",resourceVersion.getVersion()); + } + + return Message.messageToResponse(message); + } - return Message.messageToResponse(Message.ok()); + //从hdfs下载数据源，插入数据库 + @POST + @Path("/importStreamisTableMeta") + public Response importStreamisTableMeta(@Context HttpServletRequest request, @Context HttpServletResponse resp,JsonNode jsonNode ) + throws ErrorException, IOException { + Message message = new Message(); + String user = RestfulUtils.getUserName(request); + String resourceId = jsonNode.get("resourceId").asText(); + String version = jsonNode.get("version").asText(); + if (StringUtils.isBlank(resourceId) || !resourceService.checkResourceId(resourceId)) { + message = Message.error("resourceId:"+resourceId+"为空,非法或者已被删除!"); + message.setStatus(1); + return Message.messageToResponse(message); + } + if (!resourceService.checkAuthority(user, resourceId)){ + throw new BmlPermissionDeniedException("您没有权限下载此资源"); + } + //判version空,返回最新版本 + if (StringUtils.isBlank(version)){ + version = versionService.getNewestVersion(resourceId); + } + //判version不存在或者非法 + if (!versionService.checkVersion(resourceId, version)) { + message = Message.error("version:"+version+"不存在,非法或者已被删除"); + message.setStatus(1); + return Message.messageToResponse(message); + } + //判resourceId和version是否过期 + if (!resourceService.checkExpire(resourceId, version)){ + throw new BmlResourceExpiredException(resourceId); + } + + ResourceVersion resourceVersion = versionDao.findResourceVersion(resourceId, version); + long startByte = resourceVersion.getStartByte(); + long endByte = resourceVersion.getEndByte(); + String path = resourceVersion.getResource(); + Fs fileSystem = FSFactory.getFsByProxyUser(new FsPath(path), user); + fileSystem.init(new HashMap()); + InputStream inputStream = fileSystem.read(new FsPath(path)); + inputStream.skip(startByte - 1L); + logger.info("{} 下载资源 {} inputStream skipped {} bytes", new Object[]{user, resourceId, startByte - 1L}); + + try { + //将流转换为json + Gson gson = new Gson(); + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); + StringBuilder sb = new StringBuilder(); + String s = null; + while ((s = bufferedReader.readLine()) != null) { + sb.append(s); + sb.append("\n"); + } + String json = sb.toString().trim(); + //将存放入数组的数据转为map格式 + HashMap map = gson.fromJson(sb.toString(), HashMap.class); + message.data("streamisEntity",map); + } catch (IOException e) { + logger.error("用户 {} 下载资源 resourceId: {}, version:{} 出现IO异常", user, resourceId, version, e); + throw new ErrorException(73562, "抱歉,后台IO错误造成您本次下载资源失败"); + } catch (final Throwable t){ + logger.error("用户 {} 下载资源 resourceId: {}, version:{} 出现异常", user, resourceId, version, t); + throw new ErrorException(73561, "抱歉，后台服务出错导致您本次下载资源失败"); + }finally { + IOUtils.closeQuietly(inputStream); + } + return Message.messageToResponse(message); } } diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisTableEntity.java b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisTableEntity.java index 831556180..058c453e3 100644 --- a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisTableEntity.java +++ b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/entity/StreamisTableEntity.java @@ -42,4 +42,9 @@ public StreamisTableMeta getTableInfo() { public void setTableInfo(StreamisTableMeta tableInfo) { this.tableInfo = tableInfo; } + + public String toString(){ + + return null; + } } diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/DataSourceTransfer.java b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/DataSourceTransfer.java index da8906832..35a513049 100644 --- a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/DataSourceTransfer.java +++ b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/DataSourceTransfer.java @@ -9,6 +9,6 @@ public interface DataSourceTransfer { StreamisDataSourceCode transfer(StreamisTableEntity streamisTableEntity, Map labels, Map params); - StreamisTableEntity getStreamisTableMetaById(String streamisTableMetaId) throws ErrorException; + StreamisTableEntity getStreamisTableMetaById(String tableMetaId, String userName, Long dataSourceId) throws ErrorException; } diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java index bb7ffb3f4..9ac33bec4 100644 --- a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java +++ b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java @@ -2,12 +2,13 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.webank.wedatasphere.linkis.common.exception.ErrorException; -import com.webank.wedatasphere.linkis.datasource.client.response.QueryDataSourceResult; -import com.webank.wedatasphere.linkis.manager.label.entity.Label; +import com.webank.wedatasphere.linkis.datasource.client.response.GetConnectParamsByDataSourceIdResult; import com.webank.wedatasphere.linkis.manager.label.entity.engine.EngineType; import com.webank.wedatasphere.linkis.manager.label.entity.engine.RunType; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraInfo; import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceFields; import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisTableMeta; +import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceExtraInfoService; import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceFieldsService; import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisTableMetaService; import com.webank.wedatasphere.streamis.datasource.transfer.client.LinkisDataSourceClient; @@ -34,12 +35,16 @@ public class DataSourceTransferImpl implements DataSourceTransfer { @Autowired StreamisDatasourceFieldsService streamisDatasourceFieldsService; + @Autowired + StreamisDatasourceExtraInfoService streamisDatasourceExtraInfoService; + @Override public StreamisDataSourceCode transfer(StreamisTableEntity streamisTableEntity, Map labels, Map params) { String runType = (String)params.get("runType"); - String engineType = (String)params.get("engingType"); + String engineType = (String)params.get("engineType"); String version = (String)params.get("version"); + //获取表名，列名 StreamisDataSourceCode streamisDataSourceCode = new StreamisDataSourceCode(); String tableName = streamisTableEntity.getTableInfo().getTableName(); @@ -51,10 +56,15 @@ public StreamisDataSourceCode transfer(StreamisTableEntity streamisTableEntity, if(EngineType.FLINK().equals(engineType) && version.equals("1.12.2") && RunType.SQL().equals(runType)){ Map linkisDataSourceContent = streamisTableEntity.getLinkisDatasource().getLinkisDataSourceContent(); + String dataSourceType = linkisDataSourceContent.get("dataSourceType").toString(); if(dataSourceType.equals("kafka")){ + String groupId = getKaflaGroupIdInfo(streamisTableEntity.getTableInfo().getId()); + linkisDataSourceContent.put("groupId",groupId); + linkisDataSourceContent.put("topicName",tableName); streamisDataSourceCode = transferKafkaSql(tableName, cols, labels, linkisDataSourceContent); }else if(dataSourceType.equals("mysql")){ + linkisDataSourceContent.put("dbName",params.get("nodeName")); streamisDataSourceCode = transferMysql(tableName, cols, labels, linkisDataSourceContent); } } @@ -62,18 +72,18 @@ public StreamisDataSourceCode transfer(StreamisTableEntity streamisTableEntity, } public StreamisDataSourceCode transferKafkaSql(String tableName, String cols, Map labels, Map linkisDataSourceContent) { - String servers = (String)linkisDataSourceContent.get("servers"); + String servers = (String)linkisDataSourceContent.get("brokers"); String topicName = (String)linkisDataSourceContent.get("topicName"); String groupId = (String)linkisDataSourceContent.get("groupId"); String offsetMode = (String)linkisDataSourceContent.getOrDefault("offsetMode","earliest-offset"); - String formatMode = (String)linkisDataSourceContent.getOrDefault("formatMode","formatMode"); + String formatMode = (String)linkisDataSourceContent.getOrDefault("formatMode","json"); String executionCode = DataSourceTransferFlinksqlUtils.kafkaDataSourceTransfer(tableName , cols , topicName , servers , groupId , offsetMode , formatMode); StreamisDataSourceCode streamisDataSourceCode = new StreamisDataSourceCode(executionCode, labels, new HashMap()); return streamisDataSourceCode; } public StreamisDataSourceCode transferMysql(String tableName, String cols, Map labels, Map linkisDataSourceContent){ - String serverIp = (String)linkisDataSourceContent.get("serverIp"); + String serverIp = (String)linkisDataSourceContent.get("host");//serverIp String dbName = (String)linkisDataSourceContent.get("dbName"); String executionCode = DataSourceTransferFlinksqlUtils.mysqlDataSourceTransfer(tableName,cols,serverIp,dbName); StreamisDataSourceCode streamisDataSourceCode = new StreamisDataSourceCode(executionCode, labels, new HashMap()); @@ -82,7 +92,7 @@ public StreamisDataSourceCode transferMysql(String tableName, String cols, Map linkisDataSourceContent = LinkisDataSourceClient.queryConnectParams(dataSourceId, "system", userName).getConnectParams(); //拿到linkis数据源里的连接信息 - Map linkisDataSourceContent = queryDataSourceResult.getQuery_list().get(0).getConnectParams(); +// Map linkisDataSourceContent = queryDataSourceResult.getQuery_list().get(0); LinkisDataSource linkisDatasource = new LinkisDataSource(linkisDataSourceUniqueId, linkisDataSourceContent); //返回实体 @@ -106,9 +117,18 @@ public StreamisTableEntity getStreamisTableMetaById(String streamisTableMetaId) streamisTableEntity.setFields(fields); streamisTableEntity.setLinkisDatasource(linkisDatasource); } catch (Exception e) { - logger.error("传入streamisTableMetaId {} 查询元数据失败：", streamisTableMetaId, e); - throw new ErrorException(30203, "抱歉，后台查询streamis元数据失败"); + logger.error("the param streamisTableMetaId {} Failed to query metadata(查询元数据失败)：", streamisTableMetaId, e); + throw new ErrorException(30203, "Sorry, the background query for streamis metadata failed(抱歉，后台查询streamis元数据失败)"); } return streamisTableEntity; } + + + public String getKaflaGroupIdInfo(Long streamisTableMetaId){ + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.eq("streamis_table_meta_id",streamisTableMetaId); + wrapper.eq("key","kafka.group.id"); + StreamisDatasourceExtraInfo extraInfo = streamisDatasourceExtraInfoService.getOne(wrapper); + return extraInfo.getValue(); + } } diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala b/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala index 9f6e6d063..c00803557 100644 --- a/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala +++ b/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala @@ -2,9 +2,10 @@ package com.webank.wedatasphere.streamis.datasource.transfer.client import java.util.concurrent.TimeUnit -import com.webank.wedatasphere.linkis.datasource.client.impl.LinkisDataSourceRemoteClient -import com.webank.wedatasphere.linkis.datasource.client.request.QueryDataSourceAction -import com.webank.wedatasphere.linkis.datasource.client.response.QueryDataSourceResult +import com.webank.wedatasphere.linkis.datasource.client.impl.{LinkisDataSourceRemoteClient, LinkisMetaDataRemoteClient} +import com.webank.wedatasphere.linkis.datasource.client.request.{GetAllDataSourceTypesAction, GetConnectParamsByDataSourceIdAction, MetadataGetDatabasesAction, MetadataGetTablesAction, QueryDataSourceAction, QueryDataSourceEnvAction} +import com.webank.wedatasphere.linkis.datasource.client.response.{GetConnectParamsByDataSourceIdResult, MetadataGetDatabasesResult, MetadataGetTablesResult, QueryDataSourceResult} +import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.{DataSource, DataSourceEnv, DataSourceType} import com.webank.wedatasphere.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfigBuilder import com.webank.wedatasphere.streamis.datasource.transfer.conf.TransferConfiguration @@ -41,6 +42,8 @@ object LinkisDataSourceClient { val dataSourceClient = new LinkisDataSourceRemoteClient(clientConfig) + val metaDataClient = new LinkisMetaDataRemoteClient(clientConfig) + def queryDataSource(linkisDatasourceName:String) : QueryDataSourceResult = { dataSourceClient.queryDataSource(QueryDataSourceAction.builder() .setSystem("") @@ -48,8 +51,83 @@ object LinkisDataSourceClient { .setTypeId(1) .setIdentifies("") .setCurrentPage(1) + .setUser("hadoop") .setPageSize(1).build() ) } + /** + * get datasourceConnect information + * @param dataSourceId id + * @param system dssSystem + * @param user username + * @return + */ + def queryConnectParams(dataSourceId:Long,system: String,user:String) : GetConnectParamsByDataSourceIdResult = { + dataSourceClient.getConnectParams(GetConnectParamsByDataSourceIdAction.builder() + .setDataSourceId(dataSourceId) + .setSystem(system) + .setUser(user) + .build() + ) + } + /** + * get all DataSourceTypes + * @param user user + * @return + */ + def queryDataSourceTypes(user:String):java.util.List[DataSourceType] = { + dataSourceClient.getAllDataSourceTypes(GetAllDataSourceTypesAction.builder() + .setUser(user) + .build() + ).getAllDataSourceType + } + + + + def queryClusterByDataSourceType(system: String,name:String,typeId:Long,user:String):java.util.List[DataSource] = { + dataSourceClient.queryDataSource(QueryDataSourceAction.builder() + .setSystem(system) + .setName(name) + .setTypeId(typeId) + .setIdentifies("") + .setCurrentPage(1) + .setPageSize(10) + .setUser(user) + .build() + ).getAllDataSource + } + + + /** + * get DataBases list + * @param system + * @param dataSourceId + * @param user + * @return list + */ + def queryDataBasesByCuster(system: String,dataSourceId:Long,user:String):MetadataGetDatabasesResult = { + metaDataClient.getDatabases(MetadataGetDatabasesAction.builder() + .setSystem(system) + .setDataSourceId(dataSourceId) + .setUser(user) + .build() + ) + } + + def queryTablesByDataBase(system: String,dataSourceId:Long,dataBase:String , user:String):MetadataGetTablesResult = { + metaDataClient.getTables(MetadataGetTablesAction.builder() + .setSystem(system) + .setDataSourceId(dataSourceId) + .setDatabase(dataBase) + .setUser(user) + .build() + ) + } + + + + + + } From 9b2869d653118800e524812af8c36bba4954072f Mon Sep 17 00:00:00 2001 From: kongslove <42604208+kongslove@users.noreply.github.com> Date: Thu, 10 Jun 2021 16:36:53 +0800 Subject: [PATCH 068/434] add interface for get columns --- .../domain/StreamisDataSourceTableVO.java | 9 +++ .../restful/api/StreamisTableMetaApi.java | 61 ++++++++++++++----- .../DataSourceServer2managerRestfulApi.java | 7 +++ .../api/DataSourceTransferRestfulApi.java | 14 ++++- .../client/LinkisDataSourceClient.scala | 14 ++++- 5 files changed, 86 insertions(+), 19 deletions(-) diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDataSourceTableVO.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDataSourceTableVO.java index 14f1cdc08..1444fb121 100644 --- a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDataSourceTableVO.java +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDataSourceTableVO.java @@ -4,6 +4,7 @@ public class StreamisDataSourceTableVO { private String tableName; private Boolean isStreamisDataSource; + private Long streamisTableMetaId; public String getTableName() { return tableName; @@ -21,12 +22,20 @@ public void setStreamisDataSource(Boolean streamisDataSource) { isStreamisDataSource = streamisDataSource; } + public Long getStreamisTableMetaId() { + return streamisTableMetaId; + } + + public void setStreamisTableMetaId(Long streamisTableMetaId) { + this.streamisTableMetaId = streamisTableMetaId; + } @Override public String toString() { return "StreamisDataSourceTableVO{" + "tableName='" + tableName + '\'' + ", isStreamisDataSource=" + isStreamisDataSource + + ", streamisTableMetaId=" + streamisTableMetaId + '}'; } } diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java index d8088e54c..8eeb15c83 100644 --- a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java @@ -70,10 +70,10 @@ public class StreamisTableMetaApi { public Response getDataSourceCluster( HttpServletRequest req, Map json ) throws ErrorException { - Message message = null; + Message message; try{ String userName = SecurityFilter.getLoginUsername(req); - String system = json.get("system"); + String system = json.getOrDefault("system","streamis"); String name = json.get("name"); Long dataSourceTypeId = Long.valueOf(json.get("dataSourceTypeId")); List dataSourceClusters = getDataSourceClusters(system, name, dataSourceTypeId, userName); @@ -87,7 +87,7 @@ public Response getDataSourceCluster( HttpServletRequest req, Map public Response getDataSourceTypes( HttpServletRequest req, Map json ) throws ErrorException { - Message message = null; + Message message; try{ String userName = SecurityFilter.getLoginUsername(req); List dataSourceTypes = getDataSourceTypes(userName); @@ -101,10 +101,10 @@ public Response getDataSourceTypes( HttpServletRequest req, Map j public Response getDataBasesByCuster( HttpServletRequest req, Map json ) throws ErrorException { - Message message = null; + Message message; try{ String userName = SecurityFilter.getLoginUsername(req); - String system = json.get("system"); + String system = json.getOrDefault("system","streamis"); Long dataSourceTypeId = Long.valueOf(json.get("dataSourceId")); List dataBases = queryDataBasesByCuster(system, dataSourceTypeId, userName); message = Message.ok().data("dataBases",dataBases); @@ -117,10 +117,10 @@ public Response getDataBasesByCuster( HttpServletRequest req, Map public Response getTablesByDataBase( HttpServletRequest req, Map json ) throws ErrorException { - Message message = null; + Message message; try{ String userName = SecurityFilter.getLoginUsername(req); - String system = json.get("system"); + String system = json.getOrDefault("system","streamis"); Long dataSourceId = Long.valueOf(json.get("dataSourceId")); String dataBase = json.get("dataBase"); QueryWrapper wrapper = new QueryWrapper<>();; @@ -134,6 +134,9 @@ public Response getTablesByDataBase( HttpServletRequest req, Map for (StreamisTableMeta streamisTableMeta : list) { tableVO = new StreamisDataSourceTableVO(); tableVO.setTableName(table); + if(table.equals(streamisTableMeta.getTableName())){ + tableVO.setStreamisTableMetaId(streamisTableMeta.getId()); + } tableVO.setStreamisDataSource(table.equals(streamisTableMeta.getTableName())); tableList.add(tableVO); } @@ -147,15 +150,40 @@ public Response getTablesByDataBase( HttpServletRequest req, Map return Message.messageToResponse(message); } + public Response getColumnsByTable( HttpServletRequest req, Map json + ) throws ErrorException { + Message message; + try{ + String userName = SecurityFilter.getLoginUsername(req); + String system = json.getOrDefault("system","streamis"); + Long dataSourceTypeId = Long.valueOf(json.get("dataSourceId")); + String dataBase = json.get("dataBase"); + String table = json.get("table"); + String dataSourceType = json.get("dataSourceType"); + List> columns =null; + if(!"kafka".equalsIgnoreCase(dataSourceType)){ + columns = queryColumnsByTable(system, dataSourceTypeId, dataBase, table, userName); + } + message = Message.ok().data("columns",columns); + }catch (Throwable e){ + logger.error("failed to get columns list",e); + throw new ErrorException(30017, "sorry,failed to get columns list"); + } + return Message.messageToResponse(message); + } + public Response getStreamisTableMeta( HttpServletRequest req, Long streamisTableMetaId ) throws ErrorException { - Message message = null; + Message message; try{ StreamisTableMeta streamisTableMeta = streamisTableMetaService.getById(streamisTableMetaId); QueryWrapper streamisDatasourceFieldsWrapper = new QueryWrapper<>(); streamisDatasourceFieldsWrapper.eq("streamis_table_meta_id",streamisTableMetaId); List streamisDatasourceFields = streamisDatasourceFieldsService.list(streamisDatasourceFieldsWrapper); + + //查询一下 message=Message.ok().data("streamisTableMeta",streamisTableMeta).data("streamisDatasourceFields",streamisDatasourceFields); + }catch (Throwable e){ logger.error("Failed to get streamisTableMetaInfo",e); throw new ErrorException(30002, "Failed to get streamisTableMetaInfo(后台获取数据源详细信息失败)"); @@ -166,7 +194,7 @@ public Response getStreamisTableMeta( HttpServletRequest req, Long streamisTable public Response getStreamisTableMetaByNodeNames( HttpServletRequest req,JsonNode json ) throws ErrorException { - Message message = null; + Message message; QueryWrapper wrapper = null; Map> nodeTablesMap = new HashMap<>(); List list = null; @@ -196,7 +224,7 @@ public Response getStreamisTableMetaByNodeNames( HttpServletRequest req,JsonNode public Response addStreamisTableMeta( HttpServletRequest req, JsonNode json) throws ErrorException { - Message message = null; + Message message; try { String userName = SecurityFilter.getLoginUsername(req); String authorityId = json.get("authorityId").asText(); @@ -250,7 +278,7 @@ public Response addStreamisTableMeta( HttpServletRequest req, JsonNode json) th public Response updateStreamisTableMeta( HttpServletRequest req, JsonNode json) throws ErrorException { - Message message = null; + Message message; try { String userName = SecurityFilter.getLoginUsername(req); StreamisTableMeta streamisTableMeta = mapper.readValue(json.get("streamisTableMeta"), StreamisTableMeta.class); @@ -286,7 +314,7 @@ public Response updateStreamisTableMeta( HttpServletRequest req, JsonNode json) } public Response deleteStreamisTableMeta(HttpServletRequest req, Long id) throws ErrorException { - Message message = null; + Message message; try { //delete TableFields QueryWrapper streamisDatasourceFieldsQueryWrapper = new QueryWrapper<>(); @@ -317,7 +345,7 @@ public Response deleteStreamisTableMeta(HttpServletRequest req, Long id) throws public Response addStreamisTableFields( HttpServletRequest req, JsonNode json) throws ErrorException { - Message message = null; + Message message; try { StreamisDatasourceFields streamisDatasourceFields = mapper.readValue(json.get("streamisTableFields"), StreamisDatasourceFields.class); boolean result = streamisDatasourceFieldsService.save(streamisDatasourceFields); @@ -333,7 +361,7 @@ public Response addStreamisTableFields( HttpServletRequest req, JsonNode json) } public Response updateStreamisTableFields( HttpServletRequest req, JsonNode json) throws ErrorException { - Message message = null; + Message message; try { StreamisDatasourceFields streamisDatasourceFields = mapper.readValue(json.get("streamisTableFields"), StreamisDatasourceFields.class); streamisDatasourceFieldsService.updateById(streamisDatasourceFields); @@ -346,7 +374,7 @@ public Response updateStreamisTableFields( HttpServletRequest req, JsonNode jso } public Response deleteStreamisTableFields( HttpServletRequest req, Long id) throws ErrorException { - Message message = null; + Message message; String userName = SecurityFilter.getLoginUsername(req); try { logger.info(userName+" delete StreamisTableFields："+id); @@ -403,6 +431,9 @@ public List queryTablesByDataBase(String system, Long dataSourceId , Str return LinkisDataSourceClient.queryTablesByDataBase(system,dataSourceId,dataBase ,user).tables(); } + public List> queryColumnsByTable(String system, Long dataSourceId , String dataBase,String table,String user){ + return LinkisDataSourceClient.queryColumnsByTable(system, dataSourceId, dataBase, table, user).columns(); + } diff --git a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java index fcf32ceb6..fc0e5e7aa 100644 --- a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java +++ b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java @@ -51,6 +51,13 @@ public Response getTablesByDataBase(@Context HttpServletRequest req, Map json + ) throws ErrorException { + return streamisTableMetaApi.getColumnsByTable(req,json); + } + /** * 获取数据源的详细信息，包括数据源信息、表信息和字段信息 */ diff --git a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java index 46f12286e..5668298b3 100644 --- a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java +++ b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java @@ -18,6 +18,7 @@ import com.webank.wedatasphere.linkis.server.Message; import com.webank.wedatasphere.linkis.server.security.SecurityFilter; import com.webank.wedatasphere.linkis.storage.FSFactory; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraInfo; import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceFields; import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisTableMeta; import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceFieldsService; @@ -28,6 +29,8 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.type.TypeReference; import org.glassfish.jersey.media.multipart.FormDataParam; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,14 +68,21 @@ public class DataSourceTransferRestfulApi { private ResourceService resourceService; @Autowired private VersionService versionService; + ObjectMapper mapper = new ObjectMapper(); - @GET + @POST @Path("/transfer") - public Response transfer(@Context HttpServletRequest request, @QueryParam("streamisTableMetaId") String streamisTableMetaId, @QueryParam("dataSourceId") String dataSource,@QueryParam("nodeName") String nodeName, @FormDataParam("labels") Map labels) throws ErrorException { + public Response transfer(@Context HttpServletRequest request, JsonNode json) throws ErrorException, IOException { Message message; + String streamisTableMetaId =json.get("streamisTableMetaId").asText(); + String dataSource =json.get("dataSourceId").asText(); + String nodeName =json.get("nodeName").asText(); + Map labels = mapper.readValue(json.get("streamisExtraInfo"), new TypeReference>() { + }); if(StringUtils.isBlank(streamisTableMetaId)){ throw new ErrorException(30201,"streamisTableMetaId is null."); } + //@QueryParam("streamisTableMetaId") String streamisTableMetaId, @QueryParam("dataSourceId") String dataSource,@QueryParam("nodeName") String nodeName, @FormDataParam("labels") Map labels String userName = SecurityFilter.getLoginUsername(request); Map params = new HashMap<>(); try { diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala b/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala index c00803557..54c366fef 100644 --- a/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala +++ b/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala @@ -3,8 +3,8 @@ package com.webank.wedatasphere.streamis.datasource.transfer.client import java.util.concurrent.TimeUnit import com.webank.wedatasphere.linkis.datasource.client.impl.{LinkisDataSourceRemoteClient, LinkisMetaDataRemoteClient} -import com.webank.wedatasphere.linkis.datasource.client.request.{GetAllDataSourceTypesAction, GetConnectParamsByDataSourceIdAction, MetadataGetDatabasesAction, MetadataGetTablesAction, QueryDataSourceAction, QueryDataSourceEnvAction} -import com.webank.wedatasphere.linkis.datasource.client.response.{GetConnectParamsByDataSourceIdResult, MetadataGetDatabasesResult, MetadataGetTablesResult, QueryDataSourceResult} +import com.webank.wedatasphere.linkis.datasource.client.request.{GetAllDataSourceTypesAction, GetConnectParamsByDataSourceIdAction, MetadataGetColumnsAction, MetadataGetDatabasesAction, MetadataGetTablesAction, QueryDataSourceAction, QueryDataSourceEnvAction} +import com.webank.wedatasphere.linkis.datasource.client.response.{GetConnectParamsByDataSourceIdResult, MetadataGetColumnsResult, MetadataGetDatabasesResult, MetadataGetTablesResult, QueryDataSourceResult} import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.{DataSource, DataSourceEnv, DataSourceType} import com.webank.wedatasphere.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfigBuilder @@ -124,6 +124,16 @@ object LinkisDataSourceClient { .build() ) } + def queryColumnsByTable(system: String,dataSourceId:Long,dataBase:String ,table:String, user:String):MetadataGetColumnsResult = { + metaDataClient.getColumns(MetadataGetColumnsAction.builder() + .setSystem(system) + .setDataSourceId(dataSourceId) + .setDatabase(dataBase) + .setTable(table) + .setUser(user) + .build() + ) + } From ff43c110cc2b383e0ce275edae7b269a64bddee1 Mon Sep 17 00:00:00 2001 From: kongslove <42604208+kongslove@users.noreply.github.com> Date: Thu, 10 Jun 2021 16:36:53 +0800 Subject: [PATCH 069/434] add interface for get columns --- .../domain/StreamisDataSourceTableVO.java | 9 +++ .../restful/api/StreamisTableMetaApi.java | 61 ++++++++++++++----- .../DataSourceServer2managerRestfulApi.java | 7 +++ .../api/DataSourceTransferRestfulApi.java | 14 ++++- .../client/LinkisDataSourceClient.scala | 14 ++++- 5 files changed, 86 insertions(+), 19 deletions(-) diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDataSourceTableVO.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDataSourceTableVO.java index 14f1cdc08..1444fb121 100644 --- a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDataSourceTableVO.java +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisDataSourceTableVO.java @@ -4,6 +4,7 @@ public class StreamisDataSourceTableVO { private String tableName; private Boolean isStreamisDataSource; + private Long streamisTableMetaId; public String getTableName() { return tableName; @@ -21,12 +22,20 @@ public void setStreamisDataSource(Boolean streamisDataSource) { isStreamisDataSource = streamisDataSource; } + public Long getStreamisTableMetaId() { + return streamisTableMetaId; + } + + public void setStreamisTableMetaId(Long streamisTableMetaId) { + this.streamisTableMetaId = streamisTableMetaId; + } @Override public String toString() { return "StreamisDataSourceTableVO{" + "tableName='" + tableName + '\'' + ", isStreamisDataSource=" + isStreamisDataSource + + ", streamisTableMetaId=" + streamisTableMetaId + '}'; } } diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java index d8088e54c..8eeb15c83 100644 --- a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java @@ -70,10 +70,10 @@ public class StreamisTableMetaApi { public Response getDataSourceCluster( HttpServletRequest req, Map json ) throws ErrorException { - Message message = null; + Message message; try{ String userName = SecurityFilter.getLoginUsername(req); - String system = json.get("system"); + String system = json.getOrDefault("system","streamis"); String name = json.get("name"); Long dataSourceTypeId = Long.valueOf(json.get("dataSourceTypeId")); List dataSourceClusters = getDataSourceClusters(system, name, dataSourceTypeId, userName); @@ -87,7 +87,7 @@ public Response getDataSourceCluster( HttpServletRequest req, Map public Response getDataSourceTypes( HttpServletRequest req, Map json ) throws ErrorException { - Message message = null; + Message message; try{ String userName = SecurityFilter.getLoginUsername(req); List dataSourceTypes = getDataSourceTypes(userName); @@ -101,10 +101,10 @@ public Response getDataSourceTypes( HttpServletRequest req, Map j public Response getDataBasesByCuster( HttpServletRequest req, Map json ) throws ErrorException { - Message message = null; + Message message; try{ String userName = SecurityFilter.getLoginUsername(req); - String system = json.get("system"); + String system = json.getOrDefault("system","streamis"); Long dataSourceTypeId = Long.valueOf(json.get("dataSourceId")); List dataBases = queryDataBasesByCuster(system, dataSourceTypeId, userName); message = Message.ok().data("dataBases",dataBases); @@ -117,10 +117,10 @@ public Response getDataBasesByCuster( HttpServletRequest req, Map public Response getTablesByDataBase( HttpServletRequest req, Map json ) throws ErrorException { - Message message = null; + Message message; try{ String userName = SecurityFilter.getLoginUsername(req); - String system = json.get("system"); + String system = json.getOrDefault("system","streamis"); Long dataSourceId = Long.valueOf(json.get("dataSourceId")); String dataBase = json.get("dataBase"); QueryWrapper wrapper = new QueryWrapper<>();; @@ -134,6 +134,9 @@ public Response getTablesByDataBase( HttpServletRequest req, Map for (StreamisTableMeta streamisTableMeta : list) { tableVO = new StreamisDataSourceTableVO(); tableVO.setTableName(table); + if(table.equals(streamisTableMeta.getTableName())){ + tableVO.setStreamisTableMetaId(streamisTableMeta.getId()); + } tableVO.setStreamisDataSource(table.equals(streamisTableMeta.getTableName())); tableList.add(tableVO); } @@ -147,15 +150,40 @@ public Response getTablesByDataBase( HttpServletRequest req, Map return Message.messageToResponse(message); } + public Response getColumnsByTable( HttpServletRequest req, Map json + ) throws ErrorException { + Message message; + try{ + String userName = SecurityFilter.getLoginUsername(req); + String system = json.getOrDefault("system","streamis"); + Long dataSourceTypeId = Long.valueOf(json.get("dataSourceId")); + String dataBase = json.get("dataBase"); + String table = json.get("table"); + String dataSourceType = json.get("dataSourceType"); + List> columns =null; + if(!"kafka".equalsIgnoreCase(dataSourceType)){ + columns = queryColumnsByTable(system, dataSourceTypeId, dataBase, table, userName); + } + message = Message.ok().data("columns",columns); + }catch (Throwable e){ + logger.error("failed to get columns list",e); + throw new ErrorException(30017, "sorry,failed to get columns list"); + } + return Message.messageToResponse(message); + } + public Response getStreamisTableMeta( HttpServletRequest req, Long streamisTableMetaId ) throws ErrorException { - Message message = null; + Message message; try{ StreamisTableMeta streamisTableMeta = streamisTableMetaService.getById(streamisTableMetaId); QueryWrapper streamisDatasourceFieldsWrapper = new QueryWrapper<>(); streamisDatasourceFieldsWrapper.eq("streamis_table_meta_id",streamisTableMetaId); List streamisDatasourceFields = streamisDatasourceFieldsService.list(streamisDatasourceFieldsWrapper); + + //查询一下 message=Message.ok().data("streamisTableMeta",streamisTableMeta).data("streamisDatasourceFields",streamisDatasourceFields); + }catch (Throwable e){ logger.error("Failed to get streamisTableMetaInfo",e); throw new ErrorException(30002, "Failed to get streamisTableMetaInfo(后台获取数据源详细信息失败)"); @@ -166,7 +194,7 @@ public Response getStreamisTableMeta( HttpServletRequest req, Long streamisTable public Response getStreamisTableMetaByNodeNames( HttpServletRequest req,JsonNode json ) throws ErrorException { - Message message = null; + Message message; QueryWrapper wrapper = null; Map> nodeTablesMap = new HashMap<>(); List list = null; @@ -196,7 +224,7 @@ public Response getStreamisTableMetaByNodeNames( HttpServletRequest req,JsonNode public Response addStreamisTableMeta( HttpServletRequest req, JsonNode json) throws ErrorException { - Message message = null; + Message message; try { String userName = SecurityFilter.getLoginUsername(req); String authorityId = json.get("authorityId").asText(); @@ -250,7 +278,7 @@ public Response addStreamisTableMeta( HttpServletRequest req, JsonNode json) th public Response updateStreamisTableMeta( HttpServletRequest req, JsonNode json) throws ErrorException { - Message message = null; + Message message; try { String userName = SecurityFilter.getLoginUsername(req); StreamisTableMeta streamisTableMeta = mapper.readValue(json.get("streamisTableMeta"), StreamisTableMeta.class); @@ -286,7 +314,7 @@ public Response updateStreamisTableMeta( HttpServletRequest req, JsonNode json) } public Response deleteStreamisTableMeta(HttpServletRequest req, Long id) throws ErrorException { - Message message = null; + Message message; try { //delete TableFields QueryWrapper streamisDatasourceFieldsQueryWrapper = new QueryWrapper<>(); @@ -317,7 +345,7 @@ public Response deleteStreamisTableMeta(HttpServletRequest req, Long id) throws public Response addStreamisTableFields( HttpServletRequest req, JsonNode json) throws ErrorException { - Message message = null; + Message message; try { StreamisDatasourceFields streamisDatasourceFields = mapper.readValue(json.get("streamisTableFields"), StreamisDatasourceFields.class); boolean result = streamisDatasourceFieldsService.save(streamisDatasourceFields); @@ -333,7 +361,7 @@ public Response addStreamisTableFields( HttpServletRequest req, JsonNode json) } public Response updateStreamisTableFields( HttpServletRequest req, JsonNode json) throws ErrorException { - Message message = null; + Message message; try { StreamisDatasourceFields streamisDatasourceFields = mapper.readValue(json.get("streamisTableFields"), StreamisDatasourceFields.class); streamisDatasourceFieldsService.updateById(streamisDatasourceFields); @@ -346,7 +374,7 @@ public Response updateStreamisTableFields( HttpServletRequest req, JsonNode jso } public Response deleteStreamisTableFields( HttpServletRequest req, Long id) throws ErrorException { - Message message = null; + Message message; String userName = SecurityFilter.getLoginUsername(req); try { logger.info(userName+" delete StreamisTableFields："+id); @@ -403,6 +431,9 @@ public List queryTablesByDataBase(String system, Long dataSourceId , Str return LinkisDataSourceClient.queryTablesByDataBase(system,dataSourceId,dataBase ,user).tables(); } + public List> queryColumnsByTable(String system, Long dataSourceId , String dataBase,String table,String user){ + return LinkisDataSourceClient.queryColumnsByTable(system, dataSourceId, dataBase, table, user).columns(); + } diff --git a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java index fcf32ceb6..fc0e5e7aa 100644 --- a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java +++ b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceServer2managerRestfulApi.java @@ -51,6 +51,13 @@ public Response getTablesByDataBase(@Context HttpServletRequest req, Map json + ) throws ErrorException { + return streamisTableMetaApi.getColumnsByTable(req,json); + } + /** * 获取数据源的详细信息，包括数据源信息、表信息和字段信息 */ diff --git a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java index 46f12286e..5668298b3 100644 --- a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java +++ b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java @@ -18,6 +18,7 @@ import com.webank.wedatasphere.linkis.server.Message; import com.webank.wedatasphere.linkis.server.security.SecurityFilter; import com.webank.wedatasphere.linkis.storage.FSFactory; +import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraInfo; import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceFields; import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisTableMeta; import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceFieldsService; @@ -28,6 +29,8 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.type.TypeReference; import org.glassfish.jersey.media.multipart.FormDataParam; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,14 +68,21 @@ public class DataSourceTransferRestfulApi { private ResourceService resourceService; @Autowired private VersionService versionService; + ObjectMapper mapper = new ObjectMapper(); - @GET + @POST @Path("/transfer") - public Response transfer(@Context HttpServletRequest request, @QueryParam("streamisTableMetaId") String streamisTableMetaId, @QueryParam("dataSourceId") String dataSource,@QueryParam("nodeName") String nodeName, @FormDataParam("labels") Map labels) throws ErrorException { + public Response transfer(@Context HttpServletRequest request, JsonNode json) throws ErrorException, IOException { Message message; + String streamisTableMetaId =json.get("streamisTableMetaId").asText(); + String dataSource =json.get("dataSourceId").asText(); + String nodeName =json.get("nodeName").asText(); + Map labels = mapper.readValue(json.get("streamisExtraInfo"), new TypeReference>() { + }); if(StringUtils.isBlank(streamisTableMetaId)){ throw new ErrorException(30201,"streamisTableMetaId is null."); } + //@QueryParam("streamisTableMetaId") String streamisTableMetaId, @QueryParam("dataSourceId") String dataSource,@QueryParam("nodeName") String nodeName, @FormDataParam("labels") Map labels String userName = SecurityFilter.getLoginUsername(request); Map params = new HashMap<>(); try { diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala b/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala index c00803557..54c366fef 100644 --- a/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala +++ b/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala @@ -3,8 +3,8 @@ package com.webank.wedatasphere.streamis.datasource.transfer.client import java.util.concurrent.TimeUnit import com.webank.wedatasphere.linkis.datasource.client.impl.{LinkisDataSourceRemoteClient, LinkisMetaDataRemoteClient} -import com.webank.wedatasphere.linkis.datasource.client.request.{GetAllDataSourceTypesAction, GetConnectParamsByDataSourceIdAction, MetadataGetDatabasesAction, MetadataGetTablesAction, QueryDataSourceAction, QueryDataSourceEnvAction} -import com.webank.wedatasphere.linkis.datasource.client.response.{GetConnectParamsByDataSourceIdResult, MetadataGetDatabasesResult, MetadataGetTablesResult, QueryDataSourceResult} +import com.webank.wedatasphere.linkis.datasource.client.request.{GetAllDataSourceTypesAction, GetConnectParamsByDataSourceIdAction, MetadataGetColumnsAction, MetadataGetDatabasesAction, MetadataGetTablesAction, QueryDataSourceAction, QueryDataSourceEnvAction} +import com.webank.wedatasphere.linkis.datasource.client.response.{GetConnectParamsByDataSourceIdResult, MetadataGetColumnsResult, MetadataGetDatabasesResult, MetadataGetTablesResult, QueryDataSourceResult} import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.{DataSource, DataSourceEnv, DataSourceType} import com.webank.wedatasphere.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfigBuilder @@ -124,6 +124,16 @@ object LinkisDataSourceClient { .build() ) } + def queryColumnsByTable(system: String,dataSourceId:Long,dataBase:String ,table:String, user:String):MetadataGetColumnsResult = { + metaDataClient.getColumns(MetadataGetColumnsAction.builder() + .setSystem(system) + .setDataSourceId(dataSourceId) + .setDatabase(dataBase) + .setTable(table) + .setUser(user) + .build() + ) + } From c4df251a3dd55b699dea3d642e4e5a32acd430d0 Mon Sep 17 00:00:00 2001 From: kongslove <42604208+kongslove@users.noreply.github.com> Date: Tue, 15 Jun 2021 11:32:55 +0800 Subject: [PATCH 070/434] resolve circular dependency --- .../streamis-datasource-manager/pom.xml | 7 ++- .../manager/conf/TransferConfiguration.java | 24 +++++++++ .../impl/StreamisDatasourceExtraInfoDao.xml | 2 +- .../dao/impl/StreamisDatasourceExtraUiDao.xml | 2 +- .../manager/domain/StreamisTableMeta.java | 11 ++++ .../restful/api/StreamisTableMetaApi.java | 53 ++++++++++++------- .../client/LinkisDataSourceClient.scala | 4 +- .../streamis-datasource-server/pom.xml | 10 ++-- .../api/DataSourceTransferRestfulApi.java | 5 -- .../streamis-datasource-transfer/pom.xml | 10 ++-- .../service/impl/DataSourceTransferImpl.java | 3 +- 11 files changed, 87 insertions(+), 44 deletions(-) create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/conf/TransferConfiguration.java rename streamis-datasource/{streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer => streamis-datasource-manager/src/main/scala/com/webank/wedatasphere/streamis/datasource/manager}/client/LinkisDataSourceClient.scala (97%) diff --git a/streamis-datasource/streamis-datasource-manager/pom.xml b/streamis-datasource/streamis-datasource-manager/pom.xml index 6ddfed869..dbd662f4d 100644 --- a/streamis-datasource/streamis-datasource-manager/pom.xml +++ b/streamis-datasource/streamis-datasource-manager/pom.xml @@ -55,10 +55,9 @@ ${linkis.version} - com.webank.wedatasphere.streamis - streamis-datasource-transfer - 0.1.0-SNAPSHOT - compile + com.webank.wedatasphere.linkis + linkis-datasource-client + ${linkis.version} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/conf/TransferConfiguration.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/conf/TransferConfiguration.java new file mode 100644 index 000000000..b97e71464 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/conf/TransferConfiguration.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.streamis.datasource.manager.conf; + +import com.webank.wedatasphere.linkis.common.conf.CommonVars; + +public class TransferConfiguration { + public static final CommonVars SERVER_URL = CommonVars.apply("wds.streamis.datasource.client.serverurl", ""); + public static final CommonVars CONNECTION_TIMEOUT = CommonVars.apply("wds.streamis.datasource.client.connection.timeout", 30000L); + public static final CommonVars DISCOVERY_ENABLED = CommonVars.apply("wds.streamis.datasource.client.discovery.enabled", true); + public static final CommonVars DISCOVERY_FREQUENCY_PERIOD = CommonVars.apply("wds.streamis.datasource.client.discoveryfrequency.period", 1L); + public static final CommonVars LOAD_BALANCER_ENABLED = CommonVars.apply("wds.streamis.datasource.client.loadbalancer.enabled", true); + public static final CommonVars MAX_CONNECTION_SIZE = CommonVars.apply("wds.streamis.datasource.client.maxconnection.size", 5); + public static final CommonVars RETRY_ENABLED = CommonVars.apply("wds.streamis.datasource.client.retryenabled", false); + public static final CommonVars READ_TIMEOUT = CommonVars.apply("wds.streamis.datasource.client.readtimeout", 30000L); + + public static final CommonVars AUTHTOKEN_KEY = CommonVars.apply("wds.streamis.datasource.client.authtoken.key", ""); + public static final CommonVars AUTHTOKEN_VALUE = CommonVars.apply("wds.streamis.datasource.client.authtoken.value", ""); + public static final CommonVars DWS_VERSION = CommonVars.apply("wds.streamis.datasource.client.dws.version", ""); + + + + + + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraInfoDao.xml b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraInfoDao.xml index 0d8b1e773..1c28f10ba 100644 --- a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraInfoDao.xml +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraInfoDao.xml @@ -3,7 +3,7 @@ - + diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraUiDao.xml b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraUiDao.xml index 5f15d5831..e7418bd56 100644 --- a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraUiDao.xml +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraUiDao.xml @@ -3,7 +3,7 @@ - + diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisTableMeta.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisTableMeta.java index 422e55329..5d47417ac 100644 --- a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisTableMeta.java +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisTableMeta.java @@ -66,6 +66,16 @@ public class StreamisTableMeta extends Model { @TableField("node_name") private String nodeName; + private String alias; + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + public String getTableName() { return tableName; } @@ -215,6 +225,7 @@ public String toString() { ", linkisDatasourceName='" + linkisDatasourceName + '\'' + ", tableName='" + tableName + '\'' + ", nodeName='" + nodeName + '\'' + + ", alias='" + alias + '\'' + ", createBy='" + createBy + '\'' + ", createTime=" + createTime + ", updateBy='" + updateBy + '\'' + diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java index 8eeb15c83..148621719 100644 --- a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java @@ -16,22 +16,16 @@ package com.webank.wedatasphere.streamis.datasource.manager.restful.api; -import com.baomidou.mybatisplus.core.conditions.Wrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; -import com.baomidou.mybatisplus.extension.conditions.update.UpdateChainWrapper; import com.webank.wedatasphere.linkis.common.exception.ErrorException; -import com.webank.wedatasphere.linkis.datasource.client.response.MetadataGetDatabasesResult; import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSource; import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceType; import com.webank.wedatasphere.streamis.datasource.manager.domain.*; -import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceAuthorityService; -import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceExtraInfoService; -import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceFieldsService; +import com.webank.wedatasphere.streamis.datasource.manager.service.*; import com.webank.wedatasphere.linkis.server.Message; import com.webank.wedatasphere.linkis.server.security.SecurityFilter; -import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisTableMetaService; -import com.webank.wedatasphere.streamis.datasource.transfer.client.LinkisDataSourceClient; +import com.webank.wedatasphere.streamis.datasource.manager.client.LinkisDataSourceClient; import org.apache.commons.collections.CollectionUtils; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; @@ -42,8 +36,6 @@ import org.springframework.stereotype.Service; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; import java.util.*; @Service @@ -62,6 +54,10 @@ public class StreamisTableMetaApi { @Autowired StreamisDatasourceExtraInfoService streamisDatasourceExtraInfoService; + + @Autowired + StreamisDatasourceExtraUiService streamisDatasourceExtraUiService; + ObjectMapper mapper = new ObjectMapper(); @@ -164,7 +160,10 @@ public Response getColumnsByTable( HttpServletRequest req, Map js if(!"kafka".equalsIgnoreCase(dataSourceType)){ columns = queryColumnsByTable(system, dataSourceTypeId, dataBase, table, userName); } - message = Message.ok().data("columns",columns); + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.eq("datasource_type",dataSourceType); + List extraUis = streamisDatasourceExtraUiService.list(wrapper); + message = Message.ok().data("columns",columns).data("extraUis",extraUis); }catch (Throwable e){ logger.error("failed to get columns list",e); throw new ErrorException(30017, "sorry,failed to get columns list"); @@ -290,20 +289,36 @@ public Response updateStreamisTableMeta( HttpServletRequest req, JsonNode json) // update StreamisTableMeta streamisTableMetaService.updateById(streamisTableMeta); + //更新字段信息； +// StreamisDatasourceFields streamisDatasourceFields = mapper.readValue(json.get("streamisTableFields"), StreamisDatasourceFields.class); +// streamisDatasourceFieldsService.updateById(streamisDatasourceFields); + + List streamisDatasourceFields = mapper.readValue(json.get("streamisTableFields"), new TypeReference>() { + }); + if(CollectionUtils.isNotEmpty(streamisDatasourceFields)){ + boolean b = streamisDatasourceFieldsService.updateBatchById(streamisDatasourceFields); + if(!b){ + throw new ErrorException(30021, "sorry，Failed to update the streamisDatasourceFields information(更新字段信息失败)"); + } + } + List extraInfoList = mapper.readValue(json.get("streamisExtraInfo"), new TypeReference>() { }); - extraInfoList.forEach(obj-> obj.setStreamisTableMetaId(streamisTableMetaId)); - UpdateWrapper wrapper = null; - boolean result ; - for (StreamisDatasourceExtraInfo extraInfo : extraInfoList) { - wrapper.eq("key",extraInfo.getKey()).eq("streamis_table_meta_id",streamisTableMetaId).set("value",extraInfo.getValue()); - result = streamisDatasourceExtraInfoService.update(null, wrapper); - if(!result){ - throw new ErrorException(30020, String.format("sorry，Failed to update the %s information(更新%s失败)",extraInfo.getKey(),extraInfo.getKey())); + if(CollectionUtils.isNotEmpty(extraInfoList)){ + extraInfoList.forEach(obj-> obj.setStreamisTableMetaId(streamisTableMetaId)); + UpdateWrapper wrapper = null; + boolean result ; + for (StreamisDatasourceExtraInfo extraInfo : extraInfoList) { + wrapper.eq("key",extraInfo.getKey()).eq("streamis_table_meta_id",streamisTableMetaId).set("value",extraInfo.getValue()); + result = streamisDatasourceExtraInfoService.update(null, wrapper); + if(!result){ + throw new ErrorException(30020, String.format("sorry，Failed to update the %s information(更新%s失败)",extraInfo.getKey(),extraInfo.getKey())); + } } } + message = Message.ok(); } catch (Exception e) { diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala b/streamis-datasource/streamis-datasource-manager/src/main/scala/com/webank/wedatasphere/streamis/datasource/manager/client/LinkisDataSourceClient.scala similarity index 97% rename from streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala rename to streamis-datasource/streamis-datasource-manager/src/main/scala/com/webank/wedatasphere/streamis/datasource/manager/client/LinkisDataSourceClient.scala index 54c366fef..8dc770809 100644 --- a/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala +++ b/streamis-datasource/streamis-datasource-manager/src/main/scala/com/webank/wedatasphere/streamis/datasource/manager/client/LinkisDataSourceClient.scala @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.streamis.datasource.transfer.client +package com.webank.wedatasphere.streamis.datasource.manager.client import java.util.concurrent.TimeUnit @@ -8,7 +8,7 @@ import com.webank.wedatasphere.linkis.datasource.client.response.{GetConnectPara import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.{DataSource, DataSourceEnv, DataSourceType} import com.webank.wedatasphere.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfigBuilder -import com.webank.wedatasphere.streamis.datasource.transfer.conf.TransferConfiguration +import com.webank.wedatasphere.streamis.datasource.manager.conf.TransferConfiguration object LinkisDataSourceClient { //Linkis Datasource Client Config diff --git a/streamis-datasource/streamis-datasource-server/pom.xml b/streamis-datasource/streamis-datasource-server/pom.xml index f95280fb9..cc7c9e28b 100644 --- a/streamis-datasource/streamis-datasource-server/pom.xml +++ b/streamis-datasource/streamis-datasource-server/pom.xml @@ -37,11 +37,11 @@ linkis-storage ${linkis.version} - - com.webank.wedatasphere.dss - dss-server - ${dss.version} - + + + + + com.webank.wedatasphere.streamis streamis-datasource-transfer diff --git a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java index 5668298b3..d5d56fc7c 100644 --- a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java +++ b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java @@ -2,7 +2,6 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.google.gson.Gson; -import com.webank.wedatasphere.dss.server.service.BMLService; import com.webank.wedatasphere.linkis.bml.Entity.Resource; import com.webank.wedatasphere.linkis.bml.Entity.ResourceVersion; import com.webank.wedatasphere.linkis.bml.common.*; @@ -18,7 +17,6 @@ import com.webank.wedatasphere.linkis.server.Message; import com.webank.wedatasphere.linkis.server.security.SecurityFilter; import com.webank.wedatasphere.linkis.storage.FSFactory; -import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraInfo; import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceFields; import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisTableMeta; import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceFieldsService; @@ -31,14 +29,11 @@ import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; -import org.glassfish.jersey.media.multipart.FormDataParam; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import scala.tools.cmd.Opt; -import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.*; diff --git a/streamis-datasource/streamis-datasource-transfer/pom.xml b/streamis-datasource/streamis-datasource-transfer/pom.xml index d5c82d43e..9c7b148c9 100644 --- a/streamis-datasource/streamis-datasource-transfer/pom.xml +++ b/streamis-datasource/streamis-datasource-transfer/pom.xml @@ -17,11 +17,11 @@ linkis-label-common ${linkis.version} - - com.webank.wedatasphere.linkis - linkis-datasource-client - ${linkis.version} - + + + + + com.webank.wedatasphere.streamis streamis-datasource-manager diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java index 9ac33bec4..e3a78db7c 100644 --- a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java +++ b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java @@ -2,7 +2,6 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.webank.wedatasphere.linkis.common.exception.ErrorException; -import com.webank.wedatasphere.linkis.datasource.client.response.GetConnectParamsByDataSourceIdResult; import com.webank.wedatasphere.linkis.manager.label.entity.engine.EngineType; import com.webank.wedatasphere.linkis.manager.label.entity.engine.RunType; import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraInfo; @@ -11,7 +10,7 @@ import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceExtraInfoService; import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceFieldsService; import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisTableMetaService; -import com.webank.wedatasphere.streamis.datasource.transfer.client.LinkisDataSourceClient; +import com.webank.wedatasphere.streamis.datasource.manager.client.LinkisDataSourceClient; import com.webank.wedatasphere.streamis.datasource.transfer.entity.StreamisDataSourceCode; import com.webank.wedatasphere.streamis.datasource.transfer.entity.StreamisTableEntity; import com.webank.wedatasphere.streamis.datasource.transfer.service.DataSourceTransfer; From e14b4fa93cd99a6483b3b06ade7d7d1a3e5de69c Mon Sep 17 00:00:00 2001 From: kongslove <42604208+kongslove@users.noreply.github.com> Date: Tue, 15 Jun 2021 11:32:55 +0800 Subject: [PATCH 071/434] resolve circular dependency --- .../streamis-datasource-manager/pom.xml | 7 ++- .../manager/conf/TransferConfiguration.java | 24 +++++++++ .../impl/StreamisDatasourceExtraInfoDao.xml | 2 +- .../dao/impl/StreamisDatasourceExtraUiDao.xml | 2 +- .../manager/domain/StreamisTableMeta.java | 11 ++++ .../restful/api/StreamisTableMetaApi.java | 53 ++++++++++++------- .../client/LinkisDataSourceClient.scala | 4 +- .../streamis-datasource-server/pom.xml | 10 ++-- .../api/DataSourceTransferRestfulApi.java | 5 -- .../streamis-datasource-transfer/pom.xml | 10 ++-- .../service/impl/DataSourceTransferImpl.java | 3 +- 11 files changed, 87 insertions(+), 44 deletions(-) create mode 100644 streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/conf/TransferConfiguration.java rename streamis-datasource/{streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer => streamis-datasource-manager/src/main/scala/com/webank/wedatasphere/streamis/datasource/manager}/client/LinkisDataSourceClient.scala (97%) diff --git a/streamis-datasource/streamis-datasource-manager/pom.xml b/streamis-datasource/streamis-datasource-manager/pom.xml index 6ddfed869..dbd662f4d 100644 --- a/streamis-datasource/streamis-datasource-manager/pom.xml +++ b/streamis-datasource/streamis-datasource-manager/pom.xml @@ -55,10 +55,9 @@ ${linkis.version} - com.webank.wedatasphere.streamis - streamis-datasource-transfer - 0.1.0-SNAPSHOT - compile + com.webank.wedatasphere.linkis + linkis-datasource-client + ${linkis.version} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/conf/TransferConfiguration.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/conf/TransferConfiguration.java new file mode 100644 index 000000000..b97e71464 --- /dev/null +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/conf/TransferConfiguration.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.streamis.datasource.manager.conf; + +import com.webank.wedatasphere.linkis.common.conf.CommonVars; + +public class TransferConfiguration { + public static final CommonVars SERVER_URL = CommonVars.apply("wds.streamis.datasource.client.serverurl", ""); + public static final CommonVars CONNECTION_TIMEOUT = CommonVars.apply("wds.streamis.datasource.client.connection.timeout", 30000L); + public static final CommonVars DISCOVERY_ENABLED = CommonVars.apply("wds.streamis.datasource.client.discovery.enabled", true); + public static final CommonVars DISCOVERY_FREQUENCY_PERIOD = CommonVars.apply("wds.streamis.datasource.client.discoveryfrequency.period", 1L); + public static final CommonVars LOAD_BALANCER_ENABLED = CommonVars.apply("wds.streamis.datasource.client.loadbalancer.enabled", true); + public static final CommonVars MAX_CONNECTION_SIZE = CommonVars.apply("wds.streamis.datasource.client.maxconnection.size", 5); + public static final CommonVars RETRY_ENABLED = CommonVars.apply("wds.streamis.datasource.client.retryenabled", false); + public static final CommonVars READ_TIMEOUT = CommonVars.apply("wds.streamis.datasource.client.readtimeout", 30000L); + + public static final CommonVars AUTHTOKEN_KEY = CommonVars.apply("wds.streamis.datasource.client.authtoken.key", ""); + public static final CommonVars AUTHTOKEN_VALUE = CommonVars.apply("wds.streamis.datasource.client.authtoken.value", ""); + public static final CommonVars DWS_VERSION = CommonVars.apply("wds.streamis.datasource.client.dws.version", ""); + + + + + + +} diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraInfoDao.xml b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraInfoDao.xml index 0d8b1e773..1c28f10ba 100644 --- a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraInfoDao.xml +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraInfoDao.xml @@ -3,7 +3,7 @@ - + diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraUiDao.xml b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraUiDao.xml index 5f15d5831..e7418bd56 100644 --- a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraUiDao.xml +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/dao/impl/StreamisDatasourceExtraUiDao.xml @@ -3,7 +3,7 @@ - + diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisTableMeta.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisTableMeta.java index 422e55329..5d47417ac 100644 --- a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisTableMeta.java +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/domain/StreamisTableMeta.java @@ -66,6 +66,16 @@ public class StreamisTableMeta extends Model { @TableField("node_name") private String nodeName; + private String alias; + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + public String getTableName() { return tableName; } @@ -215,6 +225,7 @@ public String toString() { ", linkisDatasourceName='" + linkisDatasourceName + '\'' + ", tableName='" + tableName + '\'' + ", nodeName='" + nodeName + '\'' + + ", alias='" + alias + '\'' + ", createBy='" + createBy + '\'' + ", createTime=" + createTime + ", updateBy='" + updateBy + '\'' + diff --git a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java index 8eeb15c83..148621719 100644 --- a/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java +++ b/streamis-datasource/streamis-datasource-manager/src/main/java/com/webank/wedatasphere/streamis/datasource/manager/restful/api/StreamisTableMetaApi.java @@ -16,22 +16,16 @@ package com.webank.wedatasphere.streamis.datasource.manager.restful.api; -import com.baomidou.mybatisplus.core.conditions.Wrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; -import com.baomidou.mybatisplus.extension.conditions.update.UpdateChainWrapper; import com.webank.wedatasphere.linkis.common.exception.ErrorException; -import com.webank.wedatasphere.linkis.datasource.client.response.MetadataGetDatabasesResult; import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSource; import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceType; import com.webank.wedatasphere.streamis.datasource.manager.domain.*; -import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceAuthorityService; -import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceExtraInfoService; -import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceFieldsService; +import com.webank.wedatasphere.streamis.datasource.manager.service.*; import com.webank.wedatasphere.linkis.server.Message; import com.webank.wedatasphere.linkis.server.security.SecurityFilter; -import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisTableMetaService; -import com.webank.wedatasphere.streamis.datasource.transfer.client.LinkisDataSourceClient; +import com.webank.wedatasphere.streamis.datasource.manager.client.LinkisDataSourceClient; import org.apache.commons.collections.CollectionUtils; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; @@ -42,8 +36,6 @@ import org.springframework.stereotype.Service; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; import java.util.*; @Service @@ -62,6 +54,10 @@ public class StreamisTableMetaApi { @Autowired StreamisDatasourceExtraInfoService streamisDatasourceExtraInfoService; + + @Autowired + StreamisDatasourceExtraUiService streamisDatasourceExtraUiService; + ObjectMapper mapper = new ObjectMapper(); @@ -164,7 +160,10 @@ public Response getColumnsByTable( HttpServletRequest req, Map js if(!"kafka".equalsIgnoreCase(dataSourceType)){ columns = queryColumnsByTable(system, dataSourceTypeId, dataBase, table, userName); } - message = Message.ok().data("columns",columns); + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.eq("datasource_type",dataSourceType); + List extraUis = streamisDatasourceExtraUiService.list(wrapper); + message = Message.ok().data("columns",columns).data("extraUis",extraUis); }catch (Throwable e){ logger.error("failed to get columns list",e); throw new ErrorException(30017, "sorry,failed to get columns list"); @@ -290,20 +289,36 @@ public Response updateStreamisTableMeta( HttpServletRequest req, JsonNode json) // update StreamisTableMeta streamisTableMetaService.updateById(streamisTableMeta); + //更新字段信息； +// StreamisDatasourceFields streamisDatasourceFields = mapper.readValue(json.get("streamisTableFields"), StreamisDatasourceFields.class); +// streamisDatasourceFieldsService.updateById(streamisDatasourceFields); + + List streamisDatasourceFields = mapper.readValue(json.get("streamisTableFields"), new TypeReference>() { + }); + if(CollectionUtils.isNotEmpty(streamisDatasourceFields)){ + boolean b = streamisDatasourceFieldsService.updateBatchById(streamisDatasourceFields); + if(!b){ + throw new ErrorException(30021, "sorry，Failed to update the streamisDatasourceFields information(更新字段信息失败)"); + } + } + List extraInfoList = mapper.readValue(json.get("streamisExtraInfo"), new TypeReference>() { }); - extraInfoList.forEach(obj-> obj.setStreamisTableMetaId(streamisTableMetaId)); - UpdateWrapper wrapper = null; - boolean result ; - for (StreamisDatasourceExtraInfo extraInfo : extraInfoList) { - wrapper.eq("key",extraInfo.getKey()).eq("streamis_table_meta_id",streamisTableMetaId).set("value",extraInfo.getValue()); - result = streamisDatasourceExtraInfoService.update(null, wrapper); - if(!result){ - throw new ErrorException(30020, String.format("sorry，Failed to update the %s information(更新%s失败)",extraInfo.getKey(),extraInfo.getKey())); + if(CollectionUtils.isNotEmpty(extraInfoList)){ + extraInfoList.forEach(obj-> obj.setStreamisTableMetaId(streamisTableMetaId)); + UpdateWrapper wrapper = null; + boolean result ; + for (StreamisDatasourceExtraInfo extraInfo : extraInfoList) { + wrapper.eq("key",extraInfo.getKey()).eq("streamis_table_meta_id",streamisTableMetaId).set("value",extraInfo.getValue()); + result = streamisDatasourceExtraInfoService.update(null, wrapper); + if(!result){ + throw new ErrorException(30020, String.format("sorry，Failed to update the %s information(更新%s失败)",extraInfo.getKey(),extraInfo.getKey())); + } } } + message = Message.ok(); } catch (Exception e) { diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala b/streamis-datasource/streamis-datasource-manager/src/main/scala/com/webank/wedatasphere/streamis/datasource/manager/client/LinkisDataSourceClient.scala similarity index 97% rename from streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala rename to streamis-datasource/streamis-datasource-manager/src/main/scala/com/webank/wedatasphere/streamis/datasource/manager/client/LinkisDataSourceClient.scala index 54c366fef..8dc770809 100644 --- a/streamis-datasource/streamis-datasource-transfer/src/main/scala/com/webank/wedatasphere/streamis/datasource/transfer/client/LinkisDataSourceClient.scala +++ b/streamis-datasource/streamis-datasource-manager/src/main/scala/com/webank/wedatasphere/streamis/datasource/manager/client/LinkisDataSourceClient.scala @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.streamis.datasource.transfer.client +package com.webank.wedatasphere.streamis.datasource.manager.client import java.util.concurrent.TimeUnit @@ -8,7 +8,7 @@ import com.webank.wedatasphere.linkis.datasource.client.response.{GetConnectPara import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.{DataSource, DataSourceEnv, DataSourceType} import com.webank.wedatasphere.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfigBuilder -import com.webank.wedatasphere.streamis.datasource.transfer.conf.TransferConfiguration +import com.webank.wedatasphere.streamis.datasource.manager.conf.TransferConfiguration object LinkisDataSourceClient { //Linkis Datasource Client Config diff --git a/streamis-datasource/streamis-datasource-server/pom.xml b/streamis-datasource/streamis-datasource-server/pom.xml index f95280fb9..cc7c9e28b 100644 --- a/streamis-datasource/streamis-datasource-server/pom.xml +++ b/streamis-datasource/streamis-datasource-server/pom.xml @@ -37,11 +37,11 @@ linkis-storage ${linkis.version} - - com.webank.wedatasphere.dss - dss-server - ${dss.version} - + + + + + com.webank.wedatasphere.streamis streamis-datasource-transfer diff --git a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java index 5668298b3..d5d56fc7c 100644 --- a/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java +++ b/streamis-datasource/streamis-datasource-server/src/main/java/com/webank/wedatasphere/streamis/datasource/server/restful/api/DataSourceTransferRestfulApi.java @@ -2,7 +2,6 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.google.gson.Gson; -import com.webank.wedatasphere.dss.server.service.BMLService; import com.webank.wedatasphere.linkis.bml.Entity.Resource; import com.webank.wedatasphere.linkis.bml.Entity.ResourceVersion; import com.webank.wedatasphere.linkis.bml.common.*; @@ -18,7 +17,6 @@ import com.webank.wedatasphere.linkis.server.Message; import com.webank.wedatasphere.linkis.server.security.SecurityFilter; import com.webank.wedatasphere.linkis.storage.FSFactory; -import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraInfo; import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceFields; import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisTableMeta; import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceFieldsService; @@ -31,14 +29,11 @@ import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; -import org.glassfish.jersey.media.multipart.FormDataParam; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import scala.tools.cmd.Opt; -import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.*; diff --git a/streamis-datasource/streamis-datasource-transfer/pom.xml b/streamis-datasource/streamis-datasource-transfer/pom.xml index d5c82d43e..9c7b148c9 100644 --- a/streamis-datasource/streamis-datasource-transfer/pom.xml +++ b/streamis-datasource/streamis-datasource-transfer/pom.xml @@ -17,11 +17,11 @@ linkis-label-common ${linkis.version} - - com.webank.wedatasphere.linkis - linkis-datasource-client - ${linkis.version} - + + + + + com.webank.wedatasphere.streamis streamis-datasource-manager diff --git a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java index 9ac33bec4..e3a78db7c 100644 --- a/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java +++ b/streamis-datasource/streamis-datasource-transfer/src/main/java/com/webank/wedatasphere/streamis/datasource/transfer/service/impl/DataSourceTransferImpl.java @@ -2,7 +2,6 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.webank.wedatasphere.linkis.common.exception.ErrorException; -import com.webank.wedatasphere.linkis.datasource.client.response.GetConnectParamsByDataSourceIdResult; import com.webank.wedatasphere.linkis.manager.label.entity.engine.EngineType; import com.webank.wedatasphere.linkis.manager.label.entity.engine.RunType; import com.webank.wedatasphere.streamis.datasource.manager.domain.StreamisDatasourceExtraInfo; @@ -11,7 +10,7 @@ import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceExtraInfoService; import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisDatasourceFieldsService; import com.webank.wedatasphere.streamis.datasource.manager.service.StreamisTableMetaService; -import com.webank.wedatasphere.streamis.datasource.transfer.client.LinkisDataSourceClient; +import com.webank.wedatasphere.streamis.datasource.manager.client.LinkisDataSourceClient; import com.webank.wedatasphere.streamis.datasource.transfer.entity.StreamisDataSourceCode; import com.webank.wedatasphere.streamis.datasource.transfer.entity.StreamisTableEntity; import com.webank.wedatasphere.streamis.datasource.transfer.service.DataSourceTransfer; From 629fdf5806e6a116b82591f2f1d951f4fea964b6 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 15 Jun 2021 11:37:00 +0800 Subject: [PATCH 072/434] Optimize the pom files. --- db/streamis_ddl.sql | 4 ++-- pom.xml | 2 +- .../streamis-datasource-server/pom.xml | 15 --------------- .../streamis-job-launcher-linkis/pom.xml | 2 +- 4 files changed, 4 insertions(+), 19 deletions(-) diff --git a/db/streamis_ddl.sql b/db/streamis_ddl.sql index a04dc475d..7abf13969 100644 --- a/db/streamis_ddl.sql +++ b/db/streamis_ddl.sql @@ -23,12 +23,12 @@ CREATE TABLE `streamis_stream_bml` ( -- Table structure for streamis_stream_bml_version -- ---------------------------- DROP TABLE IF EXISTS `streamis_stream_bml_version`; -CREATE TABLE `streamis_stream_bml_version` ( +CREATE TABLE `streamis_stream_bml_version`( `id` bigint(20) NOT NULL, `bml_id` bigint(20) NULL DEFAULT NULL, `version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, `storage_path` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, - ` attribute` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `attribute` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, PRIMARY KEY (`id`) USING BTREE ) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '物料版本' ROW_FORMAT = Compact; diff --git a/pom.xml b/pom.xml index 78f1136d9..3254d891d 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ 1.0.0-RC1 - 1.0.0-RC1 + 1.0.1 0.1.0-SNAPSHOT 2.11.8 1.8 diff --git a/streamis-datasource/streamis-datasource-server/pom.xml b/streamis-datasource/streamis-datasource-server/pom.xml index f95280fb9..616d85dc5 100644 --- a/streamis-datasource/streamis-datasource-server/pom.xml +++ b/streamis-datasource/streamis-datasource-server/pom.xml @@ -17,31 +17,16 @@ linkis-rpc ${linkis.version} - - com.webank.wedatasphere.linkis - linkis-module - ${linkis.version} - com.webank.wedatasphere.linkis linkis-mybatis ${linkis.version} - - com.webank.wedatasphere.linkis - linkis-bmlserver - ${linkis.version} - com.webank.wedatasphere.linkis linkis-storage ${linkis.version} - - com.webank.wedatasphere.dss - dss-server - ${dss.version} - com.webank.wedatasphere.streamis streamis-datasource-transfer diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml index 2106fdda4..2ff7f3579 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml @@ -3,7 +3,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - streamis-jobmanager + streamis-job-launcher com.webank.wedatasphere.streamis 0.1.0-SNAPSHOT From dbad68c014c31fa6e284e5bed63db161b5245cff Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 15 Jun 2021 11:37:00 +0800 Subject: [PATCH 073/434] Optimize the pom files. --- db/streamis_ddl.sql | 4 ++-- pom.xml | 2 +- .../streamis-datasource-server/pom.xml | 15 --------------- .../streamis-job-launcher-linkis/pom.xml | 2 +- 4 files changed, 4 insertions(+), 19 deletions(-) diff --git a/db/streamis_ddl.sql b/db/streamis_ddl.sql index a04dc475d..7abf13969 100644 --- a/db/streamis_ddl.sql +++ b/db/streamis_ddl.sql @@ -23,12 +23,12 @@ CREATE TABLE `streamis_stream_bml` ( -- Table structure for streamis_stream_bml_version -- ---------------------------- DROP TABLE IF EXISTS `streamis_stream_bml_version`; -CREATE TABLE `streamis_stream_bml_version` ( +CREATE TABLE `streamis_stream_bml_version`( `id` bigint(20) NOT NULL, `bml_id` bigint(20) NULL DEFAULT NULL, `version` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, `storage_path` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, - ` attribute` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, + `attribute` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, PRIMARY KEY (`id`) USING BTREE ) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '物料版本' ROW_FORMAT = Compact; diff --git a/pom.xml b/pom.xml index 78f1136d9..3254d891d 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ 1.0.0-RC1 - 1.0.0-RC1 + 1.0.1 0.1.0-SNAPSHOT 2.11.8 1.8 diff --git a/streamis-datasource/streamis-datasource-server/pom.xml b/streamis-datasource/streamis-datasource-server/pom.xml index f95280fb9..616d85dc5 100644 --- a/streamis-datasource/streamis-datasource-server/pom.xml +++ b/streamis-datasource/streamis-datasource-server/pom.xml @@ -17,31 +17,16 @@ linkis-rpc ${linkis.version} - - com.webank.wedatasphere.linkis - linkis-module - ${linkis.version} - com.webank.wedatasphere.linkis linkis-mybatis ${linkis.version} - - com.webank.wedatasphere.linkis - linkis-bmlserver - ${linkis.version} - com.webank.wedatasphere.linkis linkis-storage ${linkis.version} - - com.webank.wedatasphere.dss - dss-server - ${dss.version} - com.webank.wedatasphere.streamis streamis-datasource-transfer diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml index 2106fdda4..2ff7f3579 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml @@ -3,7 +3,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - streamis-jobmanager + streamis-job-launcher com.webank.wedatasphere.streamis 0.1.0-SNAPSHOT From 55eeba9540b4036d8cefd2c72c0e03be611abad9 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 15 Jun 2021 12:05:11 +0800 Subject: [PATCH 074/434] Optimize compilation problem. --- .../jobmanager/manager/service/TaskMonitorService.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala index fab2ad869..29efdd83c 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala @@ -40,7 +40,8 @@ class TaskMonitorService extends Logging { def doMonitor(): Unit = { info("Try to update all StreamTasks status.") - val streamTasks = streamTaskMapper.getTasksByStatus(util.Arrays.asList(JobConf.NOT_COMPLETED_STATUS_ARRAY.map(_.getValue))) + val status = util.Arrays.asList(JobConf.NOT_COMPLETED_STATUS_ARRAY.map(c => new Integer(c.getValue)) :_*) + val streamTasks = streamTaskMapper.getTasksByStatus(status) if(streamTasks == null || streamTasks.isEmpty) { info("No StreamTasks is running, return...") return From 5094c2a7af8faed8ca4f7af3193c24dc7c1aef57 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 15 Jun 2021 12:05:11 +0800 Subject: [PATCH 075/434] Optimize compilation problem. --- .../jobmanager/manager/service/TaskMonitorService.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala index fab2ad869..29efdd83c 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala @@ -40,7 +40,8 @@ class TaskMonitorService extends Logging { def doMonitor(): Unit = { info("Try to update all StreamTasks status.") - val streamTasks = streamTaskMapper.getTasksByStatus(util.Arrays.asList(JobConf.NOT_COMPLETED_STATUS_ARRAY.map(_.getValue))) + val status = util.Arrays.asList(JobConf.NOT_COMPLETED_STATUS_ARRAY.map(c => new Integer(c.getValue)) :_*) + val streamTasks = streamTaskMapper.getTasksByStatus(status) if(streamTasks == null || streamTasks.isEmpty) { info("No StreamTasks is running, return...") return From c33ce342c0b8f9e0ebf653491d540a95c0af93c5 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Wed, 16 Jun 2021 16:35:25 +0800 Subject: [PATCH 076/434] Resolve some bugs. --- .../launcher/linkis/LinkisJobManager.scala | 12 +++++------- .../jobmanager/manager/dao/impl/StreamJobMapper.xml | 13 ++++++++----- .../manager/dao/impl/StreamTaskMapper.xml | 2 +- .../jobmanager/manager/entity/StreamJobVersion.java | 6 +++--- .../jobmanager/manager/entity/vo/JobFlowVO.java | 6 +++--- .../jobmanager/manager/service/JobService.scala | 1 + .../StreamisFlinkSqlTransformJobBuilder.scala | 2 +- .../transform/impl/ResourceConfigTransform.scala | 7 ++++--- 8 files changed, 26 insertions(+), 23 deletions(-) diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/LinkisJobManager.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/LinkisJobManager.scala index 4c2700265..320885b28 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/LinkisJobManager.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/LinkisJobManager.scala @@ -1,7 +1,5 @@ package com.webank.wedatasphere.streamis.jobmanager.launcher.linkis -import java.util - import com.webank.wedatasphere.linkis.common.utils.ClassUtils import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.entity.{LaunchJob, LinkisJobInfo} import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.exception.FlinkJobLaunchErrorException @@ -29,14 +27,14 @@ object LinkisJobManager { val SIMPLE_FLINK = "simpleFlink" - private val linkisJobManagers = new util.HashMap[String, LinkisJobManager]() + private val linkisJobManagers = ClassUtils.reflections.getSubTypesOf(classOf[LinkisJobManager]).filterNot(ClassUtils.isInterfaceOrAbstract) - .foreach { clazz => + .map { clazz => val jobManager = clazz.newInstance() - linkisJobManagers.put(jobManager.getName, jobManager) - } + jobManager.getName -> jobManager + }.toMap - def getLinkisJobManager(jobManagerType: String): LinkisJobManager = linkisJobManagers.getOrDefault(jobManagerType, + def getLinkisJobManager(jobManagerType: String): LinkisJobManager = linkisJobManagers.getOrElse(jobManagerType, throw new FlinkJobLaunchErrorException(30402, "Not exist LinkisJobManagerType " + jobManagerType)) def getLinkisJobManager: LinkisJobManager = getLinkisJobManager(SIMPLE_FLINK) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml index c7699d4c6..f5b60c446 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml @@ -41,7 +41,7 @@ - + @@ -51,7 +51,7 @@ - `id`,`project_id`,`name`,`current_task_id`,`current_version`,`current_release_time`,`org_identification`,`create_by`,`label`,`description` + `id`,`project_id`,`name`,`current_task_id`,`current_version`,`current_release_time`,`status`,`org_identification`,`create_by`,`label`,`description`,`submit_user`,`job_type` @@ -105,7 +105,7 @@ - INSERT INTO linkis_stream_job(`project_id`,`name`,`current_task_id`,`current_version`,`current_release_time`,`org_identification`,`create_by`,`label`,`description`,`job_type`) VALUES(#{projectId},#{name},#{currentTaskId},#{currentVersion},#{currentReleaseTime},#{orgIdentification},#{createBy},#{label},#{description},#{jobType}) + INSERT INTO linkis_stream_job(`project_id`,`name`,`current_task_id`,`current_version`,`current_release_time`,`org_identification`,`create_by`,`label`,`description`,`job_type`,`submit_user`,`status`) VALUES(#{projectId},#{name},#{currentTaskId},#{currentVersion},#{currentReleaseTime},#{orgIdentification},#{createBy},#{label},#{description},#{jobType},#{submitUser},#{status}) @@ -131,12 +131,15 @@ current_release_time=#{currentReleaseTime}, - - create_by=#{createBy}, + + status=#{status}, label=#{label}, + + description=#{description}, + WHERE id=#{id} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml index ef53bde27..0c3356293 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml @@ -31,7 +31,7 @@ - + diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersion.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersion.java index 6cea14739..12ef8bd2b 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersion.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersion.java @@ -9,7 +9,7 @@ public class StreamJobVersion { private String version; private String programArguments; private String bmlVersion; - private Long bmlId; + private String bmlId; public Long getId() { return id; @@ -51,11 +51,11 @@ public void setBmlVersion(String bmlVersion) { this.bmlVersion = bmlVersion; } - public Long getBmlId() { + public String getBmlId() { return bmlId; } - public void setBmlId(Long bmlId) { + public void setBmlId(String bmlId) { this.bmlId = bmlId; } } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobFlowVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobFlowVO.java index 06580a507..06dd1e143 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobFlowVO.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobFlowVO.java @@ -13,7 +13,7 @@ public class JobFlowVO { private String creator; private Long projectID; private String dssLabels; - private Long resourceId; + private String resourceId; private String bmlVersion; private String flowJson; private Boolean rootFlow; @@ -116,11 +116,11 @@ public void setRootFlow(Boolean rootFlow) { this.rootFlow = rootFlow; } - public Long getResourceId() { + public String getResourceId() { return resourceId; } - public void setResourceId(Long resourceId) { + public void setResourceId(String resourceId) { this.resourceId = resourceId; } } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala index 208a602f7..bf18f776b 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala @@ -200,6 +200,7 @@ class JobService extends Logging{ job.setName(publishRequestVO.getStreamisJobName) job.setCurrentVersion(publishRequestVO.getVersion) job.setCreateBy(publishRequestVO.getCreateBy) + job.setSubmitUser(publishRequestVO.getPublishUser) streamJobMapper.insertJob(job) jobVersion.setJobId(job.getId) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/StreamisFlinkSqlTransformJobBuilder.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/StreamisFlinkSqlTransformJobBuilder.scala index 1584477d5..bb2785449 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/StreamisFlinkSqlTransformJobBuilder.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/StreamisFlinkSqlTransformJobBuilder.scala @@ -22,7 +22,7 @@ class StreamisFlinkSqlTransformJobBuilder extends StreamisTransformJobBuilder { @Autowired private var streamJobMapper: StreamJobMapper = _ @Autowired private var configurationService: ConfigurationService = _ - override def canBuild(streamJob: StreamJob): Boolean = JOBMANAGER_FLINK_SQL.getValue == streamJob.getJobType + override def canBuild(streamJob: StreamJob): Boolean = JOBMANAGER_FLINK_SQL.getValue == streamJob.getJobType.intValue() override def build(streamJob: StreamJob): StreamisTransformJob = { val transformJob = new StreamisSqlTransformJob diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/ResourceConfigTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/ResourceConfigTransform.scala index ec462d913..c453177eb 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/ResourceConfigTransform.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/ResourceConfigTransform.scala @@ -20,13 +20,14 @@ class ResourceConfigTransform extends ConfigTransform { override protected def transform(config: ConfigKeyVO, job: LaunchJob): LaunchJob = transformConfig(config.getResourceConfig, job) - protected def transformConfig(getConfig: => Seq[ConfigRelationVO], job: LaunchJob): LaunchJob = { + protected def transformConfig(getConfig: => util.List[ConfigRelationVO], job: LaunchJob): LaunchJob = { val startupMap = new util.HashMap[String, Any] - getConfig.foreach { vo => + val configSeq = getConfig + if(configSeq != null) configSeq.foreach { vo => startupMap.put(vo.getKey, vo.getValue) } val params = if(job.getParams == null) new util.HashMap[String, Any] else job.getParams - TaskUtils.addStartupMap(params, startupMap) + if(!startupMap.isEmpty) TaskUtils.addStartupMap(params, startupMap) LaunchJob.builder().setLaunchJob(job).setParams(params).build() } From 5886affea7a47f0a5892f0daca75039107438d41 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Wed, 16 Jun 2021 16:35:25 +0800 Subject: [PATCH 077/434] Resolve some bugs. --- .../launcher/linkis/LinkisJobManager.scala | 12 +++++------- .../jobmanager/manager/dao/impl/StreamJobMapper.xml | 13 ++++++++----- .../manager/dao/impl/StreamTaskMapper.xml | 2 +- .../jobmanager/manager/entity/StreamJobVersion.java | 6 +++--- .../jobmanager/manager/entity/vo/JobFlowVO.java | 6 +++--- .../jobmanager/manager/service/JobService.scala | 1 + .../StreamisFlinkSqlTransformJobBuilder.scala | 2 +- .../transform/impl/ResourceConfigTransform.scala | 7 ++++--- 8 files changed, 26 insertions(+), 23 deletions(-) diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/LinkisJobManager.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/LinkisJobManager.scala index 4c2700265..320885b28 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/LinkisJobManager.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/LinkisJobManager.scala @@ -1,7 +1,5 @@ package com.webank.wedatasphere.streamis.jobmanager.launcher.linkis -import java.util - import com.webank.wedatasphere.linkis.common.utils.ClassUtils import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.entity.{LaunchJob, LinkisJobInfo} import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.exception.FlinkJobLaunchErrorException @@ -29,14 +27,14 @@ object LinkisJobManager { val SIMPLE_FLINK = "simpleFlink" - private val linkisJobManagers = new util.HashMap[String, LinkisJobManager]() + private val linkisJobManagers = ClassUtils.reflections.getSubTypesOf(classOf[LinkisJobManager]).filterNot(ClassUtils.isInterfaceOrAbstract) - .foreach { clazz => + .map { clazz => val jobManager = clazz.newInstance() - linkisJobManagers.put(jobManager.getName, jobManager) - } + jobManager.getName -> jobManager + }.toMap - def getLinkisJobManager(jobManagerType: String): LinkisJobManager = linkisJobManagers.getOrDefault(jobManagerType, + def getLinkisJobManager(jobManagerType: String): LinkisJobManager = linkisJobManagers.getOrElse(jobManagerType, throw new FlinkJobLaunchErrorException(30402, "Not exist LinkisJobManagerType " + jobManagerType)) def getLinkisJobManager: LinkisJobManager = getLinkisJobManager(SIMPLE_FLINK) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml index c7699d4c6..f5b60c446 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml @@ -41,7 +41,7 @@ - + @@ -51,7 +51,7 @@ - `id`,`project_id`,`name`,`current_task_id`,`current_version`,`current_release_time`,`org_identification`,`create_by`,`label`,`description` + `id`,`project_id`,`name`,`current_task_id`,`current_version`,`current_release_time`,`status`,`org_identification`,`create_by`,`label`,`description`,`submit_user`,`job_type` @@ -105,7 +105,7 @@ - INSERT INTO linkis_stream_job(`project_id`,`name`,`current_task_id`,`current_version`,`current_release_time`,`org_identification`,`create_by`,`label`,`description`,`job_type`) VALUES(#{projectId},#{name},#{currentTaskId},#{currentVersion},#{currentReleaseTime},#{orgIdentification},#{createBy},#{label},#{description},#{jobType}) + INSERT INTO linkis_stream_job(`project_id`,`name`,`current_task_id`,`current_version`,`current_release_time`,`org_identification`,`create_by`,`label`,`description`,`job_type`,`submit_user`,`status`) VALUES(#{projectId},#{name},#{currentTaskId},#{currentVersion},#{currentReleaseTime},#{orgIdentification},#{createBy},#{label},#{description},#{jobType},#{submitUser},#{status}) @@ -131,12 +131,15 @@ current_release_time=#{currentReleaseTime}, - - create_by=#{createBy}, + + status=#{status}, label=#{label}, + + description=#{description}, + WHERE id=#{id} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml index ef53bde27..0c3356293 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml @@ -31,7 +31,7 @@ - + diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersion.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersion.java index 6cea14739..12ef8bd2b 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersion.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/StreamJobVersion.java @@ -9,7 +9,7 @@ public class StreamJobVersion { private String version; private String programArguments; private String bmlVersion; - private Long bmlId; + private String bmlId; public Long getId() { return id; @@ -51,11 +51,11 @@ public void setBmlVersion(String bmlVersion) { this.bmlVersion = bmlVersion; } - public Long getBmlId() { + public String getBmlId() { return bmlId; } - public void setBmlId(Long bmlId) { + public void setBmlId(String bmlId) { this.bmlId = bmlId; } } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobFlowVO.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobFlowVO.java index 06580a507..06dd1e143 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobFlowVO.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/entity/vo/JobFlowVO.java @@ -13,7 +13,7 @@ public class JobFlowVO { private String creator; private Long projectID; private String dssLabels; - private Long resourceId; + private String resourceId; private String bmlVersion; private String flowJson; private Boolean rootFlow; @@ -116,11 +116,11 @@ public void setRootFlow(Boolean rootFlow) { this.rootFlow = rootFlow; } - public Long getResourceId() { + public String getResourceId() { return resourceId; } - public void setResourceId(Long resourceId) { + public void setResourceId(String resourceId) { this.resourceId = resourceId; } } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala index 208a602f7..bf18f776b 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/JobService.scala @@ -200,6 +200,7 @@ class JobService extends Logging{ job.setName(publishRequestVO.getStreamisJobName) job.setCurrentVersion(publishRequestVO.getVersion) job.setCreateBy(publishRequestVO.getCreateBy) + job.setSubmitUser(publishRequestVO.getPublishUser) streamJobMapper.insertJob(job) jobVersion.setJobId(job.getId) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/StreamisFlinkSqlTransformJobBuilder.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/StreamisFlinkSqlTransformJobBuilder.scala index 1584477d5..bb2785449 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/StreamisFlinkSqlTransformJobBuilder.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/StreamisFlinkSqlTransformJobBuilder.scala @@ -22,7 +22,7 @@ class StreamisFlinkSqlTransformJobBuilder extends StreamisTransformJobBuilder { @Autowired private var streamJobMapper: StreamJobMapper = _ @Autowired private var configurationService: ConfigurationService = _ - override def canBuild(streamJob: StreamJob): Boolean = JOBMANAGER_FLINK_SQL.getValue == streamJob.getJobType + override def canBuild(streamJob: StreamJob): Boolean = JOBMANAGER_FLINK_SQL.getValue == streamJob.getJobType.intValue() override def build(streamJob: StreamJob): StreamisTransformJob = { val transformJob = new StreamisSqlTransformJob diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/ResourceConfigTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/ResourceConfigTransform.scala index ec462d913..c453177eb 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/ResourceConfigTransform.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/ResourceConfigTransform.scala @@ -20,13 +20,14 @@ class ResourceConfigTransform extends ConfigTransform { override protected def transform(config: ConfigKeyVO, job: LaunchJob): LaunchJob = transformConfig(config.getResourceConfig, job) - protected def transformConfig(getConfig: => Seq[ConfigRelationVO], job: LaunchJob): LaunchJob = { + protected def transformConfig(getConfig: => util.List[ConfigRelationVO], job: LaunchJob): LaunchJob = { val startupMap = new util.HashMap[String, Any] - getConfig.foreach { vo => + val configSeq = getConfig + if(configSeq != null) configSeq.foreach { vo => startupMap.put(vo.getKey, vo.getValue) } val params = if(job.getParams == null) new util.HashMap[String, Any] else job.getParams - TaskUtils.addStartupMap(params, startupMap) + if(!startupMap.isEmpty) TaskUtils.addStartupMap(params, startupMap) LaunchJob.builder().setLaunchJob(job).setParams(params).build() } From 86e2bcba8f834ce9e4371051c337110243885b37 Mon Sep 17 00:00:00 2001 From: pengmengsi 90080 Date: Fri, 18 Jun 2021 15:16:50 +0800 Subject: [PATCH 078/434] datasource update --- .../module/datasourceToolbar/index.vue | 82 ----- .../streamis/module/tableFieldsList/index.vue | 233 +++++++------- .../apps/streamis/module/tableInfo/index.vue | 76 ++++- .../apps/streamis/module/treeSource/index.vue | 298 ++++++++++++------ .../apps/streamis/view/dataSource/index.vue | 189 +++++++++-- 5 files changed, 556 insertions(+), 322 deletions(-) delete mode 100644 web/src/apps/streamis/module/datasourceToolbar/index.vue diff --git a/web/src/apps/streamis/module/datasourceToolbar/index.vue b/web/src/apps/streamis/module/datasourceToolbar/index.vue deleted file mode 100644 index ec2f1f90c..000000000 --- a/web/src/apps/streamis/module/datasourceToolbar/index.vue +++ /dev/null @@ -1,82 +0,0 @@ - - - - - \ No newline at end of file diff --git a/web/src/apps/streamis/module/tableFieldsList/index.vue b/web/src/apps/streamis/module/tableFieldsList/index.vue index eb139b72d..5f202557f 100644 --- a/web/src/apps/streamis/module/tableFieldsList/index.vue +++ b/web/src/apps/streamis/module/tableFieldsList/index.vue @@ -14,21 +14,25 @@ - - - - - + - - - - - + diff --git a/web/src/apps/streamis/module/uploadJobJar/index.js b/web/src/apps/streamis/module/uploadJobJar/index.js new file mode 100644 index 000000000..f082c2534 --- /dev/null +++ b/web/src/apps/streamis/module/uploadJobJar/index.js @@ -0,0 +1,19 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import uploadJobJar from './index.vue'; +export default uploadJobJar; diff --git a/web/src/apps/streamis/module/uploadJobJar/index.vue b/web/src/apps/streamis/module/uploadJobJar/index.vue new file mode 100644 index 000000000..820903aa4 --- /dev/null +++ b/web/src/apps/streamis/module/uploadJobJar/index.vue @@ -0,0 +1,103 @@ + + + diff --git a/web/src/apps/streamis/module/versionDetail/index.vue b/web/src/apps/streamis/module/versionDetail/index.vue index 50d470aae..1b04e4307 100644 --- a/web/src/apps/streamis/module/versionDetail/index.vue +++ b/web/src/apps/streamis/module/versionDetail/index.vue @@ -5,8 +5,9 @@ v-model="visible" footer-hide width="1200" + @on-cancel="cancel" > -
+
@@ -16,14 +17,14 @@ export default { components: { table }, props: { visible: Boolean, - tableDatas: Array + datas: Array }, data() { return { columns: [ { title: this.$t("message.streamis.versionDetail.jobId"), - key: "jobId" + key: "id" }, { title: this.$t("message.streamis.versionDetail.version"), @@ -35,24 +36,24 @@ export default { }, { title: this.$t("message.streamis.versionDetail.createTime"), - key: "createTime" + key: "releaseTime" }, { title: this.$t("message.streamis.versionDetail.creator"), - key: "creator" + key: "createBy" }, { title: this.$t("message.streamis.versionDetail.projectID"), - key: "projectID" + key: "projectId" }, { title: this.$t("message.streamis.versionDetail.dssLabels"), - key: "dssLabels" + key: "dssEnv" }, { title: this.$t("message.streamis.versionDetail.resourceId"), - key: "resourceId" + key: "bmlId" }, { title: this.$t("message.streamis.versionDetail.bmlVersion"), @@ -75,7 +76,7 @@ export default { this.$Message.info("Clicked ok"); }, cancel() { - this.$Message.info("Clicked cancel"); + this.$emit('modalCancel'); } } }; diff --git a/web/src/apps/streamis/view/jobDetail/index.vue b/web/src/apps/streamis/view/jobDetail/index.vue index 5b3e4f28f..67e53cdf8 100644 --- a/web/src/apps/streamis/view/jobDetail/index.vue +++ b/web/src/apps/streamis/view/jobDetail/index.vue @@ -1,29 +1,51 @@ diff --git a/web/src/apps/streamis/module/uploadJobJar/index.js b/web/src/apps/streamis/module/uploadJobJar/index.js new file mode 100644 index 000000000..f082c2534 --- /dev/null +++ b/web/src/apps/streamis/module/uploadJobJar/index.js @@ -0,0 +1,19 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import uploadJobJar from './index.vue'; +export default uploadJobJar; diff --git a/web/src/apps/streamis/module/uploadJobJar/index.vue b/web/src/apps/streamis/module/uploadJobJar/index.vue new file mode 100644 index 000000000..820903aa4 --- /dev/null +++ b/web/src/apps/streamis/module/uploadJobJar/index.vue @@ -0,0 +1,103 @@ + + + diff --git a/web/src/apps/streamis/module/versionDetail/index.vue b/web/src/apps/streamis/module/versionDetail/index.vue index 50d470aae..1b04e4307 100644 --- a/web/src/apps/streamis/module/versionDetail/index.vue +++ b/web/src/apps/streamis/module/versionDetail/index.vue @@ -5,8 +5,9 @@ v-model="visible" footer-hide width="1200" + @on-cancel="cancel" > -
+
@@ -16,14 +17,14 @@ export default { components: { table }, props: { visible: Boolean, - tableDatas: Array + datas: Array }, data() { return { columns: [ { title: this.$t("message.streamis.versionDetail.jobId"), - key: "jobId" + key: "id" }, { title: this.$t("message.streamis.versionDetail.version"), @@ -35,24 +36,24 @@ export default { }, { title: this.$t("message.streamis.versionDetail.createTime"), - key: "createTime" + key: "releaseTime" }, { title: this.$t("message.streamis.versionDetail.creator"), - key: "creator" + key: "createBy" }, { title: this.$t("message.streamis.versionDetail.projectID"), - key: "projectID" + key: "projectId" }, { title: this.$t("message.streamis.versionDetail.dssLabels"), - key: "dssLabels" + key: "dssEnv" }, { title: this.$t("message.streamis.versionDetail.resourceId"), - key: "resourceId" + key: "bmlId" }, { title: this.$t("message.streamis.versionDetail.bmlVersion"), @@ -75,7 +76,7 @@ export default { this.$Message.info("Clicked ok"); }, cancel() { - this.$Message.info("Clicked cancel"); + this.$emit('modalCancel'); } } }; diff --git a/web/src/apps/streamis/view/jobDetail/index.vue b/web/src/apps/streamis/view/jobDetail/index.vue index 5b3e4f28f..67e53cdf8 100644 --- a/web/src/apps/streamis/view/jobDetail/index.vue +++ b/web/src/apps/streamis/view/jobDetail/index.vue @@ -1,29 +1,51 @@ diff --git a/web/src/apps/streamis/module/uploadJobJar/index.vue b/web/src/apps/streamis/module/uploadJobJar/index.vue index 820903aa4..04f2bd301 100644 --- a/web/src/apps/streamis/module/uploadJobJar/index.vue +++ b/web/src/apps/streamis/module/uploadJobJar/index.vue @@ -19,85 +19,127 @@ > - - + + - - - - -
- Upload file: {{ file.name }} - + +
+ {{ $t('message.streamis.uploadJar.choosedJar') }}: {{ file.name }}
- +
From 2d1d33ef63d1a3dce54b21d072aabd89e36386b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8F=B8=E6=B5=B7=E6=B4=8B?= Date: Tue, 22 Jun 2021 16:53:14 +0800 Subject: [PATCH 083/434] update upload jar --- web/src/apps/streamis/i18n/en.json | 10 ++ web/src/apps/streamis/i18n/zh.json | 12 ++ .../apps/streamis/module/jobList/index.vue | 6 +- .../streamis/module/uploadJobJar/index.vue | 134 ++++++++++++------ 4 files changed, 114 insertions(+), 48 deletions(-) diff --git a/web/src/apps/streamis/i18n/en.json b/web/src/apps/streamis/i18n/en.json index 90d2f8a2d..54e6fe13c 100644 --- a/web/src/apps/streamis/i18n/en.json +++ b/web/src/apps/streamis/i18n/en.json @@ -143,6 +143,16 @@ "variable": "Input variable value" } } + }, + "uploadJar": { + "entrypointClass": "Entrypoint Class", + "label": "Label", + "entrypointMainArgs": "Entrypoint Main Args", + "parallelism": "Parallelism", + "selectJar": "Please choose Jar", + "upload": "Upload", + "choosedJar" : "Choosed Jar", + "jarError": "File must be Jar" } } } diff --git a/web/src/apps/streamis/i18n/zh.json b/web/src/apps/streamis/i18n/zh.json index 3b10e4a9c..a17ebfbdc 100644 --- a/web/src/apps/streamis/i18n/zh.json +++ b/web/src/apps/streamis/i18n/zh.json @@ -142,6 +142,18 @@ "variable": "输入参数值" } } + }, + "uploadJar": { + "entrypointClass": "主类", + "label": "标签", + "entrypointMainArgs": "参数", + "parallelism": "并行度", + "selectJar": "请选择Jar包", + "upload": "上传", + "choosedJar" : "选中的Jar包", + "jarError": "文件必须是Jar包", + "jarEmpty": "必须选择Jar包", + "jobNameEmpty": "作业名称不能为空" } } } diff --git a/web/src/apps/streamis/module/jobList/index.vue b/web/src/apps/streamis/module/jobList/index.vue index 5588239e3..dd4ae5ea2 100644 --- a/web/src/apps/streamis/module/jobList/index.vue +++ b/web/src/apps/streamis/module/jobList/index.vue @@ -143,7 +143,7 @@ - + diff --git a/web/src/apps/streamis/module/uploadJobJar/index.vue b/web/src/apps/streamis/module/uploadJobJar/index.vue index 820903aa4..04f2bd301 100644 --- a/web/src/apps/streamis/module/uploadJobJar/index.vue +++ b/web/src/apps/streamis/module/uploadJobJar/index.vue @@ -19,85 +19,127 @@ > - - + + - - - - -
- Upload file: {{ file.name }} - + +
+ {{ $t('message.streamis.uploadJar.choosedJar') }}: {{ file.name }}
- +
From bf20ad72c931f4859d64f1da5f3ffc3135023cec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8F=B8=E6=B5=B7=E6=B4=8B?= Date: Tue, 22 Jun 2021 17:01:37 +0800 Subject: [PATCH 084/434] update upload jar --- web/src/apps/streamis/module/uploadJobJar/index.vue | 1 - 1 file changed, 1 deletion(-) diff --git a/web/src/apps/streamis/module/uploadJobJar/index.vue b/web/src/apps/streamis/module/uploadJobJar/index.vue index 04f2bd301..cfdc76cee 100644 --- a/web/src/apps/streamis/module/uploadJobJar/index.vue +++ b/web/src/apps/streamis/module/uploadJobJar/index.vue @@ -118,7 +118,6 @@ export default { .catch(e => { console.log(e) this.loading = false; - this.$emit("jarUploadSuccess"); }) } }) From 0aa7c3a2c6dbe9ac6f1c74d80aeff691ad5e1db9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8F=B8=E6=B5=B7=E6=B4=8B?= Date: Tue, 22 Jun 2021 17:01:37 +0800 Subject: [PATCH 085/434] update upload jar --- web/src/apps/streamis/module/uploadJobJar/index.vue | 1 - 1 file changed, 1 deletion(-) diff --git a/web/src/apps/streamis/module/uploadJobJar/index.vue b/web/src/apps/streamis/module/uploadJobJar/index.vue index 04f2bd301..cfdc76cee 100644 --- a/web/src/apps/streamis/module/uploadJobJar/index.vue +++ b/web/src/apps/streamis/module/uploadJobJar/index.vue @@ -118,7 +118,6 @@ export default { .catch(e => { console.log(e) this.loading = false; - this.$emit("jarUploadSuccess"); }) } }) From 6a79961ef52df909b2b3fafb17d5c85e5efa4775 Mon Sep 17 00:00:00 2001 From: ryanlei Date: Wed, 23 Jun 2021 11:00:11 +0800 Subject: [PATCH 086/434] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=8B=BC=E5=86=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/src/apps/streamis/module/jobHistory/index.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/src/apps/streamis/module/jobHistory/index.vue b/web/src/apps/streamis/module/jobHistory/index.vue index 86d6a362a..320a4bcf1 100644 --- a/web/src/apps/streamis/module/jobHistory/index.vue +++ b/web/src/apps/streamis/module/jobHistory/index.vue @@ -79,7 +79,7 @@ export default { methods: { getDatas() { const {id, version} = this.$route.params || {}; - const queries = `?jodId=${id}&version=${version}` + const queries = `?jobId=${id}&version=${version}` api .fetch( "streamis/streamJobManager/job/execute/history" + queries, @@ -88,7 +88,7 @@ export default { .then(res => { console.log(res); if(res && res.details){ - this.tableDatas = res.details; + this.tableDatas = res.details; } }) .catch(e => console.log(e)); From 9141e8e102c52f5f8aa59defc7724566fd60fccf Mon Sep 17 00:00:00 2001 From: ryanlei Date: Wed, 23 Jun 2021 11:00:11 +0800 Subject: [PATCH 087/434] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=8B=BC=E5=86=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/src/apps/streamis/module/jobHistory/index.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/src/apps/streamis/module/jobHistory/index.vue b/web/src/apps/streamis/module/jobHistory/index.vue index 86d6a362a..320a4bcf1 100644 --- a/web/src/apps/streamis/module/jobHistory/index.vue +++ b/web/src/apps/streamis/module/jobHistory/index.vue @@ -79,7 +79,7 @@ export default { methods: { getDatas() { const {id, version} = this.$route.params || {}; - const queries = `?jodId=${id}&version=${version}` + const queries = `?jobId=${id}&version=${version}` api .fetch( "streamis/streamJobManager/job/execute/history" + queries, @@ -88,7 +88,7 @@ export default { .then(res => { console.log(res); if(res && res.details){ - this.tableDatas = res.details; + this.tableDatas = res.details; } }) .catch(e => console.log(e)); From 0e72be0e21dead0b8fa3d76757e2edc49b654f3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8F=B8=E6=B5=B7=E6=B4=8B?= Date: Wed, 23 Jun 2021 12:11:48 +0800 Subject: [PATCH 088/434] up --- .../apps/streamis/module/jobList/index.vue | 301 ++++++++++-------- 1 file changed, 161 insertions(+), 140 deletions(-) diff --git a/web/src/apps/streamis/module/jobList/index.vue b/web/src/apps/streamis/module/jobList/index.vue index dd4ae5ea2..1dee6a2b1 100644 --- a/web/src/apps/streamis/module/jobList/index.vue +++ b/web/src/apps/streamis/module/jobList/index.vue @@ -24,7 +24,7 @@ :value="item" :key="index" > - {{ $t("message.streamis.jobStatus." + item) }} + {{ $t('message.streamis.jobStatus.' + item) }} @@ -39,8 +39,8 @@ :key="index" > {{ - item === "all" - ? $t("message.streamis.jobStatus." + item) + item === 'all' + ? $t('message.streamis.jobStatus.' + item) : item }} @@ -53,18 +53,31 @@ @click="handleQuery()" style="width:80px;height:30px;background:rgba(22, 155, 213, 1);margin-left: 80px;" > - {{ $t("message.streamis.formItems.queryBtn") }} + {{ $t('message.streamis.formItems.queryBtn') }}
\ No newline at end of file diff --git a/web/src/apps/streamis/view/dataSource/index.vue b/web/src/apps/streamis/view/dataSource/index.vue index fc5288d22..f60ea16cc 100644 --- a/web/src/apps/streamis/view/dataSource/index.vue +++ b/web/src/apps/streamis/view/dataSource/index.vue @@ -3,7 +3,11 @@
- +
@@ -18,7 +22,7 @@ 停止
-
+
- 保存 + 保存
@@ -43,7 +47,7 @@ 返回图形界面
-
+ +

请点击保存按钮，否则会清空您记录哦！！！

+
diff --git a/web/src/apps/streamis/module/treeSource/index.vue b/web/src/apps/streamis/module/treeSource/index.vue index e67cfee56..b82b3605e 100644 --- a/web/src/apps/streamis/module/treeSource/index.vue +++ b/web/src/apps/streamis/module/treeSource/index.vue @@ -15,16 +15,19 @@
- + + + +
@@ -38,8 +41,8 @@ {{ item }} -