Skip to content

Commit

Permalink
Merge pull request apache#23 from WeDataSphere/dev-1.3.1
Browse files Browse the repository at this point in the history 
Merge dev-1.3.1 into  dev-1.1.6-webank
  • Loading branch information
@casionone
casionone authored Sep 1, 2022
2 parents 4174334 + 7f7b8f5 commit e3b5151
Show file tree
Hide file tree
Showing 10 changed files with 308 additions and 55 deletions.
22 changes: 22 additions & 0 deletions linkis-dist/package/db/upgrade/1.3.1_schema/mysql/linkis_ddl.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

-- ----------------------------
-- add UNIQUE KEY with env_name field for linkis_ps_dm_datasource_env
-- ----------------------------

ALTER TABLE `linkis_ps_dm_datasource_env` ADD CONSTRAINT `uk_env_name` UNIQUE (`env_name`);
10 changes: 10 additions & 0 deletions ...r/server/src/main/java/org/apache/linkis/datasourcemanager/core/dao/DataSourceEnvDao.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import org.apache.linkis.datasourcemanager.common.domain.DataSourceEnv;
import org.apache.linkis.datasourcemanager.core.vo.DataSourceEnvVo;

import org.apache.ibatis.annotations.Param;

import java.util.List;

/** Data source dao */
Expand Down Expand Up @@ -70,4 +72,12 @@ public interface DataSourceEnvDao {
* @return
*/
List<DataSourceEnv> selectByPageVo(DataSourceEnvVo dataSourceEnvVo);

/**
* get DataSourceEnv by envName
*
* @param envName
* @return
*/
DataSourceEnv selectOneByName(@Param("envName") String envName);
}
1 change: 1 addition & 0 deletions ...ver/src/main/java/org/apache/linkis/datasourcemanager/core/dao/mapper/DataSouceMapper.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@
<if test="dataSourceTypeId != null and dataSourceTypeId > 0">
<![CDATA[`datasource_type_id` = #{dataSourceTypeId},]]>
</if>
<![CDATA[`parameter` = #{parameter},]]>
<![CDATA[`create_identify` = #{createIdentify,jdbcType=VARCHAR},]]>
<![CDATA[`create_system` = #{createSystem,jdbcType=VARCHAR},]]>
<choose>
Expand Down
10 changes: 10 additions & 0 deletions ...src/main/java/org/apache/linkis/datasourcemanager/core/dao/mapper/DataSourceEnvMapper.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,4 +123,14 @@
`id` = #{id}
</where>
</update>

<select id="selectOneByName" resultMap="dataSourceEnvMap">
<![CDATA[SELECT ]]>
<include refid="data_source_env_detail"/>
<![CDATA[ FROM `linkis_ps_dm_datasource_env` ]]>
<where>
`env_name` = #{envName}
</where>
<![CDATA[ LIMIT 1; ]]>
</select>
</mapper>
129 changes: 125 additions & 4 deletions ...main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceAdminRestfulApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.linkis.datasourcemanager.core.restful;

import com.google.common.collect.Lists;
import org.apache.linkis.common.exception.ErrorException;
import org.apache.linkis.datasourcemanager.common.domain.DataSourceEnv;
import org.apache.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
Expand Down Expand Up @@ -45,10 +46,7 @@
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;

import java.util.Calendar;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;

@Api(tags = "data source admin restful api")
@RestController
Expand All @@ -57,6 +55,7 @@
produces = {"application/json"})
public class DataSourceAdminRestfulApi {

private final List<String> permitSystemList = Arrays.asList("Qualitis");
@Autowired private DataSourceInfoService dataSourceInfoService;

@Autowired private DataSourceRelateService dataSourceRelateService;
Expand Down Expand Up @@ -89,13 +88,126 @@ public Message insertJsonEnv(@RequestBody DataSourceEnv dataSourceEnv, HttpServl
if (result.size() > 0) {
throw new ConstraintViolationException(result);
}
if (dataSourceInfoService.existDataSourceEnv(dataSourceEnv.getEnvName())) {
return Message.error(
"The data source env named: "
+ dataSourceEnv.getEnvName()
+ " has been existed [数据源环境: "
+ dataSourceEnv.getEnvName()
+ " 已经存在]");
}
dataSourceEnv.setCreateUser(userName);
insertDataSourceEnv(dataSourceEnv);
return Message.ok().data("insertId", dataSourceEnv.getId());
},
"Fail to insert data source environment[新增数据源环境失败]");
}

@ApiOperation(value = "insertJsonEnvBatch", notes = "insert batch json env", response = Message.class)
@ApiOperationSupport(ignoreParameters = {"dataSourceEnvList", "system"})
@RequestMapping(value = "/env/json/batch", method = RequestMethod.POST)
public Message insertJsonEnvBatch(
@RequestBody List<DataSourceEnv> dataSourceEnvList,
@RequestParam("system") String system,
HttpServletRequest req)
throws ErrorException {
String userName = SecurityFilter.getLoginUsername(req);
if (!RestfulApiHelper.isAdminUser(userName) && !permitSystemList.contains(system)) {
return Message.error("User '" + userName + "' is not admin user[非管理员用户]");
}
for (DataSourceEnv dataSourceEnv : dataSourceEnvList) {
// Bean validation
Set<ConstraintViolation<DataSourceEnv>> result =
beanValidator.validate(dataSourceEnv, Default.class);
if (result.size() > 0) {
throw new ConstraintViolationException(result);
}
if (dataSourceInfoService.existDataSourceEnv(dataSourceEnv.getEnvName())) {
return Message.error(
"The data source env named: "
+ dataSourceEnv.getEnvName()
+ " has been existed [数据源环境: "
+ dataSourceEnv.getEnvName()
+ " 已经存在]");
}
dataSourceEnv.setCreateUser(userName);
// Get key definitions in environment scope
List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(
dataSourceEnv.getDataSourceTypeId(),
DataSourceParamKeyDefinition.Scope.ENV);
dataSourceEnv.setKeyDefinitions(keyDefinitionList);
Map<String, Object> connectParams = dataSourceEnv.getConnectParams();
// Validate connect parameters
parameterValidator.validate(keyDefinitionList, connectParams);
}
dataSourceInfoService.saveBatchDataSourceEnv(dataSourceEnvList);
return RestfulApiHelper.doAndResponse(
() -> Message.ok().data("envs", dataSourceEnvList),
"Fail to insert data source environment[新增数据源环境失败]");
}

@ApiOperation(value = "insertJsonEnvBatch", notes = "update batch json env", response = Message.class)
@ApiOperationSupport(ignoreParameters = {"dataSourceEnvList", "system"})
@RequestMapping(value = "/env/json/batch", method = RequestMethod.PUT)
public Message updateEnvBatch(
@RequestBody List<DataSourceEnv> dataSourceEnvList,
@RequestParam("system") String system,
HttpServletRequest request)
throws ErrorException {
String userName = SecurityFilter.getLoginUsername(request);
if (!RestfulApiHelper.isAdminUser(userName) && !permitSystemList.contains(system)) {
return Message.error("User '" + userName + "' is not admin user[非管理员用户]");
}
for (DataSourceEnv dataSourceEnv : dataSourceEnvList) {
if (Objects.isNull(dataSourceEnv.getId())) {
return Message.error(
"Fail to update data source environment[更新数据源环境失败], "
+ "[Please check the id if exists']");
}
// Bean validation
Set<ConstraintViolation<DataSourceEnv>> result =
beanValidator.validate(dataSourceEnv, Default.class);
if (result.size() > 0) {
throw new ConstraintViolationException(result);
}
Long envId = dataSourceEnv.getId();
dataSourceEnv.setModifyUser(userName);
dataSourceEnv.setModifyTime(Calendar.getInstance().getTime());
DataSourceEnv storedDataSourceEnv = dataSourceInfoService.getDataSourceEnv(envId);
if (null == storedDataSourceEnv) {
return Message.error(
"Fail to update data source environment[更新数据源环境失败], "
+ "[Please check the id:'"
+ envId
+ " is correct ']");
}
if (!Objects.equals(dataSourceEnv.getEnvName(), storedDataSourceEnv.getEnvName())
&& dataSourceInfoService.existDataSourceEnv(dataSourceEnv.getEnvName())) {
return Message.error(
"The data source env named: "
+ dataSourceEnv.getEnvName()
+ " has been existed [数据源环境: "
+ dataSourceEnv.getEnvName()
+ " 已经存在]");
}
dataSourceEnv.setCreateUser(storedDataSourceEnv.getCreateUser());
// Get key definitions in environment scope
List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(
dataSourceEnv.getDataSourceTypeId(),
DataSourceParamKeyDefinition.Scope.ENV);
dataSourceEnv.setKeyDefinitions(keyDefinitionList);
Map<String, Object> connectParams = dataSourceEnv.getConnectParams();
// Validate connect parameters
parameterValidator.validate(keyDefinitionList, connectParams);
}
dataSourceInfoService.updateBatchDataSourceEnv(dataSourceEnvList);
return RestfulApiHelper.doAndResponse(
() -> Message.ok().data("envs", dataSourceEnvList),
"Fail to update data source environment[更新数据源环境失败]");
}

@ApiOperation(value = "getAllEnvListByDataSourceType", notes = "get all env list by data source type", response = Message.class)
@ApiImplicitParams({
@ApiImplicitParam(name = "typeId", required = true, dataType = "Long", value = "type id")
Expand Down Expand Up @@ -184,6 +296,15 @@ public Message updateJsonEnv(
+ envId
+ " is correct ']");
}
if (!Objects.equals(dataSourceEnv.getEnvName(), storedDataSourceEnv.getEnvName())
&& dataSourceInfoService.existDataSourceEnv(dataSourceEnv.getEnvName())) {
return Message.error(
"The data source env named: "
+ dataSourceEnv.getEnvName()
+ " has been existed [数据源环境: "
+ dataSourceEnv.getEnvName()
+ " 已经存在]");
}
dataSourceEnv.setCreateUser(storedDataSourceEnv.getCreateUser());
updateDataSourceEnv(dataSourceEnv, storedDataSourceEnv);
return Message.ok().data("updateId", envId);
Expand Down
14 changes: 14 additions & 0 deletions .../main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
import org.apache.linkis.datasourcemanager.common.domain.DataSourceType;
import org.apache.linkis.datasourcemanager.common.domain.DatasourceVersion;
import org.apache.linkis.datasourcemanager.common.util.json.Json;
import org.apache.linkis.datasourcemanager.core.formdata.FormDataTransformerFactory;
import org.apache.linkis.datasourcemanager.core.formdata.MultiPartFormDataTransformer;
import org.apache.linkis.datasourcemanager.core.service.DataSourceInfoService;
Expand Down Expand Up @@ -216,6 +217,14 @@ public Message updateDataSourceInJson(
+ dataSourceName
+ " 已经存在]");
}
List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
dataSource.setKeyDefinitions(keyDefinitionList);
for (DataSourceParamsHook hook : dataSourceParamsHooks) {
hook.beforePersist(dataSource.getConnectParams(), keyDefinitionList);
}
String parameter = Json.toJson(dataSource.getConnectParams(), null);
dataSource.setParameter(parameter);
dataSourceInfoService.updateDataSourceInfo(dataSource);
return Message.ok().data("updateId", dataSourceId);
},
Expand Down Expand Up @@ -728,6 +737,11 @@ private void insertDataSource(DataSource dataSource) throws ErrorException {
List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
dataSource.setKeyDefinitions(keyDefinitionList);
for (DataSourceParamsHook hook : dataSourceParamsHooks) {
hook.beforePersist(dataSource.getConnectParams(), keyDefinitionList);
}
String parameter = Json.toJson(dataSource.getConnectParams(), null);
dataSource.setParameter(parameter);
dataSourceInfoService.saveDataSourceInfo(dataSource);
}
}
37 changes: 31 additions & 6 deletions ...src/main/java/org/apache/linkis/datasourcemanager/core/service/DataSourceInfoService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.linkis.datasourcemanager.common.domain.DataSourceEnv;
import org.apache.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
import org.apache.linkis.datasourcemanager.common.domain.DatasourceVersion;
import org.apache.linkis.datasourcemanager.common.exception.JsonErrorException;
import org.apache.linkis.datasourcemanager.core.vo.DataSourceEnvVo;
import org.apache.linkis.datasourcemanager.core.vo.DataSourceVo;

Expand Down Expand Up @@ -53,15 +54,15 @@ public interface DataSourceInfoService {
* @param dataSourceId id
* @return data source entity
*/
DataSource getDataSourceInfo(Long dataSourceId);
DataSource getDataSourceInfo(Long dataSourceId) throws JsonErrorException;

/**
* Get data source for current version by data source name
*
* @param dataSourceName data source name
* @return data source entity
*/
DataSource getDataSourceInfo(String dataSourceName);
DataSource getDataSourceInfo(String dataSourceName) throws JsonErrorException;

/**
* Get data source for latest published version by data source name
Expand All @@ -77,7 +78,7 @@ public interface DataSourceInfoService {
* @param dataSourceId id
* @return data source entity
*/
DataSource getDataSourceInfo(Long dataSourceId, Long version);
DataSource getDataSourceInfo(Long dataSourceId, Long version) throws JsonErrorException;

/**
* Get data source brief information
Expand Down Expand Up @@ -117,6 +118,14 @@ public interface DataSourceInfoService {
*/
void saveDataSourceEnv(DataSourceEnv dataSourceEnv) throws ErrorException;

/**
* Batch save data source environment
*
* @param dataSourceEnvList
* @throws ErrorException
*/
void saveBatchDataSourceEnv(List<DataSourceEnv> dataSourceEnvList) throws ErrorException;

/**
* List data source environments
*
Expand Down Expand Up @@ -149,6 +158,14 @@ public interface DataSourceInfoService {
*/
void updateDataSourceEnv(DataSourceEnv updatedOne, DataSourceEnv storedOne) throws ErrorException;

/**
* Batch update data source environment
*
* @param dataSourceEnvList
* @throws ErrorException
*/
void updateBatchDataSourceEnv(List<DataSourceEnv> dataSourceEnvList) throws ErrorException;

/**
* Page query of data source environment
*
Expand Down Expand Up @@ -205,7 +222,7 @@ long insertDataSourceParameter(
* @param dataSourceId
* @return
*/
DataSource getDataSourceInfoForConnect(Long dataSourceId);
DataSource getDataSourceInfoForConnect(Long dataSourceId) throws JsonErrorException;

/**
* get datasource info for connect for published version by name, if there is a dependency
Expand All @@ -214,7 +231,7 @@ long insertDataSourceParameter(
* @param dataSourceName
* @return
*/
DataSource getDataSourceInfoForConnect(String dataSourceName);
DataSource getDataSourceInfoForConnect(String dataSourceName) throws JsonErrorException;
/**
* get datasource info for connect, if there is a dependency environment, merge datasource
* parameter and environment parameter.
Expand All @@ -223,7 +240,7 @@ long insertDataSourceParameter(
* @param version
* @return
*/
DataSource getDataSourceInfoForConnect(Long dataSourceId, Long version);
DataSource getDataSourceInfoForConnect(Long dataSourceId, Long version) throws JsonErrorException;

/**
* Check if exist data source
Expand All @@ -232,4 +249,12 @@ long insertDataSourceParameter(
* @return boolean
*/
boolean existDataSource(String dataSourceName);

/**
* Check if exist data source env
*
* @param dataSourceEnvName
* @return
*/
boolean existDataSourceEnv(String dataSourceEnvName);
}
Loading

0 comments on commit e3b5151

Please sign in to comment.