Skip to content

Commit

Permalink
[Feature][scaleph-engine-sql-gateway] add sql gateway splited service (
Browse files Browse the repository at this point in the history
…#623)

* feature: add flink sql gateway service interface

* feature: upgrade flink kubernetes operator api

* feature: upgrade flink kubernetes operator api

* feature: add dataservice module

* feature: add flink sql gateway service interface

* feature: add dataservice module

* fix: dataservice sql script error

* feature: dataservice config mapper

* feature: dataservice config controller

* feature: dataservice parameter and result map sql table

* feature: dataservice parameter and result map mapper

* feature: dataservice parameter and result map interface

* feature: dataservice parameter and result map service

* feature: add dataservice parameter and result mapping sql table

* feature: add dataservice parameter and result mapping mapper

* fix: jobId NPE

* fix: jobId NPE

* feature: add parametermap and resultmap

* feature: add parametermap and resultmap

* feature: add parametermap and resultmap

* feature: add parametermap and resultmap

* feature: add parametermapping and resultmapping

* feature: add parametermapping and resultmapping

* feature: add flink custom artifact table

* fix: docker build

* fix: docker build

* fix: docker build

* fix: flink sql gateway feitch results api page param error
  • Loading branch information
kalencaya authored Oct 11, 2023
1 parent 3cdadc5 commit 88d0f56
Show file tree
Hide file tree
Showing 77 changed files with 3,158 additions and 60 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ jobs:
cache: maven
- name: Build with Maven
timeout-minutes: 360
run: mvn -B -U -T 4 clean package --file pom.xml
run: mvn -B -U -T 4C clean package --file pom.xml
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/docker-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,13 +47,13 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Build scaleph-dist
run: mvn -B -U -T 4 clean package -Pdist -DskipTests -Dfast
run: mvn -B -U -T 4C clean package -Pdist -DskipTests -Dfast
- name: Build scaleph-ui-react-dist
run: mvn -B -U package -pl scaleph-ui-react -am -DskipTests
- name: Build scaleph-ui-react
run: cd scaleph-ui-react && npm install && npm run build --prod
- name: Build scaleph-file-fetcher
run: mvn -B -U -T 4 clean package -Pdist -DskipTests -Dfast -am --projects scaleph-file-fetcher
run: mvn -B -U -T 4C clean package -Pdist -DskipTests -Dfast -am --projects scaleph-file-fetcher
- name: Build image
uses: docker/build-push-action@v4
with:
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/docker-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,13 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Build scaleph-dist
run: mvn -B -U -T 4 clean package -Pdist -DskipTests -Dfast
run: mvn -B -U -T 4C clean package -Pdist -DskipTests -Dfast
- name: Build scaleph-ui-react-dist
run: mvn -B -U package -pl scaleph-ui-react -am -DskipTests
- name: Build scaleph-ui-react
run: cd scaleph-ui-react && npm install && npm run build --prod
- name: Build scaleph-file-fetcher
run: mvn -B -U -T 4 clean package -Pdist -DskipTests -Dfast -am --projects scaleph-file-fetcher
run: mvn -B -U -T 4C clean package -Pdist -DskipTests -Dfast -am --projects scaleph-file-fetcher
- name: Push image
uses: docker/build-push-action@v4
with:
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/release-dist-bin.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,16 @@ jobs:
name: Prepare JDK 17
uses: actions/setup-java@v3
with:
distribution: 'oracle'
distribution: temurin
java-version: 17
cache: 'maven'
cache: maven
- id: get_project_version
name: Get Project Version
run: |
echo "PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout 2>/dev/null)" >> $GITHUB_ENV
- id: run_build
name: Run Maven Build
run: mvn -B -U -T 4 clean package -Pdist -DskipTests -Dfast
run: mvn -B -U -T 4C clean package -Pdist -DskipTests -Dfast
- id: upload_assets
name: Upload binaries to release
uses: svenstaro/upload-release-action@v2
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ jobs:
- name: Build scaleph-ui-react
run: cd scaleph-ui-react && npm install && npm run build --prod
- name: Build scaleph-file-fetcher
run: mvn -B -U -T 4 clean package -Pdist -DskipTests -Dfast -am --projects scaleph-file-fetcher
run: mvn -B -U -T 4C clean package -Pdist -DskipTests -Dfast -am --projects scaleph-file-fetcher
- name: Release image
uses: docker/build-push-action@v4
with:
Expand Down
6 changes: 6 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@

<module>scaleph-dist</module>
<module>scaleph-file-fetcher</module>
<module>scaleph-dataservice</module>
</modules>

<properties>
Expand Down Expand Up @@ -241,6 +242,11 @@
<artifactId>scaleph-config</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>scaleph-dataservice</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>scaleph-datasource</artifactId>
Expand Down
4 changes: 4 additions & 0 deletions scaleph-api/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@
<groupId>${project.parent.groupId}</groupId>
<artifactId>scaleph-engine-flink-kubernetes</artifactId>
</dependency>
<dependency>
<groupId>${project.parent.groupId}</groupId>
<artifactId>scaleph-dataservice</artifactId>
</dependency>
<dependency>
<groupId>${project.parent.groupId}</groupId>
<artifactId>scaleph-datasource</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package cn.sliew.scaleph.api.config;

import cn.sliew.scaleph.api.annotation.AnonymousAccess;
import cn.sliew.scaleph.common.constant.Constants;
import cn.sliew.scaleph.common.util.SpringApplicationContextUtil;
import cn.sliew.scaleph.security.authentication.CustomAccessDeniedHandler;
import cn.sliew.scaleph.security.authentication.CustomAuthenticationEntryPoint;
Expand All @@ -37,13 +38,14 @@
import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.security.web.SecurityFilterChain;
import org.springframework.util.ObjectUtils;
import org.springframework.web.cors.CorsConfiguration;
import org.springframework.web.cors.UrlBasedCorsConfigurationSource;
import org.springframework.web.filter.CorsFilter;
import org.springframework.web.method.HandlerMethod;
import org.springframework.web.servlet.mvc.method.RequestMappingInfo;
import org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping;

import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.*;

/**
* https://spring.io/blog/2022/02/21/spring-security-without-the-websecurityconfigureradapter
Expand Down Expand Up @@ -126,4 +128,21 @@ public SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
;
return http.build();
}

/**
* fix When allowCredentials is true, allowedOrigins cannot contain the special value "*" since that cannot be set on the "Access-Control-Allow-Origin" response header.
* To allow credentials to a set of origins, list them explicitly or consider using "allowedOriginPatterns" instead
*/
@Bean
public CorsFilter corsFilter() {
final UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource();
CorsConfiguration config = new CorsConfiguration();
config.setAllowCredentials(true);
config.addAllowedOriginPattern("*");
config.setAllowedHeaders(Arrays.asList("Origin", "Content-Type", "Accept", "responseType", Constants.TOKEN_KEY));
config.setAllowedMethods(Arrays.asList("GET", "POST", "PUT", "OPTIONS", "DELETE", "PATCH"));
source.registerCorsConfiguration("/**", config);
return new CorsFilter(source);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package cn.sliew.scaleph.api.controller.dataservice;

import cn.sliew.scaleph.api.annotation.Logging;
import cn.sliew.scaleph.dataservice.service.DataserviceConfigService;
import cn.sliew.scaleph.dataservice.service.dto.DataserviceConfigDTO;
import cn.sliew.scaleph.dataservice.service.param.DataserviceConfigAddParam;
import cn.sliew.scaleph.dataservice.service.param.DataserviceConfigListParam;
import cn.sliew.scaleph.dataservice.service.param.DataserviceConfigUpdateParam;
import cn.sliew.scaleph.system.model.ResponseVO;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;

import javax.validation.Valid;
import java.text.ParseException;
import java.util.List;

@Tag(name = "数据服务-配置管理")
@RestController
@RequestMapping(path = "/api/dataservice/config")
public class DataserviceConfigController {

@Autowired
private DataserviceConfigService dataserviceConfigService;

@Logging
@GetMapping
@Operation(summary = "查询 config 列表", description = "查询 config 列表")
public ResponseEntity<ResponseVO<Page<DataserviceConfigDTO>>> get(@Valid DataserviceConfigListParam param) throws ParseException {
Page<DataserviceConfigDTO> result = dataserviceConfigService.list(param);
return new ResponseEntity<>(ResponseVO.success(result), HttpStatus.OK);
}

@Logging
@PutMapping
@Operation(summary = "新增 config", description = "新增 config")
public ResponseEntity<ResponseVO> insert(@Valid @RequestBody DataserviceConfigAddParam param) throws ParseException {
dataserviceConfigService.insert(param);
return new ResponseEntity<>(ResponseVO.success(), HttpStatus.OK);
}

@Logging
@PostMapping
@Operation(summary = "修改 config", description = "修改 config")
public ResponseEntity<ResponseVO> update(@Valid @RequestBody DataserviceConfigUpdateParam param) throws ParseException {
dataserviceConfigService.update(param);
return new ResponseEntity<>(ResponseVO.success(), HttpStatus.OK);
}

@Logging
@DeleteMapping("{id}")
@Operation(summary = "删除 config", description = "删除 config")
public ResponseEntity<ResponseVO> deleteById(@PathVariable("id") Long id) {
dataserviceConfigService.deleteById(id);
return new ResponseEntity<>(ResponseVO.success(), HttpStatus.OK);
}

@Logging
@DeleteMapping(path = "/batch")
@Operation(summary = "批量删除 config", description = "批量删除 config")
public ResponseEntity<ResponseVO> deleteBatch(@RequestBody List<Long> ids) {
dataserviceConfigService.deleteBatch(ids);
return new ResponseEntity<>(ResponseVO.success(), HttpStatus.OK);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package cn.sliew.scaleph.api.controller.dataservice;

import cn.sliew.scaleph.api.annotation.Logging;
import cn.sliew.scaleph.dataservice.service.DataserviceParameterMapService;
import cn.sliew.scaleph.dataservice.service.dto.DataserviceParameterMapDTO;
import cn.sliew.scaleph.dataservice.service.dto.DataserviceParameterMappingDTO;
import cn.sliew.scaleph.dataservice.service.param.DataserviceParameterMapAddParam;
import cn.sliew.scaleph.dataservice.service.param.DataserviceParameterMapListParam;
import cn.sliew.scaleph.dataservice.service.param.DataserviceParameterMapUpdateParam;
import cn.sliew.scaleph.dataservice.service.param.DataserviceParameterMappingReplaceParam;
import cn.sliew.scaleph.system.model.ResponseVO;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;

import javax.validation.Valid;
import java.text.ParseException;
import java.util.List;

@Tag(name = "数据服务-ParameterMap管理")
@RestController
@RequestMapping(path = "/api/dataservice/parameter-map")
public class DataserviceParameterMapController {

@Autowired
private DataserviceParameterMapService dataserviceParameterMapService;

@Logging
@GetMapping
@Operation(summary = "查询 parameterMap 列表", description = "查询 parameterMap 列表")
public ResponseEntity<ResponseVO<Page<DataserviceParameterMapDTO>>> get(@Valid DataserviceParameterMapListParam param) throws ParseException {
Page<DataserviceParameterMapDTO> result = dataserviceParameterMapService.list(param);
return new ResponseEntity<>(ResponseVO.success(result), HttpStatus.OK);
}

@Logging
@GetMapping("{parameterMapId}/mappings")
@Operation(summary = "查询 parameterMapping 列表", description = "查询 parameterMapping 列表")
public ResponseEntity<ResponseVO<List<DataserviceParameterMappingDTO>>> listMappings(@PathVariable("parameterMapId") Long parameterMapId) throws ParseException {
List<DataserviceParameterMappingDTO> result = dataserviceParameterMapService.listMappings(parameterMapId);
return new ResponseEntity<>(ResponseVO.success(result), HttpStatus.OK);
}

@Logging
@PutMapping
@Operation(summary = "新增 parameterMap", description = "新增 parameterMap")
public ResponseEntity<ResponseVO> insert(@Valid @RequestBody DataserviceParameterMapAddParam param) throws ParseException {
dataserviceParameterMapService.insert(param);
return new ResponseEntity<>(ResponseVO.success(), HttpStatus.OK);
}

@Logging
@PostMapping
@Operation(summary = "修改 parameterMap", description = "修改 parameterMap")
public ResponseEntity<ResponseVO> update(@Valid @RequestBody DataserviceParameterMapUpdateParam param) throws ParseException {
dataserviceParameterMapService.update(param);
return new ResponseEntity<>(ResponseVO.success(), HttpStatus.OK);
}

@Logging
@PostMapping("mappings/replace")
@Operation(summary = "替换 parameterMapping", description = "替换 parameterMapping")
public ResponseEntity<ResponseVO> replaceMappings(@Valid @RequestBody DataserviceParameterMappingReplaceParam param) throws ParseException {
dataserviceParameterMapService.replaceMappings(param);
return new ResponseEntity<>(ResponseVO.success(), HttpStatus.OK);
}

@Logging
@DeleteMapping("{id}")
@Operation(summary = "删除 parameterMap", description = "删除 parameterMap")
public ResponseEntity<ResponseVO> deleteById(@PathVariable("id") Long id) {
dataserviceParameterMapService.deleteById(id);
return new ResponseEntity<>(ResponseVO.success(), HttpStatus.OK);
}

@Logging
@DeleteMapping(path = "/batch")
@Operation(summary = "批量删除 parameterMap", description = "批量删除 parameterMap")
public ResponseEntity<ResponseVO> deleteBatch(@RequestBody List<Long> ids) {
dataserviceParameterMapService.deleteBatch(ids);
return new ResponseEntity<>(ResponseVO.success(), HttpStatus.OK);
}

}
Loading

0 comments on commit 88d0f56

Please sign in to comment.