Merge pull request #3032 from ingef/release

Merge Release

.github/workflows/codeql-analysis.yml

@@ -13,7 +13,10 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ develop, master, release/* ]
+    branches:
+      - develop
+      - master
+      - release
   pull_request:
     # The branches below must be a subset of the branches above
     branches: [ develop ]
@@ -59,18 +62,20 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
+      if: matrix.language != 'java'
       uses: github/codeql-action/autobuild@v2
 
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 https://git.io/JvXDl
 
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
+      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+      #    and modify them (or add more) to build your code if your project
+      #    uses a compiled language
 
-    #- run: |
-    #   make bootstrap
-    #   make release
+    - name: Build
+      if: matrix.language == 'java'
+      run: |
+        ./scripts/build_backend_version.sh "version_placeholder"
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v2

.github/workflows/create_backend_branch_image.yml

@@ -0,0 +1,53 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Create and publish a backend image of specified branches
+
+on:
+  push:
+    branches:
+      - develop
+      - master
+      - release
+      - staging/**
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}-backend
+
+jobs:
+  build-and-push-image-and-clean-ghcr:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          # Fetch all tags
+          fetch-depth: 0
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: type=raw,value={{branch}}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/create_frontend_branch_image.yml

@@ -0,0 +1,53 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Create and publish a frontend image of specified branches
+
+on:
+  push:
+    branches:
+      - develop
+      - master
+      - release
+      - staging/*
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}-frontend
+
+jobs:
+  build-and-push-image-and-clean-ghcr:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          # Fetch all tags
+          fetch-depth: 0
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: type=raw,value={{branch}}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: ./frontend
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/prepare-merge-release.yml

@@ -6,7 +6,7 @@ on:
       - release
 
 jobs:
-  bump-test:
+  merge-release:
     runs-on: ubuntu-latest
     timeout-minutes: 3
     steps:

.github/workflows/reintegrate-master.yml

@@ -6,7 +6,7 @@ on:
       - master
 
 jobs:
-  bump-test:
+  reintegrate-master:
     runs-on: ubuntu-latest
     timeout-minutes: 3
     steps:

.github/workflows/run_autodoc.yml

@@ -13,7 +13,7 @@ on:
       - "backend/src/main/java/com/bakdata/conquery/apiv1/**"
 
 jobs:
-  generate_auto_doc:
+  generate-auto-doc:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:

.github/workflows/test_backend.yml

@@ -1,12 +1,18 @@
 name: "Test Backend"
 
 on:
+  push:
+    branches:
+      # Always run on protected branches
+      - master
+      - develop
+      - release
   pull_request:
     paths:
       - "backend/**"
 
 jobs:
-  run_tests:
+  test:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:

.github/workflows/test_cypress.yml

@@ -1,12 +1,18 @@
 name: "End-To-End Tests"
 
 on:
+  push:
+    branches:
+      # Always run on protected branches
+      - master
+      - develop
+      - release
   pull_request:
     paths:
       - "**"
 
 jobs:
-  end-to-end-test:
+  test:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:

.github/workflows/test_frontend.yml

@@ -1,12 +1,18 @@
 name: "Test Frontend"
 
 on:
+  push:
+    branches:
+      # Always run on protected branches
+      - master
+      - develop
+      - release
   pull_request:
     paths:
       - "frontend/**"
 
 jobs:
-  format-lint-unittest:
+  test:
     runs-on: ubuntu-latest
     timeout-minutes: 6
     steps:

.gitignore

@@ -71,3 +71,5 @@ hs_err_pid*
 tutorial/mimic_iii_demo/data/**
 
 /node_modules
+cypress/screenshots/
+cypress/videos/

Dockerfile

@@ -46,6 +46,9 @@ ENV CLUSTER_PORT=${CLUSTER_PORT:-8082}
 ENV ADMIN_PORT=${ADMIN_PORT:-8081}
 ENV API_PORT=${API_PORT:-8080}
 
+RUN mkdir /app/logs
+VOLUME /app/logs
+
 ENTRYPOINT [ "java", "-jar", "conquery.jar" ]
 
 CMD [ "standalone" ]

README.md

@@ -1,10 +1,9 @@
 # Conquery
 *fast & efficient analysis*
 
-[![Last Release](https://img.shields.io/github/release-date/bakdata/conquery.svg?logo=github)](https://github.com/bakdata/conquery/releases/latest)
-![Code Size](https://img.shields.io/github/languages/code-size/bakdata/conquery.svg)
-[![License](https://img.shields.io/github/license/bakdata/conquery.svg)](https://github.com/bakdata/conquery/blob/develop/LICENSE)
-
+[![Last Release](https://img.shields.io/github/release-date/ingef/conquery.svg?logo=github)](https://github.com/ingef/conquery/releases/latest)
+![Code Size](https://img.shields.io/github/languages/code-size/ingef/conquery.svg)
+[![License](https://img.shields.io/github/license/ingef/conquery.svg)](https://github.com/ingef/conquery/blob/develop/LICENSE)
 
 ![conquery Screenshot](images/screenshot-v4.png)
 
@@ -32,10 +31,11 @@ Check the README in `/frontend` for details.
 ### Frontend + Backend
 
 #### Steps
+
 To test frontend and backend together you can start the setup that is used for end-to-end tests.
 
 First build the backend using `conquery/scripts/build_backend_version.sh` or download a JAR from
-the [release page](https://github.com/bakdata/conquery/releases) and place it in `conquery/executable/target/`.
+the [release page](https://github.com/ingef/conquery/releases) and place it in `conquery/executable/target/`.
 
 Build the frontend by running:
 
@@ -46,14 +46,36 @@ yarn
 yarn build
 ```
 
-You can then run `conquery/scripts/run_e2e_all.sh` to start frontend and backend, and also load the test data required by cypress end-to-end test or you can run `conquery/scripts/run_e2e_backend.sh` and `conquery/scripts/run_e2e_frontend.sh` separately without loading any data.
+You can then run `conquery/scripts/run_e2e_all.sh` to start frontend and backend, and also load the test data required
+by cypress end-to-end test or you can run `conquery/scripts/run_e2e_backend.sh`
+and `conquery/scripts/run_e2e_frontend.sh` separately without loading any data.
 
 After that, you can visit http://localhost:8081/admin-ui and explore the Admin Panel.
 
 The frontend is accessible at http://localhost:8000 as the default "superuser" implicitly. Since the backend uses a
 development authentication, you can switch users by passing another users "UserId" as the access token in the query
 string when accessing the frontend, e.g.: http://localhost:8000/?access_token=user.user2.
 
+## Configuration
+
+The configuration options for the backend are based on Java classes which reside
+under [this package](https://github.com/ingef/conquery/tree/develop/backend/src/main/java/com/bakdata/conquery/models/config)
+.
+Usually you provide configuration with in a JSON file that is referenced by the start command:
+
+```bash 
+java -jar conquery.jar standalone config.json
+```
+
+This `config.json` represents
+configuration [root class](https://github.com/ingef/conquery/blob/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java)
+.
+
+We continuously improve and extend the documentation on these classes and will provide dedicated articles on specific
+configurations, such as:
+
+- [Configure Authentication with OAuth2/OpenId Connect](./docs/authentication.md)
+
 ## Development
 
 ### Testing
@@ -69,7 +91,8 @@ To run the end-to-end test locally:
 5. Then run `yarn cypress open` to start cypress
 6. Then chose a test suite and start it.
 
-For further informations on this and other tests, please refer to the corresponding [CI configuration](https://github.com/bakdata/conquery/tree/develop/.github/workflows).
+For further informations on this and other tests, please refer to the
+corresponding [CI configuration](https://github.com/ingef/conquery/tree/develop/.github/workflows).
 
 ### Data Integration
 

backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java

@@ -184,7 +184,7 @@ private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId exe
 		if (!user.isOwner(execution)) {
 			final ManagedExecution
 					newExecution =
-					executionManager.createExecution(namespace, execution.getSubmitted(), user, execution.getDataset(), false);
+					executionManager.createExecution(execution.getSubmitted(), user, execution.getDataset(), false);
 			newExecution.setLabel(execution.getLabel());
 			newExecution.setTags(execution.getTags().clone());
 			storage.updateExecution(newExecution);
@@ -318,22 +318,6 @@ public void patchQuery(Subject subject, ManagedExecution execution, MetaDataPatc
 
 		patch.applyTo(execution, storage, subject);
 		storage.updateExecution(execution);
-
-		// TODO remove this, since we don't translate anymore
-		// Patch this query in other datasets
-		final List<Dataset> remainingDatasets = datasetRegistry.getAllDatasets();
-		remainingDatasets.remove(execution.getDataset());
-
-		for (Dataset dataset : remainingDatasets) {
-			final ManagedExecutionId id = new ManagedExecutionId(dataset.getId(), execution.getQueryId());
-			final ManagedExecution otherExecution = storage.getExecution(id);
-			if (otherExecution == null) {
-				continue;
-			}
-			log.trace("Patching {} ({}) with patch: {}", execution.getClass().getSimpleName(), id, patch);
-			patch.applyTo(otherExecution, storage, subject);
-			storage.updateExecution(execution);
-		}
 	}
 
 	public void reexecute(Subject subject, ManagedExecution query) {
@@ -399,7 +383,7 @@ public ExternalUploadResult uploadEntities(Subject subject, Dataset dataset, Ext
 				execution =
 				((ManagedQuery) namespace
 						.getExecutionManager()
-						.createExecution(namespace, query, subject.getUser(), dataset, false));
+						.createExecution(query, subject.getUser(), dataset, false));
 
 		execution.setLastResultCount((long) statistic.getResolved().size());