Backport cloud features

.env.example

@@ -72,5 +72,5 @@ DEKART_STATIC_FILES=
 REACT_APP_CUSTOM_CODE=
 DEKART_DEV_CLAIMS_EMAIL=
 DEKART_DEV_REFRESH_TOKEN=
-DEKART_DOCKER_DEV_TAG= dekart-dev
-DEKART_DOCKER_E2E_TAG= dekart-e2e
+DEKART_DOCKER_DEV_TAG=dekart-dev
+DEKART_DOCKER_E2E_TAG=dekart-e2e

.github/workflows/e2e.yaml

@@ -234,6 +234,8 @@ jobs:
             -e DEKART_SNOWFLAKE_ACCOUNT_ID=${{ secrets.SNOWFLAKE_ACCOUNT_ID }} \
             -e DEKART_SNOWFLAKE_USER=${{ secrets.SNOWFLAKE_USER }} \
             -e DEKART_SNOWFLAKE_PASSWORD=${{ secrets.SNOWFLAKE_PASSWORD }} \
+            -e DEKART_DEV_CLAIMS_EMAIL=test@gmail.com \
+            -e DEKART_REQUIRE_AMAZON_OIDC=1 \
             dekartxyz/dekart:${{ env.IMAGE_CACHE_KEY }}
       - name: Upload cypress artifacts
         if: failure()

.gitignore

@@ -45,4 +45,7 @@ cypress/screenshots
 
 # IDE
 .idea
-testtmp*
+testtmp*
+
+# .env
+.env.*

Makefile

@@ -1,10 +1,10 @@
-.PHONY: proto-build proto-docker proto docker docker-compose-up docker-compose-rm version minor patch
+.PHONY: proto-clean proto-build proto-docker proto nodetest docker-compose-up down cloudsql up-and-down
 
 # load .env
 # https://lithic.tech/blog/2020-05/makefile-dot-env
 ifneq (,$(wildcard ./.env))
     include .env
-    export
+    # export # don't export, it will be available only in this Makefile
 endif
 
 UNAME := $(shell uname -m)
@@ -184,8 +184,6 @@ docker: # build docker for local use
 
 up-and-down:
 	docker-compose  --env-file .env --profile local up; docker-compose --env-file .env --profile local down --volumes
-
-
 up:
 	docker-compose  --env-file .env --profile local up
 
@@ -195,12 +193,24 @@ down:
 cloudsql:
 	docker-compose  --env-file .env --profile cloudsql up
 
-server:
+
+define run_server
+	@set -a; \
+	. $(1); \
+	set +a; \
 	go run ./src/server/main.go
+endef
+
+# Pattern rule to match any target starting with ".env."
+server-%:
+	$(call run_server,.env.$*)
+
+# Rule for the default .env file
+server:
+	$(call run_server,.env)
 
 npm:
 	npm i --legacy-peer-deps
-
 prerelease:
 	npm version prerelease --preid=rc
 preminor:

cypress/e2e/athena/spec.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('basic query flow', () => {
   it('should make simple athena query and get ready status', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_athena_query, { force: true })
@@ -17,7 +17,7 @@ describe('basic query flow', () => {
 describe('cancelling query', () => {
   it('should cancels query', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_athena_query, { force: true })

cypress/e2e/bq/basicFlow.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('basic query flow', () => {
   it('should make simple bigquery query and get ready status', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_sql_query, { force: true })

cypress/e2e/bq/cancelQuery.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('cancelling query', () => {
   it('should cancels query', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_sql_query, { force: true })

cypress/e2e/bq/scripts.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('support bq scripts', () => {
   it('retrieve result for bigquery script', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.bigquery_script, { force: true })

cypress/e2e/bq/updateDataset.cy.js

@@ -5,7 +5,7 @@ describe('update dataset', () => {
   it('should persist kepler config when updating dataset', () => {
     // create report
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
 
     // first query
     cy.get('button:contains("Add data from...")').click()

cypress/e2e/google-oauth/testConnection.cy.js

@@ -6,7 +6,7 @@ describe('basic query flow', () => {
     cy.visit('/')
 
     // create connection
-    cy.get('button:contains("Create connection")').click()
+    cy.get('button:contains("BigQuery")').click()
     const randomConnectionName = `test-${Math.floor(Math.random() * 1000000)}`
     cy.get('div.ant-modal-title').should('contain', 'BigQuery')
     cy.get('input#connectionName').clear()

cypress/e2e/pg/cancelQuery.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('cancelling query', () => {
   it('should cancels query', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_pg_query_long, { force: true })

cypress/e2e/pg/happyPath.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('happy path', () => {
   it('should make simple postgres query and get ready status', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_pg_query, { force: true })

cypress/e2e/snowflake-s3/cancelQuery.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('cancelling query', () => {
   it('should cancels query', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_snowflake_query, { force: true })

cypress/e2e/snowflake-s3/emptyResult.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('empty path', () => {
   it('should complete query returning empty result', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type('SELECT ROUND(uniform(-90::float, 90::float, random()), 6) AS lat, ROUND(uniform(-180::float, 180::float, random()), 6) AS lon FROM TABLE(GENERATOR(ROWCOUNT => 0))', { force: true })

cypress/e2e/snowflake-s3/fork.cy.js

@@ -21,9 +21,8 @@ async function getColorAtMapCenter (win) {
 
 describe('fork', () => {
   it('should have same viz style after fork', () => {
-    let originalColor
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type('select 0 as lat, 0 as lon', { force: true })

cypress/e2e/snowflake-s3/happyPath.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('happy path', () => {
   it('should make simple snowflake query and get ready status', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_snowflake_query, { force: true })

cypress/e2e/snowflake-s3/spec.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('basic query flow', () => {
   it('should make simple snowflake query and get ready status', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_snowflake_query, { force: true })
@@ -18,7 +18,7 @@ describe('basic query flow', () => {
 describe('cancelling query', () => {
   it('should cancels query', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_snowflake_query, { force: true })

cypress/e2e/snowflake/happyPath.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('happy path', () => {
   it('should make simple snowflake query and get ready status', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('textarea').type(copy.simple_snowflake_query, { force: true })
     cy.get(`button:contains("${copy.execute}")`).click()
     cy.get(`span:contains("${copy.ready}")`, { timeout: 20000 }).should('be.visible')

cypress/e2e/snowflake/runAllQueries.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('run all queries', () => {
   it('should run all queries', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('textarea').type('SELECT ROUND(uniform(-90::float, 90::float, random()), 6) AS lat, ROUND(uniform(-180::float, 180::float, random()), 6) AS lon FROM TABLE(GENERATOR(ROWCOUNT => 1000))', { force: true })
     cy.get(`button:contains("${copy.execute}")`).click()
     cy.get(`span:contains("${copy.ready}")`, { timeout: 20000 }).should('be.visible')

cypress/fixtures/copy.json

@@ -1,5 +1,4 @@
 {
-  "create_report": "Report",
   "bigquery_query": "BigQuery query",
   "athena_query": "Athena query",
   "snowflake_query": "Snowflake query",

go.mod

@@ -23,6 +23,7 @@ require (
 )
 
 require (
+	cloud.google.com/go/secretmanager v1.10.0
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/snowflakedb/gosnowflake v1.6.22
 	github.com/stretchr/testify v1.8.1

go.sum

@@ -12,6 +12,8 @@ cloud.google.com/go/datacatalog v1.13.0 h1:4H5IJiyUE0X6ShQBqgFFZvGGcrwGVndTwUSLP
 cloud.google.com/go/iam v1.1.0 h1:67gSqaPukx7O8WLLHMa0PNs3EBGd2eE4d+psbO/CO94=
 cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk=
 cloud.google.com/go/longrunning v0.4.2 h1:WDKiiNXFTaQ6qz/G8FCOkuY9kJmOJGY67wPUC1M2RbE=
+cloud.google.com/go/secretmanager v1.10.0 h1:pu03bha7ukxF8otyPKTFdDz+rr9sE3YauS5PliDXK60=
+cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU=
 cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM=
 cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=

migrations/000023__cloud_snowflake_connection.up.sql

@@ -0,0 +1,6 @@
+ALTER TABLE connections
+ADD COLUMN connection_type INT default 1,
+ADD COLUMN snowflake_account_id text default NULL,
+ADD COLUMN snowflake_username text default NULL,
+ADD COLUMN snowflake_password_encrypted text default NULL,
+ADD COLUMN snowflake_warehouse text default NULL;

proto/dekart.proto

@@ -80,8 +80,6 @@ message GetUserStreamResponse {
     StreamOptions stream_options = 1;
     int64 connection_update = 2;
     string email = 3; // user email used to show user icon in UI
-    bool sensitive_scopes_granted = 4; // user has granted sensitive scopes
-    bool sensitive_scopes_granted_once = 5; // user has granted sensitive scopes at least once, now we request all scopes at once
 }
 
 message TestConnectionRequest {
@@ -127,6 +125,22 @@ message Connection {
     int64 updated_at = 8;
     int64 dataset_count = 9;
     bool can_store_files = 10;
+    enum ConnectionType {
+        CONNECTION_TYPE_UNSPECIFIED = 0;
+        CONNECTION_TYPE_BIGQUERY = 1;
+        CONNECTION_TYPE_SNOWFLAKE = 2;
+    }
+    ConnectionType connection_type = 11;
+    string snowflake_account_id = 12;
+    string snowflake_username = 13;
+    Secret snowflake_password = 14;
+    string snowflake_warehouse = 15;
+}
+
+message Secret {
+    string client_encrypted = 1; // encrypted with client key
+    string server_encrypted = 2; // encrypted with server key
+    int32 length = 3; // length of the password for placeholder
 }
 
 message GetUsageRequest {}
@@ -180,6 +194,10 @@ message GetEnvResponse {
             TYPE_UX_ACCESS_ERROR_INFO_HTML = 13;
             TYPE_UX_NOT_FOUND_ERROR_INFO_HTML = 14;
             TYPE_UX_SAMPLE_QUERY_SQL = 15;
+            TYPE_AES_KEY = 16;
+            TYPE_AES_IV = 17;
+            TYPE_AUTH_ENABLED = 18;
+            TYPE_USER_DEFINED_CONNECTION = 19;
         }
         Type type = 1;
         string value = 2;
@@ -191,6 +209,7 @@ message GetEnvResponse {
 message RedirectState {
     string token_json = 1;
     string error  = 2;
+    bool sensitive_scopes_granted = 3; // if true, sensitive scope are granted
 }
 
 // AuthState is used to pass state between UI, auth endpoint and Google OAuth via redirect
@@ -238,6 +257,8 @@ message Report {
     bool is_author = 9; // user is the author of the report
     int64 created_at = 10;
     int64 updated_at = 11;
+    bool is_sharable = 12; // can be shared with other users (depends where result is stored)
+    bool need_sensitive_scope = 13; // need sensitive scope to run queries and read results
 }
 
 message Dataset {
@@ -261,7 +282,7 @@ message Query {
         JOB_STATUS_PENDING = 1;
         JOB_STATUS_RUNNING = 2;
         JOB_STATUS_DONE_LEGACY = 3;  // legacy for backwards compatibility
-        JOB_STATUS_READING_RESULTS = 4; // job is done proccessing results
+        JOB_STATUS_READING_RESULTS = 4; // job is done processing results
         JOB_STATUS_DONE = 5;
     }