Skip to content

Commit

Permalink
Merge pull request #70 from technologiestiftung/staging
Browse files Browse the repository at this point in the history
staging to main
  • Loading branch information
ff6347 authored Mar 29, 2023
2 parents a88200e + 77e8a53 commit d18bb79
Show file tree
Hide file tree
Showing 8 changed files with 11,845 additions and 40 deletions.
84 changes: 53 additions & 31 deletions .github/workflows/test-harvest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,19 @@ on:
push:
branches:
- master
- staging
pull_request:
branches:
- master
- staging
workflow_dispatch:

env:
PG_SERVER: localhost
PG_PORT: 5432
PG_USER: fangorn
PG_PASS: ent
PG_DB: trees
PG_PORT: "54322"
PG_USER: postgres
PG_PASS: postgres
PG_DB: postgres
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_TEST }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_TEST }}
S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_TEST }}
Expand All @@ -24,42 +26,62 @@ env:
MAPBOXTILESET: "xyz"
MAPBOXLAYERNAME: "abc"
LOGGING: INFO
DATABASE_URL: postgresql://fangorn:ent@localhost:5432/trees?schema=public
# DATABASE_URL: postgresql://fangorn:ent@localhost:5432/trees?schema=public

jobs:
test-harvest:
runs-on: ubuntu-20.04
runs-on: ubuntu-latest
if: "!contains(github.event.head_commit.message, 'skip ci')"
services:
postgres:
image: postgis/postgis:11-2.5-alpine
env:
POSTGRES_USER: fangorn
POSTGRES_DB: trees
POSTGRES_PASSWORD: ent
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
steps:
- name: Get the source
uses: actions/checkout@v3
- name: Use Node.js 14
- name: Use Node.js from .nvmrc
uses: actions/setup-node@v2
with:
node-version: 14
- name: Get giessdenkiez-de-postgres-api
run: git clone https://github.com/technologiestiftung/giessdenkiez-de-postgres-api.git api
- name: install node dependencies
run: cd api && npm ci
- name: prisma db push
run: cd api && npm run prisma:push:dangerously
- name: prisma db seed
run: cd api && npm run prisma:seed:dangerously
node-version-file: ".nvmrc"
- name: Get the source for the api staging
uses: actions/checkout@v3
if: github.event.pull_request.base.ref == 'staging' || github.ref == 'refs/heads/staging'
with:
repository: technologiestiftung/giessdenkiez-de-postgres-api
path: api
ref: staging
- name: Get the source for the api master
uses: actions/checkout@v3
if: github.event.pull_request.base.ref == 'master' || github.ref == 'refs/heads/master'
with:
repository: technologiestiftung/giessdenkiez-de-postgres-api
path: api
- uses: supabase/setup-cli@v1
with:
version: 1.33.0
- name: build the harvester
run: cd harvester && docker build --tag technologiestiftung/giessdenkiez-de-dwd-harvester:test .
- name: Start the api
run: cd api && supabase start && cd ..
- name: run the harvester
run: docker run --env PG_SERVER='0.0.0.0' --env PG_DB --env PG_PORT --env PG_USER --env PG_PASS --env AWS_SECRET_ACCESS_KEY --env AWS_ACCESS_KEY_ID --env S3_BUCKET="${{ secrets.S3_BUCKET_TEST }}" --env MAPBOXTOKEN --env MAPBOXUSERNAME --env LOGGING --env OUTPUT --network host technologiestiftung/giessdenkiez-de-dwd-harvester:test
run: docker run --env PG_SERVER='0.0.0.0' --env PG_DB --env PG_PORT --env PG_USER --env PG_PASS --env AWS_SECRET_ACCESS_KEY --env AWS_ACCESS_KEY_ID --env S3_BUCKET --env MAPBOXTOKEN --env MAPBOXUSERNAME --env LOGGING --env OUTPUT --network host technologiestiftung/giessdenkiez-de-dwd-harvester:test
- name: stop the api
run: cd api && supabase stop && cd ..
release:
name: semantic-release
needs: [test-harvest]
runs-on: ubuntu-latest
outputs:
new-release-published: ${{ steps.semantic-release.outputs.new_release_published }}
new-release-version: ${{ steps.semantic-release.outputs.new_release_version }}
steps:
- uses: actions/checkout@v3
with:
persist-credentials: false
- uses: actions/setup-node@v3
with:
node-version: 18
- run: npm ci
- id: semantic-release
uses: cycjimmy/semantic-release-action@v2
with:
semantic_version: 18
env:
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
100 changes: 93 additions & 7 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -175,33 +175,119 @@ static

# Local .terraform directories
**/.terraform/*

# .tfstate files
*.tfstate
*.tfstate.*

# Crash log files
crash.log

# Ignore any .tfvars files that are generated automatically for each Terraform run. Most
# .tfvars files are managed as part of configuration and so should be included in
# version control.
#
# example.tfvars

# Ignore override files as they are usually used to override resources locally and so
# are not checked in
override.tf
override.tf.json
*_override.tf
*_override.tf.json

# Include override files you do wish to add to version control using negated pattern
#
# !example_override.tf

# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
# example: *tfplan*
terraform/rds/terraform.tfvars
terraform/ecs-harvester/terraform.tfvars
terraform/.vscode/settings.json
# Logs
logs
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
.pnpm-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# Snowpack dependency directory (https://snowpack.dev/)
web_modules/
# TypeScript cache
*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional stylelint cache
.stylelintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variable files
.env.development.local
.env.test.local
.env.production.local
.env.local
# parcel-bundler cache (https://parceljs.org/)
.parcel-cache
# Next.js build output
.next
out
# Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and not Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# vuepress v2.x temp and cache directory
.temp
# Docusaurus cache and generated files
.docusaurus
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
# yarn v2
.yarn/cache
.yarn/unplugged
.yarn/build-state.yml
.yarn/install-state.gz
.pnp.*
1 change: 1 addition & 0 deletions .nvmrc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
v18.13.0
11 changes: 10 additions & 1 deletion .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,16 @@
"commandCenter.border": "#15202b99",
"sash.hoverBorder": "#9ec7e2",
"statusBarItem.remoteBackground": "#77b0d6",
"statusBarItem.remoteForeground": "#15202b"
"statusBarItem.remoteForeground": "#15202b",
"editorGroup.border": "#9ec7e2",
"panel.border": "#9ec7e2",
"sideBar.border": "#9ec7e2",
"statusBar.border": "#77b0d6",
"statusBar.debuggingBackground": "#d69d77",
"statusBar.debuggingBorder": "#d69d77",
"statusBar.debuggingForeground": "#15202b",
"tab.activeBorder": "#9ec7e2",
"titleBar.border": "#77b0d6"
},
"peacock.color": "#77b0d6"
}
2 changes: 1 addition & 1 deletion harvester/harvester.py
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,7 @@ def finishGeojson(feature_list, file_name):
# create a CSV with all trees (id, lat, lng, radolan_sum)
with conn.cursor() as cur:
# WARNING: The db is still mislabeled lat <> lng
cur.execute("SELECT trees.id, trees.lat, trees.lng, trees.radolan_sum, CASE WHEN LENGTH(trees.pflanzjahr) = 4 THEN date_part('year', CURRENT_DATE) - To_Number(trees.pflanzjahr, '9999') ELSE NULL END AS age FROM trees WHERE ST_CONTAINS(ST_SetSRID((SELECT ST_EXTENT(geometry) FROM radolan_geometry), 4326), trees.geom)")
cur.execute("SELECT trees.id, trees.lat, trees.lng, trees.radolan_sum, (date_part('year', CURRENT_DATE) - trees.pflanzjahr) as age FROM trees WHERE ST_CONTAINS(ST_SetSRID (( SELECT ST_EXTENT (geometry) FROM radolan_geometry), 4326), trees.geom)")
trees = cur.fetchall()
trees_head = "id,lng,lat,radolan_sum,age"
trees_csv = trees_head
Expand Down
Loading

0 comments on commit d18bb79

Please sign in to comment.