From b10b67869429e36d13a27ec1037477c2ecfdd5da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jens=20Winter-Hu=CC=88benthal?= <jens.winter@gmail.com>
Date: Wed, 3 May 2023 22:51:48 +0200
Subject: [PATCH] create tests for prepare.py

- test default buffer distance
- test custom buffer distance
- test file creation
---
 .github/workflows/test-harvest.yml    |  25 ++++++++++++++++++++++++
 harvester/Dockerfile                  |   5 +++--
 harvester/assets/test/buffer_1000.cpg |   1 +
 harvester/assets/test/buffer_1000.dbf | Bin 0 -> 78 bytes
 harvester/assets/test/buffer_1000.prj |   1 +
 harvester/assets/test/buffer_1000.shp | Bin 0 -> 300 bytes
 harvester/assets/test/buffer_1000.shx | Bin 0 -> 108 bytes
 harvester/assets/test/buffer_2000.cpg |   1 +
 harvester/assets/test/buffer_2000.dbf | Bin 0 -> 78 bytes
 harvester/assets/test/buffer_2000.prj |   1 +
 harvester/assets/test/buffer_2000.shp | Bin 0 -> 284 bytes
 harvester/assets/test/buffer_2000.shx | Bin 0 -> 108 bytes
 harvester/assets/test/shape.dbf       | Bin 0 -> 36 bytes
 harvester/assets/test/shape.prj       |   1 +
 harvester/assets/test/shape.shp       | Bin 0 -> 304 bytes
 harvester/assets/test/shape.shx       | Bin 0 -> 108 bytes
 harvester/prepare.py                  |  25 ++++++++++++++----------
 harvester/requirements.txt            |   8 ++++----
 harvester/test_prepare.py             |  27 ++++++++++++++++++++++++++
 19 files changed, 79 insertions(+), 16 deletions(-)
 create mode 100644 harvester/assets/test/buffer_1000.cpg
 create mode 100644 harvester/assets/test/buffer_1000.dbf
 create mode 100644 harvester/assets/test/buffer_1000.prj
 create mode 100644 harvester/assets/test/buffer_1000.shp
 create mode 100644 harvester/assets/test/buffer_1000.shx
 create mode 100644 harvester/assets/test/buffer_2000.cpg
 create mode 100644 harvester/assets/test/buffer_2000.dbf
 create mode 100644 harvester/assets/test/buffer_2000.prj
 create mode 100644 harvester/assets/test/buffer_2000.shp
 create mode 100644 harvester/assets/test/buffer_2000.shx
 create mode 100644 harvester/assets/test/shape.dbf
 create mode 100644 harvester/assets/test/shape.prj
 create mode 100644 harvester/assets/test/shape.shp
 create mode 100644 harvester/assets/test/shape.shx
 create mode 100644 harvester/test_prepare.py

diff --git a/.github/workflows/test-harvest.yml b/.github/workflows/test-harvest.yml
index 0162bc5b..2862389f 100644
--- a/.github/workflows/test-harvest.yml
+++ b/.github/workflows/test-harvest.yml
@@ -29,6 +29,31 @@ env:
   # DATABASE_URL: postgresql://fangorn:ent@localhost:5432/trees?schema=public
 
 jobs:
+  test-prepare:
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.head_commit.message, 'skip ci')"
+    defaults:
+      run:
+        working-directory: harvester
+    steps:
+      - name: Get the source
+        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install g++ gdal-bin libgdal-dev libpq-dev -y
+      - name: Install python packages
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --no-cache-dir GDAL==3.4.0
+          python -m pip install --no-warn-script-location -r requirements.txt
+      - name: Test with pytest
+        run: |
+          pytest
   test-harvest:
     runs-on: ubuntu-latest
     if: "!contains(github.event.head_commit.message, 'skip ci')"
diff --git a/harvester/Dockerfile b/harvester/Dockerfile
index f7b0938c..cda5a437 100644
--- a/harvester/Dockerfile
+++ b/harvester/Dockerfile
@@ -12,14 +12,15 @@ RUN apt-get update \
   && rm -rf /var/lib/apt/lists/*
 ENV CPLUS_INCLUDE_PATH=/usr/include/gdal
 ENV C_INCLUDE_PATH=/usr/include/gdal
-# RUN pip install GDAL
 
 FROM base as builder
 
 
 # Install pip requirements
 COPY requirements.txt /app/
-RUN cd /app/ && python -m pip install --no-warn-script-location --prefix=/install -r requirements.txt
+RUN python -m pip install --upgrade pip \
+  && cd /app/ && python -m pip install --no-warn-script-location --prefix=/install -r requirements.txt \
+  && python -m pip install --prefix=/install GDAL==2.4.2
 
 FROM base as app
 COPY --from=builder /install /usr/local
diff --git a/harvester/assets/test/buffer_1000.cpg b/harvester/assets/test/buffer_1000.cpg
new file mode 100644
index 00000000..cd89cb97
--- /dev/null
+++ b/harvester/assets/test/buffer_1000.cpg
@@ -0,0 +1 @@
+ISO-8859-1
\ No newline at end of file
diff --git a/harvester/assets/test/buffer_1000.dbf b/harvester/assets/test/buffer_1000.dbf
new file mode 100644
index 0000000000000000000000000000000000000000..9ec44fc9dfd0faededac4f848b277c75414c63dd
GIT binary patch
literal 78
mcmZRsW|3oLU|?`$-~p1Dz|GSICg=xZaKm^|npXh<45R>pt^?8l

literal 0
HcmV?d00001

diff --git a/harvester/assets/test/buffer_1000.prj b/harvester/assets/test/buffer_1000.prj
new file mode 100644
index 00000000..5c6f76df
--- /dev/null
+++ b/harvester/assets/test/buffer_1000.prj
@@ -0,0 +1 @@
+PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]]
\ No newline at end of file
diff --git a/harvester/assets/test/buffer_1000.shp b/harvester/assets/test/buffer_1000.shp
new file mode 100644
index 0000000000000000000000000000000000000000..ae23522612e04d26394d212222171738906bc03b
GIT binary patch
literal 300
zcmZQzQ0HR64yL_eW?*0i%JJB{GRxPRI!f*2`*dtsw4;LNJ5#Uarj937eYGpsM>}HI
zjv~qk<RuVe7Dyo{R4dHhy_H(`9<GjdY$|f|-qne)H#g*B(e&x2j(KK{H`cF-c05(I
zDSgu<Q%9o}Yzl{0A>5gpoWVP3d9)+@e4bf*yAl3iHY=Lm2Q=@Dah25L71542EspEh
RHJUndT#(&e4&<ZT4*<MMR8s%|

literal 0
HcmV?d00001

diff --git a/harvester/assets/test/buffer_1000.shx b/harvester/assets/test/buffer_1000.shx
new file mode 100644
index 0000000000000000000000000000000000000000..a50616e02fe08a0f5d72e5fb89fa995fa1ff8a4e
GIT binary patch
literal 108
zcmZQzQ0HR64$NLKGcd3M<#_B}ndNIu9i?{ieLA)*+EGFCovGJyQ^%94zS@=RqaCqp
KM-epw@)7{-84YIu

literal 0
HcmV?d00001

diff --git a/harvester/assets/test/buffer_2000.cpg b/harvester/assets/test/buffer_2000.cpg
new file mode 100644
index 00000000..cd89cb97
--- /dev/null
+++ b/harvester/assets/test/buffer_2000.cpg
@@ -0,0 +1 @@
+ISO-8859-1
\ No newline at end of file
diff --git a/harvester/assets/test/buffer_2000.dbf b/harvester/assets/test/buffer_2000.dbf
new file mode 100644
index 0000000000000000000000000000000000000000..9ec44fc9dfd0faededac4f848b277c75414c63dd
GIT binary patch
literal 78
mcmZRsW|3oLU|?`$-~p1Dz|GSICg=xZaKm^|npXh<45R>pt^?8l

literal 0
HcmV?d00001

diff --git a/harvester/assets/test/buffer_2000.prj b/harvester/assets/test/buffer_2000.prj
new file mode 100644
index 00000000..5c6f76df
--- /dev/null
+++ b/harvester/assets/test/buffer_2000.prj
@@ -0,0 +1 @@
+PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]]
\ No newline at end of file
diff --git a/harvester/assets/test/buffer_2000.shp b/harvester/assets/test/buffer_2000.shp
new file mode 100644
index 0000000000000000000000000000000000000000..907f3360b3cf805d50a6625f0f262b6f737ba4ec
GIT binary patch
literal 284
zcmZQzQ0HR64*Fg&Gcd3M<!&)lZi}uob-be?yZ++RXvg}#Tw|5hrjA!_^L8X{h<3!T
z9YvH8$crGxERaGDs8*Q0-2PV%&08Jq$hF}<*U~0a$N9}nuFuv-JMMaPfL*>D;SQL)
zN@shfMlX+cyt!{$$*U!%j;D2695|OnJBGb;F1*l%@JAc7%^ux4Q^#<IKWpy*)uY=7
E0I~E}C;$Ke

literal 0
HcmV?d00001

diff --git a/harvester/assets/test/buffer_2000.shx b/harvester/assets/test/buffer_2000.shx
new file mode 100644
index 0000000000000000000000000000000000000000..bbf0ae416ec9ac752f15cb284edb5179d0d9152e
GIT binary patch
literal 108
zcmZQzQ0HR64$NLKGcd3M<!&)lZi}uob-be?yZ++RXvg}#Tw|5hrjA!_^L8X{h<3!T
L9Yxd#$cq2~j!X~l

literal 0
HcmV?d00001

diff --git a/harvester/assets/test/shape.dbf b/harvester/assets/test/shape.dbf
new file mode 100644
index 0000000000000000000000000000000000000000..3228c53078016bf85be5a3fd561e63b6f5156cc5
GIT binary patch
literal 36
ZcmZRsW|m`OU|>*WU<8twfLB353IH030V@Ci

literal 0
HcmV?d00001

diff --git a/harvester/assets/test/shape.prj b/harvester/assets/test/shape.prj
new file mode 100644
index 00000000..a30c00a5
--- /dev/null
+++ b/harvester/assets/test/shape.prj
@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]
\ No newline at end of file
diff --git a/harvester/assets/test/shape.shp b/harvester/assets/test/shape.shp
new file mode 100644
index 0000000000000000000000000000000000000000..fddd5ff1106ed0f6c0720979919596d964a0dbe9
GIT binary patch
literal 304
zcmZQzQ0HR64raVyW?*0i$^~@CWUmg?bV!iB?3Q8b?O?D@C3k(5rUMW2eMdzrZwKt!
zQA8Pmyd+}GVgf4Ugla`re??q&XQhR=!-osbW}8Db9R!j;c*dIn?RMM}XqchtaNyQT
zqbWf7g4$Q2g;5B%!Q2kh&+v<1es+SU!vuz9F<R!{4iBzZ<;DT|=ze>%^Pk*YpkL9|
F0|4HIO=SQ8

literal 0
HcmV?d00001

diff --git a/harvester/assets/test/shape.shx b/harvester/assets/test/shape.shx
new file mode 100644
index 0000000000000000000000000000000000000000..9e413aab714079c85fb11f321fe59070895ab34c
GIT binary patch
literal 108
zcmZQzQ0HR64$NLKGcd3M<pR26vR4OcIwZ(mcFQpJb}(3{lDoc2(}9QizN4a*w*z+V
KD56F{UJ?M~%?!r?

literal 0
HcmV?d00001

diff --git a/harvester/prepare.py b/harvester/prepare.py
index 47135e2f..2face6cd 100644
--- a/harvester/prepare.py
+++ b/harvester/prepare.py
@@ -2,15 +2,20 @@
 import geopandas
 from shapely.ops import cascaded_union
 
-berlin = geopandas.read_file("./assets/Berlin.shp")
-berlin = berlin.to_crs("epsg:3857")
-berlin_boundary = geopandas.GeoDataFrame(geopandas.GeoSeries(cascaded_union(berlin['geometry'])))
-berlin_boundary = berlin_boundary.rename(columns={0:'geometry'}).set_geometry('geometry')
+def create_buffer_shape(shape_file, buffer_distance = 2000):
+    shape = geopandas.read_file(shape_file)
+    shape = shape.to_crs("epsg:3857")
+    boundary = geopandas.GeoDataFrame(geopandas.GeoSeries(cascaded_union(shape['geometry'])))
+    boundary = boundary.rename(columns={0:'geometry'}).set_geometry('geometry')
 
-berlin_buffer = berlin_boundary.buffer(2000)
-berlin_buffer = berlin_buffer.simplify(1000)
+    buffer = boundary.buffer(buffer_distance)
+    buffer = buffer.simplify(1000)
 
-berlin_buffer = geopandas.GeoDataFrame(berlin_buffer)
-berlin_buffer = berlin_buffer.rename(columns={0:'geometry'}).set_geometry('geometry')
-berlin_buffer.crs = "epsg:3857"
-berlin_buffer.to_file("./assets/buffer.shp")
\ No newline at end of file
+    buffer = geopandas.GeoDataFrame(buffer)
+    buffer = buffer.rename(columns={0:'geometry'}).set_geometry('geometry')
+    buffer.crs = "epsg:3857"
+
+    return buffer
+
+buffer = create_buffer_shape('./assets/Berlin.shp')
+buffer.to_file("./assets/buffer.shp")
diff --git a/harvester/requirements.txt b/harvester/requirements.txt
index 092c9e86..f472c3d4 100644
--- a/harvester/requirements.txt
+++ b/harvester/requirements.txt
@@ -1,13 +1,12 @@
 attrs==19.3.0
 boto3==1.18.26
-botocore==1.21.23
+botocore==1.21.26
 click==7.1.1
 click-plugins==1.1.1
 cligj==0.5.0
 DateTime==4.3
 docutils==0.17.1
 Fiona==1.8.13.post1
-GDAL==2.4.2
 geopandas==0.7.0
 jmespath==0.9.5
 munch==2.5.0
@@ -18,9 +17,10 @@ pyproj==2.6.0
 python-dateutil==2.8.1
 python-dotenv==0.19.0
 pytz==2019.3
-s3transfer==0.3.3
+s3transfer==0.5.0
 Shapely==1.7.0
 six==1.14.0
 urllib3==1.25.8
 zope.interface==5.0.1
-requests==2.25.1
\ No newline at end of file
+requests==2.25.1
+pytest==6.2.3
\ No newline at end of file
diff --git a/harvester/test_prepare.py b/harvester/test_prepare.py
new file mode 100644
index 00000000..a4487272
--- /dev/null
+++ b/harvester/test_prepare.py
@@ -0,0 +1,27 @@
+# building a buffer shape for filtering the weather data
+import geopandas
+import geopandas.testing
+from shapely.ops import cascaded_union
+from prepare import create_buffer_shape
+import subprocess
+import os
+
+def test_create_buffer_with_default_distance():
+    buffer = create_buffer_shape('./assets/test/shape.shp')
+
+    expected_buffer_2000 = geopandas.read_file(f"./assets/test/buffer_2000.shp")
+    expected_buffer_2000.drop("FID", inplace=True, axis=1)
+
+    geopandas.testing.assert_geodataframe_equal(buffer, expected_buffer_2000)
+
+def test_create_buffer_with_custom_distance():
+    buffer_1000 = create_buffer_shape('./assets/test/shape.shp', 1000)
+
+    expected_buffer_1000 = geopandas.read_file(f"./assets/test/buffer_1000.shp")
+    expected_buffer_1000.drop("FID", inplace=True, axis=1)
+
+    geopandas.testing.assert_geodataframe_equal(buffer_1000, expected_buffer_1000)
+
+def test_buffer_file_gets_created():
+    subprocess.run(["python", "prepare.py"])
+    assert os.path.exists("./assets/buffer.shp")