create tests for prepare.py

.github/workflows/test-harvest.yml

@@ -29,6 +29,31 @@ env:
   # DATABASE_URL: postgresql://fangorn:ent@localhost:5432/trees?schema=public
 
 jobs:
+  test-prepare:
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.head_commit.message, 'skip ci')"
+    defaults:
+      run:
+        working-directory: harvester
+    steps:
+      - name: Get the source
+        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install g++ gdal-bin libgdal-dev libpq-dev -y
+      - name: Install python packages
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --no-cache-dir GDAL==3.4.0
+          python -m pip install --no-warn-script-location -r requirements.txt
+      - name: Test with pytest
+        run: |
+          pytest
   test-harvest:
     runs-on: ubuntu-latest
     if: "!contains(github.event.head_commit.message, 'skip ci')"

harvester/Dockerfile

@@ -12,14 +12,15 @@ RUN apt-get update \
   && rm -rf /var/lib/apt/lists/*
 ENV CPLUS_INCLUDE_PATH=/usr/include/gdal
 ENV C_INCLUDE_PATH=/usr/include/gdal
-# RUN pip install GDAL
 
 FROM base as builder
 
 
 # Install pip requirements
 COPY requirements.txt /app/
-RUN cd /app/ && python -m pip install --no-warn-script-location --prefix=/install -r requirements.txt
+RUN python -m pip install --upgrade pip \
+  && cd /app/ && python -m pip install --no-warn-script-location --prefix=/install -r requirements.txt \
+  && python -m pip install --prefix=/install GDAL==2.4.2
 
 FROM base as app
 COPY --from=builder /install /usr/local

harvester/assets/test/buffer_1000.cpg

@@ -0,0 +1 @@
+ISO-8859-1

harvester/assets/test/buffer_1000.prj

@@ -0,0 +1 @@
+PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]]

harvester/assets/test/buffer_2000.cpg

@@ -0,0 +1 @@
+ISO-8859-1

harvester/assets/test/buffer_2000.prj

@@ -0,0 +1 @@
+PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]]

harvester/assets/test/shape.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

harvester/prepare.py

@@ -2,15 +2,20 @@
 import geopandas
 from shapely.ops import cascaded_union
 
-berlin = geopandas.read_file("./assets/Berlin.shp")
-berlin = berlin.to_crs("epsg:3857")
-berlin_boundary = geopandas.GeoDataFrame(geopandas.GeoSeries(cascaded_union(berlin['geometry'])))
-berlin_boundary = berlin_boundary.rename(columns={0:'geometry'}).set_geometry('geometry')
+def create_buffer_shape(shape_file, buffer_distance = 2000):
+    shape = geopandas.read_file(shape_file)
+    shape = shape.to_crs("epsg:3857")
+    boundary = geopandas.GeoDataFrame(geopandas.GeoSeries(cascaded_union(shape['geometry'])))
+    boundary = boundary.rename(columns={0:'geometry'}).set_geometry('geometry')
 
-berlin_buffer = berlin_boundary.buffer(2000)
-berlin_buffer = berlin_buffer.simplify(1000)
+    buffer = boundary.buffer(buffer_distance)
+    buffer = buffer.simplify(1000)
 
-berlin_buffer = geopandas.GeoDataFrame(berlin_buffer)
-berlin_buffer = berlin_buffer.rename(columns={0:'geometry'}).set_geometry('geometry')
-berlin_buffer.crs = "epsg:3857"
-berlin_buffer.to_file("./assets/buffer.shp")
+    buffer = geopandas.GeoDataFrame(buffer)
+    buffer = buffer.rename(columns={0:'geometry'}).set_geometry('geometry')
+    buffer.crs = "epsg:3857"
+
+    return buffer
+
+buffer = create_buffer_shape('./assets/Berlin.shp')
+buffer.to_file("./assets/buffer.shp")

harvester/requirements.txt

@@ -1,13 +1,12 @@
 attrs==19.3.0
 boto3==1.18.26
-botocore==1.21.23
+botocore==1.21.26
 click==7.1.1
 click-plugins==1.1.1
 cligj==0.5.0
 DateTime==4.3
 docutils==0.17.1
 Fiona==1.8.13.post1
-GDAL==2.4.2
 geopandas==0.7.0
 jmespath==0.9.5
 munch==2.5.0
@@ -18,9 +17,10 @@ pyproj==2.6.0
 python-dateutil==2.8.1
 python-dotenv==0.19.0
 pytz==2019.3
-s3transfer==0.3.3
+s3transfer==0.5.0
 Shapely==1.7.0
 six==1.14.0
 urllib3==1.25.8
 zope.interface==5.0.1
-requests==2.25.1
+requests==2.25.1
+pytest==6.2.3

harvester/test_prepare.py

@@ -0,0 +1,27 @@
+# building a buffer shape for filtering the weather data
+import geopandas
+import geopandas.testing
+from shapely.ops import cascaded_union
+from prepare import create_buffer_shape
+import subprocess
+import os
+
+def test_create_buffer_with_default_distance():
+    buffer = create_buffer_shape('./assets/test/shape.shp')
+
+    expected_buffer_2000 = geopandas.read_file(f"./assets/test/buffer_2000.shp")
+    expected_buffer_2000.drop("FID", inplace=True, axis=1)
+
+    geopandas.testing.assert_geodataframe_equal(buffer, expected_buffer_2000)
+
+def test_create_buffer_with_custom_distance():
+    buffer_1000 = create_buffer_shape('./assets/test/shape.shp', 1000)
+
+    expected_buffer_1000 = geopandas.read_file(f"./assets/test/buffer_1000.shp")
+    expected_buffer_1000.drop("FID", inplace=True, axis=1)
+
+    geopandas.testing.assert_geodataframe_equal(buffer_1000, expected_buffer_1000)
+
+def test_buffer_file_gets_created():
+    subprocess.run(["python", "prepare.py"])
+    assert os.path.exists("./assets/buffer.shp")