diff --git a/.github/workflows/test-harvest.yml b/.github/workflows/test-harvest.yml index 0162bc5b..2862389f 100644 --- a/.github/workflows/test-harvest.yml +++ b/.github/workflows/test-harvest.yml @@ -29,6 +29,31 @@ env: # DATABASE_URL: postgresql://fangorn:ent@localhost:5432/trees?schema=public jobs: + test-prepare: + runs-on: ubuntu-latest + if: "!contains(github.event.head_commit.message, 'skip ci')" + defaults: + run: + working-directory: harvester + steps: + - name: Get the source + uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install g++ gdal-bin libgdal-dev libpq-dev -y + - name: Install python packages + run: | + python -m pip install --upgrade pip + python -m pip install --no-cache-dir GDAL==3.4.0 + python -m pip install --no-warn-script-location -r requirements.txt + - name: Test with pytest + run: | + pytest test-harvest: runs-on: ubuntu-latest if: "!contains(github.event.head_commit.message, 'skip ci')" diff --git a/harvester/Dockerfile b/harvester/Dockerfile index f7b0938c..cda5a437 100644 --- a/harvester/Dockerfile +++ b/harvester/Dockerfile @@ -12,14 +12,15 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* ENV CPLUS_INCLUDE_PATH=/usr/include/gdal ENV C_INCLUDE_PATH=/usr/include/gdal -# RUN pip install GDAL FROM base as builder # Install pip requirements COPY requirements.txt /app/ -RUN cd /app/ && python -m pip install --no-warn-script-location --prefix=/install -r requirements.txt +RUN python -m pip install --upgrade pip \ + && cd /app/ && python -m pip install --no-warn-script-location --prefix=/install -r requirements.txt \ + && python -m pip install --prefix=/install GDAL==2.4.2 FROM base as app COPY --from=builder /install /usr/local diff --git a/harvester/assets/test/buffer_1000.cpg b/harvester/assets/test/buffer_1000.cpg new file mode 100644 index 00000000..cd89cb97 --- /dev/null +++ b/harvester/assets/test/buffer_1000.cpg @@ -0,0 +1 @@ +ISO-8859-1 \ No newline at end of file diff --git a/harvester/assets/test/buffer_1000.dbf b/harvester/assets/test/buffer_1000.dbf new file mode 100644 index 00000000..9ec44fc9 Binary files /dev/null and b/harvester/assets/test/buffer_1000.dbf differ diff --git a/harvester/assets/test/buffer_1000.prj b/harvester/assets/test/buffer_1000.prj new file mode 100644 index 00000000..5c6f76df --- /dev/null +++ b/harvester/assets/test/buffer_1000.prj @@ -0,0 +1 @@ +PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]] \ No newline at end of file diff --git a/harvester/assets/test/buffer_1000.shp b/harvester/assets/test/buffer_1000.shp new file mode 100644 index 00000000..ae235226 Binary files /dev/null and b/harvester/assets/test/buffer_1000.shp differ diff --git a/harvester/assets/test/buffer_1000.shx b/harvester/assets/test/buffer_1000.shx new file mode 100644 index 00000000..a50616e0 Binary files /dev/null and b/harvester/assets/test/buffer_1000.shx differ diff --git a/harvester/assets/test/buffer_2000.cpg b/harvester/assets/test/buffer_2000.cpg new file mode 100644 index 00000000..cd89cb97 --- /dev/null +++ b/harvester/assets/test/buffer_2000.cpg @@ -0,0 +1 @@ +ISO-8859-1 \ No newline at end of file diff --git a/harvester/assets/test/buffer_2000.dbf b/harvester/assets/test/buffer_2000.dbf new file mode 100644 index 00000000..9ec44fc9 Binary files /dev/null and b/harvester/assets/test/buffer_2000.dbf differ diff --git a/harvester/assets/test/buffer_2000.prj b/harvester/assets/test/buffer_2000.prj new file mode 100644 index 00000000..5c6f76df --- /dev/null +++ b/harvester/assets/test/buffer_2000.prj @@ -0,0 +1 @@ +PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]] \ No newline at end of file diff --git a/harvester/assets/test/buffer_2000.shp b/harvester/assets/test/buffer_2000.shp new file mode 100644 index 00000000..907f3360 Binary files /dev/null and b/harvester/assets/test/buffer_2000.shp differ diff --git a/harvester/assets/test/buffer_2000.shx b/harvester/assets/test/buffer_2000.shx new file mode 100644 index 00000000..bbf0ae41 Binary files /dev/null and b/harvester/assets/test/buffer_2000.shx differ diff --git a/harvester/assets/test/shape.dbf b/harvester/assets/test/shape.dbf new file mode 100644 index 00000000..3228c530 Binary files /dev/null and b/harvester/assets/test/shape.dbf differ diff --git a/harvester/assets/test/shape.prj b/harvester/assets/test/shape.prj new file mode 100644 index 00000000..a30c00a5 --- /dev/null +++ b/harvester/assets/test/shape.prj @@ -0,0 +1 @@ +GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]] \ No newline at end of file diff --git a/harvester/assets/test/shape.shp b/harvester/assets/test/shape.shp new file mode 100644 index 00000000..fddd5ff1 Binary files /dev/null and b/harvester/assets/test/shape.shp differ diff --git a/harvester/assets/test/shape.shx b/harvester/assets/test/shape.shx new file mode 100644 index 00000000..9e413aab Binary files /dev/null and b/harvester/assets/test/shape.shx differ diff --git a/harvester/prepare.py b/harvester/prepare.py index 47135e2f..2face6cd 100644 --- a/harvester/prepare.py +++ b/harvester/prepare.py @@ -2,15 +2,20 @@ import geopandas from shapely.ops import cascaded_union -berlin = geopandas.read_file("./assets/Berlin.shp") -berlin = berlin.to_crs("epsg:3857") -berlin_boundary = geopandas.GeoDataFrame(geopandas.GeoSeries(cascaded_union(berlin['geometry']))) -berlin_boundary = berlin_boundary.rename(columns={0:'geometry'}).set_geometry('geometry') +def create_buffer_shape(shape_file, buffer_distance = 2000): + shape = geopandas.read_file(shape_file) + shape = shape.to_crs("epsg:3857") + boundary = geopandas.GeoDataFrame(geopandas.GeoSeries(cascaded_union(shape['geometry']))) + boundary = boundary.rename(columns={0:'geometry'}).set_geometry('geometry') -berlin_buffer = berlin_boundary.buffer(2000) -berlin_buffer = berlin_buffer.simplify(1000) + buffer = boundary.buffer(buffer_distance) + buffer = buffer.simplify(1000) -berlin_buffer = geopandas.GeoDataFrame(berlin_buffer) -berlin_buffer = berlin_buffer.rename(columns={0:'geometry'}).set_geometry('geometry') -berlin_buffer.crs = "epsg:3857" -berlin_buffer.to_file("./assets/buffer.shp") \ No newline at end of file + buffer = geopandas.GeoDataFrame(buffer) + buffer = buffer.rename(columns={0:'geometry'}).set_geometry('geometry') + buffer.crs = "epsg:3857" + + return buffer + +buffer = create_buffer_shape('./assets/Berlin.shp') +buffer.to_file("./assets/buffer.shp") diff --git a/harvester/requirements.txt b/harvester/requirements.txt index 092c9e86..f472c3d4 100644 --- a/harvester/requirements.txt +++ b/harvester/requirements.txt @@ -1,13 +1,12 @@ attrs==19.3.0 boto3==1.18.26 -botocore==1.21.23 +botocore==1.21.26 click==7.1.1 click-plugins==1.1.1 cligj==0.5.0 DateTime==4.3 docutils==0.17.1 Fiona==1.8.13.post1 -GDAL==2.4.2 geopandas==0.7.0 jmespath==0.9.5 munch==2.5.0 @@ -18,9 +17,10 @@ pyproj==2.6.0 python-dateutil==2.8.1 python-dotenv==0.19.0 pytz==2019.3 -s3transfer==0.3.3 +s3transfer==0.5.0 Shapely==1.7.0 six==1.14.0 urllib3==1.25.8 zope.interface==5.0.1 -requests==2.25.1 \ No newline at end of file +requests==2.25.1 +pytest==6.2.3 \ No newline at end of file diff --git a/harvester/test_prepare.py b/harvester/test_prepare.py new file mode 100644 index 00000000..a4487272 --- /dev/null +++ b/harvester/test_prepare.py @@ -0,0 +1,27 @@ +# building a buffer shape for filtering the weather data +import geopandas +import geopandas.testing +from shapely.ops import cascaded_union +from prepare import create_buffer_shape +import subprocess +import os + +def test_create_buffer_with_default_distance(): + buffer = create_buffer_shape('./assets/test/shape.shp') + + expected_buffer_2000 = geopandas.read_file(f"./assets/test/buffer_2000.shp") + expected_buffer_2000.drop("FID", inplace=True, axis=1) + + geopandas.testing.assert_geodataframe_equal(buffer, expected_buffer_2000) + +def test_create_buffer_with_custom_distance(): + buffer_1000 = create_buffer_shape('./assets/test/shape.shp', 1000) + + expected_buffer_1000 = geopandas.read_file(f"./assets/test/buffer_1000.shp") + expected_buffer_1000.drop("FID", inplace=True, axis=1) + + geopandas.testing.assert_geodataframe_equal(buffer_1000, expected_buffer_1000) + +def test_buffer_file_gets_created(): + subprocess.run(["python", "prepare.py"]) + assert os.path.exists("./assets/buffer.shp")