diff --git a/openFlowML/requirements.txt b/openFlowML/requirements.txt index a8ff797..b9f6341 100644 --- a/openFlowML/requirements.txt +++ b/openFlowML/requirements.txt @@ -5,6 +5,7 @@ numpy>=1.17.3 scipy>=1.5.0 tensorflow==2.13 u8darts>=0.29.0 +geopandas>=1.0.1 #onnx==1.14.1 #tf2onnx>=1.15.1 diff --git a/openFlowML/soilmoisture2.py b/openFlowML/soilmoisture2.py index 28b22f8..fab550a 100644 --- a/openFlowML/soilmoisture2.py +++ b/openFlowML/soilmoisture2.py @@ -8,7 +8,7 @@ from dask.distributed import Client import argparse import logging -from get_poly import get_huc8_polygon, simplify_polygon +from openFlowML.utils.get_poly import get_huc8_polygon, simplify_polygon # Configure logging logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') diff --git a/openFlowML/get_coordinates.py b/openFlowML/utils/get_coordinates.py similarity index 52% rename from openFlowML/get_coordinates.py rename to openFlowML/utils/get_coordinates.py index e116e50..9251c83 100644 --- a/openFlowML/get_coordinates.py +++ b/openFlowML/utils/get_coordinates.py @@ -1,5 +1,6 @@ import requests import argparse +import sys def get_usgs_coordinates(site_number): base_url = "https://waterdata.usgs.gov/nwis/inventory" @@ -16,17 +17,21 @@ def get_usgs_coordinates(site_number): response = requests.get(base_url, params=params) lines = response.text.splitlines() - # Filter out comment lines and retrieve relevant data - data_lines = [line for line in lines if not line.startswith('#')] - data = data_lines[2] # Since we're skipping the first two lines after comments - - fields = data.split('\t') - site_no, station_nm, dec_lat_va, dec_long_va = fields[0], fields[1], fields[2], fields[3] - - return { - 'latitude': dec_lat_va, - 'longitude': dec_long_va - } + try: + # Filter out comment lines and retrieve relevant data + data_lines = [line for line in lines if not line.startswith('#')] + if len(data_lines) < 3: + return None + data = next((line for line in data_lines[2:] if line.split('\t')[0] == site_number), None) + if data is None: + return None + fields = data.split('\t') + return { + 'latitude': fields[2], + 'longitude': fields[3] + } + except Exception: + return None def get_dwr_coordinates(abbrev): base_url = "https://dwr.state.co.us/Rest/GET/api/v2/surfacewater/surfacewaterstations" @@ -41,27 +46,32 @@ def get_dwr_coordinates(abbrev): response = requests.get(base_url, params=params) lines = response.text.splitlines() - # Filter out comment lines and retrieve relevant data - data_lines = [line for line in lines if not line.startswith('#')] - data = data_lines[2] # Since we're skipping the first two lines after comments - - fields = data.split('\t') - station_nm, dec_lat_va, dec_long_va = fields[0], fields[1], fields[2] - - return { - 'latitude': dec_lat_va, - 'longitude': dec_long_va - } + try: + # Filter out comment lines and retrieve relevant data + data_lines = [line for line in lines if not line.startswith('#')] + if len(data_lines) < 3: + return None + data = data_lines[2] + fields = data.split('\t') + return { + 'latitude': fields[1], + 'longitude': fields[2] + } + except Exception: + return None def main(): parser = argparse.ArgumentParser(description='Fetch latitude and longitude for a given USGS site number.') parser.add_argument('site_type', type=str, help='site type (ex: usgs, dwr)') parser.add_argument('site_number', type=str, help='site number') args = parser.parse_args() - if args.site_type.lowercase == 'usgs': - result = get_usgs_coordinates(args.site_type, args.site_number) - elif args.site_type.lowercase == 'dwr': - result = get_dwr_coordinates(args.site_type, args.site_number) + if args.site_type.lower() == 'usgs': + result = get_usgs_coordinates(args.site_number) + elif args.site_type.lower() == 'dwr': + result = get_dwr_coordinates(args.site_number) + else: + print("Invalid source. Use 'usgs' or 'dwr'.") + sys.exit(1) print(result) if __name__ == '__main__': diff --git a/openFlowML/get_poly.py b/openFlowML/utils/get_poly.py similarity index 100% rename from openFlowML/get_poly.py rename to openFlowML/utils/get_poly.py diff --git a/openFlowML/ml_utils.py b/openFlowML/utils/ml_utils.py similarity index 100% rename from openFlowML/ml_utils.py rename to openFlowML/utils/ml_utils.py diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..9df1ecd --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +pythonpath = openFlowML \ No newline at end of file diff --git a/tests/test_coords.py b/tests/test_coords.py new file mode 100644 index 0000000..8dd55ea --- /dev/null +++ b/tests/test_coords.py @@ -0,0 +1,134 @@ +import pytest +import requests_mock +from utils.get_coordinates import get_usgs_coordinates, get_dwr_coordinates, main + +@pytest.fixture +def mock_requests(): + with requests_mock.Mocker() as m: + yield m + +@pytest.fixture +def usgs_mock_response(): + return ''' +# comment line +# another comment line +site_no station_nm dec_lat_va dec_long_va +12345678 Test Station 40.123456 -105.654321 +07094500 ARKANSAS RIVER AT PARKDALE, CO 38.4872189 -105.373604 +''' + +@pytest.fixture +def dwr_mock_response(): + return ''' +# comment line +# another comment line +abbrev latitude longitude +TESTABBR 39.987654 -104.123456 +''' + +@pytest.fixture +def mock_sys_argv(monkeypatch): + def _mock_argv(args): + monkeypatch.setattr('sys.argv', ['script_name'] + args) + return _mock_argv + +def test_get_usgs_coordinates(mock_requests, usgs_mock_response): + mock_requests.get('https://waterdata.usgs.gov/nwis/inventory', text=usgs_mock_response) + + result = get_usgs_coordinates('12345678') + expected = { + 'latitude': '40.123456', + 'longitude': '-105.654321' + } + assert result == expected + +def test_get_usgs_coordinates_specific_station(mock_requests, usgs_mock_response): + mock_requests.get('https://waterdata.usgs.gov/nwis/inventory', text=usgs_mock_response) + + result = get_usgs_coordinates('07094500') + expected = { + 'latitude': '38.4872189', + 'longitude': '-105.373604' + } + assert result == expected + +def test_get_usgs_coordinates_invalid_station(mock_requests): + mock_requests.get('https://waterdata.usgs.gov/nwis/inventory', text='') + + result = get_usgs_coordinates('invalid_station') + assert result is None + +def test_get_usgs_coordinates_api_error(mock_requests): + mock_requests.get('https://waterdata.usgs.gov/nwis/inventory', status_code=500) + + result = get_usgs_coordinates('12345678') + assert result is None + +def test_get_dwr_coordinates(mock_requests, dwr_mock_response): + mock_requests.get('https://dwr.state.co.us/Rest/GET/api/v2/surfacewater/surfacewaterstations', text=dwr_mock_response) + + result = get_dwr_coordinates('TESTABBR') + expected = { + 'latitude': '39.987654', + 'longitude': '-104.123456' + } + assert result == expected + +def test_get_dwr_coordinates_invalid_abbrev(mock_requests): + mock_requests.get('https://dwr.state.co.us/Rest/GET/api/v2/surfacewater/surfacewaterstations', text='') + + result = get_dwr_coordinates('INVALID') + assert result is None + +def test_get_dwr_coordinates_api_error(mock_requests): + mock_requests.get('https://dwr.state.co.us/Rest/GET/api/v2/surfacewater/surfacewaterstations', status_code=500) + + result = get_dwr_coordinates('TESTABBR') + assert result is None + +def test_main_usgs(mock_sys_argv, monkeypatch, capsys): + mock_sys_argv(['usgs', '07094500']) + + def mock_get_usgs_coordinates(site_number): + return {'latitude': '38.4872189', 'longitude': '-105.373604'} + + monkeypatch.setattr('utils.get_coordinates.get_usgs_coordinates', mock_get_usgs_coordinates) + + main() + + captured = capsys.readouterr() + assert captured.out.strip() == "{'latitude': '38.4872189', 'longitude': '-105.373604'}" + +def test_main_dwr(mock_sys_argv, monkeypatch, capsys): + mock_sys_argv(['dwr', 'TESTABBR']) + + def mock_get_dwr_coordinates(abbrev): + return {'latitude': '39.987654', 'longitude': '-104.123456'} + + monkeypatch.setattr('utils.get_coordinates.get_dwr_coordinates', mock_get_dwr_coordinates) + + main() + + captured = capsys.readouterr() + assert captured.out.strip() == "{'latitude': '39.987654', 'longitude': '-104.123456'}" + +def test_main_invalid_source(mock_sys_argv, capsys): + mock_sys_argv(['invalid', 'TESTABBR']) + + with pytest.raises(SystemExit): + main() + + captured = capsys.readouterr() + assert "Invalid source. Use 'usgs' or 'dwr'." in captured.out + +def test_main_missing_arguments(mock_sys_argv, capsys): + mock_sys_argv([]) + + with pytest.raises(SystemExit): + main() + + captured = capsys.readouterr() + assert "error: the following arguments are required: site_type, site_number" in captured.err + +if __name__ == "__main__": + pytest.main() \ No newline at end of file diff --git a/tests/test_dwr_flows.py b/tests/test_dwr_flows.py index 59c05f3..464f106 100644 --- a/tests/test_dwr_flows.py +++ b/tests/test_dwr_flows.py @@ -1,5 +1,3 @@ -from .test_utils import set_ml_pypath -set_ml_pypath() import pytest from datetime import datetime, timedelta from get_CODWR_flow import get_historical_data diff --git a/tests/test_poly.py b/tests/test_poly.py index 5a35a8c..2f86b5b 100644 --- a/tests/test_poly.py +++ b/tests/test_poly.py @@ -1,6 +1,4 @@ -from test_utils import set_ml_pypath -set_ml_pypath() -from get_poly import * +from utils.get_poly import get_huc8_polygon, simplify_polygon, main # type: ignore import pytest import requests_mock diff --git a/tests/test_soil_moisture.py b/tests/test_soil_moisture.py index c094f34..bbad67c 100644 --- a/tests/test_soil_moisture.py +++ b/tests/test_soil_moisture.py @@ -1,5 +1,3 @@ -from test_utils import set_ml_pypath -set_ml_pypath() import pytest import requests_mock from nasa_moisture import search_smap_data, get_smap_soil_moisture_data, process_smap_data, get_smap_timeseries, main diff --git a/tests/test_swe.py b/tests/test_swe.py index b9ce508..d6e8023 100644 --- a/tests/test_swe.py +++ b/tests/test_swe.py @@ -1,5 +1,3 @@ -from .test_utils import set_ml_pypath -set_ml_pypath() import pytest from datetime import datetime, timedelta import get_swe # Assuming get_swe is your script with the main function diff --git a/tests/test_usgs_flows.py b/tests/test_usgs_flows.py index 1cbf1b5..99973e6 100644 --- a/tests/test_usgs_flows.py +++ b/tests/test_usgs_flows.py @@ -1,5 +1,3 @@ -from .test_utils import set_ml_pypath -set_ml_pypath() from get_flow import get_daily_flow_data import pytest from datetime import datetime, timedelta diff --git a/tests/test_utils.py b/tests/test_utils.py deleted file mode 100644 index 5b6a6bf..0000000 --- a/tests/test_utils.py +++ /dev/null @@ -1,10 +0,0 @@ -import sys -import os - -def set_ml_pypath(): - """ - Appends the project's openFlowML directory to the PYTHONPATH. - This helps Python to find the main package `my_project` for imports. - """ - sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'openFlowML'))) - print(f'Test path set: {sys.path}') \ No newline at end of file diff --git a/tests/test_vegitation.py b/tests/test_vegitation.py index 85d98d5..ad57751 100644 --- a/tests/test_vegitation.py +++ b/tests/test_vegitation.py @@ -1,11 +1,9 @@ -from test_utils import set_ml_pypath -set_ml_pypath() import pytest import requests_mock from get_vegdri import get_vegdri_data import requests import json -from get_poly import simplify_polygon, get_huc8_polygon +from utils.get_poly import simplify_polygon, get_huc8_polygon @pytest.fixture def mock_response():