From 3989fd779af551737952647430cd2c12ea66edc3 Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Wed, 31 Oct 2018 15:51:59 +1300 Subject: [PATCH] :white_check_mark: Test csv data download a bit better Actually download a file (sample with header only) and check that the header line matches. Remove the file when test is done. --- data_prep.ipynb | 9 ++++++--- test_ipynb.ipynb | 19 ++++++++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/data_prep.ipynb b/data_prep.ipynb index 26be5dd..dc031da 100644 --- a/data_prep.ipynb +++ b/data_prep.ipynb @@ -78,11 +78,14 @@ "outputs": [], "source": [ "def download_to_path(path:str, url:str):\n", - " \"\"\"\n", + " r\"\"\"\n", " Download from a url to a path\n", " \n", - " >>> download_to_path(path=\"highres/2017_Antarctica_Basler.csv\", url=\"https://data.cresis.ku.edu/data/rds/2017_Antarctica_Basler/csv_good/2017_Antarctica_Basler.csv\").headers['Content-Length']\n", - " '64'\n", + " >>> download_to_path(path=\"highres/2017_Antarctica_Basler.csv\", url=\"https://data.cresis.ku.edu/data/rds/2017_Antarctica_Basler/csv_good/2017_Antarctica_Basler.csv\")\n", + " \n", + " >>> open('highres/2017_Antarctica_Basler.csv').readlines()\n", + " ['LAT,LON,UTCTIMESOD,THICK,ELEVATION,FRAME,SURFACE,BOTTOM,QUALITY\\n']\n", + " >>> os.remove(path=\"highres/2017_Antarctica_Basler.csv\")\n", " \"\"\"\n", " #if not os.path.exists(path=path):\n", " r = requests.get(url=url, stream=True)\n", diff --git a/test_ipynb.ipynb b/test_ipynb.ipynb index bcc4618..42c6508 100644 --- a/test_ipynb.ipynb +++ b/test_ipynb.ipynb @@ -75,18 +75,27 @@ "output_type": "stream", "text": [ "Trying:\n", - " download_to_path(path=\"highres/2017_Antarctica_Basler.csv\", url=\"https://data.cresis.ku.edu/data/rds/2017_Antarctica_Basler/csv_good/2017_Antarctica_Basler.csv\").headers['Content-Length']\n", + " download_to_path(path=\"highres/2017_Antarctica_Basler.csv\", url=\"https://data.cresis.ku.edu/data/rds/2017_Antarctica_Basler/csv_good/2017_Antarctica_Basler.csv\")\n", "Expecting:\n", - " '64'\n", + " \n", + "ok\n", + "Trying:\n", + " open('highres/2017_Antarctica_Basler.csv').readlines()\n", + "Expecting:\n", + " ['LAT,LON,UTCTIMESOD,THICK,ELEVATION,FRAME,SURFACE,BOTTOM,QUALITY\\n']\n", + "ok\n", + "Trying:\n", + " os.remove(path=\"highres/2017_Antarctica_Basler.csv\")\n", + "Expecting nothing\n", "ok\n", "3 items had no tests:\n", " data_prep\n", " data_prep.get_window_bounds\n", " data_prep.selective_tile\n", "1 items passed all tests:\n", - " 1 tests in data_prep.download_to_path\n", - "1 tests in 4 items.\n", - "1 passed and 0 failed.\n", + " 3 tests in data_prep.download_to_path\n", + "3 tests in 4 items.\n", + "3 passed and 0 failed.\n", "Test passed.\n" ] }