tests: skip many external data tests

README.md

@@ -136,6 +136,13 @@ To run the tests:
 pytest
 ```
 
+Many tests are skipped by default because access external data, which makes them slow and a bit flakey.
+To run these tests:
+
+```shell
+pytest --external-data
+```
+
 ### Updating static resources
 
 There are several .json files in the source tree that are used to populate

tests/conftest.py

@@ -0,0 +1,30 @@
+from typing import Any
+
+import pytest
+from pytest import Config, Parser
+
+
+def pytest_addoption(parser: Parser) -> None:
+    parser.addoption(
+        "--external-data",
+        action="store_true",
+        default=False,
+        help="run tests that require external data",
+    )
+
+
+def pytest_configure(config: Config) -> None:
+    config.addinivalue_line(
+        "markers",
+        "external_data: marks tests requiring external data, "
+        "and disables them by default (enable with --external-data)",
+    )
+
+
+def pytest_collection_modifyitems(config: Config, items: Any) -> None:
+    if config.getoption("--external-data"):
+        return
+    skip_network_access = pytest.mark.skip(reason="need --external-data option to run")
+    for item in items:
+        if "external_data" in item.keywords:
+            item.add_marker(skip_network_access)

tests/ocean_heat_content/test_commands.py

@@ -1,5 +1,6 @@
 from pathlib import Path
 
+import pytest
 from pystac import Collection, ItemCollection
 
 from tests import run_command, test_data
@@ -16,6 +17,7 @@ def test_create_collection(tmp_path: Path) -> None:
     collection.validate()
 
 
+@pytest.mark.external_data
 def test_create_items(tmp_path: Path) -> None:
     destination = tmp_path / "item-collection.json"
     infile = test_data.get_external_data("heat_content_anomaly_0-2000_yearly.nc")
@@ -38,6 +40,7 @@ def test_download() -> None:
     assert result.exit_code == 0
 
 
+@pytest.mark.external_data
 def test_cogify(tmp_path: Path) -> None:
     path = test_data.get_external_data("heat_content_anomaly_0-2000_yearly.nc")
     result = run_command(f"noaa-cdr ocean-heat-content cogify {path} -o {tmp_path}")

tests/ocean_heat_content/test_stac.py

@@ -9,7 +9,6 @@
 from pystac.extensions.projection import ProjectionExtension
 from pystac.extensions.raster import RasterExtension
 from pystac.extensions.scientific import ScientificExtension
-
 from stactools.noaa_cdr.ocean_heat_content import cog, stac
 
 from .. import test_data
@@ -51,6 +50,7 @@ def test_create_collection() -> None:
     collection.validate_all()
 
 
+@pytest.mark.external_data
 def test_create_items_one_netcdf(tmp_path: Path) -> None:
     path = test_data.get_external_data("heat_content_anomaly_0-2000_yearly.nc")
     items = stac.create_items([path], str(tmp_path))
@@ -88,6 +88,7 @@ def test_create_items_one_netcdf(tmp_path: Path) -> None:
         item.validate()
 
 
+@pytest.mark.external_data
 def test_create_items_one_netcdf_cog_hrefs(tmp_path: Path) -> None:
     path = test_data.get_external_data("heat_content_anomaly_0-2000_yearly.nc")
     items = stac.create_items([path], str(tmp_path))
@@ -104,6 +105,7 @@ def test_create_items_one_netcdf_cog_hrefs(tmp_path: Path) -> None:
     assert len(new_items) == len(items)
 
 
+@pytest.mark.external_data
 def test_create_items_two_netcdfs_same_items(tmp_path: Path) -> None:
     paths = [
         test_data.get_external_data("heat_content_anomaly_0-2000_yearly.nc"),
@@ -118,6 +120,7 @@ def test_create_items_two_netcdfs_same_items(tmp_path: Path) -> None:
         item.validate()
 
 
+@pytest.mark.external_data
 def test_create_items_two_netcdfs_different_items() -> None:
     paths = [
         test_data.get_external_data("heat_content_anomaly_0-2000_yearly.nc"),
@@ -133,6 +136,7 @@ def test_create_items_two_netcdfs_different_items() -> None:
         item.validate()
 
 
+@pytest.mark.external_data
 def test_create_items_one_netcdf_latest_only(tmp_path: Path) -> None:
     path = test_data.get_external_data("heat_content_anomaly_0-2000_yearly.nc")
     items = stac.create_items([path], str(tmp_path), latest_only=True)
@@ -155,6 +159,7 @@ def test_create_items_one_netcdf_latest_only(tmp_path: Path) -> None:
         ("mean_total_steric_sea_level_anomaly_0-2000_yearly.nc", 17),
     ],
 )
+@pytest.mark.external_data
 def test_cogify(tmp_path: Path, infile: str, num_cogs: int) -> None:
     external_data_path = test_data.get_external_data(infile)
     cogs = cog.cogify(external_data_path, str(tmp_path))
@@ -165,6 +170,7 @@ def test_cogify(tmp_path: Path, infile: str, num_cogs: int) -> None:
         assert Path(c.asset().href).exists()
 
 
+@pytest.mark.external_data
 def test_cogify_href(tmp_path: Path) -> None:
     href = (
         "https://www.ncei.noaa.gov/data/oceans/ncei/archive/data"
@@ -178,6 +184,7 @@ def test_cogify_href(tmp_path: Path) -> None:
         assert Path(c.asset().href).exists()
 
 
+@pytest.mark.external_data
 def test_cogify_href_no_output_directory() -> None:
     href = (
         "https://www.ncei.noaa.gov/data/oceans/ncei/archive/data"
@@ -187,12 +194,14 @@ def test_cogify_href_no_output_directory() -> None:
         cog.cogify(href)
 
 
+@pytest.mark.external_data
 def test_unitless(tmp_path: Path) -> None:
     path = test_data.get_external_data("mean_salinity_anomaly_0-2000_yearly.nc")
     cogs = cog.cogify(path, str(tmp_path))
     assert "unit" not in cogs[0].asset().extra_fields["raster:bands"][0]
 
 
+@pytest.mark.external_data
 def test_cogify_cog_href(tmp_path: Path) -> None:
     path = test_data.get_external_data("heat_content_anomaly_0-2000_yearly.nc")
     cogs = cog.cogify(path, str(tmp_path))
@@ -222,6 +231,7 @@ def test_cogify_cog_href(tmp_path: Path) -> None:
         ("heat_content_anomaly_0-2000_seasonal.nc", 2005, "seasonal", 2000),
     ],
 )
+@pytest.mark.external_data
 def test_create_netcdf_item(
     infile: str, year: int, interval: str, max_depth: int
 ) -> None:

tests/sea_surface_temperature_optimum_interpolation/test_commands.py

@@ -1,5 +1,6 @@
 from pathlib import Path
 
+import pytest
 from pystac import Collection, Item
 
 from tests import run_command, test_data
@@ -19,6 +20,7 @@ def test_create_collection(tmp_path: Path) -> None:
     collection.validate()
 
 
+@pytest.mark.external_data
 def test_create_item(tmp_path: Path) -> None:
     destination = tmp_path / "item.json"
     infile = test_data.get_external_data("oisst-avhrr-v02r01.20220913.nc")
@@ -33,6 +35,7 @@ def test_create_item(tmp_path: Path) -> None:
     item.validate()
 
 
+@pytest.mark.external_data
 def test_create_item_with_cogs(tmp_path: Path) -> None:
     path = test_data.get_external_data("oisst-avhrr-v02r01.20220913.nc")
     result = run_command(

tests/sea_surface_temperature_optimum_interpolation/test_stac.py

@@ -1,12 +1,13 @@
 import datetime
 from pathlib import Path
 
+import pytest
+import stactools.noaa_cdr.stac
 from dateutil.tz import tzutc
 from pystac.extensions.projection import ProjectionExtension
 from pystac.extensions.raster import RasterExtension
-
-import stactools.noaa_cdr.stac
 from stactools.noaa_cdr.sea_surface_temperature_optimum_interpolation import stac
+
 from tests import test_data
 
 
@@ -20,6 +21,7 @@ def test_create_collection() -> None:
     collection.validate()
 
 
+@pytest.mark.external_data
 def test_create_item() -> None:
     path = test_data.get_external_data("oisst-avhrr-v02r01.20220913.nc")
     item = stac.create_item(path)
@@ -58,6 +60,7 @@ def test_create_item() -> None:
     item.validate()
 
 
+@pytest.mark.external_data
 def test_add_items(tmp_path: Path) -> None:
     path = test_data.get_external_data("oisst-avhrr-v02r01.20220913.nc")
     item = stac.create_item(path)

tests/sea_surface_temperature_whoi/test_commands.py

@@ -1,10 +1,12 @@
 from pathlib import Path
 
+import pytest
 from pystac import Collection, ItemCollection
 
 from .. import run_command, test_data
 
 
+@pytest.mark.external_data
 def test_create_cog_items(tmp_path: Path) -> None:
     path = test_data.get_external_data(
         "SEAFLUX-OSB-CDR_V02R00_SST_D20210831_C20211223.nc"

tests/sea_surface_temperature_whoi/test_stac.py

@@ -1,13 +1,14 @@
 from pathlib import Path
 
+import pytest
 from pystac.extensions.raster import RasterExtension
 from pystac.extensions.scientific import ScientificExtension
-
 from stactools.noaa_cdr.sea_surface_temperature_whoi import stac
 
 from .. import test_data
 
 
+@pytest.mark.external_data
 def test_create_cog_items(tmp_path: Path) -> None:
     path = test_data.get_external_data(
         "SEAFLUX-OSB-CDR_V02R00_SST_D20210831_C20211223.nc"

tests/test_dataset.py

@@ -1,11 +1,11 @@
 import pytest
 import xarray
-
 from stactools.noaa_cdr import dataset
 
 from . import test_data
 
 
+@pytest.mark.external_data
 def test_data_variable_name() -> None:
     path = test_data.get_external_data("heat_content_anomaly_0-2000_yearly.nc")
     with xarray.open_dataset(path, decode_times=False) as ds:
@@ -23,6 +23,7 @@ def test_data_variable_name() -> None:
             dataset.data_variable_name(ds)
 
 
+@pytest.mark.external_data
 def test_data_variable_names() -> None:
     path = test_data.get_external_data("oisst-avhrr-v02r01.20220913.nc")
     with xarray.open_dataset(path) as ds: