Update test and add comment to workflow

.github/workflows/run_dandi_read_tests.yml

@@ -1,15 +1,15 @@
 name: Run DANDI read tests
 on:
+  # NOTE this is disabled until we can run this systematically instead of randomly
+  # so we don't get constant error notifications and waste compute cycles
+  # See https://github.com/NeurodataWithoutBorders/pynwb/issues/1804
   # schedule:
   #   - cron: '0 6 * * *'  # once per day at 1am ET
   workflow_dispatch:
 
 jobs:
   run-tests:
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash -l {0}  # necessary for conda
     steps:
       - name: Cancel non-latest runs
         uses: styfle/cancel-workflow-action@0.11.0
@@ -22,19 +22,14 @@ jobs:
           submodules: 'recursive'
           fetch-depth: 0  # tags are required for versioneer to determine the version
 
-      - name: Set up Conda
-        uses: conda-incubator/setup-miniconda@v2
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
-          auto-update-conda: true
-          activate-environment: ros3
-          environment-file: environment-ros3.yml
-          python-version: "3.11"
-          channels: conda-forge
-          auto-activate-base: false
+          python-version: '3.11'
 
       - name: Install run dependencies
         run: |
-          python -m pip install dandi pytest
+          python -m pip install dandi fsspec requests aiohttp pytest
           python -m pip uninstall -y pynwb  # uninstall pynwb
           python -m pip install -e .
           python -m pip list
@@ -47,4 +42,4 @@ jobs:
 
       - name: Run DANDI read tests
         run: |
-          python tests/read_dandi/test_read_dandi.py
+          python tests/read_dandi/read_dandi.py

tests/read_dandi/test_read_dandi.py → tests/read_dandi/read_first_nwb_asset.py

@@ -1,5 +1,7 @@
-"""Test reading NWB files from the DANDI Archive using ROS3."""
+"""Test reading NWB files from the DANDI Archive using fsspec."""
 from dandi.dandiapi import DandiAPIClient
+import fsspec
+import h5py
 import random
 import sys
 import traceback
@@ -10,16 +12,21 @@
 # NOTE: do not name the function with "test_" prefix, otherwise pytest
 # will try to run it as a test
 
+# TODO read dandisets systematically, not randomly
+# see https://github.com/NeurodataWithoutBorders/pynwb/issues/1804
+
 def read_first_nwb_asset():
-    """Test reading the first NWB asset from a random selection of 50 dandisets that uses NWB."""
-    num_dandisets_to_read = 50
+    """Test reading the first NWB asset from a random selection of 2 dandisets that uses NWB."""
+    num_dandisets_to_read = 2
     client = DandiAPIClient()
     dandisets = list(client.get_dandisets())
     random.shuffle(dandisets)
     dandisets_to_read = dandisets[:num_dandisets_to_read]
     print("Reading NWB files from the following dandisets:")
     print([d.get_raw_metadata()["identifier"] for d in dandisets_to_read])
 
+    fs = fsspec.filesystem("http")
+
     failed_reads = dict()
     for i, dandiset in enumerate(dandisets_to_read):
         dandiset_metadata = dandiset.get_raw_metadata()
@@ -47,8 +54,10 @@ def read_first_nwb_asset():
         s3_url = first_asset.get_content_url(follow_redirects=1, strip_query=True)
 
         try:
-            with NWBHDF5IO(path=s3_url, driver="ros3") as io:
-                io.read()
+            with fs.open(s3_url, "rb") as f:
+                with h5py.File(f) as file:
+                    with NWBHDF5IO(file=file) as io:
+                        io.read()
         except Exception as e:
             print(traceback.format_exc())
             failed_reads[dandiset] = e