Compile tests with riscof run (#16)

* Strip riscof template to bare-minimum. Move into src tree.

* Implement test compilation in riscof_kriscv.py

* Build arch tests with riscof rather than in test_conformance.py

* Set Version: 0.1.10

* Set Version: 0.1.11

* test_conformance.py: Assert that all test input files exists

* Add tests/README.md

* Remove needless -name '*' from find command

* Update riscof/README.md

---------

Co-authored-by: devops <devops@runtimeverification.com>

.gitignore

@@ -1,5 +1,4 @@
 /dist/
 __pycache__/
 .coverage
-tests/riscof/work.lock
-tests/riscof/work
+tests/riscv-arch-test-compiled

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "tests/riscv-arch-test"]
-	path = tests/riscof/riscv-arch-test
+	path = tests/riscv-arch-test
 	url = git@github.com:riscv-non-isa/riscv-arch-test.git

Makefile

@@ -41,9 +41,14 @@ test-all: poetry-install
 test-unit: poetry-install
 	$(POETRY_RUN) pytest src/tests/unit --maxfail=1 --verbose $(TEST_ARGS)
 
-test-integration: poetry-install
+test-integration: tests/riscv-arch-test-compiled poetry-install
 	$(POETRY_RUN) pytest src/tests/integration --maxfail=1 --verbose --durations=0 --numprocesses=4 --dist=worksteal $(TEST_ARGS)
 
+RISCOF_DIRS = $(shell find src/tests/integration/riscof -type d)
+RISCOF_FILES = $(shell find src/tests/integration/riscof -type f)
+
+tests/riscv-arch-test-compiled: tests/riscv-arch-test $(RISCOF_DIRS) $(RISCOF_FILES)
+	$(POETRY_RUN) riscof run --suite tests/riscv-arch-test/riscv-test-suite --env tests/riscv-arch-test/riscv-test-suite/env --config src/tests/integration/riscof/config.ini --work-dir tests/riscv-arch-test-compiled --no-ref-run
 
 # Coverage
 

package/version

@@ -1 +1 @@
-0.1.10
+0.1.11

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "kriscv"
-version = "0.1.10"
+version = "0.1.11"
 description = "K tooling for the RISC-V architecture"
 authors = [
     "Runtime Verification, Inc. <contact@runtimeverification.com>",

tests/riscof/README.md → src/tests/integration/riscof/README.md

@@ -1,13 +1,7 @@
 # RISC-V Architectural Tests
-This directory contains the RISC-V Architectural Test Suite and associated configuration.
+This directory contains the configuration for the `riscof` tool used to execute the RISC-V Architectural Test Suite.
 
 ## Contents
-### riscv-arch-test
-The submodule containing the actual test files under `riscv-arch-test/riscv-test-suite`.
-
-### failing.txt
-A list of tests that are known to fail with the current implementation. Each line gives the path to a failing test file relative to the `riscv-semantics` repo root.
-
 ### config.ini
 The [riscof configuration file](https://riscof.readthedocs.io/en/1.24.0/inputs.html?highlight=config.ini#config-ini-syntax) specifying the paths to our DUT and golden reference plugins. 
 
@@ -26,32 +20,11 @@ The [DUT plugin](https://riscof.readthedocs.io/en/1.24.0/plugins.html) for our R
    - Specification of the platform as described in [https://riscv-config.readthedocs.io/en/3.3.1/yaml-specs.html#platform-yaml-spec](https://riscv-config.readthedocs.io/en/3.3.1/yaml-specs.html#platform-yaml-spec)
 - riscof_kriscv.py
    - The actual plugin implementation for initializing, building, and running the test suite  with riscof.
-   - Currently under construction.
-
-### sail_cSim
-Analogous to `kriscv`, but for the golden reference [Sail RISC-V model](https://github.com/riscv/sail-riscv). Taken from [https://gitlab.com/incoresemi/riscof-plugins/-/tree/master/sail_cSim](https://gitlab.com/incoresemi/riscof-plugins/-/tree/master/sail_cSim)
-
-### work
-The test working directory generated during a test run. After a test run, this will contain:
- - kriscv_isa_checked.yaml
-   - Completion of `kriscv_isa.yaml` with all defaults made explicit.
- - kriscv_platform_checked.yaml
-   - Completion of `kriscv_platform.yaml` with all defaults made explicit.  
- - database.yaml
-    -  A database of all tests in the test suite and their associated info pulled from the various `RVTEST` macros.
-- test_list.yaml
-   - The list of supported tests to run based on the `kriscv` ISA and platform specs.
-- rv(32|64)_(i|e)...
-   - The working directory for each individual test.
-   - Follows the same directory structure as the tests in `riscv-arch-test/riscv-test-suite`, but each `.S` file is replaced by a directory of the same name containing the compiled `.elf` and disassembled `.diss` files.
-
-### work.lock
-A lock file used to control concurrent accesses to the `work` directory by different test workers.
+   - Currently only supports compiling but not executing the tests.
 
 ## Running the Tests
 ### PyTest
 These tests are run as part of `make test-integration` under `src/tests/integration/test_conformance.py`. Tests are selected based on the riscof-generated `test_list.yaml`. Failing tests under `failing.txt` will automatically be skipped.
 
 ### riscof
 The full riscof DUT plugin is still under construction, so execution with riscof is currently unsupported.
-

src/tests/integration/riscof/config.ini

@@ -0,0 +1,11 @@
+[RISCOF]
+ReferencePlugin=kriscv
+ReferencePluginPath=src/tests/integration/riscof/kriscv
+DUTPlugin=kriscv
+DUTPluginPath=src/tests/integration/riscof/kriscv
+
+[kriscv]
+pluginpath=src/tests/integration/riscof/kriscv
+ispec=src/tests/integration/riscof/kriscv/kriscv_isa.yaml
+pspec=src/tests/integration/riscof/kriscv/kriscv_platform.yaml
+target_run=0

src/tests/integration/riscof/kriscv/riscof_kriscv.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+from importlib.metadata import version
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import riscof.utils as utils  # type: ignore
+from riscof.pluginTemplate import pluginTemplate  # type: ignore
+
+if TYPE_CHECKING:
+    from typing import Any
+
+logger = logging.getLogger()
+
+
+class kriscv(pluginTemplate):  # noqa N801
+    __model__ = 'kriscv'
+    __version__ = version('kriscv')
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+
+        config = kwargs.get('config')
+        if config is None:
+            logger.error('Config for kriscv is missing.')
+            raise SystemExit
+        self.num_jobs = str(config['jobs'] if 'jobs' in config else 1)
+        self.pluginpath = os.path.abspath(config['pluginpath'])
+        self.isa_spec = os.path.abspath(config['ispec'])
+        self.platform_spec = os.path.abspath(config['pspec'])
+        self.target_run = ('target_run' not in config) or config['target_run'] == '1'
+        if self.target_run:
+            logger.error(
+                "Executing tests with riscof not yet supported. Set 'target_run=0' in config.ini to just compile the tests"
+            )
+            raise SystemExit
+
+    def initialise(self, suite: str, workdir: str, env: str) -> None:
+        self.suite = suite
+        self.workdir = workdir
+        self.compile_cmd = 'riscv64-unknown-elf-gcc -march={march} -mabi={mabi} -static -mcmodel=medany -fvisibility=hidden -nostdlib -nostartfiles'
+        self.compile_cmd += f' -T {self.pluginpath}/env/link.ld -I {self.pluginpath}/env/ -I {env}'
+        self.compile_cmd += ' {flags} {input} {output}'
+        self.objdump_cmd = 'riscv64-unknown-elf-objdump --no-addresses --no-show-raw-insn -D {input} > {output}'
+
+    def build(self, isa_yaml: str, platform_yaml: str) -> None:
+        ispec = utils.load_yaml(isa_yaml)['hart0']
+        self.mabi = _mabi(ispec['ISA'])
+        _check_exec_exists('riscv64-unknown-elf-gcc')
+        _check_exec_exists('riscv64-unknown-elf-objdump')
+        _check_exec_exists('make')
+
+    def runTests(self, testlist: dict[str, dict[str, Any]]) -> None:  # noqa N802
+        name = self.name[:-1]  # riscof includes an extra : on the end of the name for some reason
+        make = utils.makeUtil(makefilePath=os.path.join(self.workdir, f'Makefile.{name}'))
+        make.makeCommand = f'make -j {self.num_jobs}'
+        for entry in testlist.values():
+            test_path = entry['test_path']
+            march = entry['isa'].lower()
+            macro_flags = ' '.join(f'-D{macro}' for macro in entry['macros'])
+            test_name = Path(test_path).stem
+            compile_asm_cmd = self.compile_cmd.format(
+                march=march, mabi=self.mabi, flags=macro_flags + ' -S', input=test_path, output=f'> {test_name}.s'
+            )
+            compile_elf_cmd = self.compile_cmd.format(
+                march=march,
+                mabi=self.mabi,
+                flags=macro_flags,
+                input=test_path,
+                output=f'-o {test_name}.elf',
+            )
+            objdump_cmd = self.objdump_cmd.format(
+                input=f'{test_name}.elf',
+                output=f'{test_name}.disass',
+            )
+            work_dir = entry['work_dir']
+            execute = f'@cd {work_dir}; {compile_asm_cmd}; {compile_elf_cmd}; {objdump_cmd}'
+            make.add_target(execute, tname=test_name)
+        make.execute_all(self.workdir)
+
+
+def _mabi(spec_isa: str) -> str:
+    if '64I' in spec_isa:
+        return 'lp64'
+    if '64E' in spec_isa:
+        return 'lp64e'
+    if '32I' in spec_isa:
+        return 'ilp32'
+    if '32E' in spec_isa:
+        return 'ilp32e'
+    logger.error(f'Bad ISA string in ISA YAML: {spec_isa}')
+    raise SystemExit
+
+
+def _check_exec_exists(executable: str) -> None:
+    if shutil.which(executable) is None:
+        logger.error(f'{executable}: executable not found. Please check environment setup.')
+        raise SystemExit

src/tests/integration/test_conformance.py

@@ -1,15 +1,11 @@
 from __future__ import annotations
 
 import logging
-import os
-import shutil
-import subprocess
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 import pytest
 import yaml
-from filelock import FileLock
 
 from ..utils import REPO_ROOT
 
@@ -19,130 +15,31 @@
 _LOGGER: Final = logging.getLogger(__name__)
 _LOG_FORMAT: Final = '%(levelname)s %(asctime)s %(name)s - %(message)s'
 
-RISCOF: Final = REPO_ROOT / 'tests' / 'riscof'
+RISCOF: Final = Path(__file__).parent / 'riscof'
 
 
 def _skipped_tests() -> set[Path]:
-    return {REPO_ROOT / test_path for test_path in (RISCOF / 'failing.txt').read_text().splitlines()}
+    return {REPO_ROOT / test_path for test_path in (REPO_ROOT / 'tests' / 'failing.txt').read_text().splitlines()}
 
 
-# Failing tests pulled from tests/riscof/failing.txt
 SKIPPED_TESTS: Final = _skipped_tests()
 
-ARCH_SUITE_DIR: Final = RISCOF / 'riscv-arch-test' / 'riscv-test-suite'
-
-
-def _mod_time(path: Path) -> float:
-    """Get the modification time of a directory, accounting for all its recursive contents."""
-    if path.is_file():
-        return os.path.getmtime(path)
-    return max(
-        os.path.getmtime(entry)
-        for root, _, files in os.walk(path)
-        for entry in [os.path.join(root, name) for name in files] + [root]
-    )
-
-
-def _test_list() -> Path:
-    """Get the path to test_list.yaml, regenerating if needed."""
-    work_dir = RISCOF / 'work'
-    test_list = work_dir / 'test_list.yaml'
-    config = RISCOF / 'config.ini'
-    # Lock file to avoid every worker trying to regenerate the test list when using pytest-xdist
-    with FileLock(RISCOF / 'work.lock'):
-        if not test_list.exists() or _mod_time(test_list) < max(_mod_time(ARCH_SUITE_DIR), _mod_time(config)):
-            if work_dir.exists():
-                shutil.rmtree(work_dir)
-            subprocess.run(
-                [
-                    'riscof',
-                    'testlist',
-                    f'--suite={ARCH_SUITE_DIR}',
-                    f'--env={ARCH_SUITE_DIR / "env"}',
-                    f'--config={config}',
-                    f'--work-dir={work_dir}',
-                ],
-                check=True,
-            )
-    assert test_list.exists()
-    return test_list
-
-
-TEST_LIST: Final = _test_list()
+ARCH_SUITE_DIR: Final = REPO_ROOT / 'tests' / 'riscv-arch-test' / 'riscv-test-suite'
+ARCH_TEST_COMPILED_DIR: Final = REPO_ROOT / 'tests' / 'riscv-arch-test-compiled'
+TEST_LIST: Final = ARCH_TEST_COMPILED_DIR / 'test_list.yaml'
 ALL_ARCH_TESTS: Final = tuple(yaml.safe_load(TEST_LIST.read_text()).values())
 ARCH_TESTS: Final = tuple(test for test in ALL_ARCH_TESTS if test['test_path'] not in SKIPPED_TESTS)
 REST_ARCH_TESTS: Final = tuple(test for test in ALL_ARCH_TESTS if test['test_path'] in SKIPPED_TESTS)
 
 
-def _isa_spec() -> dict[str, Any]:
-    isa_yaml = RISCOF / 'kriscv' / 'kriscv_isa.yaml'
-    return yaml.safe_load(isa_yaml.read_text())['hart0']
-
-
-ISA_SPEC: Final = _isa_spec()
-
-
-def _mabi() -> str:
-    spec_isa = ISA_SPEC['ISA']
-    if '64I' in spec_isa:
-        return 'lp64'
-    if '64E' in spec_isa:
-        return 'lp64e'
-    if '32I' in spec_isa:
-        return 'ilp32'
-    if '32E' in spec_isa:
-        return 'ilp32e'
-    raise AssertionError(f'Bad ISA spec: {spec_isa}')
-
-
-def _compile_test(test: dict[str, Any]) -> Path:
-    """Compile the test for the given test entry from test_list.yaml.
-
-    Produces the compiled <test>.elf file and returns the path to its disassembly <test>.diss."
-    """
-    test_file = Path(test['test_path'])
-    _LOGGER.info(f'Running test: {test_file}')
-    march = test['isa'].lower()
-    plugin_env = RISCOF / 'kriscv' / 'env'
-    plugin_link = plugin_env / 'link.ld'
-    suite_env = ARCH_SUITE_DIR / 'env'
-    work_dir = Path(test['work_dir'])
-    elf_output = work_dir / (work_dir.stem + '.elf')
-    diss_output = work_dir / (work_dir.stem + '.diss')
-    compile_cmd = [
-        'riscv64-unknown-elf-gcc',
-        f'-march={march}',
-        f'-mabi={_mabi()}',
-        '-static',
-        '-mcmodel=medany',
-        '-fvisibility=hidden',
-        '-nostdlib',
-        '-nostartfiles',
-        '-T',
-        f'{plugin_link}',
-        '-I',
-        f'{plugin_env}',
-        '-I',
-        f'{suite_env}',
-        f'{test_file}',
-        '-o',
-        f'{elf_output}',
-    ]
-    compile_cmd += [f'-D{macro}' for macro in test['macros']]
-    subprocess.run(
-        compile_cmd,
-        check=True,
-    )
-    assert elf_output.exists()
-    with open(diss_output, 'w') as out:
-        subprocess.run(
-            ['riscv64-unknown-elf-objdump', '--no-addresses', '--no-show-raw-insn', '-D', elf_output], stdout=out
-        )
-    return diss_output
-
+def _test(test_dir: Path, name: str, isa: str) -> None:
+    asm = test_dir / f'{name}.s'
+    elf = test_dir / f'{name}.elf'
+    disass = test_dir / f'{name}.disass'
 
-def _test(diss_file: Path) -> None:
-    """Execute a compiled test given its disassembled ELF file."""
+    assert asm.exists()
+    assert elf.exists()
+    assert disass.exists()
 
 
 @pytest.mark.parametrize(
@@ -151,8 +48,11 @@ def _test(diss_file: Path) -> None:
     ids=[str(Path(test['test_path']).relative_to(ARCH_SUITE_DIR)) for test in ARCH_TESTS],
 )
 def test_arch(test_entry: dict[str, Any]) -> None:
-    diss_file = _compile_test(test_entry)
-    _test(diss_file)
+    _test(
+        Path(test_entry['work_dir']) / 'dut',
+        Path(test_entry['test_path']).relative_to(ARCH_SUITE_DIR).stem,
+        test_entry['isa'].lower(),
+    )
 
 
 @pytest.mark.skip(reason='failing arch tests')
@@ -162,5 +62,8 @@ def test_arch(test_entry: dict[str, Any]) -> None:
     ids=[str(Path(test['test_path']).relative_to(ARCH_SUITE_DIR)) for test in REST_ARCH_TESTS],
 )
 def test_rest_arch(test_entry: dict[str, Any]) -> None:
-    diss_file = _compile_test(test_entry)
-    _test(diss_file)
+    _test(
+        Path(test_entry['work_dir']) / 'dut',
+        Path(test_entry['test_path']).relative_to(ARCH_SUITE_DIR).stem,
+        test_entry['isa'].lower(),
+    )