Merge pull request #324 from daniel-thom/retest

Use ReTest.jl for tests

.gitignore

@@ -7,6 +7,7 @@ _*.jl
 *.jl.mem
 
 ./logging_config.toml
+test/*.log
 
 ## Autogenerated code during the documentation process
 generated*.md
@@ -39,6 +40,8 @@ docs/site/
 Manifest.toml
 .vscode
 *.h5
+# ctags
+tags
 
 ################################################################################
 #                              Operating systems                               #

docs/src/dev_guide/tests.md

@@ -1,32 +1,70 @@
 # Running Tests
 
+## Standard test execution
 Unit tests can be executed in the REPL by executing the following:
 
 ```julia
 julia> ] test
 ```
 
-The unit test module supports several customizations to aid development and
-debug. For instance, runnning a specific test file
+## Interactive test execution
+While developing code and tests it can be convenient to run a subset of tests.
+You can do this with a combination of TestEnv.jl and ReTest.jl.
 
-  - Run a subset of tests in the REPL:
+**Note**: Per recommendations from the developers of TestEnv.jl, install the package
+in your global julia environment. Do the same for Revise.jl.
 
+```
+$ julia
+julia> ]
+(@v1.10) pkg> add TestEnv Revise
+```
+
+Start the environment with the InfrastructureSystems.jl environment.
+```
+$ julia --project
+```
+
+Load the test environment.
+```
+julia> using TestEnv
+julia> TestEnv.activate()
+```
+
+Load the tests through ReTest.jl and Revise.jl.
 ```julia
-julia> push!(ARGS, "<test_filename_without_.jl>")
-julia> include("test/runtests.jl")
+julia> include("test/load_tests.jl")
 ```
 
-  - Change logging level(s):
+Run all tests.
+```julia
+julia> run_tests()
+```
+
+Run a subset of tests with a regular expression. This pattern matches multiple testset definitions.
+The `run_tests` function forwards all arguments and keyword arguments to `ReTest.retest`.
+```
+julia> run_tests(r"Test.*components")
+```
+
+Refer to the [ReTest documentation](https://juliatesting.github.io/ReTest.jl/stable/) for more
+information. 
+
+## Change logging levels
+
+```julia
+julia> InfrastructureSystems.make_logging_config_file("logging_config.toml")
+julia> ENV["SIENNA_LOGGING_CONFIG"] = "logging_config.toml"
+```
 
+Edit the file to suit your preferences and rerun.
 ```julia
-julia> IS.make_logging_config_file("logging_config.toml")
-julia> ENV["SIIP_LOGGING_CONFIG"] = "logging_config.toml"
-# Edit the file to suit your preferences.
-julia> include("test/runtests.jl")
+julia> run_tests()
 ```
 
 **Note** that you can filter out noisy log groups in this file.
 
+## Noisy log messages
 The unit test module appends a summary of all log message counts to the log
 file.  If a message is logged too frequently then consider tagging that message
 with maxlog=X to suppress it.

src/utils/logging.jl

@@ -14,7 +14,7 @@ const LOG_GROUPS = (
     LOG_GROUP_SYSTEM,
     LOG_GROUP_TIME_SERIES,
 )
-const SIIP_LOGGING_CONFIG_FILENAME =
+const SIENNA_LOGGING_CONFIG_FILENAME =
     joinpath(dirname(pathof(InfrastructureSystems)), "utils", "logging_config.toml")
 
 const LOG_LEVELS = Dict(
@@ -145,7 +145,7 @@ function LoggingConfiguration(config_filename)
 end
 
 function make_logging_config_file(filename = "logging_config.toml"; force = false)
-    cp(SIIP_LOGGING_CONFIG_FILENAME, filename; force = force)
+    cp(SIENNA_LOGGING_CONFIG_FILENAME, filename; force = force)
     println("Created $filename")
     return
 end

src/utils/logging_config.toml

@@ -3,7 +3,7 @@ console_level = "Info"
 console_stream = "stderr"
 file = true
 file_level = "Info"
-filename = "siip_log.txt"
+filename = "sienna_log.txt"
 file_mode = "w+"
 set_global = true
 

src/utils/test.jl

@@ -95,22 +95,6 @@ function TestEvent2(val::Int)
     return TestEvent2(RecorderEventCommon("TestEvent2"), val)
 end
 
-function runtests(args...)
-    test_prefix = "test_"
-    for arg in args
-        if !startswith(arg, test_prefix)
-            arg = test_prefix * arg
-        end
-        push!(ARGS, arg)
-    end
-
-    try
-        include("test/runtests.jl")
-    finally
-        empty!(ARGS)
-    end
-end
-
 struct TestSupplemental <: SupplementalAttribute
     value::Float64
     component_uuids::ComponentUUIDs

test/InfrastructureSystemsTests.jl

@@ -0,0 +1,89 @@
+module InfrastructureSystemsTests
+
+using ReTest
+using Logging
+import Dates
+import TerminalLoggers: TerminalLogger
+import TimeSeries
+import UUIDs
+import JSON3
+import HDF5
+using DataStructures: SortedDict
+using DataFrames
+using Random
+using ProgressLogging
+
+import InfrastructureSystems
+
+import Aqua
+Aqua.test_unbound_args(InfrastructureSystems)
+Aqua.test_undefined_exports(InfrastructureSystems)
+Aqua.test_ambiguities(InfrastructureSystems)
+Aqua.test_stale_deps(InfrastructureSystems)
+Aqua.test_deps_compat(InfrastructureSystems)
+
+const IS = InfrastructureSystems
+const BASE_DIR =
+    abspath(joinpath(dirname(Base.find_package("InfrastructureSystems")), ".."))
+const DATA_DIR = joinpath(BASE_DIR, "test", "data")
+const FORECASTS_DIR = joinpath(DATA_DIR, "time_series")
+
+const LOG_FILE = "infrastructure-systems.log"
+
+include("common.jl")
+
+for filename in readdir(joinpath(BASE_DIR, "test"))
+    if startswith(filename, "test_") && endswith(filename, ".jl")
+        include(filename)
+    end
+end
+
+function get_logging_level_from_env(env_name::String, default)
+    level = get(ENV, env_name, default)
+    return IS.get_logging_level(level)
+end
+
+function run_tests(args...; kwargs...)
+    logger = global_logger()
+    try
+        logging_config_filename = get(ENV, "SIENNA_LOGGING_CONFIG", nothing)
+        if logging_config_filename !== nothing
+            config = IS.LoggingConfiguration(logging_config_filename)
+        else
+            config = IS.LoggingConfiguration(;
+                filename = LOG_FILE,
+                file_level = get_logging_level_from_env("SIENNA_FILE_LOG_LEVEL", "Info"),
+                console_level = get_logging_level_from_env(
+                    "SIENNA_CONSOLE_LOG_LEVEL",
+                    "Error",
+                ),
+            )
+        end
+        console_logger = TerminalLogger(config.console_stream, config.console_level)
+
+        IS.open_file_logger(config.filename, config.file_level) do file_logger
+            levels = (Logging.Info, Logging.Warn, Logging.Error)
+            multi_logger =
+                IS.MultiLogger([console_logger, file_logger], IS.LogEventTracker(levels))
+            global_logger(multi_logger)
+
+            if !isempty(config.group_levels)
+                IS.set_group_levels!(multi_logger, config.group_levels)
+            end
+
+            @time retest(args...; kwargs...)
+            @test length(IS.get_log_events(multi_logger.tracker, Logging.Error)) == 0
+            @info IS.report_log_summary(multi_logger)
+        end
+    finally
+        # Guarantee that the global logger is reset.
+        global_logger(logger)
+        nothing
+    end
+end
+
+export run_tests
+
+end
+
+using .InfrastructureSystemsTests

test/Project.toml

@@ -14,6 +14,7 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
 ProgressLogging = "33c8b6b6-d38a-422a-b730-caa89a2f386c"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+ReTest = "e0db7c4e-2690-44b9-bad6-7687da720f89"
 TerminalLoggers = "5d786b92-1e48-4d6f-9151-6b4477ca9bed"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 TimeSeries = "9e3dc215-6440-5c97-bce1-76c03772f85e"

test/load_tests.jl

@@ -0,0 +1,13 @@
+using Revise
+
+# Copied from https://juliatesting.github.io/ReTest.jl/stable/#Working-with-Revise
+function recursive_includet(filename)
+    already_included = copy(Revise.included_files)
+    includet(filename)
+    newly_included = setdiff(Revise.included_files, already_included)
+    for (mod, file) in newly_included
+        Revise.track(mod, file)
+    end
+end
+
+recursive_includet("InfrastructureSystemsTests.jl")

test/runtests.jl

@@ -1,121 +1,4 @@
-using Test
-using Logging
-import Dates
-import TerminalLoggers: TerminalLogger
-import TimeSeries
-import UUIDs
-import JSON3
-import HDF5
-using DataStructures: SortedDict
-using DataFrames
-using Random
-using ProgressLogging
+using InfrastructureSystems
 
-import InfrastructureSystems
-
-import Aqua
-Aqua.test_unbound_args(InfrastructureSystems)
-Aqua.test_undefined_exports(InfrastructureSystems)
-Aqua.test_ambiguities(InfrastructureSystems)
-Aqua.test_stale_deps(InfrastructureSystems)
-Aqua.test_deps_compat(InfrastructureSystems)
-
-const IS = InfrastructureSystems
-const BASE_DIR =
-    abspath(joinpath(dirname(Base.find_package("InfrastructureSystems")), ".."))
-const DATA_DIR = joinpath(BASE_DIR, "test", "data")
-const FORECASTS_DIR = joinpath(DATA_DIR, "time_series")
-
-const LOG_FILE = "infrastructure-systems.log"
-
-include("common.jl")
-
-"""
-Copied @includetests from https://github.com/ssfrr/TestSetExtensions.jl.
-Ideally, we could import and use TestSetExtensions.  Its functionality was broken by changes
-in Julia v0.7.  Refer to https://github.com/ssfrr/TestSetExtensions.jl/pull/7.
-"""
-
-"""
-Includes the given test files, given as a list without their ".jl" extensions.
-If none are given it will scan the directory of the calling file and include all
-the julia files.
-"""
-macro includetests(testarg...)
-    if length(testarg) == 0
-        tests = []
-    elseif length(testarg) == 1
-        tests = testarg[1]
-    else
-        error("@includetests takes zero or one argument")
-    end
-
-    quote
-        tests = $tests
-        rootfile = @__FILE__
-        if length(tests) == 0
-            tests = readdir(dirname(rootfile))
-            tests = filter(
-                f ->
-                    startswith(f, "test_") && endswith(f, ".jl") && f != basename(rootfile),
-                tests,
-            )
-        else
-            tests = map(f -> string(f, ".jl"), tests)
-        end
-        println()
-        for test in tests
-            print(splitext(test)[1], ": ")
-            include(test)
-            println()
-        end
-    end
-end
-
-function get_logging_level_from_env(env_name::String, default)
-    level = get(ENV, env_name, default)
-    return IS.get_logging_level(level)
-end
-
-function run_tests()
-    logging_config_filename = get(ENV, "SIIP_LOGGING_CONFIG", nothing)
-    if logging_config_filename !== nothing
-        config = IS.LoggingConfiguration(logging_config_filename)
-    else
-        config = IS.LoggingConfiguration(;
-            filename = LOG_FILE,
-            file_level = Logging.Info,
-            console_level = Logging.Error,
-        )
-    end
-    console_logger = TerminalLogger(config.console_stream, config.console_level)
-
-    IS.open_file_logger(config.filename, config.file_level) do file_logger
-        levels = (Logging.Info, Logging.Warn, Logging.Error)
-        multi_logger =
-            IS.MultiLogger([console_logger, file_logger], IS.LogEventTracker(levels))
-        global_logger(multi_logger)
-
-        if !isempty(config.group_levels)
-            IS.set_group_levels!(multi_logger, config.group_levels)
-        end
-
-        @time @testset "Begin Systems tests" begin
-            @includetests ARGS
-        end
-
-        @test length(IS.get_log_events(multi_logger.tracker, Logging.Error)) == 0
-
-        @info IS.report_log_summary(multi_logger)
-    end
-end
-
-logger = global_logger()
-
-try
-    run_tests()
-finally
-    # Guarantee that the global logger is reset.
-    global_logger(logger)
-    nothing
-end
+include("InfrastructureSystemsTests.jl")
+run_tests()

test/test_time_series.jl

@@ -1322,7 +1322,7 @@ end
     @test TimeSeries.timestamp(IS.get_data(fcast))[1] == start_time
 end
 
-@testset "Test time_series from" begin
+@testset "Test time_series to" begin
     data = create_system_data(; with_time_series = true)
     time_series = get_all_time_series(data)[1]
     for end_time in (