diff --git a/.gitignore b/.gitignore index de1231e3d..ba48d126b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ _*.jl *.jl.mem ./logging_config.toml +test/*.log ## Autogenerated code during the documentation process generated*.md @@ -39,6 +40,8 @@ docs/site/ Manifest.toml .vscode *.h5 +# ctags +tags ################################################################################ # Operating systems # diff --git a/docs/src/dev_guide/tests.md b/docs/src/dev_guide/tests.md index 65cfbc22e..02168a5f8 100644 --- a/docs/src/dev_guide/tests.md +++ b/docs/src/dev_guide/tests.md @@ -1,32 +1,70 @@ # Running Tests +## Standard test execution Unit tests can be executed in the REPL by executing the following: ```julia julia> ] test ``` -The unit test module supports several customizations to aid development and -debug. For instance, runnning a specific test file +## Interactive test execution +While developing code and tests it can be convenient to run a subset of tests. +You can do this with a combination of TestEnv.jl and ReTest.jl. - - Run a subset of tests in the REPL: +**Note**: Per recommendations from the developers of TestEnv.jl, install the package +in your global julia environment. Do the same for Revise.jl. +``` +$ julia +julia> ] +(@v1.10) pkg> add TestEnv Revise +``` + +Start the environment with the InfrastructureSystems.jl environment. +``` +$ julia --project +``` + +Load the test environment. +``` +julia> using TestEnv +julia> TestEnv.activate() +``` + +Load the tests through ReTest.jl and Revise.jl. ```julia -julia> push!(ARGS, "") -julia> include("test/runtests.jl") +julia> include("test/load_tests.jl") ``` - - Change logging level(s): +Run all tests. +```julia +julia> run_tests() +``` + +Run a subset of tests with a regular expression. This pattern matches multiple testset definitions. +The `run_tests` function forwards all arguments and keyword arguments to `ReTest.retest`. +``` +julia> run_tests(r"Test.*components") +``` + +Refer to the [ReTest documentation](https://juliatesting.github.io/ReTest.jl/stable/) for more +information. + +## Change logging levels + +```julia +julia> InfrastructureSystems.make_logging_config_file("logging_config.toml") +julia> ENV["SIENNA_LOGGING_CONFIG"] = "logging_config.toml" +``` +Edit the file to suit your preferences and rerun. ```julia -julia> IS.make_logging_config_file("logging_config.toml") -julia> ENV["SIIP_LOGGING_CONFIG"] = "logging_config.toml" -# Edit the file to suit your preferences. -julia> include("test/runtests.jl") +julia> run_tests() ``` **Note** that you can filter out noisy log groups in this file. +## Noisy log messages The unit test module appends a summary of all log message counts to the log file. If a message is logged too frequently then consider tagging that message with maxlog=X to suppress it. diff --git a/src/utils/logging.jl b/src/utils/logging.jl index 0d8345598..3955ba8b3 100644 --- a/src/utils/logging.jl +++ b/src/utils/logging.jl @@ -14,7 +14,7 @@ const LOG_GROUPS = ( LOG_GROUP_SYSTEM, LOG_GROUP_TIME_SERIES, ) -const SIIP_LOGGING_CONFIG_FILENAME = +const SIENNA_LOGGING_CONFIG_FILENAME = joinpath(dirname(pathof(InfrastructureSystems)), "utils", "logging_config.toml") const LOG_LEVELS = Dict( @@ -145,7 +145,7 @@ function LoggingConfiguration(config_filename) end function make_logging_config_file(filename = "logging_config.toml"; force = false) - cp(SIIP_LOGGING_CONFIG_FILENAME, filename; force = force) + cp(SIENNA_LOGGING_CONFIG_FILENAME, filename; force = force) println("Created $filename") return end diff --git a/src/utils/logging_config.toml b/src/utils/logging_config.toml index 51b37ce84..b18b0216a 100644 --- a/src/utils/logging_config.toml +++ b/src/utils/logging_config.toml @@ -3,7 +3,7 @@ console_level = "Info" console_stream = "stderr" file = true file_level = "Info" -filename = "siip_log.txt" +filename = "sienna_log.txt" file_mode = "w+" set_global = true diff --git a/src/utils/test.jl b/src/utils/test.jl index 21d35d220..d618abc0c 100644 --- a/src/utils/test.jl +++ b/src/utils/test.jl @@ -95,22 +95,6 @@ function TestEvent2(val::Int) return TestEvent2(RecorderEventCommon("TestEvent2"), val) end -function runtests(args...) - test_prefix = "test_" - for arg in args - if !startswith(arg, test_prefix) - arg = test_prefix * arg - end - push!(ARGS, arg) - end - - try - include("test/runtests.jl") - finally - empty!(ARGS) - end -end - struct TestSupplemental <: SupplementalAttribute value::Float64 component_uuids::ComponentUUIDs diff --git a/test/InfrastructureSystemsTests.jl b/test/InfrastructureSystemsTests.jl new file mode 100644 index 000000000..9f2d0dc44 --- /dev/null +++ b/test/InfrastructureSystemsTests.jl @@ -0,0 +1,89 @@ +module InfrastructureSystemsTests + +using ReTest +using Logging +import Dates +import TerminalLoggers: TerminalLogger +import TimeSeries +import UUIDs +import JSON3 +import HDF5 +using DataStructures: SortedDict +using DataFrames +using Random +using ProgressLogging + +import InfrastructureSystems + +import Aqua +Aqua.test_unbound_args(InfrastructureSystems) +Aqua.test_undefined_exports(InfrastructureSystems) +Aqua.test_ambiguities(InfrastructureSystems) +Aqua.test_stale_deps(InfrastructureSystems) +Aqua.test_deps_compat(InfrastructureSystems) + +const IS = InfrastructureSystems +const BASE_DIR = + abspath(joinpath(dirname(Base.find_package("InfrastructureSystems")), "..")) +const DATA_DIR = joinpath(BASE_DIR, "test", "data") +const FORECASTS_DIR = joinpath(DATA_DIR, "time_series") + +const LOG_FILE = "infrastructure-systems.log" + +include("common.jl") + +for filename in readdir(joinpath(BASE_DIR, "test")) + if startswith(filename, "test_") && endswith(filename, ".jl") + include(filename) + end +end + +function get_logging_level_from_env(env_name::String, default) + level = get(ENV, env_name, default) + return IS.get_logging_level(level) +end + +function run_tests(args...; kwargs...) + logger = global_logger() + try + logging_config_filename = get(ENV, "SIENNA_LOGGING_CONFIG", nothing) + if logging_config_filename !== nothing + config = IS.LoggingConfiguration(logging_config_filename) + else + config = IS.LoggingConfiguration(; + filename = LOG_FILE, + file_level = get_logging_level_from_env("SIENNA_FILE_LOG_LEVEL", "Info"), + console_level = get_logging_level_from_env( + "SIENNA_CONSOLE_LOG_LEVEL", + "Error", + ), + ) + end + console_logger = TerminalLogger(config.console_stream, config.console_level) + + IS.open_file_logger(config.filename, config.file_level) do file_logger + levels = (Logging.Info, Logging.Warn, Logging.Error) + multi_logger = + IS.MultiLogger([console_logger, file_logger], IS.LogEventTracker(levels)) + global_logger(multi_logger) + + if !isempty(config.group_levels) + IS.set_group_levels!(multi_logger, config.group_levels) + end + + @time retest(args...; kwargs...) + @test length(IS.get_log_events(multi_logger.tracker, Logging.Error)) == 0 + @info IS.report_log_summary(multi_logger) + end + finally + # Guarantee that the global logger is reset. + global_logger(logger) + nothing + end +end + +export run_tests + +end + +using .InfrastructureSystemsTests diff --git a/test/Project.toml b/test/Project.toml index f5bd669f0..f866feda1 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -14,6 +14,7 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" ProgressLogging = "33c8b6b6-d38a-422a-b730-caa89a2f386c" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +ReTest = "e0db7c4e-2690-44b9-bad6-7687da720f89" TerminalLoggers = "5d786b92-1e48-4d6f-9151-6b4477ca9bed" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimeSeries = "9e3dc215-6440-5c97-bce1-76c03772f85e" diff --git a/test/load_tests.jl b/test/load_tests.jl new file mode 100644 index 000000000..4ae299c2f --- /dev/null +++ b/test/load_tests.jl @@ -0,0 +1,13 @@ +using Revise + +# Copied from https://juliatesting.github.io/ReTest.jl/stable/#Working-with-Revise +function recursive_includet(filename) + already_included = copy(Revise.included_files) + includet(filename) + newly_included = setdiff(Revise.included_files, already_included) + for (mod, file) in newly_included + Revise.track(mod, file) + end +end + +recursive_includet("InfrastructureSystemsTests.jl") diff --git a/test/runtests.jl b/test/runtests.jl index c17ed87cc..026fbc5d7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,121 +1,4 @@ -using Test -using Logging -import Dates -import TerminalLoggers: TerminalLogger -import TimeSeries -import UUIDs -import JSON3 -import HDF5 -using DataStructures: SortedDict -using DataFrames -using Random -using ProgressLogging +using InfrastructureSystems -import InfrastructureSystems - -import Aqua -Aqua.test_unbound_args(InfrastructureSystems) -Aqua.test_undefined_exports(InfrastructureSystems) -Aqua.test_ambiguities(InfrastructureSystems) -Aqua.test_stale_deps(InfrastructureSystems) -Aqua.test_deps_compat(InfrastructureSystems) - -const IS = InfrastructureSystems -const BASE_DIR = - abspath(joinpath(dirname(Base.find_package("InfrastructureSystems")), "..")) -const DATA_DIR = joinpath(BASE_DIR, "test", "data") -const FORECASTS_DIR = joinpath(DATA_DIR, "time_series") - -const LOG_FILE = "infrastructure-systems.log" - -include("common.jl") - -""" -Copied @includetests from https://github.com/ssfrr/TestSetExtensions.jl. -Ideally, we could import and use TestSetExtensions. Its functionality was broken by changes -in Julia v0.7. Refer to https://github.com/ssfrr/TestSetExtensions.jl/pull/7. -""" - -""" -Includes the given test files, given as a list without their ".jl" extensions. -If none are given it will scan the directory of the calling file and include all -the julia files. -""" -macro includetests(testarg...) - if length(testarg) == 0 - tests = [] - elseif length(testarg) == 1 - tests = testarg[1] - else - error("@includetests takes zero or one argument") - end - - quote - tests = $tests - rootfile = @__FILE__ - if length(tests) == 0 - tests = readdir(dirname(rootfile)) - tests = filter( - f -> - startswith(f, "test_") && endswith(f, ".jl") && f != basename(rootfile), - tests, - ) - else - tests = map(f -> string(f, ".jl"), tests) - end - println() - for test in tests - print(splitext(test)[1], ": ") - include(test) - println() - end - end -end - -function get_logging_level_from_env(env_name::String, default) - level = get(ENV, env_name, default) - return IS.get_logging_level(level) -end - -function run_tests() - logging_config_filename = get(ENV, "SIIP_LOGGING_CONFIG", nothing) - if logging_config_filename !== nothing - config = IS.LoggingConfiguration(logging_config_filename) - else - config = IS.LoggingConfiguration(; - filename = LOG_FILE, - file_level = Logging.Info, - console_level = Logging.Error, - ) - end - console_logger = TerminalLogger(config.console_stream, config.console_level) - - IS.open_file_logger(config.filename, config.file_level) do file_logger - levels = (Logging.Info, Logging.Warn, Logging.Error) - multi_logger = - IS.MultiLogger([console_logger, file_logger], IS.LogEventTracker(levels)) - global_logger(multi_logger) - - if !isempty(config.group_levels) - IS.set_group_levels!(multi_logger, config.group_levels) - end - - @time @testset "Begin Systems tests" begin - @includetests ARGS - end - - @test length(IS.get_log_events(multi_logger.tracker, Logging.Error)) == 0 - - @info IS.report_log_summary(multi_logger) - end -end - -logger = global_logger() - -try - run_tests() -finally - # Guarantee that the global logger is reset. - global_logger(logger) - nothing -end +include("InfrastructureSystemsTests.jl") +run_tests() diff --git a/test/test_time_series.jl b/test/test_time_series.jl index 962f4a7c4..728654ca8 100644 --- a/test/test_time_series.jl +++ b/test/test_time_series.jl @@ -1322,7 +1322,7 @@ end @test TimeSeries.timestamp(IS.get_data(fcast))[1] == start_time end -@testset "Test time_series from" begin +@testset "Test time_series to" begin data = create_system_data(; with_time_series = true) time_series = get_all_time_series(data)[1] for end_time in (