From b72aebcb10ceb95ffde6ef7bd7a6e5e5080bdaf0 Mon Sep 17 00:00:00 2001 From: GabrielKS <23368820+GabrielKS@users.noreply.github.com> Date: Tue, 20 Feb 2024 16:10:04 -0700 Subject: [PATCH 1/3] Move FunctionData from PowerSystems to InfrastructureSystems The files `function_data.jl` and `test_function_data.jl` are copied from https://github.com/NREL-Sienna/PowerSystems.jl/tree/4a5375b9db694c1d64de0bec87234008d0c27640; see history there. --- src/InfrastructureSystems.jl | 1 + src/function_data.jl | 222 +++++++++++++++++++++++++++++++++++ test/test_function_data.jl | 102 ++++++++++++++++ 3 files changed, 325 insertions(+) create mode 100644 src/function_data.jl create mode 100644 test/test_function_data.jl diff --git a/src/InfrastructureSystems.jl b/src/InfrastructureSystems.jl index cf612b6d4..ee79b7cea 100644 --- a/src/InfrastructureSystems.jl +++ b/src/InfrastructureSystems.jl @@ -147,6 +147,7 @@ include("validation.jl") include("utils/print.jl") include("utils/test.jl") include("units.jl") +include("function_data.jl") include("deprecated.jl") end # module diff --git a/src/function_data.jl b/src/function_data.jl new file mode 100644 index 000000000..1f8f8cf7e --- /dev/null +++ b/src/function_data.jl @@ -0,0 +1,222 @@ +abstract type FunctionData end + +""" +Structure to represent the underlying data of linear functions. Principally used for +the representation of cost functions `f(x) = proportional_term*x`. + +# Arguments + - `proportional_term::Float64`: the proportional term in the function + `f(x) = proportional_term*x` +""" +struct LinearFunctionData <: FunctionData + proportional_term::Float64 +end + +get_proportional_term(fd::LinearFunctionData) = fd.proportional_term + +""" +Structure to represent the underlying data of quadratic polynomial functions. Principally +used for the representation of cost functions +`f(x) = quadratic_term*x^2 + proportional_term*x + constant_term`. + +# Arguments + - `quadratic_term::Float64`: the quadratic term in the represented function + - `proportional_term::Float64`: the proportional term in the represented function + - `constant_term::Float64`: the constant term in the represented function +""" +struct QuadraticFunctionData <: FunctionData + quadratic_term::Float64 + proportional_term::Float64 + constant_term::Float64 +end + +get_quadratic_term(fd::QuadraticFunctionData) = fd.quadratic_term +get_proportional_term(fd::QuadraticFunctionData) = fd.proportional_term +get_constant_term(fd::QuadraticFunctionData) = fd.constant_term + +""" +Structure to represent the underlying data of higher order polynomials. Principally used for +the representation of cost functions where +`f(x) = sum_{i in keys(coefficients)} coefficients[i]*x^i`. + +# Arguments + - `coefficients::Dict{Int, Float64}`: values are coefficients, keys are degrees to which + the coefficients apply (0 for the constant term, 2 for the squared term, etc.) +""" +struct PolynomialFunctionData <: FunctionData + coefficients::Dict{Int, Float64} +end + +get_coefficients(fd::PolynomialFunctionData) = fd.coefficients + +function _validate_piecewise_x(x_coords) + # TODO currently there exist cases where we are constructing a PiecewiseLinearPointData + # with only one point (e.g., `calculate_variable_cost` within + # `power_system_table_data.jl`) -- what does this represent? + # (length(x_coords) >= 2) || + # throw(ArgumentError("Must specify at least two x-coordinates")) + issorted(x_coords) || throw(ArgumentError("Piecewise x-coordinates must be ascending")) +end + +""" +Structure to represent pointwise piecewise linear data. Principally used for the +representation of cost functions where the points store quantities (x, y), such as (MW, \$). +The curve starts at the first point given, not the origin. + +# Arguments + - `points::Vector{Tuple{Float64, Float64}}`: the points (x, y) that define the function +""" +struct PiecewiseLinearPointData <: FunctionData + points::Vector{Tuple{Float64, Float64}} + + function PiecewiseLinearPointData(points) + _validate_piecewise_x(first.(points)) + return new(points) + end +end + +"Get the points that define the piecewise data" +get_points(data::PiecewiseLinearPointData) = data.points + +"Get the x-coordinates of the points that define the piecewise data" +get_x_coords(data::PiecewiseLinearPointData) = first.(get_points(data)) + +function _get_slopes(vc::Vector{NTuple{2, Float64}}) + slopes = Vector{Float64}(undef, length(vc) - 1) + (prev_x, prev_y) = vc[1] + for (i, (comp_x, comp_y)) in enumerate(vc[2:end]) + slopes[i] = (comp_y - prev_y) / (comp_x - prev_x) + (prev_x, prev_y) = (comp_x, comp_y) + end + return slopes +end + +_get_x_lengths(x_coords) = x_coords[2:end] .- x_coords[1:(end - 1)] + +""" +Calculates the slopes of the line segments defined by the PiecewiseLinearPointData, +returning one fewer slope than the number of underlying points. +""" +function get_slopes(pwl::PiecewiseLinearPointData) + return _get_slopes(get_points(pwl)) +end + +""" +Structure to represent the underlying data of slope piecewise linear data. Principally used +for the representation of cost functions where the points store quantities (x, dy/dx), such +as (MW, \$/MW). + +# Arguments + - `x_coords::Vector{Float64}`: the x-coordinates of the endpoints of the segments + - `y0::Float64`: the y-coordinate of the data at the first x-coordinate + - `slopes::Vector{Float64}`: the slopes of the segments: `slopes[1]` is the slope between + `x_coords[1]` and `x_coords[2]`, etc. +""" +struct PiecewiseLinearSlopeData <: FunctionData + x_coords::Vector{Float64} + y0::Float64 + slopes::Vector{Float64} + + function PiecewiseLinearSlopeData(x_coords, y0, slopes) + _validate_piecewise_x(x_coords) + (length(slopes) == length(x_coords) - 1) || + throw(ArgumentError("Must specify one fewer slope than x-coordinates")) + return new(x_coords, y0, slopes) + end +end + +"Get the slopes that define the PiecewiseLinearSlopeData" +get_slopes(data::PiecewiseLinearSlopeData) = data.slopes + +"Get the x-coordinates of the points that define the piecewise data" +get_x_coords(data::PiecewiseLinearSlopeData) = data.x_coords + +"Get the y-coordinate of the data at the first x-coordinate" +get_y0(data::PiecewiseLinearSlopeData) = data.y0 + +"Calculate the endpoints of the segments in the PiecewiseLinearSlopeData" +function get_points(data::PiecewiseLinearSlopeData) + slopes = get_slopes(data) + x_coords = get_x_coords(data) + points = Vector{Tuple{Float64, Float64}}(undef, length(x_coords)) + running_y = get_y0(data) + points[1] = (x_coords[1], running_y) + for (i, (prev_slope, this_x, dx)) in + enumerate(zip(slopes, x_coords[2:end], get_x_lengths(data))) + running_y += prev_slope * dx + points[i + 1] = (this_x, running_y) + end + return points +end + +""" +Calculates the x-length of each segment of a piecewise curve. +""" +function get_x_lengths( + pwl::Union{PiecewiseLinearPointData, PiecewiseLinearSlopeData}, +) + return _get_x_lengths(get_x_coords(pwl)) +end + +Base.length(pwl::Union{PiecewiseLinearPointData, PiecewiseLinearSlopeData}) = + length(get_x_coords(pwl)) - 1 + +Base.getindex(pwl::PiecewiseLinearPointData, ix::Int) = + getindex(get_points(pwl), ix) + +Base.:(==)(a::PiecewiseLinearPointData, b::PiecewiseLinearPointData) = + get_points(a) == get_points(b) + +Base.:(==)(a::PiecewiseLinearSlopeData, b::PiecewiseLinearSlopeData) = + (get_x_coords(a) == get_x_coords(b)) && + (get_y0(a) == get_y0(b)) && + (get_slopes(a) == get_slopes(b)) + +Base.:(==)(a::PolynomialFunctionData, b::PolynomialFunctionData) = + get_coefficients(a) == get_coefficients(b) + +function _slope_convexity_check(slopes::Vector{Float64}) + for ix in 1:(length(slopes) - 1) + if slopes[ix] > slopes[ix + 1] + @debug slopes + return false + end + end + return true +end + +""" +Returns True/False depending on the convexity of the underlying data +""" +is_convex(pwl::Union{PiecewiseLinearPointData, PiecewiseLinearSlopeData}) = + _slope_convexity_check(get_slopes(pwl)) + +# kwargs-only constructors for deserialization +LinearFunctionData(; proportional_term) = LinearFunctionData(proportional_term) + +QuadraticFunctionData(; quadratic_term, proportional_term, constant_term) = + QuadraticFunctionData(quadratic_term, proportional_term, constant_term) + +PolynomialFunctionData(; coefficients) = PolynomialFunctionData(coefficients) + +PiecewiseLinearPointData(; points) = PiecewiseLinearPointData(points) + +PiecewiseLinearSlopeData(; x_coords, y0, slopes) = + PiecewiseLinearSlopeData(x_coords, y0, slopes) + +serialize(val::FunctionData) = serialize_struct(val) + +function deserialize_struct(T::Type{PolynomialFunctionData}, val::Dict) + data = deserialize_to_dict(T, val) + data[Symbol("coefficients")] = + Dict( + (k isa String ? parse(Int, k) : k, v) + for (k, v) in data[Symbol("coefficients")] + ) + return T(; data...) +end + +deserialize(T::Type{<:FunctionData}, val::Dict) = deserialize_struct(T, val) + +deserialize(::Type{FunctionData}, val::Dict) = + throw(ArgumentError("FunctionData is abstract, must specify a concrete subtype")) diff --git a/test/test_function_data.jl b/test/test_function_data.jl new file mode 100644 index 000000000..ec52ed268 --- /dev/null +++ b/test/test_function_data.jl @@ -0,0 +1,102 @@ +get_test_function_data() = [ + IS.LinearFunctionData(5), + IS.QuadraticFunctionData(2, 3, 4), + IS.PolynomialFunctionData(Dict(0 => 3.0, 1 => 1.0, 3 => 4.0)), + IS.PiecewiseLinearPointData([(1, 1), (3, 5), (5, 10)]), + IS.PiecewiseLinearSlopeData([1, 3, 5], 1, [2, 2.5]), +] + +@testset "Test FunctionData validation" begin + @test_throws ArgumentError IS.PiecewiseLinearPointData([(2, 1), (1, 1)]) + @test_throws ArgumentError IS.PiecewiseLinearSlopeData([2, 1], 1, [1]) +end + +@testset "Test FunctionData trivial getters" begin + ld = IS.LinearFunctionData(5) + @test IS.get_proportional_term(ld) == 5 + + qd = IS.QuadraticFunctionData(2, 3, 4) + @test IS.get_quadratic_term(qd) == 2 + @test IS.get_proportional_term(qd) == 3 + @test IS.get_constant_term(qd) == 4 + + pd = IS.PolynomialFunctionData(Dict(0 => 3.0, 1 => 1.0, 3 => 4.0)) + coeffs = IS.get_coefficients(pd) + @test length(coeffs) == 3 + @test coeffs[0] === 3.0 && coeffs[1] === 1.0 && coeffs[3] === 4.0 + + yd = IS.PiecewiseLinearPointData([(1, 1), (3, 5)]) + @test IS.get_points(yd) == [(1, 1), (3, 5)] + + dd = IS.PiecewiseLinearSlopeData([1, 3, 5], 2, [3, 6]) + @test IS.get_x_coords(dd) == [1, 3, 5] + @test IS.get_y0(dd) == 2 + @test IS.get_slopes(dd) == [3, 6] +end + +@testset "Test FunctionData calculations" begin + @test length(IS.PiecewiseLinearPointData([(0, 0), (1, 1), (1.1, 2)])) == 2 + @test length(IS.PiecewiseLinearSlopeData([1, 1.1, 1.2], 1, [1.1, 10])) == 2 + + @test IS.PiecewiseLinearPointData([(0, 0), (1, 1), (1.1, 2)])[2] == (1, 1) + @test IS.get_x_coords(IS.PiecewiseLinearPointData([(0, 0), (1, 1), (1.1, 2)])) == [0, 1, 1.1] + + # Tests our overridden Base.:(==) + @test all(get_test_function_data() .== get_test_function_data()) + + @test all(isapprox.( + IS.get_slopes(IS.PiecewiseLinearPointData([(0, 0), (10, 31.4)])), [3.14])) + @test isapprox( + IS.get_slopes(IS.PiecewiseLinearPointData([(0, 0), (1, 1), (1.1, 2), (1.2, 3)])), + [1, 10, 10]) + @test isapprox( + IS.get_slopes(IS.PiecewiseLinearPointData([(0, 0), (1, 1), (1.1, 2)])), + [1, 10]) + + @test isapprox( + collect.(IS.get_points(IS.PiecewiseLinearSlopeData([1, 3, 5], 1, [2.5, 10]))), + collect.([(1, 1), (3, 6), (5, 26)]), + ) + + @test isapprox( + IS.get_x_lengths(IS.PiecewiseLinearPointData([(1, 1), (1.1, 2), (1.2, 3)])), + [0.1, 0.1]) + @test isapprox( + IS.get_x_lengths(IS.PiecewiseLinearSlopeData([1, 1.1, 1.2], 1, [1.1, 10])), + [0.1, 0.1]) + + @test IS.is_convex(IS.PiecewiseLinearSlopeData([0, 1, 1.1, 1.2], 1, [1.1, 10, 10])) + @test !IS.is_convex(IS.PiecewiseLinearSlopeData([0, 1, 1.1, 1.2], 1, [1.1, 10, 9])) + @test IS.is_convex(IS.PiecewiseLinearPointData([(0, 0), (1, 1), (1.1, 2), (1.2, 3)])) + @test !IS.is_convex(IS.PiecewiseLinearPointData([(0, 0), (1, 1), (1.1, 2), (5, 3)])) +end + +@testset "Test FunctionData piecewise point/slope conversion" begin + rng = Random.Xoshiro(47) # Set random seed for determinism + n_tests = 100 + n_points = 10 + for _ in 1:n_tests + rand_x = sort(rand(rng, n_points)) + rand_y = rand(rng, n_points) + pointwise = IS.PiecewiseLinearPointData(collect(zip(rand_x, rand_y))) + slopewise = IS.PiecewiseLinearSlopeData( + IS.get_x_coords(pointwise), + first(IS.get_points(pointwise))[2], + IS.get_slopes(pointwise)) + pointwise_2 = IS.PiecewiseLinearPointData(IS.get_points(slopewise)) + @test isapprox( + collect.(IS.get_points(pointwise_2)), collect.(IS.get_points(pointwise))) + end +end + +@testset "Test FunctionData serialization round trip" begin + for fd in get_test_function_data() + for do_jsonify in (false, true) + serialized = IS.serialize(fd) + do_jsonify && (serialized = JSON3.read(JSON3.write(serialized), Dict)) + @test typeof(serialized) <: AbstractDict + deserialized = IS.deserialize(typeof(fd), serialized) + @test deserialized == fd + end + end +end From 142aafbe1d4cb77b34c750ea00ade414da4720f5 Mon Sep 17 00:00:00 2001 From: GabrielKS <23368820+GabrielKS@users.noreply.github.com> Date: Tue, 20 Feb 2024 16:22:52 -0700 Subject: [PATCH 2/3] Run formatter --- test/test_function_data.jl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/test_function_data.jl b/test/test_function_data.jl index ec52ed268..b30d32886 100644 --- a/test/test_function_data.jl +++ b/test/test_function_data.jl @@ -39,13 +39,16 @@ end @test length(IS.PiecewiseLinearSlopeData([1, 1.1, 1.2], 1, [1.1, 10])) == 2 @test IS.PiecewiseLinearPointData([(0, 0), (1, 1), (1.1, 2)])[2] == (1, 1) - @test IS.get_x_coords(IS.PiecewiseLinearPointData([(0, 0), (1, 1), (1.1, 2)])) == [0, 1, 1.1] + @test IS.get_x_coords(IS.PiecewiseLinearPointData([(0, 0), (1, 1), (1.1, 2)])) == + [0, 1, 1.1] # Tests our overridden Base.:(==) @test all(get_test_function_data() .== get_test_function_data()) - @test all(isapprox.( - IS.get_slopes(IS.PiecewiseLinearPointData([(0, 0), (10, 31.4)])), [3.14])) + @test all( + isapprox.( + IS.get_slopes(IS.PiecewiseLinearPointData([(0, 0), (10, 31.4)])), [3.14]), + ) @test isapprox( IS.get_slopes(IS.PiecewiseLinearPointData([(0, 0), (1, 1), (1.1, 2), (1.2, 3)])), [1, 10, 10]) From dff57ce14a0ad0655e0fe9a564b1c784024d0b34 Mon Sep 17 00:00:00 2001 From: GabrielKS <23368820+GabrielKS@users.noreply.github.com> Date: Wed, 21 Feb 2024 23:47:20 -0700 Subject: [PATCH 3/3] Adapt Forecasts to use FunctionData, reduce code duplication --- src/InfrastructureSystems.jl | 2 +- src/common.jl | 2 - src/deterministic.jl | 33 +-- src/hdf5_time_series_storage.jl | 89 +++--- src/probabilistic.jl | 33 +-- src/scenarios.jl | 17 +- src/utils/utils.jl | 54 +++- test/test_time_series.jl | 486 ++++++++++++-------------------- 8 files changed, 305 insertions(+), 411 deletions(-) diff --git a/src/InfrastructureSystems.jl b/src/InfrastructureSystems.jl index ee79b7cea..5c0c18371 100644 --- a/src/InfrastructureSystems.jl +++ b/src/InfrastructureSystems.jl @@ -108,6 +108,7 @@ include("utils/generate_structs.jl") include("utils/lazy_dict_from_iterator.jl") include("utils/logging.jl") include("utils/stdout_redirector.jl") +include("function_data.jl") include("utils/utils.jl") include("internal.jl") include("time_series_storage.jl") @@ -147,7 +148,6 @@ include("validation.jl") include("utils/print.jl") include("utils/test.jl") include("units.jl") -include("function_data.jl") include("deprecated.jl") end # module diff --git a/src/common.jl b/src/common.jl index c4c82931c..f805b7ba7 100644 --- a/src/common.jl +++ b/src/common.jl @@ -23,6 +23,4 @@ struct HashMismatchError <: Exception end const CONSTANT = Float64 -const POLYNOMIAL = Tuple{Float64, Float64} -const PWL = Vector{Tuple{Float64, Float64}} const LIM_TOL = 1e-6 diff --git a/src/deterministic.jl b/src/deterministic.jl index 11c71425d..c60f61e0a 100644 --- a/src/deterministic.jl +++ b/src/deterministic.jl @@ -1,11 +1,7 @@ """ mutable struct Deterministic <: AbstractDeterministic name::String - data::Union{ - SortedDict{Dates.DateTime, Vector{CONSTANT}}, - SortedDict{Dates.DateTime, Vector{POLYNOMIAL}}, - SortedDict{Dates.DateTime, Vector{PWL}}, - } + data::SortedDict resolution::Dates.Period scaling_factor_multiplier::Union{Nothing, Function} internal::InfrastructureSystemsInternal @@ -16,7 +12,7 @@ A deterministic forecast for a particular data field in a Component. # Arguments - `name::String`: user-defined name - - `data::Union{SortedDict{Dates.DateTime, Vector{CONSTANT}}, SortedDict{Dates.DateTime, Vector{POLYNOMIAL}}, SortedDict{Dates.DateTime, Vector{PWL}}}`: timestamp - scalingfactor + - `data::SortedDict`: timestamp - scalingfactor - `resolution::Dates.Period`: forecast resolution - `scaling_factor_multiplier::Union{Nothing, Function}`: Applicable when the time series data are scaling factors. Called on the associated component to convert the values. @@ -26,11 +22,7 @@ mutable struct Deterministic <: AbstractDeterministic "user-defined name" name::String "timestamp - scalingfactor" - data::Union{ - SortedDict{Dates.DateTime, Vector{CONSTANT}}, - SortedDict{Dates.DateTime, Vector{POLYNOMIAL}}, - SortedDict{Dates.DateTime, Vector{PWL}}, - } + data::SortedDict # TODO handle typing here in a more principled fashion "forecast resolution" resolution::Dates.Period "Applicable when the time series data are scaling factors. Called on the associated component to convert the values." @@ -202,19 +194,13 @@ function Deterministic(forecast::Deterministic, data) return Deterministic(; vals...) end -convert_data(data::AbstractDict{Dates.DateTime, Vector{T}}) where {T} = - SortedDict{Dates.DateTime, Vector{CONSTANT}}(data...) -convert_data(data::AbstractDict{Dates.DateTime, Vector{T}}) where {T <: Tuple} = - SortedDict{Dates.DateTime, Vector{POLYNOMIAL}}(data...) -convert_data(data::AbstractDict{Dates.DateTime, Vector{Vector{T}}}) where {T <: Tuple} = - SortedDict{Dates.DateTime, Vector{PWL}}(data...) convert_data( - data::Union{ - SortedDict{Dates.DateTime, Vector{CONSTANT}}, - SortedDict{Dates.DateTime, Vector{POLYNOMIAL}}, - SortedDict{Dates.DateTime, Vector{PWL}}, - }, -) = data + data::AbstractDict{<:Any, Vector{T}}, +) where {T <: Union{CONSTANT, FunctionData}} = + SortedDict{Dates.DateTime, Vector{T}}(data...) +convert_data( + data::SortedDict{Dates.DateTime, Vector{T}}, +) where {T <: Union{CONSTANT, FunctionData}} = data # Workaround for a bug/limitation in SortedDict. If a user tries to construct # SortedDict(i => ones(2) for i in 1:2) @@ -271,6 +257,7 @@ Set [`Deterministic`](@ref) `internal`. """ set_internal!(value::Deterministic, val) = value.internal = val +# TODO handle typing here in a more principled fashion eltype_data(forecast::Deterministic) = eltype_data_common(forecast) get_count(forecast::Deterministic) = get_count_common(forecast) get_horizon(forecast::Deterministic) = get_horizon_common(forecast) diff --git a/src/hdf5_time_series_storage.jl b/src/hdf5_time_series_storage.jl index 4e9a94649..d61075798 100644 --- a/src/hdf5_time_series_storage.jl +++ b/src/hdf5_time_series_storage.jl @@ -236,19 +236,9 @@ Return a String for the data type of the forecast data, this implementation avoi """ function get_data_type(ts::TimeSeriesData) data_type = eltype_data(ts) - if data_type <: CONSTANT - return "CONSTANT" - elseif data_type == POLYNOMIAL - return "POLYNOMIAL" - elseif data_type == PWL - return "PWL" - elseif data_type <: Integer - # We currently don't convert integers stored in TimeSeries.TimeArrays to floats. - # This is a workaround. - return "CONSTANT" - else - error("$data_type is not supported in forecast data") - end + ((data_type <: CONSTANT) || (data_type <: Integer)) && (return "CONSTANT") + (data_type <: FunctionData) && (return strip_module_name(data_type)) + error("$data_type is not supported in forecast data") end function _write_time_series_attributes!( @@ -302,7 +292,17 @@ function _read_time_series_attributes( return data end -const _TYPE_DICT = Dict("CONSTANT" => CONSTANT, "POLYNOMIAL" => POLYNOMIAL, "PWL" => PWL) +# TODO I suspect this could be designed better using reflection even without the security risks of eval discussed above +const _TYPE_DICT = Dict( + strip_module_name(st) => st for st in [ + LinearFunctionData, + QuadraticFunctionData, + PolynomialFunctionData, + PiecewiseLinearPointData, + PiecewiseLinearSlopeData, + ] +) +_TYPE_DICT["CONSTANT"] = CONSTANT function _read_time_series_attributes_common(storage::Hdf5TimeSeriesStorage, path, rows) initial_timestamp = @@ -492,12 +492,27 @@ end function get_hdf_array( dataset, - type::Type{POLYNOMIAL}, + ::Type{LinearFunctionData}, attributes::Dict{String, Any}, rows::UnitRange{Int}, columns::UnitRange{Int}, ) - data = SortedDict{Dates.DateTime, Vector{POLYNOMIAL}}() + data = get_hdf_array(dataset, CONSTANT, attributes, rows, columns) + return SortedDict{Dates.DateTime, Vector{LinearFunctionData}}( + k => LinearFunctionData.(v) for (k, v) in data + ) +end + +_quadratic_from_tuple((a, b)::Tuple{Float64, Float64}) = QuadraticFunctionData(a, b, 0.0) + +function get_hdf_array( + dataset, + type::Type{QuadraticFunctionData}, + attributes::Dict{String, Any}, + rows::UnitRange{Int}, + columns::UnitRange{Int}, +) + data = SortedDict{Dates.DateTime, Vector{QuadraticFunctionData}}() initial_timestamp = attributes["start_time"] interval = attributes["interval"] start_time = initial_timestamp + interval * (columns.start - 1) @@ -515,12 +530,12 @@ end function get_hdf_array( dataset, - type::Type{PWL}, + type::Type{PiecewiseLinearPointData}, attributes::Dict{String, Any}, rows::UnitRange{Int}, columns::UnitRange{Int}, ) - data = SortedDict{Dates.DateTime, Vector{PWL}}() + data = SortedDict{Dates.DateTime, Vector{PiecewiseLinearPointData}}() initial_timestamp = attributes["start_time"] interval = attributes["interval"] start_time = initial_timestamp + interval * (columns.start - 1) @@ -536,17 +551,21 @@ function get_hdf_array( return data end -function get_hdf_array(dataset, type::Type{<:CONSTANT}, rows::UnitRange{Int}) +function get_hdf_array( + dataset, + type::Union{Type{<:CONSTANT}, Type{LinearFunctionData}}, + rows::UnitRange{Int}, +) data = retransform_hdf_array(dataset[rows], type) return data end -function get_hdf_array(dataset, type::Type{POLYNOMIAL}, rows::UnitRange{Int}) +function get_hdf_array(dataset, type::Type{QuadraticFunctionData}, rows::UnitRange{Int}) data = retransform_hdf_array(dataset[rows, :, :], type) return data end -function get_hdf_array(dataset, type::Type{PWL}, rows::UnitRange{Int}) +function get_hdf_array(dataset, type::Type{PiecewiseLinearPointData}, rows::UnitRange{Int}) data = retransform_hdf_array(dataset[rows, :, :, :], type) return data end @@ -555,47 +574,51 @@ function retransform_hdf_array(data::Array, ::Type{<:CONSTANT}) return data end -function retransform_hdf_array(data::Array, T::Type{POLYNOMIAL}) +function retransform_hdf_array(data::Array, ::Type{LinearFunctionData}) + return LinearFunctionData.(data) +end + +function retransform_hdf_array(data::Array, T::Type{QuadraticFunctionData}) row, column, tuple_length = get_data_dims(data, T) if isnothing(column) - t_data = Array{POLYNOMIAL}(undef, row) + t_data = Array{Tuple{Float64, Float64}}(undef, row) for r in 1:row t_data[r] = tuple(data[r, 1:tuple_length]...) end else - t_data = Array{POLYNOMIAL}(undef, row, column) + t_data = Array{Tuple{Float64, Float64}}(undef, row, column) for r in 1:row, c in 1:column t_data[r, c] = tuple(data[r, c, 1:tuple_length]...) end end - return t_data + return _quadratic_from_tuple.(t_data) end -function retransform_hdf_array(data::Array, T::Type{PWL}) +function retransform_hdf_array(data::Array, T::Type{PiecewiseLinearPointData}) row, column, tuple_length, array_length = get_data_dims(data, T) if isnothing(column) - t_data = Array{PWL}(undef, row) + t_data = Array{Vector{Tuple{Float64, Float64}}}(undef, row) for r in 1:row - tuple_array = Array{POLYNOMIAL}(undef, array_length) + tuple_array = Array{Tuple{Float64, Float64}}(undef, array_length) for l in 1:array_length tuple_array[l] = tuple(data[r, 1:tuple_length, l]...) end t_data[r] = tuple_array end else - t_data = Array{PWL}(undef, row, column) + t_data = Array{Vector{Tuple{Float64, Float64}}}(undef, row, column) for r in 1:row, c in 1:column - tuple_array = Array{POLYNOMIAL}(undef, array_length) + tuple_array = Array{Tuple{Float64, Float64}}(undef, array_length) for l in 1:array_length tuple_array[l] = tuple(data[r, c, 1:tuple_length, l]...) end t_data[r, c] = tuple_array end end - return t_data + return PiecewiseLinearPointData.(t_data) end -function get_data_dims(data::Array, ::Type{POLYNOMIAL}) +function get_data_dims(data::Array, ::Type{QuadraticFunctionData}) if length(size(data)) == 2 row, tuple_length = size(data) return (row, nothing, tuple_length) @@ -606,7 +629,7 @@ function get_data_dims(data::Array, ::Type{POLYNOMIAL}) end end -function get_data_dims(data::Array, ::Type{PWL}) +function get_data_dims(data::Array, ::Type{PiecewiseLinearPointData}) if length(size(data)) == 3 row, tuple_length, array_length = size(data) return (row, nothing, tuple_length, array_length) diff --git a/src/probabilistic.jl b/src/probabilistic.jl index eaf778bd0..60ed5434f 100644 --- a/src/probabilistic.jl +++ b/src/probabilistic.jl @@ -3,11 +3,7 @@ name::String resolution::Dates.Period percentiles::Vector{Float64} - data::Union{ - SortedDict{Dates.DateTime, Matrix{CONSTANT}}, - SortedDict{Dates.DateTime, Matrix{POLYNOMIAL}}, - SortedDict{Dates.DateTime, Matrix{PWL}}, - } + data::SortedDict scaling_factor_multiplier::Union{Nothing, Function} internal::InfrastructureSystemsInternal end @@ -19,7 +15,7 @@ A Probabilistic forecast for a particular data field in a Component. - `name::String`: user-defined name - `resolution::Dates.Period`: forecast resolution - `percentiles::Vector{Float64}`: Percentiles for the probabilistic forecast - - `data::Union{SortedDict{Dates.DateTime, Matrix{CONSTANT}}, SortedDict{Dates.DateTime, Matrix{POLYNOMIAL}}, SortedDict{Dates.DateTime, Matrix{PWL}}}`: timestamp - scalingfactor + - `data::SortedDict`: timestamp - scalingfactor - `scaling_factor_multiplier::Union{Nothing, Function}`: Applicable when the time series data are scaling factors. Called on the associated component to convert the values. - `internal::InfrastructureSystemsInternal` @@ -28,11 +24,7 @@ mutable struct Probabilistic <: Forecast "user-defined name" name::String "timestamp - scalingfactor" - data::Union{ - SortedDict{Dates.DateTime, Matrix{CONSTANT}}, - SortedDict{Dates.DateTime, Matrix{POLYNOMIAL}}, - SortedDict{Dates.DateTime, Matrix{PWL}}, - } + data::SortedDict # TODO see note in Deterministic "Percentiles for the probabilistic forecast" percentiles::Vector{Float64} "forecast resolution" @@ -203,19 +195,13 @@ function ProbabilisticMetadata(time_series::Probabilistic) ) end -convert_data(data::AbstractDict{Dates.DateTime, Matrix{T}}) where {T} = - SortedDict{Dates.DateTime, Matrix{CONSTANT}}(data...) -convert_data(data::AbstractDict{Dates.DateTime, Matrix{T}}) where {T <: Tuple} = - SortedDict{Dates.DateTime, Matrix{POLYNOMIAL}}(data...) -convert_data(data::AbstractDict{Dates.DateTime, Matrix{Matrix{T}}}) where {T <: Tuple} = - SortedDict{Dates.DateTime, Matrix{PWL}}(data...) convert_data( - data::Union{ - SortedDict{Dates.DateTime, Matrix{CONSTANT}}, - SortedDict{Dates.DateTime, Matrix{POLYNOMIAL}}, - SortedDict{Dates.DateTime, Matrix{PWL}}, - }, -) = data + data::AbstractDict{<:Any, Matrix{T}}, +) where {T <: Union{CONSTANT, FunctionData}} = + SortedDict{Dates.DateTime, Matrix{T}}(data...) +convert_data( + data::SortedDict{Dates.DateTime, Matrix{T}}, +) where {T <: Union{CONSTANT, FunctionData}} = data """ Get [`Probabilistic`](@ref) `name`. @@ -285,6 +271,7 @@ function get_horizon(forecast::Probabilistic) return size(first(values(get_data(forecast))))[1] end +# TODO see Deterministic eltype_data(forecast::Probabilistic) = eltype_data_common(forecast) get_count(forecast::Probabilistic) = get_count_common(forecast) get_initial_times(forecast::Probabilistic) = get_initial_times_common(forecast) diff --git a/src/scenarios.jl b/src/scenarios.jl index aae6d922e..5564b2f6b 100644 --- a/src/scenarios.jl +++ b/src/scenarios.jl @@ -3,11 +3,7 @@ name::String resolution::Dates.Period scenario_count::Int64 - data::Union{ - SortedDict{Dates.DateTime, Matrix{CONSTANT}}, - SortedDict{Dates.DateTime, Matrix{POLYNOMIAL}}, - SortedDict{Dates.DateTime, Matrix{PWL}}, - } + data::SortedDict scaling_factor_multiplier::Union{Nothing, Function} internal::InfrastructureSystemsInternal end @@ -19,7 +15,7 @@ A Discrete Scenario Based time series for a particular data field in a Component - `name::String`: user-defined name - `resolution::Dates.Period`: forecast resolution - `scenario_count::Int64`: Number of scenarios - - `data::Union{SortedDict{Dates.DateTime, Matrix{CONSTANT}}, SortedDict{Dates.DateTime, Matrix{POLYNOMIAL}}, SortedDict{Dates.DateTime, Matrix{PWL}}}`: timestamp - scalingfactor + - `data::SortedDict`: timestamp - scalingfactor - `scaling_factor_multiplier::Union{Nothing, Function}`: Applicable when the time series data are scaling factors. Called on the associated component to convert the values. - `internal::InfrastructureSystemsInternal` @@ -28,11 +24,7 @@ mutable struct Scenarios <: Forecast "user-defined name" name::String "timestamp - scalingfactor" - data::Union{ - SortedDict{Dates.DateTime, Matrix{CONSTANT}}, - SortedDict{Dates.DateTime, Matrix{POLYNOMIAL}}, - SortedDict{Dates.DateTime, Matrix{PWL}}, - } + data::SortedDict # TODO see note in Deterministic "Number of scenarios" scenario_count::Int64 "forecast resolution" @@ -51,7 +43,7 @@ function Scenarios(; normalization_factor = 1.0, internal = InfrastructureSystemsInternal(), ) - data = handle_normalization_factor(convert_data(data), normalization_factor) + data = handle_normalization_factor(data, normalization_factor) return Scenarios( name, data, @@ -237,6 +229,7 @@ Set [`Scenarios`](@ref) `internal`. """ set_internal!(value::Scenarios, val) = value.internal = val +# TODO see Deterministic eltype_data(forecast::Scenarios) = eltype_data_common(forecast) get_count(forecast::Scenarios) = get_count_common(forecast) get_initial_times(forecast::Scenarios) = get_initial_times_common(forecast) diff --git a/src/utils/utils.jl b/src/utils/utils.jl index 9a836d039..ffec2b37f 100644 --- a/src/utils/utils.jl +++ b/src/utils/utils.jl @@ -437,7 +437,36 @@ function transform_array_for_hdf(data::Vector{<:Real}) return data end -function transform_array_for_hdf(data::SortedDict{Dates.DateTime, Vector{POLYNOMIAL}}) +transform_array_for_hdf(data::SortedDict{Dates.DateTime, Vector{LinearFunctionData}}) = + transform_array_for_hdf( + SortedDict{Dates.DateTime, Vector{CONSTANT}}( + k => get_proportional_term.(v) for (k, v) in data + ), + ) + +transform_array_for_hdf(data::Vector{LinearFunctionData}) = + transform_array_for_hdf(get_proportional_term.(data)) + +transform_array_for_hdf(data::Vector{PolynomialFunctionData}) = + throw(ArgumentError("Not yet implemented for PolynomialFunctionData")) + +transform_array_for_hdf(data::SortedDict{Dates.DateTime, Vector{PolynomialFunctionData}}) = + throw(ArgumentError("Not yet implemented for PolynomialFunctionData")) + +function transform_array_for_hdf( + data::SortedDict{Dates.DateTime, Vector{QuadraticFunctionData}}, +) + all(get_constant_term.(vcat(values(data)...)) .== 0) || + throw( + ArgumentError( + "Not yet implemented for nonzero constant term ($(get_constant_term.(vcat(values(data)...))))", + ), + ) + data = SortedDict( + k => + [(get_quadratic_term(q), (get_proportional_term(q))) for q in v] for + (k, v) in data + ) lin_cost = hcat(values(data)...) rows, cols = size(lin_cost) @assert_op length(first(lin_cost)) == 2 @@ -451,7 +480,10 @@ function transform_array_for_hdf(data::SortedDict{Dates.DateTime, Vector{POLYNOM return t_lin_cost end -function transform_array_for_hdf(data::Vector{POLYNOMIAL}) +function transform_array_for_hdf(data::Vector{QuadraticFunctionData}) + all(get_constant_term.(data) .== 0) || + throw(ArgumentError("Not yet implemented for nonzero constant term")) + data = [(get_quadratic_term(q), (get_proportional_term(q))) for q in data] rows = length(data) @assert_op length(first(data)) == 2 t_lin_cost = Array{Float64}(undef, rows, 1, 2) @@ -464,8 +496,10 @@ function transform_array_for_hdf(data::Vector{POLYNOMIAL}) return t_lin_cost end -function transform_array_for_hdf(data::SortedDict{Dates.DateTime, Vector{PWL}}) - quad_cost = hcat(values(data)...) +function transform_array_for_hdf( + data::SortedDict{Dates.DateTime, Vector{PiecewiseLinearPointData}}, +) + quad_cost = hcat([get_points.(v) for v in values(data)]...) rows, cols = size(quad_cost) tuple_length = length(first(quad_cost)) @assert_op length(first(first(quad_cost))) == 2 @@ -481,7 +515,10 @@ function transform_array_for_hdf(data::SortedDict{Dates.DateTime, Vector{PWL}}) return t_quad_cost end -function transform_array_for_hdf(data::Vector{PWL}) +# TODO: old code here does not properly handle data with different numbers of points +# TODO: remove duplication +function transform_array_for_hdf(data::Vector{PiecewiseLinearPointData}) + data = get_points.(data) rows = length(data) tuple_length = length(first(data)) @assert_op length(first(first(data))) == 2 @@ -496,3 +533,10 @@ function transform_array_for_hdf(data::Vector{PWL}) end return t_quad_cost end + +transform_array_for_hdf( + data::SortedDict{Dates.DateTime, Vector{T}}) where {T <: FunctionData} = + throw(ArgumentError("Not currently implemented for $T")) + +transform_array_for_hdf(data::Vector{T}) where {T <: FunctionData} = + throw(ArgumentError("Not currently implemented for $T")) diff --git a/test/test_time_series.jl b/test/test_time_series.jl index 3822cd189..e92a9b4b3 100644 --- a/test/test_time_series.jl +++ b/test/test_time_series.jl @@ -130,12 +130,14 @@ end other_time = initial_time + resolution name = "test" horizon = 24 - polynomial_cost = repeat([(999.0, 1.0)], 24) + linear_cost = repeat([IS.LinearFunctionData(3.14)], 24) + data_linear = SortedDict(initial_time => linear_cost, other_time => linear_cost) + polynomial_cost = repeat([IS.QuadraticFunctionData(999.0, 1.0, 0.0)], 24) data_polynomial = SortedDict(initial_time => polynomial_cost, other_time => polynomial_cost) - pwl_cost = repeat([repeat([(999.0, 1.0)], 5)], 24) + pwl_cost = repeat([IS.PiecewiseLinearPointData(repeat([(999.0, 1.0)], 5))], 24) data_pwl = SortedDict(initial_time => pwl_cost, other_time => pwl_cost) - for d in [data_polynomial, data_pwl] + for d in [data_linear, data_polynomial, data_pwl] @testset "Add deterministic from $(typeof(d))" begin sys = IS.SystemData() component_name = "Component1" @@ -148,6 +150,16 @@ end end end + data_ts_linear = Dict( + initial_time => TimeSeries.TimeArray( + range(initial_time; length = horizon, step = resolution), + linear_cost, + ), + other_time => TimeSeries.TimeArray( + range(other_time; length = horizon, step = resolution), + linear_cost, + ), + ) data_ts_polynomial = Dict( initial_time => TimeSeries.TimeArray( range(initial_time; length = horizon, step = resolution), @@ -168,7 +180,7 @@ end pwl_cost, ), ) - for d in [data_ts_polynomial, data_ts_pwl] + for d in [data_ts_linear, data_ts_polynomial, data_ts_pwl] @testset "Add deterministic from $(typeof(d))" begin sys = IS.SystemData() component_name = "Component1" @@ -260,21 +272,7 @@ end @test IS.get_initial_timestamp(forecast) == initial_time end -@testset "Test add SingleTimeSeries" begin - sys = IS.SystemData() - name = "Component1" - component = IS.TestComponent(name, 5) - IS.add_component!(sys, component) - - initial_time = Dates.DateTime("2020-09-01") - resolution = Dates.Hour(1) - - data = TimeSeries.TimeArray( - range(initial_time; length = 365, step = resolution), - ones(365), - ) - data = IS.SingleTimeSeries(; data = data, name = "test_c") - IS.add_time_series!(sys, component, data) +function _test_add_single_time_series_helper(component, initial_time) ts1 = IS.get_time_series( IS.SingleTimeSeries, component, @@ -298,6 +296,26 @@ end start_time = initial_time + Dates.Day(1), ) @test length(IS.get_data(ts3)) == 341 +end + +@testset "Test add SingleTimeSeries" begin + sys = IS.SystemData() + name = "Component1" + component = IS.TestComponent(name, 5) + IS.add_component!(sys, component) + + initial_time = Dates.DateTime("2020-09-01") + resolution = Dates.Hour(1) + + data = TimeSeries.TimeArray( + range(initial_time; length = 365, step = resolution), + ones(365), + ) + data = IS.SingleTimeSeries(; data = data, name = "test_c") + IS.add_time_series!(sys, component, data) + + _test_add_single_time_series_helper(component, initial_time) + #Throws errors @test_throws ArgumentError IS.get_time_series( IS.SingleTimeSeries, @@ -670,7 +688,7 @@ end @test length(IS.get_components(IS.TestComponent, sys)) == 0 end -@testset "Test add SingleTimeSeries with Polynomial Cost" begin +function _test_add_single_time_series_type(test_value, type_name) sys = IS.SystemData() name = "Component1" component = IS.TestComponent(name, 5) @@ -678,83 +696,38 @@ end initial_time = Dates.DateTime("2020-09-01") resolution = Dates.Hour(1) - other_time = initial_time + resolution - polynomial_cost = repeat([(999.0, 1.0)], 365) - data_polynomial = TimeSeries.TimeArray( - range(initial_time; length = 365, step = resolution), - polynomial_cost, - ) - data = IS.SingleTimeSeries(; data = data_polynomial, name = "test_c") + data_series = + TimeSeries.TimeArray( + range(initial_time; length = 365, step = resolution), + test_value, + ) + data = IS.SingleTimeSeries(; data = data_series, name = "test_c") IS.add_time_series!(sys, component, data) ts = IS.get_time_series(IS.SingleTimeSeries, component, "test_c";) - @test IS.get_data_type(ts) == "POLYNOMIAL" - @test reshape(TimeSeries.values(IS.get_data(ts)), 365) == - TimeSeries.values(data_polynomial) - ts1 = IS.get_time_series( - IS.SingleTimeSeries, - component, - "test_c"; - start_time = initial_time, - len = 12, - ) - @test length(IS.get_data(ts1)) == 12 - ts2 = IS.get_time_series( - IS.SingleTimeSeries, - component, - "test_c"; - start_time = initial_time + Dates.Day(1), - len = 12, - ) - @test length(IS.get_data(ts2)) == 12 - ts3 = IS.get_time_series( - IS.SingleTimeSeries, - component, - "test_c"; - start_time = initial_time + Dates.Day(1), - ) - @test length(IS.get_data(ts3)) == 341 + @test IS.get_data_type(ts) == type_name + @test reshape(TimeSeries.values(IS.get_data(ts)), 365) == TimeSeries.values(data_series) + _test_add_single_time_series_helper(component, initial_time) end -@testset "Test add SingleTimeSeries with PWL Cost" begin - sys = IS.SystemData() - name = "Component1" - component = IS.TestComponent(name, 5) - IS.add_component!(sys, component) - - initial_time = Dates.DateTime("2020-09-01") - resolution = Dates.Hour(1) - other_time = initial_time + resolution - pwl_cost = repeat([repeat([(999.0, 1.0)], 5)], 365) - data_pwl = - TimeSeries.TimeArray(range(initial_time; length = 365, step = resolution), pwl_cost) - data = IS.SingleTimeSeries(; data = data_pwl, name = "test_c") - IS.add_time_series!(sys, component, data) - ts = IS.get_time_series(IS.SingleTimeSeries, component, "test_c";) - @test IS.get_data_type(ts) == "PWL" - @test reshape(TimeSeries.values(IS.get_data(ts)), 365) == TimeSeries.values(data_pwl) - ts1 = IS.get_time_series( - IS.SingleTimeSeries, - component, - "test_c"; - start_time = initial_time, - len = 12, +@testset "Test add SingleTimeSeries with LinearFunctionData Cost" begin + _test_add_single_time_series_type( + repeat([IS.LinearFunctionData(3.14)], 365), + "LinearFunctionData", ) - @test length(IS.get_data(ts1)) == 12 - ts2 = IS.get_time_series( - IS.SingleTimeSeries, - component, - "test_c"; - start_time = initial_time + Dates.Day(1), - len = 12, +end + +@testset "Test add SingleTimeSeries with QuadraticFunctionData Cost" begin + _test_add_single_time_series_type( + repeat([IS.QuadraticFunctionData(999.0, 1.0, 0.0)], 365), + "QuadraticFunctionData", ) - @test length(IS.get_data(ts2)) == 12 - ts3 = IS.get_time_series( - IS.SingleTimeSeries, - component, - "test_c"; - start_time = initial_time + Dates.Day(1), +end + +@testset "Test add SingleTimeSeries with PiecewiseLinearPointData Cost" begin + _test_add_single_time_series_type( + repeat([IS.PiecewiseLinearPointData(repeat([(999.0, 1.0)], 5))], 365), + "PiecewiseLinearPointData", ) - @test length(IS.get_data(ts3)) == 341 end @testset "Test read_time_series_file_metadata" begin @@ -1465,109 +1438,45 @@ end @test IS.get_forecast_total_period(sys) == IS.get_total_period(forecast) end -@testset "Test get_time_series options" begin - for in_memory in (true, false) - sys = IS.SystemData(; time_series_in_memory = in_memory) - name = "Component1" - component = IS.TestComponent(name, 5) - IS.add_component!(sys, component) - - # Set baseline parameters for the rest of the tests. - resolution = Dates.Minute(5) - interval = Dates.Hour(1) - initial_timestamp = Dates.DateTime("2020-09-01") - initial_times = collect(range(initial_timestamp; length = 24, step = interval)) - name = "test" - horizon = 24 - data = SortedDict(it => ones(horizon) * i for (i, it) in enumerate(initial_times)) - - forecast = IS.Deterministic(name, data, resolution) - IS.add_time_series!(sys, component, forecast) - @test IS.get_forecast_window_count(sys) == length(data) - - f2 = IS.get_time_series(IS.Deterministic, component, name) - @test IS.get_count(f2) == length(data) - @test IS.get_initial_timestamp(f2) == initial_times[1] - for (i, window) in enumerate(IS.iterate_windows(f2)) - @test TimeSeries.values(window) == data[initial_times[i]] - end - - offset = 12 - count = 5 - it = initial_times[offset] - f2 = IS.get_time_series( - IS.Deterministic, - component, - name; - start_time = it, - count = count, - ) - @test IS.get_initial_timestamp(f2) == it - @test IS.get_count(f2) == count - @test IS.get_horizon(f2) == horizon - for (i, window) in enumerate(IS.iterate_windows(f2)) - @test TimeSeries.values(window) == data[initial_times[i + offset - 1]] - end +# TODO something like this could be much more widespread to reduce code duplication +default_time_params = ( + interval = Dates.Hour(1), + initial_timestamp = Dates.DateTime("2020-09-01"), + initial_times = collect( + range(Dates.DateTime("2020-09-01"); length = 24, step = Dates.Hour(1)), + ), + horizon = 24, +) + +function _test_get_time_series_option_type(test_data, in_memory, extended) + sys = IS.SystemData(; time_series_in_memory = in_memory) + name = "Component1" + component = IS.TestComponent(name, 5) + IS.add_component!(sys, component) - horizon -= 1 - f2 = IS.get_time_series( - IS.Deterministic, - component, - name; - start_time = it, - count = count, - len = horizon, - ) - @test IS.get_initial_timestamp(f2) == it - @test IS.get_count(f2) == count - @test IS.get_horizon(f2) == horizon - for (i, window) in enumerate(IS.iterate_windows(f2)) - @test TimeSeries.values(window) == - data[initial_times[i + offset - 1]][1:horizon] - end + # Set baseline parameters for the rest of the tests. + resolution = Dates.Minute(5) + name = "test" - @test_throws ArgumentError IS.get_time_series( - IS.Deterministic, - component, - name; - start_time = it + Dates.Minute(1), - ) + forecast = if extended + IS.Deterministic(; data = test_data, name = name, resolution = resolution) + else + IS.Deterministic(name, test_data, resolution) end -end - -@testset "Test get_time_series options for Polynomial Cost" begin - for in_memory in (true, false) - sys = IS.SystemData(; time_series_in_memory = in_memory) - name = "Component1" - component = IS.TestComponent(name, 5) - IS.add_component!(sys, component) - - # Set baseline parameters for the rest of the tests. - resolution = Dates.Minute(5) - interval = Dates.Hour(1) - initial_timestamp = Dates.DateTime("2020-09-01") - initial_times = collect(range(initial_timestamp; length = 24, step = interval)) - name = "test" - horizon = 24 - data_polynomial = SortedDict{Dates.DateTime, Vector{IS.POLYNOMIAL}}( - it => repeat([(999.0, 1.0 * i)], 24) for (i, it) in enumerate(initial_times) - ) - - forecast = - IS.Deterministic(; data = data_polynomial, name = name, resolution = resolution) - IS.add_time_series!(sys, component, forecast) - @test IS.get_forecast_window_count(sys) == length(data_polynomial) + IS.add_time_series!(sys, component, forecast) + @test IS.get_forecast_window_count(sys) == length(test_data) - f2 = IS.get_time_series(IS.Deterministic, component, name) - @test IS.get_count(f2) == length(data_polynomial) - @test IS.get_initial_timestamp(f2) == initial_times[1] - for (i, window) in enumerate(IS.iterate_windows(f2)) - @test TimeSeries.values(window) == data_polynomial[initial_times[i]] - end + f2 = IS.get_time_series(IS.Deterministic, component, name) + @test IS.get_count(f2) == length(test_data) + @test IS.get_initial_timestamp(f2) == default_time_params.initial_times[1] + for (i, window) in enumerate(IS.iterate_windows(f2)) + @test TimeSeries.values(window) == test_data[default_time_params.initial_times[i]] + end + if extended offset = 1 count = 1 - it = initial_times[offset] + it = default_time_params.initial_times[offset] f2 = IS.get_time_series( IS.Deterministic, component, @@ -1577,144 +1486,97 @@ end ) @test IS.get_initial_timestamp(f2) == it @test IS.get_count(f2) == count - @test IS.get_horizon(f2) == horizon + @test IS.get_horizon(f2) == default_time_params.horizon for (i, window) in enumerate(IS.iterate_windows(f2)) @test TimeSeries.values(window) == - data_polynomial[initial_times[i + offset - 1]] + test_data[default_time_params.initial_times[i + offset - 1]] end + end - offset = 12 - count = 5 - it = initial_times[offset] - f2 = IS.get_time_series( - IS.Deterministic, - component, - name; - start_time = it, - count = count, - ) - @test IS.get_initial_timestamp(f2) == it - @test IS.get_count(f2) == count - @test IS.get_horizon(f2) == horizon - for (i, window) in enumerate(IS.iterate_windows(f2)) - @test TimeSeries.values(window) == - data_polynomial[initial_times[i + offset - 1]] - end + offset = 12 + count = 5 + it = default_time_params.initial_times[offset] + f2 = IS.get_time_series( + IS.Deterministic, + component, + name; + start_time = it, + count = count, + ) + @test IS.get_initial_timestamp(f2) == it + @test IS.get_count(f2) == count + @test IS.get_horizon(f2) == default_time_params.horizon + for (i, window) in enumerate(IS.iterate_windows(f2)) + @test TimeSeries.values(window) == + test_data[default_time_params.initial_times[i + offset - 1]] + end - horizon -= 1 - f2 = IS.get_time_series( - IS.Deterministic, - component, - name; - start_time = it, - count = count, - len = horizon, - ) - @test IS.get_initial_timestamp(f2) == it - @test IS.get_count(f2) == count - @test IS.get_horizon(f2) == horizon - for (i, window) in enumerate(IS.iterate_windows(f2)) - @test TimeSeries.values(window) == - data_polynomial[initial_times[i + offset - 1]][1:horizon] - end + f2 = IS.get_time_series( + IS.Deterministic, + component, + name; + start_time = it, + count = count, + len = default_time_params.horizon - 1, + ) + @test IS.get_initial_timestamp(f2) == it + @test IS.get_count(f2) == count + @test IS.get_horizon(f2) == default_time_params.horizon - 1 + for (i, window) in enumerate(IS.iterate_windows(f2)) + @test TimeSeries.values(window) == + test_data[default_time_params.initial_times[i + offset - 1]][1:(default_time_params.horizon - 1)] + end - @test_throws ArgumentError IS.get_time_series( - IS.Deterministic, - component, - name; - start_time = it + Dates.Minute(1), + @test_throws ArgumentError IS.get_time_series( + IS.Deterministic, + component, + name; + start_time = it + Dates.Minute(1), + ) +end +@testset "Test get_time_series options" begin + for in_memory in (true, false) + _test_get_time_series_option_type( + SortedDict( + it => ones(default_time_params.horizon) * i for + (i, it) in enumerate(default_time_params.initial_times) + ), + in_memory, + false, ) end end -@testset "Test get_time_series options for PWL Cost" begin - #for in_memory in (true, false) - for in_memory in [false] - sys = IS.SystemData(; time_series_in_memory = in_memory) - name = "Component1" - component = IS.TestComponent(name, 5) - IS.add_component!(sys, component) - - # Set baseline parameters for the rest of the tests. - resolution = Dates.Minute(5) - interval = Dates.Hour(1) - initial_timestamp = Dates.DateTime("2020-09-01") - initial_times = collect(range(initial_timestamp; length = 24, step = interval)) - name = "test" - horizon = 24 - data_pwl = SortedDict{Dates.DateTime, Vector{IS.PWL}}( - it => repeat([repeat([(999.0, 1.0 * i)], 5)], 24) for - (i, it) in enumerate(initial_times) - ) - - forecast = IS.Deterministic(; data = data_pwl, name = name, resolution = resolution) - IS.add_time_series!(sys, component, forecast) - @test IS.get_forecast_window_count(sys) == length(data_pwl) - - f2 = IS.get_time_series(IS.Deterministic, component, name) - @test IS.get_count(f2) == length(data_pwl) - @test IS.get_initial_timestamp(f2) == initial_times[1] - for (i, window) in enumerate(IS.iterate_windows(f2)) - @test TimeSeries.values(window) == data_pwl[initial_times[i]] - end - - offset = 1 - count = 1 - it = initial_times[offset] - f2 = IS.get_time_series( - IS.Deterministic, - component, - name; - start_time = it, - count = count, - ) - @test IS.get_initial_timestamp(f2) == it - @test IS.get_count(f2) == count - @test IS.get_horizon(f2) == horizon - for (i, window) in enumerate(IS.iterate_windows(f2)) - @test TimeSeries.values(window) == data_pwl[initial_times[i + offset - 1]] - end - - offset = 12 - count = 5 - it = initial_times[offset] - f2 = IS.get_time_series( - IS.Deterministic, - component, - name; - start_time = it, - count = count, - ) - @test IS.get_initial_timestamp(f2) == it - @test IS.get_count(f2) == count - @test IS.get_horizon(f2) == horizon - for (i, window) in enumerate(IS.iterate_windows(f2)) - @test TimeSeries.values(window) == data_pwl[initial_times[i + offset - 1]] - end +@testset "Test get_time_series options for LinearFunctionData Cost" begin + for in_memory in (true, false) + _test_get_time_series_option_type( + SortedDict{Dates.DateTime, Vector{IS.LinearFunctionData}}( + it => repeat([IS.LinearFunctionData(3.14 * i)], 24) for + (i, it) in enumerate(default_time_params.initial_times) + ), in_memory, true) + end +end - horizon -= 1 - f2 = IS.get_time_series( - IS.Deterministic, - component, - name; - start_time = it, - count = count, - len = horizon, - ) - @test IS.get_initial_timestamp(f2) == it - @test IS.get_count(f2) == count - @test IS.get_horizon(f2) == horizon - for (i, window) in enumerate(IS.iterate_windows(f2)) - @test TimeSeries.values(window) == - data_pwl[initial_times[i + offset - 1]][1:horizon] - end +@testset "Test get_time_series options for QuadraticFunctionData Cost" begin + for in_memory in (true, false) + _test_get_time_series_option_type( + SortedDict{Dates.DateTime, Vector{IS.QuadraticFunctionData}}( + it => repeat([IS.QuadraticFunctionData(999.0, 1.0 * i, 0.0)], 24) for + (i, it) in enumerate(default_time_params.initial_times) + ), in_memory, true) + end +end - @test_throws ArgumentError IS.get_time_series( - IS.Deterministic, - component, - name; - start_time = it + Dates.Minute(1), - ) +@testset "Test get_time_series options for PiecewiseLinearPointData Cost" begin + for in_memory in (true, false) + _test_get_time_series_option_type( + SortedDict{Dates.DateTime, Vector{IS.PiecewiseLinearPointData}}( + it => repeat( + [IS.PiecewiseLinearPointData(repeat([(999.0, 1.0 * i)], 5))], + 24, + ) for + (i, it) in enumerate(default_time_params.initial_times) + ), in_memory, true) end end