Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix performance issue with reading time series #410

Merged
merged 2 commits into from
Oct 31, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 1 addition & 12 deletions src/single_time_series.jl
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ function SingleTimeSeries(;
return SingleTimeSeries(
name,
data,
_get_resolution(data),
get_resolution(data),
scaling_factor_multiplier,
internal,
)
Expand Down Expand Up @@ -72,17 +72,6 @@ function SingleTimeSeries(
)
end

function _get_resolution(data::TimeSeries.TimeArray)
if length(data) < 2
throw(
ConflictingInputsError(
"Resolution can't be inferred from the data. Please select an appropiate constructor.",
),
)
end
return TimeSeries.timestamp(data)[2] - TimeSeries.timestamp(data)[1]
end

"""
Construct SingleTimeSeries from a TimeArray or DataFrame.

Expand Down
26 changes: 22 additions & 4 deletions src/utils/utils.jl
Original file line number Diff line number Diff line change
Expand Up @@ -385,11 +385,18 @@
Return the resolution from a TimeArray.
"""
function get_resolution(ts::TimeSeries.TimeArray)
if length(ts) < 2
throw(ConflictingInputsError("Resolution can't be inferred from the data."))

Check warning on line 389 in src/utils/utils.jl

View check run for this annotation

Codecov / codecov/patch

src/utils/utils.jl#L389

Added line #L389 was not covered by tests
end

timestamps = TimeSeries.timestamp(ts)
return timestamps[2] - timestamps[1]
end

function check_resolution(ts::TimeSeries.TimeArray)
tstamps = TimeSeries.timestamp(ts)
timediffs = unique([tstamps[ix] - tstamps[ix - 1] for ix in 2:length(tstamps)])

res = []

for timediff in timediffs
if mod(timediff, Dates.Millisecond(Dates.Day(1))) == Dates.Millisecond(0)
push!(res, Dates.Day(timediff / Dates.Millisecond(Dates.Day(1))))
Expand Down Expand Up @@ -419,14 +426,25 @@
return TimeSeries.timestamp(data)[1]
end

function get_module(module_name)
# Looking up modules with Base.root_module is slow; cache them.
const g_cached_modules = Dict{String, Module}()

function get_module(module_name::AbstractString)
cached_module = get(g_cached_modules, module_name, nothing)
if !isnothing(cached_module)
return cached_module
end

# root_module cannot find InfrastructureSystems if it hasn't been installed by the
# user (but has been installed as a dependency to another package).
return if module_name == "InfrastructureSystems"
mod = if module_name == "InfrastructureSystems"
InfrastructureSystems
else
Base.root_module(Base.__toplevel__, Symbol(module_name))
end

g_cached_modules[module_name] = mod
return mod
end

get_type_from_strings(module_name, type) =
Expand Down
19 changes: 19 additions & 0 deletions test/test_utils.jl
Original file line number Diff line number Diff line change
Expand Up @@ -53,3 +53,22 @@ IS.get_name(::FakeTimeSeries) = "fake"
sprint(show, MIME("text/plain"), summary(FakeTimeSeries())),
)
end

@testset "Test check_resolution" begin
initial_time1 = Dates.DateTime("2020-09-01")
len1 = 24
resolution1 = Dates.Hour(1)
timestamps1 = range(initial_time1; length = len1, step = resolution1)
ta1 = TimeSeries.TimeArray(timestamps1, rand(len1))
IS.check_resolution(ta1) == resolution1

initial_time2 = Dates.DateTime("2020-09-02")
len2 = 12
resolution2 = Dates.Minute(5)
timestamps2 = range(initial_time2; length = len2, step = resolution2)
ta2 = TimeSeries.TimeArray(timestamps2, rand(len2))
IS.check_resolution(ta2) == resolution2

ta3 = vcat(ta1, ta2)
@test_throws IS.DataFormatError IS.check_resolution(ta3)
end
Loading