diff --git a/PRAS.jl/examples/pras_adequacy_metrics.jl b/PRAS.jl/examples/pras_adequacy_metrics.jl new file mode 100644 index 00000000..6ab242e4 --- /dev/null +++ b/PRAS.jl/examples/pras_adequacy_metrics.jl @@ -0,0 +1,167 @@ +# # Interpreting Resource Adequacy Metrics +# +# In practice, no single metric fully captures system adequacy. Instead, +# multiple complementary metrics should be considered together to understand +# the frequency, distribution and severity of shortfall events. +# ([NERC (2018)](https://www.nerc.com/globalassets/who-we-are/standing-committees/rstc/pawg/probabilistic_adequacy_and_measures_report.pdf), +# [EPRI](https://www.epri.com/research/products/3002027833), +# [Stephen et al. 2022](https://doi.org/10.1109/PMAPS53380.2022.9810615)). +# +# For this reason, PRAS provides multiple result specifications and derived +# metrics that allow different aspects of system risk to be evaluated +# consistently. + +# ## Event-Based Interpretation +# +# A useful way to interpret adequacy metrics is through the concept of +# **event-periods**. +# +# - An **event-period** occurs when shortfall exists in a given simulation time step +# - An **event-day** occurs when at least one event-period occurs within a day +# +# An **adequacy event** is a set of event-periods that are contiguous at the +# highest available temporal resolution +# ([Stephen et al. 2022](https://doi.org/10.1109/PMAPS53380.2022.9810615)). +# +# This distinction is important because different metrics count different +# quantities: +# +# - **LOLE** counts event-periods +# - **LOLD** counts event-days +# - **LOLEv** counts adequacy events +# +# These metrics are related, but they are not interchangeable. +# +#md # !!! note +#md # In PRAS the time resolution of LOLE is determined by the +#md # simulation timestamps of the system and is not assumed to always be hourly. + +# Another important reason to use multiple metrics, as described in +# ([Stephen et al. 2022](https://doi.org/10.1109/PMAPS53380.2022.9810615)), +# is that systems with similar shortfall magnitudes or counts of event-periods +# can exhibit very different temporal patterns. +# +# We can consider a simple example of two cases next: +# +# **Case A**: One day with 10 hours of shortfall +# +# **Case B**: Ten days with 1 hour of shortfall each +# +# | Metric | Case A | Case B | +# |------|--------|--------| +# | LOLE | same | same | +# | EUE | same | same | +# | LOLD | 1 | 10 | +# +# As we can see in the table above, even though LOLE and EUE are identical in this case, +# LOLD reveals that shortfall events are more dispersed in Case B. +# + +# Because event-periods may be distributed across many days, a system with the +# same number of shortfall periods can have very different numbers of event-days. +# As a result, exact conversions between hourly and daily adequacy +# criteria are not generally possible +# ([Stephen et al. 2022](https://doi.org/10.1109/PMAPS53380.2022.9810615)). + +# This behavior is reflected in PRAS results, where LOLE and LOLD provide +# complementary views of how shortfall events are distributed in time. + +# ## Mathematical Interpretation +# +# In PRAS, adequacy metrics can be interpreted from Monte Carlo shortfall +# samples. +# +# Using the following notation: +# +# - ``r`` indexes regions +# - ``t`` indexes timestamps +# - ``d`` indexes calendar days +# - ``s`` indexes Monte Carlo samples +# - ``e`` indexes adequacy events +# - ``S_{r,t,s}`` denotes the shortfall in region ``r``, at timestamp ``t``, +# in Monte Carlo sample ``s`` +# - ``T(d)`` is the set of timestamps in day ``d`` +# +# the adequacy metrics can be expressed as expectations over Monte Carlo samples: +# +# ### LOLE +# +# LOLE counts the expected number of event-periods with shortfall: + +# ```math +# \mathrm{LOLE} = +# \mathbb{E}\left[\sum_t +# \mathbf{1}\left(\sum_r S_{r,t,s} > 0\right)\right] +# ``` +# +# +# ### LOLD +# +# LOLD counts the expected number of days containing at least one shortfall: + +# ```math +# \mathrm{LOLD} = \mathbb{E}\left[\sum_d I_{d,s}\right] +# ``` +# +# where: + +# ```math +# I_{d,s} = +# \begin{cases} +# 1 & \text{if } \exists t \in T(d) \text{ such that } \sum_r S_{r,t,s} > 0 \\ +# 0 & \text{otherwise} +# \end{cases} +# ``` +# +# +# ### LOLEv +# +# LOLEv counts the expected number of adequacy events: + +# ```math +# \mathrm{LOLEv} = \mathbb{E}\left[\sum_e J_{e,s}\right] +# ``` +# +# where: + +# ```math +# J_{e,s} = +# \begin{cases} +# 1 & \text{if adequacy event } e \text{ occurs in sample } s \\ +# 0 & \text{otherwise} +# \end{cases} +# ``` + +# ## Analysis with PRAS +# +# We revisit the RTS-GMLC with increased system load to induce shortfall +# which was described in ref(@id pras_walkthrough) + +using PRAS +sys = PRAS.rts_gmlc() +sys.regions.load .+= 700.0 + +shortfall_samples, = assess( + sys, + SequentialMonteCarlo(samples=100, seed=1), + ShortfallSamples(), +) + +# And print the metrics we discussed above: +println(LOLE(shortfall_samples)) +println(LOLD(shortfall_samples)) + +# LOLE describes how many simulation periods experience shortfall, while LOLD +# describes how many days contain at least one such period. +# +# In the RTS example above, the system has approximately 85 shortfall hours +# but only 25.8 shortfall days. This indicates that shortfall events are +# temporally clustered, meaning that multiple shortfall hours tend to occur within the +# same day rather than being evenly distributed across the year. + + +# ## References +# +# - [NERC (2018), *Probabilistic Adequacy and Measures Technical Reference Report*](https://www.nerc.com/globalassets/who-we-are/standing-committees/rstc/pawg/probabilistic_adequacy_and_measures_report.pdf) +# - [EPRI, *Resource Adequacy Gap Assessment: Resource Adequacy Assessment Framework*](https://www.epri.com/research/products/3002027833) +# - [Stephen et al. (2022), *Clarifying the Interpretation and Use of the LOLE Resource Adequacy Metric*](https://doi.org/10.1109/PMAPS53380.2022.9810615) \ No newline at end of file diff --git a/PRAS.jl/examples/pras_walkthrough.jl b/PRAS.jl/examples/pras_walkthrough.jl index 081f982a..d9f26b1c 100644 --- a/PRAS.jl/examples/pras_walkthrough.jl +++ b/PRAS.jl/examples/pras_walkthrough.jl @@ -176,3 +176,28 @@ println("Surplus in") # performed on the subset of samples in which the event was observed, using the # `ShortfallSamples`, `UtilizationSamples`, and # `StorageEnergySamples` result specifications instead. + + +# ## Export Aggregate Results + +# After exploring the simulation outputs, we may want to save the +# aggregate results for reporting or further post-processing. + +# Rather than querying individual metrics (e.g., LOLE, EUE, NEUE) +# one by one, PRAS provides a utility to export all aggregate +# system-level and region-level results in a single step. + +using PRASFiles + +output_path = saveshortfall(shortfall, sys, "pras_output"); +println("Results exported to: ", output_path) + +# This creates a timestamped directory containing a `pras_results.json` +# file with: +# - system-level metrics (LOLE, EUE, NEUE) +# - region-level metrics +# - load and capacity summaries +# - horizon timestamps + +# Note that only aggregate results are exported. Sample-level data +# from the Monte Carlo simulation are not included. \ No newline at end of file diff --git a/PRASCore.jl/src/Results/Results.jl b/PRASCore.jl/src/Results/Results.jl index 3e4e3300..42f64ff2 100644 --- a/PRASCore.jl/src/Results/Results.jl +++ b/PRASCore.jl/src/Results/Results.jl @@ -5,6 +5,7 @@ import OnlineStats: Series import OnlineStatsBase: EqualWeight, Mean, Variance, value import Printf: @sprintf import StatsBase: mean, std, stderror +import Dates: Date import ..Systems: SystemModel, ZonedDateTime, Period, PowerUnit, EnergyUnit, conversionfactor, @@ -12,7 +13,7 @@ import ..Systems: SystemModel, ZonedDateTime, Period, export # Metrics - ReliabilityMetric, LOLE, EUE, NEUE, + ReliabilityMetric, LOLE, EUE, NEUE, LOLD, val, stderror, # Result specifications @@ -80,6 +81,15 @@ NEUE(x::AbstractShortfallResult, r::AbstractString, ::Colon) = NEUE(x::AbstractShortfallResult, ::Colon, ::Colon) = NEUE.(x, x.regions.names, permutedims(x.timestamps)) +LOLD(x::AbstractShortfallResult, ::Colon, d::Date) = + LOLD.(x, x.regions.names, d) + +LOLD(x::AbstractShortfallResult, r::AbstractString, ::Colon) = + LOLD.(x, r, _unique_days(x.timestamps)) + +LOLD(x::AbstractShortfallResult, ::Colon, ::Colon) = + LOLD.(x, x.regions.names, permutedims(_unique_days(x.timestamps))) + include("Shortfall.jl") include("ShortfallSamples.jl") diff --git a/PRASCore.jl/src/Results/ShortfallSamples.jl b/PRASCore.jl/src/Results/ShortfallSamples.jl index 99106433..e5a2ab8b 100644 --- a/PRASCore.jl/src/Results/ShortfallSamples.jl +++ b/PRASCore.jl/src/Results/ShortfallSamples.jl @@ -15,6 +15,7 @@ shortfall, = assess(sys, SequentialMonteCarlo(samples=10), ShortfallSamples()) period = ZonedDateTime(2020, 1, 1, 0, tz"UTC") +day = Date(2020, 1, 1) samples = shortfall["Region A", period] @@ -25,19 +26,23 @@ samples = shortfall["Region A", period] eue = EUE(shortfall) lole = LOLE(shortfall) neue = NEUE(shortfall) +lold = LOLD(shortfall) # Regional risk metrics regional_eue = EUE(shortfall, "Region A") regional_lole = LOLE(shortfall, "Region A") regional_neue = NEUE(shortfall, "Region A") +regional_lold = LOLD(shortfall, "Region A") # Period-specific risk metrics period_eue = EUE(shortfall, period) period_lolp = LOLE(shortfall, period) +day_lold = LOLD(shortfall, day) # Region- and period-specific risk metrics period_eue = EUE(shortfall, "Region A", period) period_lolp = LOLE(shortfall, "Region A", period) +regional_day_lold = LOLD(shortfall, "Region A", day) ``` Note that this result specification requires large amounts of memory for @@ -194,3 +199,68 @@ function finalize( system.regions, system.timestamps, acc.shortfall) end + +function _count_dropped_days_by_sample(flags_by_period_sample, day_ids) + nperiods, nsamples = size(flags_by_period_sample) + counts = zeros(Int, nsamples) + + isempty(day_ids) && return counts + + for s in 1:nsamples + current_day = day_ids[1] + day_has_shortfall = false + dropped_days = 0 + + for t in 1:nperiods + if day_ids[t] != current_day + dropped_days += day_has_shortfall + current_day = day_ids[t] + day_has_shortfall = false + end + + day_has_shortfall |= flags_by_period_sample[t, s] + end + + dropped_days += day_has_shortfall + counts[s] = dropped_days + end + + return counts +end + +function LOLD(x::ShortfallSamplesResult) + flags = dropdims(sum(x.shortfall, dims=1) .> 0, dims=1) + daycounts = _count_dropped_days_by_sample(flags, _day_ids(x.timestamps)) + return LOLD{_ndays(x.timestamps)}(MeanEstimate(daycounts)) +end + +function LOLD(x::ShortfallSamplesResult, r::AbstractString) + i_r = findfirstunique(x.regions.names, r) + flags = Matrix(view(x.shortfall, i_r, :, :) .> 0) + daycounts = _count_dropped_days_by_sample(flags, _day_ids(x.timestamps)) + return LOLD{_ndays(x.timestamps)}(MeanEstimate(daycounts)) +end + +function LOLD(x::ShortfallSamplesResult, d::Date) + dayrange = _day_range(x.timestamps, d) + + # day_slice has shape: region × day_timestamps × sample + day_slice = view(x.shortfall, :, dayrange, :) + + # For each sample, event occurs if any region has any shortfall in any timestep of the day + eventdays = vec(dropdims(any(day_slice .> 0, dims=(1, 2)), dims=(1, 2))) + + return LOLD{1}(MeanEstimate(eventdays)) +end + +function LOLD(x::ShortfallSamplesResult, r::AbstractString, d::Date) + i_r = findfirstunique(x.regions.names, r) + dayrange = _day_range(x.timestamps, d) + + region_day_slice = view(x.shortfall, i_r, dayrange, :) + + # For each sample, did this region have any shortfall during the day? + eventdays = vec(dropdims(any(region_day_slice .> 0, dims=1), dims=1)) + + return LOLD{1}(MeanEstimate(eventdays)) +end diff --git a/PRASCore.jl/src/Results/metrics.jl b/PRASCore.jl/src/Results/metrics.jl index f44c7096..33075599 100644 --- a/PRASCore.jl/src/Results/metrics.jl +++ b/PRASCore.jl/src/Results/metrics.jl @@ -164,3 +164,32 @@ function Base.show(io::IO, x::NEUE) print(io, "NEUE = ", x.neue, " ppm") end + + +""" + LOLD + +`LOLD` reports loss of load days over a particular time period +and regional extent. + +Contains both the estimated value itself as well as the standard error +of that estimate, which can be extracted with `val` and `stderror`, +respectively. +""" +struct LOLD{D} <: ReliabilityMetric + lold::MeanEstimate + + function LOLD{D}(lold::MeanEstimate) where {D} + val(lold) >= 0 || throw(DomainError(val(lold), + "$(val(lold)) is not a valid expected count of event-days")) + new{D}(lold) + end +end + +val(x::LOLD) = val(x.lold) +stderror(x::LOLD) = stderror(x.lold) + +function Base.show(io::IO, x::LOLD{D}) where {D} + print(io, "LOLD = ", x.lold, " event-day/", + D == 1 ? "day" : string(D) * "days") +end diff --git a/PRASCore.jl/src/Results/utils.jl b/PRASCore.jl/src/Results/utils.jl index 41782995..95934e34 100644 --- a/PRASCore.jl/src/Results/utils.jl +++ b/PRASCore.jl/src/Results/utils.jl @@ -40,3 +40,54 @@ function findfirstunique(a::AbstractVector{T}, i::T) where T i_idx === nothing && throw(BoundsError(a)) return i_idx end + +function _day_ids(timestamps) + n = length(timestamps) + ids = Vector{Int}(undef, n) + + n == 0 && return ids + + current_day = Date(first(timestamps)) + current_id = 1 + ids[1] = current_id + + for i in 2:n + d = Date(timestamps[i]) + if d != current_day + current_day = d + current_id += 1 + end + ids[i] = current_id + end + + return ids +end + +function _ndays(timestamps) + if isempty(timestamps) + return 0 + else + return last(_day_ids(timestamps)) + end +end + +function _unique_days(timestamps) + return unique(Date.(timestamps)) +end + +function _day_range(timestamps, d::Date) + n = length(timestamps) + n == 0 && throw(ArgumentError("date $(d) is not in the simulation horizon (empty horizon)")) + + first_i = findfirst(t -> Date(t) == d, timestamps) + first_i === nothing && throw(ArgumentError( + "date $(d) is not in the simulation horizon ($(Date(first(timestamps))) to $(Date(last(timestamps))))" + )) + + last_i = first_i + while last_i < n && Date(timestamps[last_i + 1]) == d + last_i += 1 + end + + return first_i:last_i +end \ No newline at end of file diff --git a/PRASCore.jl/test/Simulations/runtests.jl b/PRASCore.jl/test/Simulations/runtests.jl index 3c518634..2511ef9a 100644 --- a/PRASCore.jl/test/Simulations/runtests.jl +++ b/PRASCore.jl/test/Simulations/runtests.jl @@ -590,5 +590,112 @@ @test isapprox(sum(dr_shortfall_samples["Region 1",dts[5]])/1e4,TestData.threenode_dr_shortfall_samples/1e4, rtol=0.01) end + @testset "LOLD" begin + @testset "Whole-horizon equals sum over days" begin + for x in (shortfall2_1a, shortfall2_1a5, shortfall2_1b, shortfall2_3) + days = unique(Date.(x.timestamps)) + + @test isapprox( + val(LOLD(x)), + sum(val(LOLD(x, d)) for d in days) + ) + + for r in x.regions.names + @test isapprox( + val(LOLD(x, r)), + sum(val(LOLD(x, r, d)) for d in days) + ) + end + end + end + + @testset "Single-day query matches direct sample calculation" begin + for x in (shortfall2_1a, shortfall2_1a5, shortfall2_1b, shortfall2_3) + days = unique(Date.(x.timestamps)) + + # test first, middle, and last day + testdays = unique([first(days), days[cld(length(days), 2)], last(days)]) + + for d in testdays + mask = Date.(x.timestamps) .== d + + # system-wide day event by sample: + # did any region have any shortfall in any timestep of this day? + manual_system = vec(any(dropdims(sum(x.shortfall[:, mask, :], dims=1), dims=1) .> 0, dims=1)) + expected_system = MeanEstimate(manual_system) + + @test LOLD(x, d) ≈ LOLD{1}(expected_system) + + for r in x.regions.names + i_r = findfirst(isequal(r), x.regions.names) + + # region-day event by sample: + # did this region have any shortfall in any timestep of this day? + manual_region = vec(any(x.shortfall[i_r, mask, :] .> 0, dims=1)) + expected_region = MeanEstimate(manual_region) + + @test LOLD(x, r, d) ≈ LOLD{1}(expected_region) + end + end + end + end + + @testset "Broadcast helpers match scalar calls" begin + for x in (shortfall2_1a, shortfall2_1a5, shortfall2_1b, shortfall2_3) + days = unique(Date.(x.timestamps)) + + # region over all days + for r in x.regions.names + @test all(LOLD(x, r, :) .≈ LOLD.(Ref(x), r, days)) + end + + # all regions for one day + for d in (first(days), days[cld(length(days), 2)], last(days)) + @test all(LOLD(x, :, d) .≈ LOLD.(Ref(x), x.regions.names, d)) + end + + # full region x day grid + @test all(LOLD(x, :, :) .≈ LOLD.(Ref(x), x.regions.names, permutedims(days))) + end + end + + @testset "System day risk dominates regional day risk" begin + for x in (shortfall2_1a, shortfall2_1a5, shortfall2_1b, shortfall2_3) + for d in unique(Date.(x.timestamps)) + system_val = val(LOLD(x, d)) + for r in x.regions.names + @test system_val >= val(LOLD(x, r, d)) + end + end + end + end + + @testset "Invalid day throws useful error" begin + for x in (shortfall2_1a, shortfall2_1a5, shortfall2_1b, shortfall2_3) + bad_day = first(unique(Date.(x.timestamps))) - Day(1) + + err = try + LOLD(x, bad_day) + nothing + catch e + e + end + + @test err isa ArgumentError + @test occursin("simulation horizon", sprint(showerror, err)) + + err = try + LOLD(x, first(x.regions.names), bad_day) + nothing + catch e + e + end + + @test err isa ArgumentError + @test occursin("simulation horizon", sprint(showerror, err)) + end + end + end + end diff --git a/PRASFiles.jl/src/PRASFiles.jl b/PRASFiles.jl/src/PRASFiles.jl index 1baed2da..49320453 100644 --- a/PRASFiles.jl/src/PRASFiles.jl +++ b/PRASFiles.jl/src/PRASFiles.jl @@ -4,7 +4,7 @@ import PRASCore.Systems: SystemModel, Regions, Interfaces, Generators, Storages, GeneratorStorages, DemandResponses, Lines, timeunits, powerunits, energyunits, unitsymbol -import PRASCore.Results: EUE, LOLE, NEUE, ShortfallResult, ShortfallSamplesResult, AbstractShortfallResult, Result +import PRASCore.Results: EUE, LOLE, NEUE, LOLD, ShortfallResult, ShortfallSamplesResult, AbstractShortfallResult, Result import StatsBase: mean import Dates: @dateformat_str, format, now import TimeZones: ZonedDateTime diff --git a/PRASFiles.jl/src/Results/utils.jl b/PRASFiles.jl/src/Results/utils.jl index c06c35e6..4e243d9a 100644 --- a/PRASFiles.jl/src/Results/utils.jl +++ b/PRASFiles.jl/src/Results/utils.jl @@ -58,11 +58,25 @@ function NEUEResult(shortfall::AbstractShortfallResult; region::Union{Nothing, S ) end +struct LOLDResult + mean::Float64 + stderror::Float64 +end + +function LOLDResult(shortfall::ShortfallSamplesResult; region::Union{Nothing,String}=nothing) + lold = (region === nothing) ? LOLD(shortfall) : LOLD(shortfall, region) + return LOLDResult( + lold.lold.estimate, + lold.lold.standarderror, + ) +end + struct RegionResult name::String eue::EUEResult lole::LOLEResult neue::NEUEResult + lold::Union{Nothing,LOLDResult} load::Vector{Int64} peak_load::Float64 capacity::Dict{String,Vector{Int64}} @@ -78,6 +92,7 @@ struct SystemResult eue::EUEResult lole::LOLEResult neue::NEUEResult + lold::Union{Nothing,LOLDResult} region_results::Vector{RegionResult} end @@ -97,10 +112,24 @@ function get_nsamples(shortfall::ShortfallSamplesResult) return size(shortfall.shortfall,3) end +const _lold_warned = Ref(false) +function get_lold_result(shortfall::ShortfallResult; region::Union{Nothing,String}=nothing) + if !_lold_warned[] + @info "LOLD is not implemented for ShortfallResult and will not be included in the JSON export. Use ShortfallSamplesResult to compute LOLD." + _lold_warned[] = true + end + return nothing +end + +function get_lold_result(shortfall::ShortfallSamplesResult; region::Union{Nothing,String}=nothing) + return LOLDResult(shortfall; region=region) +end + # Define structtypes for different structs defined above StructType(::Type{TypeParams}) = Struct() StructType(::Type{EUEResult}) = Struct() StructType(::Type{NEUEResult}) = Struct() StructType(::Type{LOLEResult}) = Struct() +StructType(::Type{LOLDResult}) = Struct() StructType(::Type{RegionResult}) = OrderedStruct() StructType(::Type{SystemResult}) = OrderedStruct() \ No newline at end of file diff --git a/PRASFiles.jl/src/Results/write.jl b/PRASFiles.jl/src/Results/write.jl index 40b21e7c..d8cee986 100644 --- a/PRASFiles.jl/src/Results/write.jl +++ b/PRASFiles.jl/src/Results/write.jl @@ -28,6 +28,7 @@ function generate_systemresult(shortfall::AbstractShortfallResult, pras_sys::Sys EUEResult(shortfall, region = reg_name), LOLEResult(shortfall, region = reg_name), NEUEResult(shortfall, region = reg_name), + get_lold_result(shortfall, region = reg_name), pras_sys.regions.load[idx,:], peak_load, capacity, @@ -46,6 +47,7 @@ function generate_systemresult(shortfall::AbstractShortfallResult, pras_sys::Sys EUEResult(shortfall), LOLEResult(shortfall), NEUEResult(shortfall), + get_lold_result(shortfall), region_results, ) @@ -77,7 +79,8 @@ function saveshortfall( pras_sys::SystemModel, outfile::String, ) - + _lold_warned[] = false + dt_now = format(now(), "dd-u-yy-H-M-S") export_location = joinpath(outfile, dt_now) if ~(isdir(export_location)) diff --git a/PRASFiles.jl/test/runtests.jl b/PRASFiles.jl/test/runtests.jl index 2249c6dd..5b6c8f4f 100644 --- a/PRASFiles.jl/test/runtests.jl +++ b/PRASFiles.jl/test/runtests.jl @@ -54,6 +54,8 @@ using JSON3 @test exp_results_1.region_results[1].lole.mean == PRASCore.LOLE(shortfall, exp_results_1.region_results[1].name).lole.estimate @test exp_results_1.region_results[1].eue.mean == PRASCore.EUE(shortfall, exp_results_1.region_results[1].name).eue.estimate @test exp_results_1.region_results[1].neue.mean == PRASCore.NEUE(shortfall, exp_results_1.region_results[1].name).neue.estimate + @test exp_results_1.lold === nothing + @test exp_results_1.region_results[1].lold === nothing shortfall_samples = results[2]; exp_location_2 = PRASFiles.saveshortfall(shortfall_samples, rts_sys, path); @@ -65,6 +67,11 @@ using JSON3 @test exp_results_2.region_results[1].lole.mean == PRASCore.LOLE(shortfall_samples, exp_results_2.region_results[1].name).lole.estimate @test exp_results_2.region_results[1].eue.mean == PRASCore.EUE(shortfall_samples, exp_results_2.region_results[1].name).eue.estimate @test exp_results_2.region_results[1].neue.mean == PRASCore.NEUE(shortfall_samples, exp_results_2.region_results[1].name).neue.estimate + @test exp_results_2.lold.mean == PRASCore.LOLD(shortfall_samples).lold.estimate + @test exp_results_2.lold.stderror == PRASCore.LOLD(shortfall_samples).lold.standarderror + region_name = exp_results_2.region_results[1].name + @test exp_results_2.region_results[1].lold.mean == PRASCore.LOLD(shortfall_samples, region_name).lold.estimate + @test exp_results_2.region_results[1].lold.stderror == PRASCore.LOLD(shortfall_samples, region_name).lold.standarderror @test exp_results_1.lole.mean ≈ exp_results_2.lole.mean @test exp_results_1.eue.mean ≈ exp_results_2.eue.mean @@ -73,6 +80,10 @@ using JSON3 @test exp_results_1.region_results[1].eue.mean ≈ exp_results_2.region_results[1].eue.mean @test exp_results_1.region_results[1].neue.mean ≈ exp_results_2.region_results[1].neue.mean + @test exp_results_1.lold === nothing + @test exp_results_2.lold !== nothing + @test exp_results_2.region_results[1].lold !== nothing + surplus = results[3] @test_throws "saveshortfall is not implemented for" PRASFiles.saveshortfall(surplus, rts_sys, path) end diff --git a/docs/Project.toml b/docs/Project.toml index c39c6e12..3a670c94 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -4,6 +4,7 @@ Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" LiveServer = "16fef848-5104-11e9-1b77-fb7a48bbb589" Measures = "442fdcdd-2543-5da2-b0f3-8c86c306513e" +PRAS = "05348d26-1c52-11e9-35e3-9d51842d34b9" PRASCapacityCredits = "2e1a2ed5-e89d-4cd3-bc86-c0e88a73d3a3" PRASCore = "c5c32b99-e7c3-4530-a685-6f76e19f7fe2" PRASFiles = "a2806276-6d43-4ef5-91c0-491704cd7cf1" diff --git a/docs/make.jl b/docs/make.jl index b05cd341..c3583558 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -4,6 +4,8 @@ using PRASFiles using PRASCapacityCredits using Literate +ENV["GKSwstype"] = "100" # Prevent GR from opening gksqt GUI + # Building examples was inspired by COSMO.jl repo @info "Building example problems..." @@ -23,7 +25,8 @@ end makedocs( sitename = "PRAS", format = Documenter.HTML( - prettyurls = true, + prettyurls = haskey(ENV, "GITHUB_ACTIONS"), + size_threshold = nothing, canonical = "https://natlabrockies.github.io/PRAS/stable" ), modules = [PRASCore, PRASFiles, PRASCapacityCredits], @@ -44,6 +47,7 @@ makedocs( "Tutorials" => [ "PRAS 101 Walkthrough" => "examples/pras_walkthrough.md", "Demand Response Walkthrough" => "examples/demand_response_walkthrough.md", + "Interpreting Adequacy Metrics" => "examples/pras_adequacy_metrics.md", ], "Extending PRAS" => "extending.md", # "Contributing" => "contributing.md", diff --git a/docs/src/PRASCore/api.md b/docs/src/PRASCore/api.md index a555af07..7ee1ec04 100644 --- a/docs/src/PRASCore/api.md +++ b/docs/src/PRASCore/api.md @@ -23,6 +23,7 @@ Order = [:function,:type] PRASCore.Results.LOLE PRASCore.Results.EUE PRASCore.Results.NEUE +PRASCore.Results.LOLD PRASCore.Results.Shortfall PRASCore.Results.ShortfallSamples PRASCore.Results.Surplus