From 8c1a0beb2ef2ec12e7c57debc052a1b6b5ebec0d Mon Sep 17 00:00:00 2001 From: Peplinski Date: Wed, 17 Sep 2025 10:31:57 -0400 Subject: [PATCH 01/35] Corrects distribution cost description --- src/results/results_formulas.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/results/results_formulas.csv b/src/results/results_formulas.csv index dc4201f2..ce4bfd25 100644 --- a/src/results/results_formulas.csv +++ b/src/results/results_formulas.csv @@ -100,7 +100,7 @@ bus,plcurt_min,MinHourly(plcurt),MWCurtailed,Minimum hourly load power curtailed bus,electricity_cost,"SumHourlyWeighted(plserv,lmp_elserv)",Dollars,Total cost of electricity served bus,electricity_price,electricity_cost / elserv_total,DollarsPerMWhServed,Average cost of electricity served bus,merchandising_surplus_total,SumHourly(merchandising_surplus),Dollars,Total merchandising surplus from selling electricity for a higher price at one end of a line than another. Lines that are split across the region add half of their merchandising surplus to each region. -bus,distribution_cost_total,"SumHourlyWeighted(plserv, distribution_cost)",Dollars,Total cost to consumers per MWh of served power for the transmission and distribution of the power. +bus,distribution_cost_total,"SumHourlyWeighted(plserv, distribution_cost)",Dollars,Total cost to consumers for the transmission and distribution of power. bus,unserved_load_cost_total, "SumHourlyWeighted(plcurt, voll)",Dollars,Total cost of unserved load. branch,eflow_total,SumHourlyWeighted(pflow),MWhFlow,Total energy flowing in this branch branch,pflow_hourly_min,MinHourly(pflow),MWFlow,Minimum sum of power flowing in these branches From 9645839d58e808e0e60196cab4eaf91587118870 Mon Sep 17 00:00:00 2001 From: Peplinski Date: Wed, 17 Sep 2025 10:32:53 -0400 Subject: [PATCH 02/35] Sets up option for retail price --- src/E4ST.jl | 1 + src/io/data.jl | 1 + src/results/retail_price.jl | 70 ++++++++++++++++++++++ src/types/modifications/ResultsTemplate.jl | 7 ++- 4 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 src/results/retail_price.jl diff --git a/src/E4ST.jl b/src/E4ST.jl index af804b6a..16d59940 100644 --- a/src/E4ST.jl +++ b/src/E4ST.jl @@ -106,6 +106,7 @@ include("results/process.jl") include("results/aggregate.jl") include("results/welfare.jl") include("results/util.jl") +include("results/retail_price.jl") # Include postprocessing include("post/post.jl") diff --git a/src/io/data.jl b/src/io/data.jl index 86f45f22..5758c1aa 100644 --- a/src/io/data.jl +++ b/src/io/data.jl @@ -49,6 +49,7 @@ function read_data!(config, data) promote_cols!(data) setup_results_formulas!(config, data) setup_welfare!(config, data) + setup_retail_price!(config, data) # Save the data to file as specified. if get(config, :save_data, true) diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl new file mode 100644 index 00000000..afdcb949 --- /dev/null +++ b/src/results/retail_price.jl @@ -0,0 +1,70 @@ +""" + setup_retail_price!(config, data) + +Sets up the retail price structure. +Add in welfare terms for our standard welfare results and for several welfare checks. + +Welfare Checks: +* `system_cost_check` - This is the "system cost" and the delta of total system cost should equal the delta of the sum of user, producer, and government revenue. +* `electricity_payments` - This should sum to zero to check whether electricity payments equals electricity revenue paid to producers. +* `net_rev_prelim_check` - This is meant to check that producer `net_total_revenue_prelim` is being calculated correctly, particularly when there are reserve requirements. The sum of this check should equal the sum of `net_total_revenue_prelim` for gen and storage. +""" +function setup_retail_price!(config, data) + retail_price = OrderedDict{Symbol, OrderedDict{Symbol,OrderedDict{Symbol,Function}}}() + data[:retail_price] = retail_price + + + add_price_term!(data, :retail_price, :bus, :electricity_cost, +) + add_price_term!(data, :retail_price, :bus, :distribution_cost_total, +) + add_price_term!(data, :retail_price, :bus, :merchandising_surplus_total, -) + add_price_term!(data, :retail_price, :gen, :cost_of_service_rebate, -) + add_price_term!(data, :retail_price, :gen, :net_production_cost, +) + add_price_term!(data, :retail_price, :storage, :net_production_cost, +) + if haskey(config, :mods) && haskey(config[:mods], :baa_reserve_requirement) + add_price_term!(data, :retail_price, :bus, :baa_reserve_requirement_cost, +) + add_price_term!(data, :retail_price, :bus, :baa_reserve_requirement_merchandising_surplus_total, -) + end + +end +export setup_retail_price! + +""" + get_retail_price(data) -> retail_price::OrderedDict{Symbol, OrderedDict{Symbol,OrderedDict{Symbol,Function}}} +""" +function get_retail_price(data) + return data[:retail_price]::OrderedDict{Symbol, OrderedDict{Symbol,OrderedDict{Symbol,Function}}} +end +export get_retail_price +""" + add_price_term!(data, price_type::Symbol, table_name::Symbol, result_name::Symbol, oper) +""" +function add_price_term!(data, price_type::Symbol, table_name::Symbol, result_name::Symbol, oper::Function) + retail_price = get_retail_price(data) + subretail_price = get!(retail_price, price_type) do + OrderedDict{Symbol, OrderedDict{Symbol, Function}}() + end + subretail_price = get!(subretail_price, table_name) do + OrderedDict{Symbol, Function}() + end + + + get(subretail_price, result_name, oper) == oper || @warn "Changing price sign for price[$price_type][$table_name][$result_name] to $oper" + subretail_price[result_name] = oper +end +export add_price_term! + +function compute_retail_price(data, price_type::Symbol, idxs...) + value = 0.0 + retail_price = get_retail_price(data) + table_names = retail_price[price_type] + for (table_name, result_names) in table_names + for (result_name, result_sign) in result_names + res = compute_result(data, table_name, result_name, idxs...) |> result_sign + value += res + end + end + egen_total = compute_result(data, :gen, :egen_total, idxs...) + return value/egen_total +end + +export compute_retail_price \ No newline at end of file diff --git a/src/types/modifications/ResultsTemplate.jl b/src/types/modifications/ResultsTemplate.jl index 2e97026b..456e7af1 100644 --- a/src/types/modifications/ResultsTemplate.jl +++ b/src/types/modifications/ResultsTemplate.jl @@ -101,8 +101,13 @@ function modify_results!(m::ResultsTemplate, config, data) idxs = parse_comparisons(row) yr_idxs = parse_year_idxs(row.filter_years) hr_idxs = parse_hour_idxs(row.filter_hours) - if table_name == Symbol("") + if table_name == Symbol("welfare") return compute_welfare(data, result_name, idxs, yr_idxs, hr_idxs) + elseif table_name == Symbol("retail_price") + if hr_idxs !== Colon() + error("Hourly retail prices are not available.") + end + return compute_retail_price(data, result_name, idxs, yr_idxs, hr_idxs) else try return compute_result(data, table_name, result_name, idxs, yr_idxs, hr_idxs) From 4d4948a8c18577feb5260ffa803e051986b34c51 Mon Sep 17 00:00:00 2001 From: Peplinski Date: Thu, 18 Sep 2025 11:43:54 -0400 Subject: [PATCH 03/35] Sets up generic ResultsTemplate method for cross-table results --- src/results/retail_price.jl | 41 +++++--- src/results/util.jl | 5 + src/types/modifications/ResultsTemplate.jl | 108 ++++++++++++++++++--- 3 files changed, 128 insertions(+), 26 deletions(-) diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl index afdcb949..dab6be18 100644 --- a/src/results/retail_price.jl +++ b/src/results/retail_price.jl @@ -2,29 +2,39 @@ setup_retail_price!(config, data) Sets up the retail price structure. -Add in welfare terms for our standard welfare results and for several welfare checks. +Add in retail price terms to calculate retail electricity rates in \$/MWh. -Welfare Checks: -* `system_cost_check` - This is the "system cost" and the delta of total system cost should equal the delta of the sum of user, producer, and government revenue. -* `electricity_payments` - This should sum to zero to check whether electricity payments equals electricity revenue paid to producers. -* `net_rev_prelim_check` - This is meant to check that producer `net_total_revenue_prelim` is being calculated correctly, particularly when there are reserve requirements. The sum of this check should equal the sum of `net_total_revenue_prelim` for gen and storage. +The relevant price terms are: +* `electricity_cost` +* `distribution_cost_total` +* `merchandising_surplus_total` +* `cost_of_service_rebate` +* `net_production_cost` +* `baa_reserve_requirement_cost` +* `baa_reserve_requiremetn_merchandising_surplus_total` +Reference the results formulas for more detailed descriptions of each of these terms. + +The results template can calculate annual rates by specified region, but is not set up for hourly rates. """ function setup_retail_price!(config, data) retail_price = OrderedDict{Symbol, OrderedDict{Symbol,OrderedDict{Symbol,Function}}}() data[:retail_price] = retail_price - - add_price_term!(data, :retail_price, :bus, :electricity_cost, +) - add_price_term!(data, :retail_price, :bus, :distribution_cost_total, +) - add_price_term!(data, :retail_price, :bus, :merchandising_surplus_total, -) - add_price_term!(data, :retail_price, :gen, :cost_of_service_rebate, -) - add_price_term!(data, :retail_price, :gen, :net_production_cost, +) - add_price_term!(data, :retail_price, :storage, :net_production_cost, +) + # price terms for average electricity rate + add_price_term!(data, :avg_elec_rate, :bus, :electricity_cost, +) + add_price_term!(data, :avg_elec_rate, :bus, :distribution_cost_total, +) + add_price_term!(data, :avg_elec_rate, :bus, :merchandising_surplus_total, -) + add_price_term!(data, :avg_elec_rate, :gen, :cost_of_service_rebate, -) + add_price_term!(data, :avg_elec_rate, :storage, :cost_of_service_rebate, -) + add_price_term!(data, :avg_elec_rate, :gen, :net_production_cost, +) + add_price_term!(data, :avg_elec_rate, :storage, :net_production_cost, +) if haskey(config, :mods) && haskey(config[:mods], :baa_reserve_requirement) - add_price_term!(data, :retail_price, :bus, :baa_reserve_requirement_cost, +) - add_price_term!(data, :retail_price, :bus, :baa_reserve_requirement_merchandising_surplus_total, -) + add_price_term!(data, :avg_elec_rate, :bus, :baa_reserve_requirement_cost, +) + add_price_term!(data, :avg_elec_rate, :bus, :baa_reserve_requirement_merchandising_surplus_total, -) end + # future work: calculate electricity rates by end-use sector + end export setup_retail_price! @@ -63,8 +73,9 @@ function compute_retail_price(data, price_type::Symbol, idxs...) value += res end end + # divide by total generation to get dollars per MWh egen_total = compute_result(data, :gen, :egen_total, idxs...) return value/egen_total end -export compute_retail_price \ No newline at end of file +export compute_retail_price diff --git a/src/results/util.jl b/src/results/util.jl index 8e8f1145..a1377d92 100644 --- a/src/results/util.jl +++ b/src/results/util.jl @@ -155,3 +155,8 @@ function unweight_hourly(data, v::Vector{<:Container}, s=+) return [s(v[i][y,h]) / w[h] for i in 1:length(v), y in 1:ny, h in 1:nh] end export unweight_hourly + +function get_cross_table(data, table_name) + return data[table_name]::OrderedDict{Symbol, OrderedDict{Symbol,OrderedDict{Symbol,Function}}} +end +export get_cross_table diff --git a/src/types/modifications/ResultsTemplate.jl b/src/types/modifications/ResultsTemplate.jl index 456e7af1..ed29c93c 100644 --- a/src/types/modifications/ResultsTemplate.jl +++ b/src/types/modifications/ResultsTemplate.jl @@ -8,6 +8,7 @@ This is a mod that outputs computed results, given a `file` representing the tem * `file` - the file pointing to a table specifying which results to calculate * `name` - the name of the mod, do not need to specify in a config file * `col_sort` - the column(s) to sort by. Defaults to the order in which they were originally specified. +* `cross_table` - indicates that the result is pulling results from multiple tables. Defaults to false. The `file` should represent a csv table with the following columns: * `table_name` - the name of the table being aggregated. i.e. `gen`, `bus`, etc. If you leave it empty, it will call `compute_welfare` instead of `compute_result` @@ -22,8 +23,9 @@ struct ResultsTemplate <: Modification file::String name::Symbol table::DataFrame + cross_table::Bool col_sort - function ResultsTemplate(;file, name, col_sort=:initial_order) + function ResultsTemplate(;file, name, cross_table=false, col_sort=:initial_order) table = read_table(file) force_table_types!(table, name, :table_name=>Symbol, @@ -36,7 +38,7 @@ struct ResultsTemplate <: Modification hasproperty(table, col_name) || continue force_table_types!(table, name, col_name=>String) end - return new(file, name, table, col_sort) + return new(file, name, table, cross_table, col_sort) end end @@ -53,7 +55,17 @@ export AggregationTemplate mod_rank(::Type{<:ResultsTemplate}) = 5.0 fieldnames_for_yaml(::Type{ResultsTemplate}) = (:file,) +# dispatches the single or cross-table results method based on cross_table argument function modify_results!(m::ResultsTemplate, config, data) + if m.cross_table == true + modify_results!(m::ResultsTemplate, Val(:true), config, data) + elseif m.cross_table == false + modify_results!(m::ResultsTemplate, Val(:false), config, data) + end +end + +# method for single-table result +function modify_results!(m::ResultsTemplate, ::Val{:false}, config, data) table = copy(m.table) table.initial_order = 1:nrow(table) @@ -61,6 +73,7 @@ function modify_results!(m::ResultsTemplate, config, data) # for any rows that are not a pair, separate into multiple rows not_pair_idx = findfirst(not_a_full_filter, eachrow(table)) + while not_pair_idx !== nothing row = table[not_pair_idx, :] filter_col_idx = findfirst(filter_col->not_a_full_filter(row[filter_col]), filter_cols) @@ -101,20 +114,93 @@ function modify_results!(m::ResultsTemplate, config, data) idxs = parse_comparisons(row) yr_idxs = parse_year_idxs(row.filter_years) hr_idxs = parse_hour_idxs(row.filter_hours) + + try + return compute_result(data, table_name, result_name, idxs, yr_idxs, hr_idxs) + catch e + @warn "No single-table results formula found for table $table_name and result $result_name" + return 0.0 + end + + end + sort!(table, m.col_sort) + select!(table, Not(:initial_order)) + CSV.write(get_out_path(config, string(m.name, ".csv")), table) + results = get_results(data) + results[m.name] = table + return +end + +# cross-table results +function modify_results!(m::ResultsTemplate, ::Val{:true}, config, data) + table = copy(m.table) + table.initial_order = 1:nrow(table) + + filter_cols = setdiff(propertynames(table), [:table_name, :result_name]) + + # for any rows that are not a pair, separate into multiple rows + not_pair_idx = findfirst(not_a_full_filter, eachrow(table)) + while not_pair_idx !== nothing + row = table[not_pair_idx, :] + filter_col_idx = findfirst(filter_col->not_a_full_filter(row[filter_col]), filter_cols) + col_to_expand = filter_cols[filter_col_idx] + table_name = row[:table_name] + result_name = row[:result_name] + + if col_to_expand == :filter_hours + area = row.filter_hours + hours_table_col = get_table_col(data, :hours, area) + subareas = Base.sort!(String.(string.(unique(hours_table_col))), by=hours_sortby) + elseif col_to_expand == :filter_years && row[col_to_expand] == ":" + area = :years + subareas = data[area] + else + area = row[col_to_expand] + table_names = get_cross_table(data, table_name)[result_name] + all(hasproperty(get_table(data, t), area) for (t, _) in table_names) || error("Some tables are missing property $(area)") + data_table_col = get_table_col(data, first(keys(table_names)), area) + subareas = sort!(unique(data_table_col)) + end + + row_dict = Dict(pairs(row)) + for subarea in subareas + # Add a row right after the original row + row_dict[col_to_expand] = "$area=>$subarea" + insert!(table, not_pair_idx+1, row_dict) + end + + deleteat!(table, not_pair_idx) + + # Find the next index that is not a pair, to be expanded + not_pair_idx = findfirst(not_a_full_filter, eachrow(table)) + end + + @info "Calculating results for $(nrow(table)) rows in ResultsTemplate $(m.name)" + results_formulas = get_results_formulas(data) + table.value = map(eachrow(table)) do row + table_name = row.table_name + result_name = row.result_name + idxs = parse_comparisons(row) + yr_idxs = parse_year_idxs(row.filter_years) + hr_idxs = parse_hour_idxs(row.filter_hours) + if table_name == Symbol("welfare") - return compute_welfare(data, result_name, idxs, yr_idxs, hr_idxs) - elseif table_name == Symbol("retail_price") if hr_idxs !== Colon() - error("Hourly retail prices are not available.") + @warn "Hourly welfare calculations are not set up." + return 0.0 + else + return compute_welfare(data, result_name, idxs, yr_idxs, hr_idxs) end - return compute_retail_price(data, result_name, idxs, yr_idxs, hr_idxs) - else - try - return compute_result(data, table_name, result_name, idxs, yr_idxs, hr_idxs) - catch e - @warn "No results formula found for table $table_name and result $result_name" + elseif table_name == Symbol("retail_price") + if hr_idxs !== Colon() + @warn "Hourly retail price calculations are not set up." return 0.0 + else + return compute_retail_price(data, result_name, idxs, yr_idxs, hr_idxs) end + else + @warn "No cross-table results formula found for table $table_name and result $result_name" + return 0.0 end end sort!(table, m.col_sort) From 86a1badeefb23a52cea00ab30c1bde929ecb326d Mon Sep 17 00:00:00 2001 From: Peplinski Date: Mon, 29 Sep 2025 12:19:52 -0400 Subject: [PATCH 04/35] Adjusts retail rate set up for COS regions --- src/results/results_formulas.csv | 2 ++ src/results/retail_price.jl | 12 ++++++++---- src/types/modifications/GenerationStandard.jl | 18 ++++++++++++++++-- src/types/modifications/ReserveRequirement.jl | 4 ++-- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/results/results_formulas.csv b/src/results/results_formulas.csv index ce4bfd25..dc4b5f4e 100644 --- a/src/results/results_formulas.csv +++ b/src/results/results_formulas.csv @@ -71,6 +71,7 @@ gen,emission_cost,0,Dollars,Cost for paying all emissions prices. gen,emission_cost_per_mwh,emission_cost / egen_total,DollarsPerMWhGenerated,"Average cost, per MWh, for paying emission prices" gen,net_pol_cost_for_egus,emission_cap_cost + emission_cost - gs_rebate - invest_subsidy - ptc_subsidy,Dollars,Net cost for all generators from all policy types. gen,net_pol_cost_for_egus_per_mwh,net_pol_cost_for_egus / egen_total,DollarsPerMWhGenerated,"Average cost for all generators from all policy types, per MWh." +gen,net_pol_prod_cost_for_egus,emission_cost - invest_subsidy - ptc_subsidy,Dollars,Net cost for all generators from policy types that impact the production cost. gen,net_government_revenue,emission_cap_cost + emission_cost - invest_subsidy - ptc_subsidy - past_invest_subsidy_total,Dollars,Net government revenue earned from generators gen,going_forward_cost,production_cost + net_pol_cost_for_egus,Dollars,Total going forward cost gen,total_cost_prelim,going_forward_cost + past_invest_cost_total - past_invest_subsidy_total,Dollars,"Total cost of production, including past investment costs and subsidies for investments still within their economic lifetimes, before adjusting for cost-of-service rebates" @@ -102,6 +103,7 @@ bus,electricity_price,electricity_cost / elserv_total,DollarsPerMWhServed,Averag bus,merchandising_surplus_total,SumHourly(merchandising_surplus),Dollars,Total merchandising surplus from selling electricity for a higher price at one end of a line than another. Lines that are split across the region add half of their merchandising surplus to each region. bus,distribution_cost_total,"SumHourlyWeighted(plserv, distribution_cost)",Dollars,Total cost to consumers for the transmission and distribution of power. bus,unserved_load_cost_total, "SumHourlyWeighted(plcurt, voll)",Dollars,Total cost of unserved load. +bus,gs_payment,0,Dollars,Cost of required credits for clean/renewable generation for all generation standards (RPS's and CES's) for the qualifying demand at each given bus. branch,eflow_total,SumHourlyWeighted(pflow),MWhFlow,Total energy flowing in this branch branch,pflow_hourly_min,MinHourly(pflow),MWFlow,Minimum sum of power flowing in these branches branch,pflow_hourly_max,MaxHourly(pflow),MWFlow,Maximum sum of power flowing in these branches diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl index dab6be18..84d39536 100644 --- a/src/results/retail_price.jl +++ b/src/results/retail_price.jl @@ -22,12 +22,16 @@ function setup_retail_price!(config, data) # price terms for average electricity rate add_price_term!(data, :avg_elec_rate, :bus, :electricity_cost, +) + # per MW cost adder for distribution costs add_price_term!(data, :avg_elec_rate, :bus, :distribution_cost_total, +) + # merchandising suplus is from selling electricity for higher price at one end of line than another add_price_term!(data, :avg_elec_rate, :bus, :merchandising_surplus_total, -) + # if the difference between revenue and total costs is positive, customers in COS regions get a rebate + # total cost includes production costs, net policy costs, gs_rebate, and the net of past investment costs and subsidies add_price_term!(data, :avg_elec_rate, :gen, :cost_of_service_rebate, -) add_price_term!(data, :avg_elec_rate, :storage, :cost_of_service_rebate, -) - add_price_term!(data, :avg_elec_rate, :gen, :net_production_cost, +) - add_price_term!(data, :avg_elec_rate, :storage, :net_production_cost, +) + add_price_term!(data, :avg_elec_rate, :bus, :gs_payment, +) + if haskey(config, :mods) && haskey(config[:mods], :baa_reserve_requirement) add_price_term!(data, :avg_elec_rate, :bus, :baa_reserve_requirement_cost, +) add_price_term!(data, :avg_elec_rate, :bus, :baa_reserve_requirement_merchandising_surplus_total, -) @@ -74,8 +78,8 @@ function compute_retail_price(data, price_type::Symbol, idxs...) end end # divide by total generation to get dollars per MWh - egen_total = compute_result(data, :gen, :egen_total, idxs...) - return value/egen_total + elserv_total = compute_result(data, :bus, :elserv_total, idxs...) + return value/elserv_total end export compute_retail_price diff --git a/src/types/modifications/GenerationStandard.jl b/src/types/modifications/GenerationStandard.jl index 756faeb2..18d2fcc6 100644 --- a/src/types/modifications/GenerationStandard.jl +++ b/src/types/modifications/GenerationStandard.jl @@ -2,8 +2,10 @@ struct GenerationStandard{T} <: Policy A generation standard (also refered to as a portfolio standard) is a constraint on generation where a portion of generation from certain generators must meet the a portion of the load in a specified region. -This encompasses RPSs, CESs, and technology carveouts. +This encompasses RPSs, CESs, and technology carveouts. To assign the credit (the portion of generation that can contribute) to generators, the [`Crediting`](@ref) type is used. +In certain instances, there can be a mismatch between the location of the qualifying load and the location of the generation that is used to meet the load. To handle this, gs_rebate is calculated as the payment +that a generators get for providing generation for qualifying load while gs_payment is calculated as the payment that would be associated with the qualifying load. ### Keyword Arguments * `name` - Name of the policy @@ -179,7 +181,7 @@ function modify_results!(pol::GenerationStandard, config, data) if !hasproperty(bus, :pl_gs) pl_gs_bus = get_raw_result(data, :pl_gs_bus) el_gs_bus = weight_hourly(data, pl_gs_bus) - + add_table_col!(data, :bus, :pl_gs, pl_gs_bus, MWServed, "Served Load Power that qualifies for generation standards") add_table_col!(data, :bus, :el_gs, el_gs_bus, MWhServed, "Served Load Energy that qualifies for generation standards") @@ -191,16 +193,28 @@ function modify_results!(pol::GenerationStandard, config, data) gen_idxs = get_row_idxs(gen, parse_comparisons(pol.gen_filters)) add_table_col!(data, :gen, prc_name, Container[ByNothing(0.0) for i in 1:nrow(gen)], DollarsPerMWhGenerated, "Policy price based on shadow price of $(pol.name) (converted to DollarsPerMWhGenerated) multiplied by the credit.") + add_table_col!(data, :bus, prc_name, Container[ByNothing(0.0) for i in 1:nrow(bus)], DollarsPerMWhGenerated, "Policy cost based on shadow price of $(pol.name) (converted to DollarsPerMWhGenerated).") # set to shadow_prc * crediting for i in gen_idxs gen[i, prc_name] = -(shadow_prc) .* gen[i, pol.name] end + for (k,d) in pol.load_targets + targets = d[:targets] + filters = d[:filters] + bus_idxs = get_row_idxs(bus, parse_comparisons(d[:filters])) + # set to shadow_prc for bus + for i in bus_idxs + bus[i, prc_name] = -(shadow_prc) + end + end # policy cost, price * credit * generation add_results_formula!(data, :gen, cost_name, "SumHourlyWeighted($(prc_name), pgen)", Dollars, "Cost of $(pol.name) based on the shadow price on the constraint and the generator credit level.") add_to_results_formula!(data, :gen, :gs_rebate, cost_name) + add_results_formula!(data, :bus, cost_name, "SumHourlyWeighted($(prc_name), plserv)", Dollars, "Cost of $(pol.name) based on the shadow price on the constraint and the generator credit level.") + add_to_results_formula!(data, :bus, :gs_payment, cost_name) end export modify_results! diff --git a/src/types/modifications/ReserveRequirement.jl b/src/types/modifications/ReserveRequirement.jl index 15182751..0666df54 100644 --- a/src/types/modifications/ReserveRequirement.jl +++ b/src/types/modifications/ReserveRequirement.jl @@ -454,7 +454,7 @@ function modify_results!(mod::ReserveRequirement, config, data) add_results_formula!(data, :gen, rebate_price_result_name, "$(rebate_result_name)/pcap_qual_$(mod.name)",DollarsPerMWCapacity, "The per MW of qualifying capacity price of the rebate receive by EGU's from the $(mod.name) reserve requirement.") # Add it to net_total_revenue_prelim - add_to_results_formula!(data, :gen, :net_total_revenue_prelim, "+ $rebate_result_name") + add_to_results_formula!(data, :gen, :net_total_revenue_prelim, "$rebate_result_name") # Subtract the cost from user surplus add_welfare_term!(data, :user, :bus, cost_result_name, -) @@ -490,7 +490,7 @@ function modify_results!(mod::ReserveRequirement, config, data) add_results_formula!(data, :storage, rebate_price_result_name, "$(rebate_result_name)/pcap_qual_$(mod.name)",DollarsPerMWCapacity, "The per MW price of the rebate receive by EGU's from the $(mod.name) reserve requirement.") # Add it to net_total_revenue_prelim - add_to_results_formula!(data, :storage, :net_total_revenue_prelim, "+ $rebate_result_name") + add_to_results_formula!(data, :storage, :net_total_revenue_prelim, "$rebate_result_name") end # Add merchandising surplus if applicable From a05cf66795bb5141c24e3e280760be45ffe36a99 Mon Sep 17 00:00:00 2001 From: Peplinski Date: Thu, 9 Oct 2025 17:49:23 -0400 Subject: [PATCH 05/35] Adds past_invest option to config file to work with aggregated gen model --- src/io/config.jl | 1 + src/io/data.jl | 143 ++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 131 insertions(+), 13 deletions(-) diff --git a/src/io/config.jl b/src/io/config.jl index 3ad13941..7356f02b 100644 --- a/src/io/config.jl +++ b/src/io/config.jl @@ -79,6 +79,7 @@ function summarize_config() (:require_optimal, false, true, "Whether or not to require whether or not the model is solved to optimality. If set to true and the optimizer terminates with a suboptimal termination status, [`run_e4st`](@ref) returns after optimizing, without parsing results, etc."), (:model_string_names, false, false, "Whether or not to allow the model to have string names. Defaults to `false` for memory savings. Can be helpful to turn on for debugging, especially if you are encountering an infeasible model"), (:yearly_objective_scalars, false, 1, "The amount to scale the objective by for each year, defaults to 1 for each year."), + (:past_invest_file, false, 1, "Gen file used to calculate past investment costs. Only necessary when an aggregated gen table is used.") ) diff --git a/src/io/data.jl b/src/io/data.jl index 5758c1aa..eafa452b 100644 --- a/src/io/data.jl +++ b/src/io/data.jl @@ -85,6 +85,7 @@ function read_data_files!(config, data) read_table!(config, data, :load_add_file=>:load_add, optional=true) read_table!(config, data, :build_gen_file => :build_gen, optional=true) read_table!(config, data, :gentype_genfuel_file => :genfuel, optional=true) + read_table!(config, data, :past_invest_file => :past_invest, optional=true) end export read_data_files! @@ -180,6 +181,7 @@ function setup_data!(config, data) setup_table!(config, data, :nominal_load) setup_table!(config, data, :gen) # needs to come after build_gen setup for newgens setup_table!(config, data, :af_table) + setup_table!(config, data, :past_invest) end export setup_data! @@ -516,20 +518,22 @@ function setup_table!(config, data, ::Val{:gen}) # Make columns as needed hasproperty(gen, :past_invest_cost) || (gen.past_invest_cost = zeros(nrow(gen))) hasproperty(gen, :past_invest_subsidy) || (gen.past_invest_subsidy = zeros(nrow(gen))) - z = Container(0.0) - to_container!(gen, :past_invest_cost) - to_container!(gen, :past_invest_subsidy) - for (idx_g, g) in enumerate(eachrow(gen)) - if g.build_status == "unbuilt" - if any(!=(0), g.past_invest_cost) || any(!=(0), g.past_invest_subsidy) - @warn "Generator $idx_g is unbuilt yet has past capex cost/subsidy, setting to zero" - g.past_invest_cost = z - g.past_invest_subsidy = z + if !haskey(config,:past_invest_file) + z = Container(0.0) + to_container!(gen, :past_invest_cost) + to_container!(gen, :past_invest_subsidy) + for (idx_g, g) in enumerate(eachrow(gen)) + if g.build_status == "unbuilt" + if any(!=(0), g.past_invest_cost) || any(!=(0), g.past_invest_subsidy) + @warn "Generator $idx_g is unbuilt yet has past capex cost/subsidy, setting to zero" + g.past_invest_cost = z + g.past_invest_subsidy = z + end + else + past_invest_percentages = get_past_invest_percentages(g, years) + g.past_invest_cost = g.past_invest_cost .* past_invest_percentages + g.past_invest_subsidy = g.past_invest_subsidy .* past_invest_percentages end - else - past_invest_percentages = get_past_invest_percentages(g, years) - g.past_invest_cost = g.past_invest_cost .* past_invest_percentages - g.past_invest_subsidy = g.past_invest_subsidy .* past_invest_percentages end end @@ -574,6 +578,74 @@ function setup_table!(config, data, ::Val{:gen}) return gen end export setup_table! +""" + setup_table!(config, data, ::Val{:past_invest}) + +Sets up the invest cost gen table. This is necessary when gens are aggregated so that capex for built gens can be found. +Creates age column which is a ByYear column. Unbuilt generators have a negative age before year_on. +""" +function setup_table!(config, data, ::Val{:past_invest}) + if !haskey(config, :past_invest) + return + end + bus = get_table(data, :bus) + past_invest = get_table(data, :past_invest) + years = get_years(data) + + # Set up year_unbuilt before setting up new gens. Plus we will want to save the column + hasproperty(past_invest, :year_unbuilt) || (past_invest.year_unbuilt = map(y->add_to_year(y, -1), past_invest.year_on)) + + # Set up past capex cost and subsidy to be for built generators only + # Make columns as needed + hasproperty(past_invest, :past_invest_cost) || (past_invest.past_invest_cost = zeros(nrow(past_invest))) + hasproperty(past_invest, :past_invest_subsidy) || (past_invest.past_invest_subsidy = zeros(nrow(past_invest))) + z = Container(0.0) + to_container!(past_invest, :past_invest_cost) + to_container!(past_invest, :past_invest_subsidy) + for (idx_g, g) in enumerate(eachrow(past_invest)) + if g.build_status == "unbuilt" + if any(!=(0), g.past_invest_cost) || any(!=(0), g.past_invest_subsidy) + @warn "Generator $idx_g is unbuilt yet has past capex cost/subsidy, setting to zero" + g.past_invest_cost = z + g.past_invest_subsidy = z + end + else + past_invest_percentages = get_past_invest_percentages(g, years) + g.past_invest_cost = g.past_invest_cost .* past_invest_percentages + g.past_invest_subsidy = g.past_invest_subsidy .* past_invest_percentages + end + end + + + original_cols = propertynames(past_invest) + data[:past_invest_table_original_cols] = original_cols + + #removes capex_obj if read in from previous sim + :capex_obj in propertynames(data[:past_invest]) && select!(data[:past_invest], Not(:capex_obj)) + + #set build_status to 'built' for all gens marked 'new'. This marks gens built in a previous sim as 'built'. + b = "built" # pre-allocate + transform!(past_invest, :build_status => ByRow(s->isnew(s) ? b : s) => :build_status) # transform in-place + + # Set the pcap_max to be equal to pcap0 for built generators + past_invest.pcap_max = map(row->isbuilt(row) ? row.pcap0 : row.pcap_max, eachrow(past_invest)) + past_invest.pcap0 = map(row->isbuilt(row) ? row.pcap0 : 0.0, eachrow(past_invest)) + + + ### Add age column as by ByYear based on year_on + years = year2float.(get_years(data)) + gen_age = Container[ByNothing(0.0) for i in 1:nrow(past_invest)] + for idx_g in 1:nrow(past_invest) + year_on = year2float(past_invest[idx_g, :year_on]) + g_age = [year - year_on for year in years] + gen_age[idx_g] = ByYear(g_age) + end + + add_table_col!(data, :past_invest, :age, gen_age, NumYears, "The age of the generator in each simulation year, given as a byYear container. Negative age is given for gens before their year_on.") + + return past_invest +end + """ join_bus_columns!(data, table_name) @@ -978,6 +1050,51 @@ function summarize_table(::Val{:gen}) return df end +@doc """ + summarize_table(::Val{:past_invest}) + +$(table2markdown(summarize_table(Val(:past_invest)))) +""" +function summarize_table(::Val{:past_invest}) + df = TableSummary() + push!(df, + (:bus_idx, Int64, NA, true, "The index of the `bus` table that the generator corresponds to"), + (:status, Bool, NA, false, "Whether or not the generator is in service"), + (:build_status, String15, NA, true, "Whether the generator is `built`, `new`, `unbuilt`, or `unretrofitted`. All generators marked `new` when the gen file is read in will be changed to `built`. Can also be changed to `retired_exog` or `retired_endog` after the simulation is run. See [`update_build_status!`](@ref). Note that `unretrofitted` means it is a [`Retrofit`](@ref) option based on a `built` generator."), + (:build_type, AbstractString, NA, true, "Whether the generator is 'real', 'exog' (exogenously built), or 'endog' (endogenously built)"), + (:build_id, AbstractString, NA, true, "Identifier of the build row. For pre-existing generators not specified in the build file, this is usually left empty"), + (:year_on, YearString, Year, true, "The first year of operation for the generator. (For new gens this is also the year it was built)"), + (:year_unbuilt,YearString, Year, false, "The latest year the generator was known not to be built. Defaults to year_on - 1. Used for past capex accounting."), + (:econ_life, Float64, NumYears, true, "The number of years in the economic lifetime of the generator."), + (:year_off, YearString, Year, true, "The first year that the generator is no longer operating in the simulation, computed from the simulation. Leave as y9999 if an existing generator that has not been retired in the simulation yet."), + (:year_shutdown, YearString, Year, true, "The forced (exogenous) shutdown year for the generator. Often equal to the year_on plus the econ_life"), + (:genfuel, AbstractString, NA, true, "The fuel type that the generator uses"), + (:gentype, String, NA, true, "The generation technology type that the generator uses"), + (:pcap_inv, Float64, MWCapacity, true, "Original invested nameplate power generation capacity for the generator. This is the original invested capacity of exogenously built generators (even if there have been retirements ), and the original invested capacity in year_on for endogenously built generators."), + (:pcap0, Float64, MWCapacity, true, "Nameplate power generation capacity for the generator at the start of the simulation"), + (:pcap_min, Float64, MWCapacity, true, "Minimum nameplate power generation capacity of the generator (normally set to zero to allow for retirement)"), + (:pcap_max, Float64, MWCapacity, true, "Maximum nameplate power generation capacity of the generator"), + (:vom, Float64, DollarsPerMWhGenerated, true, "Variable operation and maintenance cost per MWh of generation"), + (:fuel_price, Float64, DollarsPerMMBtu, false, "Fuel cost per MMBtu of fuel used. `heat_rate` column also necessary when supplying `fuel_price`"), + (:heat_rate, Float64, MMBtuPerMWhGenerated, false, "Heat rate, or MMBtu of fuel consumed per MWh electricity generated (0 for generators that don't use combustion)"), + (:fom, Float64, DollarsPerMWCapacityPerHour, true, "Hourly fixed operation and maintenance cost for a MW of generation capacity"), + (:capex, Float64, DollarsPerMWBuiltCapacityPerHour, true, "Hourly capital expenditures for a MW of generation capacity. For already-built generators, this is not accounted for in the optimization or accounting. For accounting for investment costs and subsidies in built generators, use `past_invest_cost` and `past_invest_subsidy`"), + (:transmission_capex, Float64, DollarsPerMWBuiltCapacityPerHour, true, "Hourly capital expenditures for the transmission supporting a MW of generation capacity"), + (:routine_capex, Float64, DollarsPerMWCapacityPerHour, true, "Routine capital expenditures for a MW of discharge capacity"), + (:past_invest_cost, Float64, DollarsPerMWCapacityPerHour, false, "Investment costs per MW of initial capacity per hour, for past investments"), + (:past_invest_subsidy, Float64, DollarsPerMWCapacityPerHour, false, "Investment subsidies from govt. per MW of initial capacity per hour, for past investments"), + (:cf_min, Float64, MWhGeneratedPerMWhCapacity, false, "The minimum capacity factor, or operable ratio of power generation to capacity for the generator to operate. Take care to ensure this is not above the hourly availability factor in any of the hours, or else the model may be infeasible. Set to zero by default."), + (:cf_max, Float64, MWhGeneratedPerMWhCapacity, false, "The maximum capacity factor, or operable ratio of power generation to capacity for the generator to operate"), + (:cf_hist, Float64, MWhGeneratedPerMWhCapacity, false, "The historical capacity factor for the generator, or the gentype if no previous data is available. Primarily used to calculate estimate policy value (PTC and EmissionPrice capex_adj)"), + (:af, Float64, MWhGeneratedPerMWhCapacity, false, "The availability factor, or maximum available ratio of pewer generation to nameplate capacity for the generator."), + (:emis_co2, Float64, ShortTonsPerMWhGenerated, false, "The emission rate per MWh of CO2"), + (:capt_co2_percent, Float64, NA, false, "The percentage of co2 emissions captured, to be sequestered."), + (:chp_co2_multi,Float64,NA,false,"The percentage of CO2 emissions from CHP attributed to the power generation. Used to calculate CO2e"), + (:reg_factor, Float64, NA, true, "The percentage of generation that dispatches to a cost-of-service regulated market"), + ) + return df +end + @doc """ summarize_table(::Val{:bus}) From a9a0e3705b9b539e6e1aec2f8358b8be43fd6718 Mon Sep 17 00:00:00 2001 From: Peplinski Date: Thu, 9 Oct 2025 19:05:35 -0400 Subject: [PATCH 06/35] Adjustments so that past_invest_cost_total can be included in retail rate calculation for an aggregated model --- src/io/data.jl | 6 +++--- src/results/formulas.jl | 16 ++++++++++++++++ src/results/results_formulas.csv | 3 +++ src/results/retail_price.jl | 5 +++++ 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/io/data.jl b/src/io/data.jl index eafa452b..941992ac 100644 --- a/src/io/data.jl +++ b/src/io/data.jl @@ -518,10 +518,10 @@ function setup_table!(config, data, ::Val{:gen}) # Make columns as needed hasproperty(gen, :past_invest_cost) || (gen.past_invest_cost = zeros(nrow(gen))) hasproperty(gen, :past_invest_subsidy) || (gen.past_invest_subsidy = zeros(nrow(gen))) + z = Container(0.0) + to_container!(gen, :past_invest_cost) + to_container!(gen, :past_invest_subsidy) if !haskey(config,:past_invest_file) - z = Container(0.0) - to_container!(gen, :past_invest_cost) - to_container!(gen, :past_invest_subsidy) for (idx_g, g) in enumerate(eachrow(gen)) if g.build_status == "unbuilt" if any(!=(0), g.past_invest_cost) || any(!=(0), g.past_invest_subsidy) diff --git a/src/results/formulas.jl b/src/results/formulas.jl index e83ad8a4..000fd1be 100644 --- a/src/results/formulas.jl +++ b/src/results/formulas.jl @@ -801,6 +801,22 @@ function (f::CostOfServiceRebate)(data, table, idxs, yr_idxs, hr_idxs) end export CostOfServiceRebate +struct CostOfServicePastCost <: Function + table_name::Symbol +end +function (f::CostOfServicePastCost)(data, table, idxs, yr_idxs, hr_idxs) + reg_factor = table.reg_factor::Vector{Float64} + res = 0.0 + for i in idxs + rf = reg_factor[i] + rev_prelim = compute_result(data, f.table_name, :past_invest_cost_total, i, yr_idxs, hr_idxs) + prod = rf * rev_prelim + res += prod + end + return res + # return sum0(reg_factor[i] * compute_result(data, f.table_name, :net_total_revenue_prelim, i, yr_idxs, hr_idxs) for i in idxs) +end +export CostOfServicePastCost function _sum(v1, idxs) res = 0.0 diff --git a/src/results/results_formulas.csv b/src/results/results_formulas.csv index dc4b5f4e..01d9d0a3 100644 --- a/src/results/results_formulas.csv +++ b/src/results/results_formulas.csv @@ -107,3 +107,6 @@ bus,gs_payment,0,Dollars,Cost of required credits for clean/renewable generation branch,eflow_total,SumHourlyWeighted(pflow),MWhFlow,Total energy flowing in this branch branch,pflow_hourly_min,MinHourly(pflow),MWFlow,Minimum sum of power flowing in these branches branch,pflow_hourly_max,MaxHourly(pflow),MWFlow,Maximum sum of power flowing in these branches +past_invest,past_invest_cost_total,"SumHourlyWeighted(past_invest_cost, pcap_inv)",Dollars,"Investment costs from past investments. This only applies to generators built prior to the simulation. This includes the full annualized investment cost (""invest_cost""), times the percentage likelihood that the generator would still be within its the economic lifetime for the year calculated, given that endogenously built generators can be built in a range of years" +past_invest,past_invest_subsidy_total,"SumHourlyWeighted(past_invest_subsidy, pcap_inv)",Dollars,"Investment subsidies from past investments. This only applies to generators built prior to the simulation. This includes the full annualized investment subsidy (""invest_subsidy""), times the percentage likelihood that the generator would still be within its the economic lifetime for the year calculated, given that endogenously built generators can be built in a range of years" +past_invest,cost_of_service_past_costs,CostOfServicePastCost(past_invest),Dollars,"This is a specially calculated result, which is the sum of past_invest_cost_total * reg_factor for each generator" \ No newline at end of file diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl index 84d39536..c6f785bb 100644 --- a/src/results/retail_price.jl +++ b/src/results/retail_price.jl @@ -32,6 +32,11 @@ function setup_retail_price!(config, data) add_price_term!(data, :avg_elec_rate, :storage, :cost_of_service_rebate, -) add_price_term!(data, :avg_elec_rate, :bus, :gs_payment, +) + # ToDo: include past subsidies, but these may be hard to track down + if haskey(config, :past_invest_file) + add_price_term!(data, :avg_elec_rate, :past_invest, :cost_of_service_past_costs, +) + end + if haskey(config, :mods) && haskey(config[:mods], :baa_reserve_requirement) add_price_term!(data, :avg_elec_rate, :bus, :baa_reserve_requirement_cost, +) add_price_term!(data, :avg_elec_rate, :bus, :baa_reserve_requirement_merchandising_surplus_total, -) From f749cdd5d3518c2981b4fd3a5326a12145b94d0c Mon Sep 17 00:00:00 2001 From: Peplinski Date: Fri, 10 Oct 2025 16:55:20 -0400 Subject: [PATCH 07/35] Moves get_cross_table function to correct location --- src/io/data.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/io/data.jl b/src/io/data.jl index 941992ac..6c7a3481 100644 --- a/src/io/data.jl +++ b/src/io/data.jl @@ -1246,6 +1246,11 @@ end export get_table export get_table +function get_cross_table(data, table_name) + return data[table_name]::OrderedDict{Symbol, OrderedDict{Symbol,OrderedDict{Symbol,Function}}} +end +export get_cross_table + """ get_subtable(table::DataFrame, conditions...) From 326f07f99ac441b8918c80441cfb48d1b8daeeae Mon Sep 17 00:00:00 2001 From: Peplinski Date: Fri, 10 Oct 2025 16:57:06 -0400 Subject: [PATCH 08/35] Moves get_cross_table function to the correct location --- src/results/util.jl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/results/util.jl b/src/results/util.jl index a1377d92..a9d9459b 100644 --- a/src/results/util.jl +++ b/src/results/util.jl @@ -156,7 +156,3 @@ function unweight_hourly(data, v::Vector{<:Container}, s=+) end export unweight_hourly -function get_cross_table(data, table_name) - return data[table_name]::OrderedDict{Symbol, OrderedDict{Symbol,OrderedDict{Symbol,Function}}} -end -export get_cross_table From 6cf66c4d5af5f76ebf3cbbab8d5571e353e12586 Mon Sep 17 00:00:00 2001 From: Peplinski Date: Fri, 10 Oct 2025 16:58:20 -0400 Subject: [PATCH 09/35] Option to add calibrator values to retail price calculation set up --- src/results/retail_price.jl | 50 ++++++++++++++++++++++ src/types/modifications/ResultsTemplate.jl | 11 +++-- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl index c6f785bb..001be49a 100644 --- a/src/results/retail_price.jl +++ b/src/results/retail_price.jl @@ -87,4 +87,54 @@ function compute_retail_price(data, price_type::Symbol, idxs...) return value/elserv_total end +function compute_retail_price(data, price_type::Symbol, calibrator_file::String, idxs, yr_idxs, hr_idxs) + value = 0.0 + retail_price = get_retail_price(data) + table_names = retail_price[price_type] + for (table_name, result_names) in table_names + for (result_name, result_sign) in result_names + res = compute_result(data, table_name, result_name, idxs, yr_idxs, hr_idxs) |> result_sign + value += res + end + end + # divide by total generation to get dollars per MWh + elserv_total = compute_result(data, :bus, :elserv_total, idxs, yr_idxs, hr_idxs) + retail_price = value/elserv_total + + cal = get_calibrator_value(calibrator_file, idxs, yr_idxs, hr_idxs) + retail_price = retail_price + cal + return retail_price +end + export compute_retail_price + +function get_calibrator_value(calibrator_file, idxs, yr_idxs, hr_idxs) + # adjust retail price with calibrator values + cal_table = read_table(calibrator_file) + + if isempty(idxs) + area = "" + subarea = "" + elseif length(idxs)==1 && idxs[1] isa Pair + area = idxs[1].first + subarea = idxs[1].second + else + error("Retail price calibrator is not set up to handle multiple filters.") + end + + @assert yr_idxs != Any[] "Retail price calibrator is not set up to handle average retail rate across years." + year = yr_idxs + + @assert hr_idxs == Colon() "Retail price calibrator is not set up to handle hourly retail rates." + + cal_values =[] + for (i, row) in enumerate(eachrow(cal_table)) + if row.area == "" && row.subarea == "" && !isempty(idxs) + push!(cal_values, row[year]) + elseif row.area == area && row.subarea ==subarea + push!(cal_values, row[year]) + end + end + + return sum(cal_values) +end \ No newline at end of file diff --git a/src/types/modifications/ResultsTemplate.jl b/src/types/modifications/ResultsTemplate.jl index ed29c93c..969f3ec1 100644 --- a/src/types/modifications/ResultsTemplate.jl +++ b/src/types/modifications/ResultsTemplate.jl @@ -24,8 +24,9 @@ struct ResultsTemplate <: Modification name::Symbol table::DataFrame cross_table::Bool + calibrator_file::String col_sort - function ResultsTemplate(;file, name, cross_table=false, col_sort=:initial_order) + function ResultsTemplate(;file, name, cross_table=false, calibrator_file="", col_sort=:initial_order) table = read_table(file) force_table_types!(table, name, :table_name=>Symbol, @@ -38,7 +39,7 @@ struct ResultsTemplate <: Modification hasproperty(table, col_name) || continue force_table_types!(table, name, col_name=>String) end - return new(file, name, table, cross_table, col_sort) + return new(file, name, table, cross_table, calibrator_file, col_sort) end end @@ -196,7 +197,11 @@ function modify_results!(m::ResultsTemplate, ::Val{:true}, config, data) @warn "Hourly retail price calculations are not set up." return 0.0 else - return compute_retail_price(data, result_name, idxs, yr_idxs, hr_idxs) + if isempty(m.calibrator_file) + return compute_retail_price(data, result_name, idxs, yr_idxs, hr_idxs) + else + return compute_retail_price(data, result_name, m.calibrator_file, idxs, yr_idxs, hr_idxs) + end end else @warn "No cross-table results formula found for table $table_name and result $result_name" From 2b8ed3bfa5a9b5ce9b9b43a6f8ee6a9c06dfac3a Mon Sep 17 00:00:00 2001 From: Peplinski Date: Fri, 10 Oct 2025 19:48:04 -0400 Subject: [PATCH 10/35] Sets up mod that calcultes the calibrator values for retail price --- src/E4ST.jl | 1 + .../modifications/RetailPriceCalibration.jl | 76 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 src/types/modifications/RetailPriceCalibration.jl diff --git a/src/E4ST.jl b/src/E4ST.jl index 16d59940..7175399c 100644 --- a/src/E4ST.jl +++ b/src/E4ST.jl @@ -73,6 +73,7 @@ include("types/modifications/GenHashID.jl") include("types/modifications/LeftJoinCols.jl") include("types/modifications/CapacityConstraint.jl") include("types/modifications/PerfectForesight.jl") +include("types/modifications/RetailPriceCalibration.jl") # Include Policies include("types/policies/ITC.jl") diff --git a/src/types/modifications/RetailPriceCalibration.jl b/src/types/modifications/RetailPriceCalibration.jl new file mode 100644 index 00000000..2f590d21 --- /dev/null +++ b/src/types/modifications/RetailPriceCalibration.jl @@ -0,0 +1,76 @@ +""" + struct RetailPriceCalibration <: Modification + +Arguments/keyword arguments: +* name::Symbol +* file - file with retail price calibrator values. +""" +struct RetailPriceCalibration <: Modification + name::Symbol + res_name::Symbol + file::String +end +RetailPriceCalibration(; name, res_name, file) = RetailPriceCalibration(name, res_name, file) +export RetailPriceCalibration + +mod_rank(::Type{<:RetailPriceCalibration}) = 5.0 + +""" + modify_results!(mod::RetailPriceCalibration, config, data) +""" +function modify_results!(mod::RetailPriceCalibration, config, data) + println("in the modification for retail price calibration") + results = get_results(data) + haskey(results, mod.res_name) || (@warn "Missing $(mod.res_name) in data, skipping calibration."; return) + retail_price = results[mod.res_name] + nyr = get_num_years(data) + years = get_years(data) + + + filter_cols = setdiff(propertynames(retail_price), [:table_name, :result_name]) + + ref_price_table = read_table(mod.file) + stack_cols = intersect(names(ref_price_table), years) + # retail_price.ref_price .= 0 + + for i in 1:nrow(ref_price_table) + println("in the rows") + row = ref_price_table[i, :] + + get(row, :status, true) || continue + # shape = Float64[row[i_yr] for i_yr in yr_idx:(yr_idx + nyr - 1)] + + filters = parse_comparisons(row) + # ref_price_table.filter + + tmp_retail_price = deepcopy(retail_price) + for filter in filters + filter!(row -> any(c -> row[c] == filter, filter_cols), tmp_retail_price) + end + println(filters) + println(tmp_retail_price) + stacked_row = stack(DataFrame(row), stack_cols, variable_name=:filter_years,value_name=:ref_price) + + stacked_row.filter_years .= "years=>" .* string.(stacked_row.filter_years) + + tmp_retail_price = innerjoin(tmp_retail_price, stacked_row, on = :filter_years) + # println(tmp_retail_price) + + end + + # CSV.write(get_out_path(config, string(m.name, ".csv")), table) +end +export modify_results! + +# function extract_results(m::WelfareTable, config, data) +# results = get_results(data) +# haskey(results, m.name) || modify_results!(m, config, data) +# return get_result(data, m.name) +# end + +# function combine_results(m::WelfareTable, post_config, post_data) + +# res = join_sim_tables(post_data, :value) + +# CSV.write(get_out_path(post_config, "$(m.name)_combined.csv"), res) +# end From 1fdb8cfff4cf0155121e422c7489c3d58a78912e Mon Sep 17 00:00:00 2001 From: Peplinski Date: Wed, 15 Oct 2025 00:26:16 -0400 Subject: [PATCH 11/35] Moves helpful functions in ResultsTemplate to util --- src/results/util.jl | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/results/util.jl b/src/results/util.jl index a9d9459b..39baf337 100644 --- a/src/results/util.jl +++ b/src/results/util.jl @@ -156,3 +156,33 @@ function unweight_hourly(data, v::Vector{<:Container}, s=+) end export unweight_hourly +function hours_sortby(s::T) where T + if endswith(s, r"h\d+") + m = match(r"h(\d+)", s) + return lpad(m.captures[1], 4, '0') |> T + else + return s |> T + end +end +export hours_sortby + +function not_a_full_filter(row::DataFrameRow) + not_a_full_filter(row.filter_years) && return true + not_a_full_filter(row.filter_hours) && return true + for i in 1:1000 + col_name = "filter$i" + hasproperty(row, col_name) || break + not_a_full_filter(row[col_name]) && return true + end + return false +end + +function not_a_full_filter(s::AbstractString) + isempty(s) && return false + all(isnumeric, s) && return false + contains(s, "=>") && return false + startswith(s, "[") && return false + startswith(s, "y2") && return false + return true +end +export not_a_full_filter \ No newline at end of file From 0e3d93dcac9bac4ce6d4aa952beb8d04be692282 Mon Sep 17 00:00:00 2001 From: Peplinski Date: Wed, 15 Oct 2025 00:27:22 -0400 Subject: [PATCH 12/35] Creates a RetailPrice mod that has option to calculate calibrator values or calibrate --- src/E4ST.jl | 1 + src/results/retail_price.jl | 64 ++++++- src/types/modifications/ResultsTemplate.jl | 58 +++--- src/types/modifications/RetailPrice.jl | 180 ++++++++++++++++++ .../modifications/RetailPriceCalibration.jl | 54 +++--- 5 files changed, 304 insertions(+), 53 deletions(-) create mode 100644 src/types/modifications/RetailPrice.jl diff --git a/src/E4ST.jl b/src/E4ST.jl index 7175399c..5ecb78cd 100644 --- a/src/E4ST.jl +++ b/src/E4ST.jl @@ -74,6 +74,7 @@ include("types/modifications/LeftJoinCols.jl") include("types/modifications/CapacityConstraint.jl") include("types/modifications/PerfectForesight.jl") include("types/modifications/RetailPriceCalibration.jl") +include("types/modifications/RetailPrice.jl") # Include Policies include("types/policies/ITC.jl") diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl index 001be49a..2b439b25 100644 --- a/src/results/retail_price.jl +++ b/src/results/retail_price.jl @@ -87,7 +87,7 @@ function compute_retail_price(data, price_type::Symbol, idxs...) return value/elserv_total end -function compute_retail_price(data, price_type::Symbol, calibrator_file::String, idxs, yr_idxs, hr_idxs) +function compute_retail_price(data, price_type::Symbol, ref_price_file::String, idxs, yr_idxs, hr_idxs) value = 0.0 retail_price = get_retail_price(data) table_names = retail_price[price_type] @@ -101,13 +101,69 @@ function compute_retail_price(data, price_type::Symbol, calibrator_file::String, elserv_total = compute_result(data, :bus, :elserv_total, idxs, yr_idxs, hr_idxs) retail_price = value/elserv_total - cal = get_calibrator_value(calibrator_file, idxs, yr_idxs, hr_idxs) - retail_price = retail_price + cal - return retail_price + ref_value, area, subarea, year = compute_calibrator_value(ref_price_file, idxs, yr_idxs, hr_idxs, retail_price) + return retail_price, [area, subarea, year, retail_price - ref_value] end +# function compute_retail_price(data, price_type::Symbol, calibrator_file::String, idxs, yr_idxs, hr_idxs) +# value = 0.0 +# retail_price = get_retail_price(data) +# table_names = retail_price[price_type] +# for (table_name, result_names) in table_names +# for (result_name, result_sign) in result_names +# res = compute_result(data, table_name, result_name, idxs, yr_idxs, hr_idxs) |> result_sign +# value += res +# end +# end +# # divide by total generation to get dollars per MWh +# elserv_total = compute_result(data, :bus, :elserv_total, idxs, yr_idxs, hr_idxs) +# retail_price = value/elserv_total +# if calibrator_file == "calibrator_file" +# cal = get_calibrator_value(calibrator_file, idxs, yr_idxs, hr_idxs) +# retail_price = retail_price + cal +# return retail_price +# elseif calibrator_file == "retail_price_file" +# ref_value, area, subarea = compute_calibrator_value(ref_price_file, retail_price, idxs, yr_idxs, hr_idxs) +# return retail_price, [area, subarea, retail_price - ref_value] +# end +# end + export compute_retail_price +function compute_calibrator_value(ref_price_file, idxs, yr_idxs, hr_idxs, retail_price) + # get corresponding price values + ref_price_table = read_table(ref_price_file) + + if isempty(idxs) + area = "" + subarea = "" + elseif length(idxs)==1 && idxs[1] isa Pair + area = idxs[1].first + subarea = idxs[1].second + else + error("Retail price calibrator is not set up to handle multiple filters.") + end + + year = yr_idxs + + @assert hr_idxs == Colon() "Retail price calibrator is not set up to handle hourly retail rates." + + ref_values = [] + for (i, row) in enumerate(eachrow(ref_price_table)) + if row.area == area && row.subarea ==subarea + push!(ref_values, row[year]) + end + end + + if length(ref_values) > 1 + error("Retail price calibator is not set up to handle multiple referenc prices") + elseif isempty(ref_values) + push!(ref_values, retail_price) + end + + return sum(ref_values), area, subarea, year +end + function get_calibrator_value(calibrator_file, idxs, yr_idxs, hr_idxs) # adjust retail price with calibrator values cal_table = read_table(calibrator_file) diff --git a/src/types/modifications/ResultsTemplate.jl b/src/types/modifications/ResultsTemplate.jl index 969f3ec1..15ddedec 100644 --- a/src/types/modifications/ResultsTemplate.jl +++ b/src/types/modifications/ResultsTemplate.jl @@ -45,6 +45,10 @@ end export ResultsTemplate +# Outer constructor to allow positional arguments if needed +ResultsTemplate(file::String, name::Symbol, table::DataFrame, cross_table::Bool, calibrator_file::String, col_sort) = + ResultsTemplate(file=file, name=name, cross_table=cross_table, calibrator_file=calibrator_file, col_sort=col_sort) + # Deal with backwards compatibility const AggregationTemplate = ResultsTemplate SYM2TYPE[:AggregationTemplate] = ResultsTemplate @@ -216,14 +220,14 @@ function modify_results!(m::ResultsTemplate, ::Val{:true}, config, data) return end -function hours_sortby(s::T) where T - if endswith(s, r"h\d+") - m = match(r"h(\d+)", s) - return lpad(m.captures[1], 4, '0') |> T - else - return s |> T - end -end +# function hours_sortby(s::T) where T +# if endswith(s, r"h\d+") +# m = match(r"h(\d+)", s) +# return lpad(m.captures[1], 4, '0') |> T +# else +# return s |> T +# end +# end function extract_results(m::ResultsTemplate, config, data) results = get_results(data) @@ -239,22 +243,22 @@ function combine_results(m::ResultsTemplate, post_config, post_data) CSV.write(get_out_path(post_config, "$(m.name)_combined.csv"), res) end -function not_a_full_filter(row::DataFrameRow) - not_a_full_filter(row.filter_years) && return true - not_a_full_filter(row.filter_hours) && return true - for i in 1:1000 - col_name = "filter$i" - hasproperty(row, col_name) || break - not_a_full_filter(row[col_name]) && return true - end - return false -end - -function not_a_full_filter(s::AbstractString) - isempty(s) && return false - all(isnumeric, s) && return false - contains(s, "=>") && return false - startswith(s, "[") && return false - startswith(s, "y2") && return false - return true -end \ No newline at end of file +# function not_a_full_filter(row::DataFrameRow) +# not_a_full_filter(row.filter_years) && return true +# not_a_full_filter(row.filter_hours) && return true +# for i in 1:1000 +# col_name = "filter$i" +# hasproperty(row, col_name) || break +# not_a_full_filter(row[col_name]) && return true +# end +# return false +# end + +# function not_a_full_filter(s::AbstractString) +# isempty(s) && return false +# all(isnumeric, s) && return false +# contains(s, "=>") && return false +# startswith(s, "[") && return false +# startswith(s, "y2") && return false +# return true +# end \ No newline at end of file diff --git a/src/types/modifications/RetailPrice.jl b/src/types/modifications/RetailPrice.jl new file mode 100644 index 00000000..0c246cca --- /dev/null +++ b/src/types/modifications/RetailPrice.jl @@ -0,0 +1,180 @@ + +""" + RetailPrice(;file, name, col_sort=:initial_order) <: Modification + +This is a mod that outputs computed results, given a `file` representing the template of the things to be aggregated. `name` is simply the name of the modification, and will be used as the root for the filename that the aggregated information is saved to. This can be used for computing results or welfare. + +## Keyword Arguments +* `file` - the file pointing to a table specifying which results to calculate +* `name` - the name of the mod, do not need to specify in a config file +* `col_sort` - the column(s) to sort by. Defaults to the order in which they were originally specified. +* `cross_table` - indicates that the result is pulling results from multiple tables. Defaults to false. + +The `file` should represent a csv table with the following columns: +* `table_name` - the name of the table being aggregated. i.e. `gen`, `bus`, etc. If you leave it empty, it will call `compute_welfare` instead of `compute_result` +* `result_name` - the name of the column in the table being aggregated. Note that the column must have a Unit accessible via [`get_table_col_unit`](@ref). +* `filter_` - the filtering conditions for the rows of the table. I.e. `filter1`. See [`parse_comparisons`](@ref) for information on what types of filters could be provided. +* `filter_years` - the filtering conditions for the years to be aggregated. See [`parse_year_idxs`](@ref) for information on the year filters. +* `filter_hours` - the filtering conditions for the hours to be aggregated. See [`parse_hour_idxs`](@ref) for information on the hour filters. + +Note that, for the `filter_` or `filter_hours` columns, if a column name of the data table (or hours table) is given, new rows will be created for each unique value of that column. I.e. if a value of `gentype` is given, there will be made a new row for `gentype=>coal`, `gentype=>ng`, etc. +""" + +# struct RetailPrice <: Modification +# base::Union{ResultsTemplate, Nothing} # will hold ResultsTemplate after init +# params::Dict{Symbol,Any} # store file, name, col_sort +# calibrator_file::String +# end + +# # Constructor: store parameters only, no table built yet +# function RetailPrice(; file, name=nothing, calibrator_file="", col_sort=:initial_order) +# params = Dict(:file => file, :name => name, :col_sort => col_sort) +# return RetailPrice(nothing, params, calibrator_file) +# end + +# # Lazy initializer: builds ResultsTemplate and forces types +# function init_base!(m::RetailPrice) +# println("init here") +# if m.base === nothing +# m.base = ResultsTemplate( +# file = m.params[:file], +# name = m.params[:name], +# cross_table = false, +# calibrator_file = m.calibrator_file, +# col_sort = m.params[:col_sort] +# ) +# end +# end + + +# export RetailPrice + +struct RetailPrice <: Modification + file::String + name::Symbol + table::DataFrame + calibrator_file::String + col_sort + function RetailPrice(;file, name, calibrator_file="", col_sort=:initial_order) + table = read_table(file) + force_table_types!(table, name, + :table_name=>Symbol, + :result_name=>Symbol, + :filter_years=>String, + :filter_hours=>String, + ) + for i in 1:1000 + col_name = "filter$i" + hasproperty(table, col_name) || continue + force_table_types!(table, name, col_name=>String) + end + return new(file, name, table, calibrator_file, col_sort) + end +end + +export RetailPrice + + +mod_rank(::Type{<:RetailPrice}) = 5.0 + +fieldnames_for_yaml(::Type{RetailPrice}) = (:file,) + + +function modify_results!(m::RetailPrice, config, data) + table = copy(m.table) + table.initial_order = 1:nrow(table) + + filter_cols = setdiff(propertynames(table), [:table_name, :result_name]) + + # for any rows that are not a pair, separate into multiple rows + not_pair_idx = findfirst(not_a_full_filter, eachrow(table)) + while not_pair_idx !== nothing + row = table[not_pair_idx, :] + filter_col_idx = findfirst(filter_col->not_a_full_filter(row[filter_col]), filter_cols) + col_to_expand = filter_cols[filter_col_idx] + table_name = row[:table_name] + result_name = row[:result_name] + + if col_to_expand == :filter_hours + area = row.filter_hours + hours_table_col = get_table_col(data, :hours, area) + subareas = Base.sort!(String.(string.(unique(hours_table_col))), by=hours_sortby) + elseif col_to_expand == :filter_years && row[col_to_expand] == ":" + area = :years + subareas = data[area] + else + area = row[col_to_expand] + table_names = get_cross_table(data, table_name)[result_name] + all(hasproperty(get_table(data, t), area) for (t, _) in table_names) || error("Some tables are missing property $(area)") + data_table_col = get_table_col(data, first(keys(table_names)), area) + subareas = sort!(unique(data_table_col)) + end + + row_dict = Dict(pairs(row)) + for subarea in subareas + # Add a row right after the original row + row_dict[col_to_expand] = "$area=>$subarea" + insert!(table, not_pair_idx+1, row_dict) + end + + deleteat!(table, not_pair_idx) + + # Find the next index that is not a pair, to be expanded + not_pair_idx = findfirst(not_a_full_filter, eachrow(table)) + end + + @info "Calculating results for $(nrow(table)) rows in RetailPrice $(m.name)" + results_formulas = get_results_formulas(data) + cal_table = DataFrame(area = String[], subarea = [], year=[], value = []) + #to do: check that the each ref price has a corresponding retail rate or else provide warning + # table.value = map(eachrow(table)) do row + if !hasproperty(table, :value) + table.value = Vector{Union{Missing, Float64}}(missing, nrow(table)) + end + for i in 1:nrow(table) + table_name = table[i, :table_name] + result_name = table[i,:result_name] + idxs = parse_comparisons(table[i,:]) + yr_idxs = parse_year_idxs(table[i,:filter_years]) + hr_idxs = parse_hour_idxs(table[i,:filter_hours]) + + if hr_idxs !== Colon() + @warn "Hourly retail price calculations are not set up." + return 0.0 + else + if isempty(m.calibrator_file) + println(idxs) + val = compute_retail_price(data, result_name, idxs, yr_idxs, hr_idxs) + else + # val = compute_retail_price(data, result_name, m.calibrator_file, idxs, yr_idxs, hr_idxs) + val, cal_row = compute_retail_price(data, result_name, m.calibrator_file, idxs, yr_idxs, hr_idxs) + push!(cal_table, cal_row) + end + end + table.value[i]= val + end + sort!(table, m.col_sort) + select!(table, Not(:initial_order)) + CSV.write(get_out_path(config, string(m.name, ".csv")), table) + results = get_results(data) + results[m.name] = table + + CSV.write(get_out_path(config, string(m.name, "_cals.csv")), cal_table) + return +end + + +# function extract_results(m::RetailPrice, config, data) +# results = get_results(data) +# # haskey(results, m.name) || modify_results!(m, config, data) +# modify_results!(m, config, data) +# return get_result(data, m.name) +# end + +# function combine_results(m::RetailPrice, post_config, post_data) + +# res = join_sim_tables(post_data, :value) + +# CSV.write(get_out_path(post_config, "$(m.name)_combined.csv"), res) +# end + diff --git a/src/types/modifications/RetailPriceCalibration.jl b/src/types/modifications/RetailPriceCalibration.jl index 2f590d21..2703d916 100644 --- a/src/types/modifications/RetailPriceCalibration.jl +++ b/src/types/modifications/RetailPriceCalibration.jl @@ -25,39 +25,49 @@ function modify_results!(mod::RetailPriceCalibration, config, data) retail_price = results[mod.res_name] nyr = get_num_years(data) years = get_years(data) + bus = get_table(data,:bus) - filter_cols = setdiff(propertynames(retail_price), [:table_name, :result_name]) - + filter_cols = setdiff(propertynames(retail_price), [:table_name, :result_name, :filter_years, :filter_hours, :value]) + for filter in filter_cols + if !isempty(retail_price[!,filter]) + println(filter) + unique_firsts = unique(first.(split.(retail_price[!,filter], "=>"))) + println(unique_firsts) + end + end + + println(filter_cols) ref_price_table = read_table(mod.file) - stack_cols = intersect(names(ref_price_table), years) + stack_cols = Symbol.(intersect(names(ref_price_table), years)) # retail_price.ref_price .= 0 - for i in 1:nrow(ref_price_table) - println("in the rows") - row = ref_price_table[i, :] + # for i in 1:nrow(ref_price_table) + # row = ref_price_table[i,:] - get(row, :status, true) || continue - # shape = Float64[row[i_yr] for i_yr in yr_idx:(yr_idx + nyr - 1)] + # get(row, :status, true) || continue + # # shape = Float64[row[i_yr] for i_yr in yr_idx:(yr_idx + nyr - 1)] - filters = parse_comparisons(row) - # ref_price_table.filter + # filters = parse_comparisons(row) + # println(filters) + # # ref_price_table.filter - tmp_retail_price = deepcopy(retail_price) - for filter in filters - filter!(row -> any(c -> row[c] == filter, filter_cols), tmp_retail_price) - end - println(filters) - println(tmp_retail_price) - stacked_row = stack(DataFrame(row), stack_cols, variable_name=:filter_years,value_name=:ref_price) + # tmp_retail_price = deepcopy(retail_price) + # for filter in filters + # filter = string(first(filter)filter2string(filter) + # filter!(row -> any(c -> row[c] == filter, filter_cols), tmp_retail_price) + # end + + # stacked_row = DataFrames.stack(ref_price_table[i:i, vcat(:area, :subarea, stack_cols)], + # stack_cols, [:area, :subarea], variable_name=:filter_years,value_name=:ref_price) - stacked_row.filter_years .= "years=>" .* string.(stacked_row.filter_years) + # stacked_row.filter_years .= "years=>" .* string.(stacked_row.filter_years) - tmp_retail_price = innerjoin(tmp_retail_price, stacked_row, on = :filter_years) - # println(tmp_retail_price) + # tmp_retail_price = innerjoin(tmp_retail_price, stacked_row, on = [:filter_years]) + # println(tmp_retail_price) - end - + # end + # to do: the price may need to be energy demand/consumption # CSV.write(get_out_path(config, string(m.name, ".csv")), table) end export modify_results! From a6e601d0d4d356b9e0ef67ad7aab4fb5773f405b Mon Sep 17 00:00:00 2001 From: Peplinski Date: Wed, 15 Oct 2025 00:48:33 -0400 Subject: [PATCH 13/35] Adjusts kwargs for retail price calibrator --- src/results/retail_price.jl | 42 ++++++++++++------------ src/types/modifications/RetailPrice.jl | 45 ++++++-------------------- 2 files changed, 29 insertions(+), 58 deletions(-) diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl index 2b439b25..547f7d7c 100644 --- a/src/results/retail_price.jl +++ b/src/results/retail_price.jl @@ -105,28 +105,25 @@ function compute_retail_price(data, price_type::Symbol, ref_price_file::String, return retail_price, [area, subarea, year, retail_price - ref_value] end -# function compute_retail_price(data, price_type::Symbol, calibrator_file::String, idxs, yr_idxs, hr_idxs) -# value = 0.0 -# retail_price = get_retail_price(data) -# table_names = retail_price[price_type] -# for (table_name, result_names) in table_names -# for (result_name, result_sign) in result_names -# res = compute_result(data, table_name, result_name, idxs, yr_idxs, hr_idxs) |> result_sign -# value += res -# end -# end -# # divide by total generation to get dollars per MWh -# elserv_total = compute_result(data, :bus, :elserv_total, idxs, yr_idxs, hr_idxs) -# retail_price = value/elserv_total -# if calibrator_file == "calibrator_file" -# cal = get_calibrator_value(calibrator_file, idxs, yr_idxs, hr_idxs) -# retail_price = retail_price + cal -# return retail_price -# elseif calibrator_file == "retail_price_file" -# ref_value, area, subarea = compute_calibrator_value(ref_price_file, retail_price, idxs, yr_idxs, hr_idxs) -# return retail_price, [area, subarea, retail_price - ref_value] -# end -# end +function compute_retail_price(data, price_type::Symbol, cal::Bool, calibrator_file::String, idxs, yr_idxs, hr_idxs) + value = 0.0 + retail_price = get_retail_price(data) + table_names = retail_price[price_type] + for (table_name, result_names) in table_names + for (result_name, result_sign) in result_names + res = compute_result(data, table_name, result_name, idxs, yr_idxs, hr_idxs) |> result_sign + value += res + end + end + # divide by total generation to get dollars per MWh + elserv_total = compute_result(data, :bus, :elserv_total, idxs, yr_idxs, hr_idxs) + retail_price = value/elserv_total + + cal = get_calibrator_value(calibrator_file, idxs, yr_idxs, hr_idxs) + retail_price = retail_price + cal + return retail_price + +end export compute_retail_price @@ -144,6 +141,7 @@ function compute_calibrator_value(ref_price_file, idxs, yr_idxs, hr_idxs, retail error("Retail price calibrator is not set up to handle multiple filters.") end + @assert yr_idxs != Any[] "Retail price calibrator is not set up to handle average retail rate across years." year = yr_idxs @assert hr_idxs == Colon() "Retail price calibrator is not set up to handle hourly retail rates." diff --git a/src/types/modifications/RetailPrice.jl b/src/types/modifications/RetailPrice.jl index 0c246cca..f6d30b28 100644 --- a/src/types/modifications/RetailPrice.jl +++ b/src/types/modifications/RetailPrice.jl @@ -20,42 +20,15 @@ The `file` should represent a csv table with the following columns: Note that, for the `filter_` or `filter_hours` columns, if a column name of the data table (or hours table) is given, new rows will be created for each unique value of that column. I.e. if a value of `gentype` is given, there will be made a new row for `gentype=>coal`, `gentype=>ng`, etc. """ -# struct RetailPrice <: Modification -# base::Union{ResultsTemplate, Nothing} # will hold ResultsTemplate after init -# params::Dict{Symbol,Any} # store file, name, col_sort -# calibrator_file::String -# end - -# # Constructor: store parameters only, no table built yet -# function RetailPrice(; file, name=nothing, calibrator_file="", col_sort=:initial_order) -# params = Dict(:file => file, :name => name, :col_sort => col_sort) -# return RetailPrice(nothing, params, calibrator_file) -# end - -# # Lazy initializer: builds ResultsTemplate and forces types -# function init_base!(m::RetailPrice) -# println("init here") -# if m.base === nothing -# m.base = ResultsTemplate( -# file = m.params[:file], -# name = m.params[:name], -# cross_table = false, -# calibrator_file = m.calibrator_file, -# col_sort = m.params[:col_sort] -# ) -# end -# end - - -# export RetailPrice - struct RetailPrice <: Modification file::String name::Symbol table::DataFrame calibrator_file::String + cal_table:: Bool + cal:: Bool col_sort - function RetailPrice(;file, name, calibrator_file="", col_sort=:initial_order) + function RetailPrice(;file, name, calibrator_file="", cal_table=false, cal=true, col_sort=:initial_order) table = read_table(file) force_table_types!(table, name, :table_name=>Symbol, @@ -68,7 +41,7 @@ struct RetailPrice <: Modification hasproperty(table, col_name) || continue force_table_types!(table, name, col_name=>String) end - return new(file, name, table, calibrator_file, col_sort) + return new(file, name, table, calibrator_file, cal_table, cal, col_sort) end end @@ -142,13 +115,13 @@ function modify_results!(m::RetailPrice, config, data) @warn "Hourly retail price calculations are not set up." return 0.0 else - if isempty(m.calibrator_file) - println(idxs) - val = compute_retail_price(data, result_name, idxs, yr_idxs, hr_idxs) - else - # val = compute_retail_price(data, result_name, m.calibrator_file, idxs, yr_idxs, hr_idxs) + if m.cal_table == true val, cal_row = compute_retail_price(data, result_name, m.calibrator_file, idxs, yr_idxs, hr_idxs) push!(cal_table, cal_row) + elseif m.cal == true + val = compute_retail_price(data, result_name, m.cal, m.calibrator_file, idxs, yr_idxs, hr_idxs) + else + val = compute_retail_price(data, result_name, idxs, yr_idxs, hr_idxs) end end table.value[i]= val From 4ae6c4fb5b95a58c47d02f2e8b4d7d5dda4bf187 Mon Sep 17 00:00:00 2001 From: Peplinski Date: Wed, 15 Oct 2025 11:28:04 -0400 Subject: [PATCH 14/35] Warns if retail price row does not have corresponding calibrator --- src/results/retail_price.jl | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl index 547f7d7c..b268ab1d 100644 --- a/src/results/retail_price.jl +++ b/src/results/retail_price.jl @@ -16,6 +16,14 @@ Reference the results formulas for more detailed descriptions of each of these t The results template can calculate annual rates by specified region, but is not set up for hourly rates. """ + +# to do: option for 1 year or all years +# to do: divide revenue by consumption? +# to do: state and national calibration +# to do: update docstrings +# to do: improve dispatch for cal table vs cal +# adjust calibrator table to be read in with right format, or change function to read in correctly + function setup_retail_price!(config, data) retail_price = OrderedDict{Symbol, OrderedDict{Symbol,OrderedDict{Symbol,Function}}}() data[:retail_price] = retail_price @@ -101,7 +109,7 @@ function compute_retail_price(data, price_type::Symbol, ref_price_file::String, elserv_total = compute_result(data, :bus, :elserv_total, idxs, yr_idxs, hr_idxs) retail_price = value/elserv_total - ref_value, area, subarea, year = compute_calibrator_value(ref_price_file, idxs, yr_idxs, hr_idxs, retail_price) + ref_value, area, subarea, year = get_ref_price(ref_price_file, idxs, yr_idxs, hr_idxs, retail_price) return retail_price, [area, subarea, year, retail_price - ref_value] end @@ -127,7 +135,7 @@ end export compute_retail_price -function compute_calibrator_value(ref_price_file, idxs, yr_idxs, hr_idxs, retail_price) +function get_ref_price(ref_price_file, idxs, yr_idxs, hr_idxs, retail_price) # get corresponding price values ref_price_table = read_table(ref_price_file) @@ -156,6 +164,7 @@ function compute_calibrator_value(ref_price_file, idxs, yr_idxs, hr_idxs, retail if length(ref_values) > 1 error("Retail price calibator is not set up to handle multiple referenc prices") elseif isempty(ref_values) + @warn "There is no reference retail price for area $(area) and subarea $(subarea). This region will not be calibrated." push!(ref_values, retail_price) end @@ -189,6 +198,13 @@ function get_calibrator_value(calibrator_file, idxs, yr_idxs, hr_idxs) push!(cal_values, row[year]) end end + + if length(cal_values) > 1 + error("Retail price calibator is not set up to handle multiple referenc prices") + elseif isempty(cal_values) + @warn "There is no calibrator value for area $(area) and subarea $(subarea). This region will not be calibrated." + push!(cal_values, 0) + end return sum(cal_values) end \ No newline at end of file From e03be1bbbc231a4300141401c70ebbd4bb776842 Mon Sep 17 00:00:00 2001 From: Peplinski Date: Wed, 15 Oct 2025 14:46:43 -0400 Subject: [PATCH 15/35] Specialized methods for different calibraton modes --- src/results/retail_price.jl | 14 ++- src/types/modifications/RetailPrice.jl | 155 ++++++++++++++++++++----- 2 files changed, 134 insertions(+), 35 deletions(-) diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl index b268ab1d..88acf9e6 100644 --- a/src/results/retail_price.jl +++ b/src/results/retail_price.jl @@ -80,7 +80,11 @@ function add_price_term!(data, price_type::Symbol, table_name::Symbol, result_na end export add_price_term! -function compute_retail_price(data, price_type::Symbol, idxs...) +function compute_retail_price(m, data, price_type::Symbol, idxs, yr_idxs, hr_idxs) + compute_retail_price((Val(Symbol(m.cal_mode))), m, data, price_type, idxs, yr_idxs, hr_idxs) +end + +function compute_retail_price(::Val{:none}, m, data, price_type::Symbol, idxs...) value = 0.0 retail_price = get_retail_price(data) table_names = retail_price[price_type] @@ -95,7 +99,7 @@ function compute_retail_price(data, price_type::Symbol, idxs...) return value/elserv_total end -function compute_retail_price(data, price_type::Symbol, ref_price_file::String, idxs, yr_idxs, hr_idxs) +function compute_retail_price(::Val{:get_cal_values}, m, data, price_type::Symbol, idxs, yr_idxs, hr_idxs) value = 0.0 retail_price = get_retail_price(data) table_names = retail_price[price_type] @@ -109,11 +113,11 @@ function compute_retail_price(data, price_type::Symbol, ref_price_file::String, elserv_total = compute_result(data, :bus, :elserv_total, idxs, yr_idxs, hr_idxs) retail_price = value/elserv_total - ref_value, area, subarea, year = get_ref_price(ref_price_file, idxs, yr_idxs, hr_idxs, retail_price) + ref_value, area, subarea, year = get_ref_price(m.calibrator_file, idxs, yr_idxs, hr_idxs, retail_price) return retail_price, [area, subarea, year, retail_price - ref_value] end -function compute_retail_price(data, price_type::Symbol, cal::Bool, calibrator_file::String, idxs, yr_idxs, hr_idxs) +function compute_retail_price(::Val{:calibrate}, m, data, price_type::Symbol, idxs, yr_idxs, hr_idxs) value = 0.0 retail_price = get_retail_price(data) table_names = retail_price[price_type] @@ -127,7 +131,7 @@ function compute_retail_price(data, price_type::Symbol, cal::Bool, calibrator_fi elserv_total = compute_result(data, :bus, :elserv_total, idxs, yr_idxs, hr_idxs) retail_price = value/elserv_total - cal = get_calibrator_value(calibrator_file, idxs, yr_idxs, hr_idxs) + cal = get_calibrator_value(m.calibrator_file, idxs, yr_idxs, hr_idxs) retail_price = retail_price + cal return retail_price diff --git a/src/types/modifications/RetailPrice.jl b/src/types/modifications/RetailPrice.jl index f6d30b28..45f556b1 100644 --- a/src/types/modifications/RetailPrice.jl +++ b/src/types/modifications/RetailPrice.jl @@ -24,11 +24,10 @@ struct RetailPrice <: Modification file::String name::Symbol table::DataFrame + cal_mode:: String calibrator_file::String - cal_table:: Bool - cal:: Bool col_sort - function RetailPrice(;file, name, calibrator_file="", cal_table=false, cal=true, col_sort=:initial_order) + function RetailPrice(;file, name, calibrator_file="", cal_mode="none", col_sort=:initial_order) table = read_table(file) force_table_types!(table, name, :table_name=>Symbol, @@ -41,7 +40,10 @@ struct RetailPrice <: Modification hasproperty(table, col_name) || continue force_table_types!(table, name, col_name=>String) end - return new(file, name, table, calibrator_file, cal_table, cal, col_sort) + if cal_mode != "none" + isempty(calibrator_file) && error("Calibrator file required when cal_mode is set to $(cal_mode).") + end + return new(file, name, table, cal_mode, calibrator_file, col_sort) end end @@ -96,14 +98,73 @@ function modify_results!(m::RetailPrice, config, data) not_pair_idx = findfirst(not_a_full_filter, eachrow(table)) end + get_retail_price(m, config, data, table) + + # # what should go in the function + # cal_table = DataFrame(area = String[], subarea = [], year=[], value = []) + + # if !hasproperty(table, :value) + # table.value = Vector{Union{Missing, Float64}}(missing, nrow(table)) + # end + + # for i in 1:nrow(table) + # table_name = table[i, :table_name] + # result_name = table[i,:result_name] + # idxs = parse_comparisons(table[i,:]) + # yr_idxs = parse_year_idxs(table[i,:filter_years]) + # hr_idxs = parse_hour_idxs(table[i,:filter_hours]) + + # if hr_idxs !== Colon() + # @warn "Hourly retail price calculations are not set up." + # return 0.0 + # else + # if m.cal_table == true + # val, cal_row = compute_retail_price(data, result_name, m.calibrator_file, idxs, yr_idxs, hr_idxs) + # push!(cal_table, cal_row) + # elseif m.cal == true + # val = compute_retail_price(data, result_name, m.cal, m.calibrator_file, idxs, yr_idxs, hr_idxs) + # else + # val = compute_retail_price(data, result_name, idxs, yr_idxs, hr_idxs) + # end + # end + # table.value[i]= val + # end + # sort!(table, m.col_sort) + # select!(table, Not(:initial_order)) + # CSV.write(get_out_path(config, string(m.name, ".csv")), table) + # results = get_results(data) + # results[m.name] = table + + # CSV.write(get_out_path(config, string(m.name, "_cals.csv")), cal_table) + return +end + + +# function extract_results(m::RetailPrice, config, data) +# results = get_results(data) +# # haskey(results, m.name) || modify_results!(m, config, data) +# modify_results!(m, config, data) +# return get_result(data, m.name) +# end + +# function combine_results(m::RetailPrice, post_config, post_data) + +# res = join_sim_tables(post_data, :value) + +# CSV.write(get_out_path(post_config, "$(m.name)_combined.csv"), res) +# end + +function get_retail_price(m::RetailPrice, config, data, table) @info "Calculating results for $(nrow(table)) rows in RetailPrice $(m.name)" - results_formulas = get_results_formulas(data) - cal_table = DataFrame(area = String[], subarea = [], year=[], value = []) - #to do: check that the each ref price has a corresponding retail rate or else provide warning - # table.value = map(eachrow(table)) do row + get_retail_price((Val(Symbol(m.cal_mode))), m, config, data, table) +end + +function get_retail_price(::Val{:none}, m, config, data, table) + if !hasproperty(table, :value) table.value = Vector{Union{Missing, Float64}}(missing, nrow(table)) end + for i in 1:nrow(table) table_name = table[i, :table_name] result_name = table[i,:result_name] @@ -115,14 +176,7 @@ function modify_results!(m::RetailPrice, config, data) @warn "Hourly retail price calculations are not set up." return 0.0 else - if m.cal_table == true - val, cal_row = compute_retail_price(data, result_name, m.calibrator_file, idxs, yr_idxs, hr_idxs) - push!(cal_table, cal_row) - elseif m.cal == true - val = compute_retail_price(data, result_name, m.cal, m.calibrator_file, idxs, yr_idxs, hr_idxs) - else - val = compute_retail_price(data, result_name, idxs, yr_idxs, hr_idxs) - end + val = compute_retail_price(m, data, result_name, idxs, yr_idxs, hr_idxs) end table.value[i]= val end @@ -131,23 +185,64 @@ function modify_results!(m::RetailPrice, config, data) CSV.write(get_out_path(config, string(m.name, ".csv")), table) results = get_results(data) results[m.name] = table - - CSV.write(get_out_path(config, string(m.name, "_cals.csv")), cal_table) - return end +function get_retail_price(::Val{:get_cal_values}, m::RetailPrice, config, data, table) + cal_table = DataFrame(area = String[], subarea = [], year=[], value = []) -# function extract_results(m::RetailPrice, config, data) -# results = get_results(data) -# # haskey(results, m.name) || modify_results!(m, config, data) -# modify_results!(m, config, data) -# return get_result(data, m.name) -# end - -# function combine_results(m::RetailPrice, post_config, post_data) + if !hasproperty(table, :value) + table.value = Vector{Union{Missing, Float64}}(missing, nrow(table)) + end -# res = join_sim_tables(post_data, :value) + for i in 1:nrow(table) + table_name = table[i, :table_name] + result_name = table[i,:result_name] + idxs = parse_comparisons(table[i,:]) + yr_idxs = parse_year_idxs(table[i,:filter_years]) + hr_idxs = parse_hour_idxs(table[i,:filter_hours]) + + if hr_idxs !== Colon() + @warn "Hourly retail price calculations are not set up." + return 0.0 + else + val, cal_row = compute_retail_price(m, data, result_name, idxs, yr_idxs, hr_idxs) + push!(cal_table, cal_row) + end + table.value[i]= val + end + sort!(table, m.col_sort) + select!(table, Not(:initial_order)) + CSV.write(get_out_path(config, string(m.name, ".csv")), table) + results = get_results(data) + results[m.name] = table -# CSV.write(get_out_path(post_config, "$(m.name)_combined.csv"), res) -# end + CSV.write(get_out_path(config, string(m.name, "_cals.csv")), cal_table) +end + +function get_retail_price(::Val{:calibrate}, m, config, data, table) + if !hasproperty(table, :value) + table.value = Vector{Union{Missing, Float64}}(missing, nrow(table)) + end + + for i in 1:nrow(table) + table_name = table[i, :table_name] + result_name = table[i,:result_name] + idxs = parse_comparisons(table[i,:]) + yr_idxs = parse_year_idxs(table[i,:filter_years]) + hr_idxs = parse_hour_idxs(table[i,:filter_hours]) + + if hr_idxs !== Colon() + @warn "Hourly retail price calculations are not set up." + return 0.0 + else + val = compute_retail_price(m, data, result_name, idxs, yr_idxs, hr_idxs) + end + table.value[i]= val + end + sort!(table, m.col_sort) + select!(table, Not(:initial_order)) + CSV.write(get_out_path(config, string(m.name, ".csv")), table) + results = get_results(data) + results[m.name] = table +end From 6cfb9dea6094b1908035774565aeb47fd29abb4a Mon Sep 17 00:00:00 2001 From: Peplinski Date: Wed, 15 Oct 2025 15:28:51 -0400 Subject: [PATCH 16/35] Option to apply ref value to all years --- src/results/retail_price.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl index 88acf9e6..35778a9c 100644 --- a/src/results/retail_price.jl +++ b/src/results/retail_price.jl @@ -160,8 +160,10 @@ function get_ref_price(ref_price_file, idxs, yr_idxs, hr_idxs, retail_price) ref_values = [] for (i, row) in enumerate(eachrow(ref_price_table)) - if row.area == area && row.subarea ==subarea - push!(ref_values, row[year]) + if row.area == area && row.subarea ==subarea && row.year == year + push!(ref_values, row["ref_price"]) + elseif row.area == area && row.subarea ==subarea && isempty(row.year) # if no year provided, ref price is used for all years + push!(ref_values, row["ref_price"]) end end From 113db585c1a497f992962594ac771017d2b875cc Mon Sep 17 00:00:00 2001 From: Peplinski Date: Wed, 15 Oct 2025 18:56:33 -0400 Subject: [PATCH 17/35] Second calibration step to ensure average price for larger region matches ref price --- src/results/retail_price.jl | 34 ++++++++++---- src/types/modifications/RetailPrice.jl | 65 +++++++++++--------------- 2 files changed, 52 insertions(+), 47 deletions(-) diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl index 35778a9c..8cf3a2cb 100644 --- a/src/results/retail_price.jl +++ b/src/results/retail_price.jl @@ -15,13 +15,15 @@ The relevant price terms are: Reference the results formulas for more detailed descriptions of each of these terms. The results template can calculate annual rates by specified region, but is not set up for hourly rates. + + +There are three specialized methods for calculating the retail rate that depend on the cal_mode argument in the RetailPrice mod. If cal_mode is set to `none` +there will be no calibration steps. If it is set to `get_cal_values`, the function will find the difference between the estimated retail rates +and the true retail rate to use as a calibrator value. If cal_mode is set to `calibrate`, the corresponding calibrator vaulue will be added to the retail price. """ -# to do: option for 1 year or all years -# to do: divide revenue by consumption? -# to do: state and national calibration + # to do: update docstrings -# to do: improve dispatch for cal table vs cal # adjust calibrator table to be read in with right format, or change function to read in correctly function setup_retail_price!(config, data) @@ -50,8 +52,6 @@ function setup_retail_price!(config, data) add_price_term!(data, :avg_elec_rate, :bus, :baa_reserve_requirement_merchandising_surplus_total, -) end - # future work: calculate electricity rates by end-use sector - end export setup_retail_price! @@ -113,8 +113,23 @@ function compute_retail_price(::Val{:get_cal_values}, m, data, price_type::Symbo elserv_total = compute_result(data, :bus, :elserv_total, idxs, yr_idxs, hr_idxs) retail_price = value/elserv_total - ref_value, area, subarea, year = get_ref_price(m.calibrator_file, idxs, yr_idxs, hr_idxs, retail_price) - return retail_price, [area, subarea, year, retail_price - ref_value] + ref_price_table = read_table(m.calibrator_file) + + ref_value, area, subarea, year = get_ref_price(ref_price_table, idxs, yr_idxs, hr_idxs, retail_price) + + subset = filter(row -> row.area == "" && row.subarea == "", ref_price_table) + + if nrow(subset) == 0 + @warn "No full model reference price row. Outputting calibration values without a full adjustment." + elserv_ratio = 0 + elseif nrow(subset) > 1 + error("Multiple full model reference price rows.") + else + elserv_total_all = compute_result(data, :bus, :elserv_total, :, yr_idxs, hr_idxs) + elserv_ratio = elserv_total/elserv_total_all + end + + return retail_price, [area, subarea, year, ref_value, retail_price, ref_value - retail_price, elserv_total, elserv_ratio] end function compute_retail_price(::Val{:calibrate}, m, data, price_type::Symbol, idxs, yr_idxs, hr_idxs) @@ -139,9 +154,8 @@ end export compute_retail_price -function get_ref_price(ref_price_file, idxs, yr_idxs, hr_idxs, retail_price) +function get_ref_price(ref_price_table, idxs, yr_idxs, hr_idxs, retail_price) # get corresponding price values - ref_price_table = read_table(ref_price_file) if isempty(idxs) area = "" diff --git a/src/types/modifications/RetailPrice.jl b/src/types/modifications/RetailPrice.jl index 45f556b1..24aea177 100644 --- a/src/types/modifications/RetailPrice.jl +++ b/src/types/modifications/RetailPrice.jl @@ -100,42 +100,6 @@ function modify_results!(m::RetailPrice, config, data) get_retail_price(m, config, data, table) - # # what should go in the function - # cal_table = DataFrame(area = String[], subarea = [], year=[], value = []) - - # if !hasproperty(table, :value) - # table.value = Vector{Union{Missing, Float64}}(missing, nrow(table)) - # end - - # for i in 1:nrow(table) - # table_name = table[i, :table_name] - # result_name = table[i,:result_name] - # idxs = parse_comparisons(table[i,:]) - # yr_idxs = parse_year_idxs(table[i,:filter_years]) - # hr_idxs = parse_hour_idxs(table[i,:filter_hours]) - - # if hr_idxs !== Colon() - # @warn "Hourly retail price calculations are not set up." - # return 0.0 - # else - # if m.cal_table == true - # val, cal_row = compute_retail_price(data, result_name, m.calibrator_file, idxs, yr_idxs, hr_idxs) - # push!(cal_table, cal_row) - # elseif m.cal == true - # val = compute_retail_price(data, result_name, m.cal, m.calibrator_file, idxs, yr_idxs, hr_idxs) - # else - # val = compute_retail_price(data, result_name, idxs, yr_idxs, hr_idxs) - # end - # end - # table.value[i]= val - # end - # sort!(table, m.col_sort) - # select!(table, Not(:initial_order)) - # CSV.write(get_out_path(config, string(m.name, ".csv")), table) - # results = get_results(data) - # results[m.name] = table - - # CSV.write(get_out_path(config, string(m.name, "_cals.csv")), cal_table) return end @@ -188,7 +152,7 @@ function get_retail_price(::Val{:none}, m, config, data, table) end function get_retail_price(::Val{:get_cal_values}, m::RetailPrice, config, data, table) - cal_table = DataFrame(area = String[], subarea = [], year=[], value = []) + cal_table = DataFrame(area = String[], subarea = [], year=[], ref_price =[], retail_price = [], cal_value = [], elserv_total = [], elserv_ratio=[]) if !hasproperty(table, :value) table.value = Vector{Union{Missing, Float64}}(missing, nrow(table)) @@ -216,6 +180,7 @@ function get_retail_price(::Val{:get_cal_values}, m::RetailPrice, config, data, results = get_results(data) results[m.name] = table + full_cal!(m, data, table, cal_table) CSV.write(get_out_path(config, string(m.name, "_cals.csv")), cal_table) end @@ -246,3 +211,29 @@ function get_retail_price(::Val{:calibrate}, m, config, data, table) results = get_results(data) results[m.name] = table end + +function full_cal!(m, data, table, cal_table) + ref_price_table = read_table(m.calibrator_file) + subset = filter(row -> row.area == "" && row.subarea == "", ref_price_table) + + if nrow(subset) == 0 + @warn "No full model reference price row. Outputting calibration values without a full adjustment." + elseif nrow(subset) > 1 + error("Multiple full model reference price rows.") + else + avg_price_ref = subset[1, :ref_price] + end + + avg_price = sum(cal_table.ref_price .* cal_table.elserv_ratio) + + for row in eachrow(cal_table[(cal_table.area .!= "") .& (cal_table.subarea .!= ""), :]) + println(row[:subarea]) + cal = (avg_price_ref - avg_price) * row[:elserv_ratio] + println(row[:cal_value]) + println(row[:retail_price]) + println(row[:ref_price]) + row[:cal_value] += cal + end + + return cal_table +end \ No newline at end of file From eafd974e16cfed9dc84d5b8b66b281433569414c Mon Sep 17 00:00:00 2001 From: Peplinski Date: Wed, 15 Oct 2025 23:28:40 -0400 Subject: [PATCH 18/35] Removes a previous version of retail price mod --- .../modifications/RetailPriceCalibration.jl | 86 ------------------- 1 file changed, 86 deletions(-) delete mode 100644 src/types/modifications/RetailPriceCalibration.jl diff --git a/src/types/modifications/RetailPriceCalibration.jl b/src/types/modifications/RetailPriceCalibration.jl deleted file mode 100644 index 2703d916..00000000 --- a/src/types/modifications/RetailPriceCalibration.jl +++ /dev/null @@ -1,86 +0,0 @@ -""" - struct RetailPriceCalibration <: Modification - -Arguments/keyword arguments: -* name::Symbol -* file - file with retail price calibrator values. -""" -struct RetailPriceCalibration <: Modification - name::Symbol - res_name::Symbol - file::String -end -RetailPriceCalibration(; name, res_name, file) = RetailPriceCalibration(name, res_name, file) -export RetailPriceCalibration - -mod_rank(::Type{<:RetailPriceCalibration}) = 5.0 - -""" - modify_results!(mod::RetailPriceCalibration, config, data) -""" -function modify_results!(mod::RetailPriceCalibration, config, data) - println("in the modification for retail price calibration") - results = get_results(data) - haskey(results, mod.res_name) || (@warn "Missing $(mod.res_name) in data, skipping calibration."; return) - retail_price = results[mod.res_name] - nyr = get_num_years(data) - years = get_years(data) - bus = get_table(data,:bus) - - - filter_cols = setdiff(propertynames(retail_price), [:table_name, :result_name, :filter_years, :filter_hours, :value]) - for filter in filter_cols - if !isempty(retail_price[!,filter]) - println(filter) - unique_firsts = unique(first.(split.(retail_price[!,filter], "=>"))) - println(unique_firsts) - end - end - - println(filter_cols) - ref_price_table = read_table(mod.file) - stack_cols = Symbol.(intersect(names(ref_price_table), years)) - # retail_price.ref_price .= 0 - - # for i in 1:nrow(ref_price_table) - # row = ref_price_table[i,:] - - # get(row, :status, true) || continue - # # shape = Float64[row[i_yr] for i_yr in yr_idx:(yr_idx + nyr - 1)] - - # filters = parse_comparisons(row) - # println(filters) - # # ref_price_table.filter - - # tmp_retail_price = deepcopy(retail_price) - # for filter in filters - # filter = string(first(filter)filter2string(filter) - # filter!(row -> any(c -> row[c] == filter, filter_cols), tmp_retail_price) - # end - - # stacked_row = DataFrames.stack(ref_price_table[i:i, vcat(:area, :subarea, stack_cols)], - # stack_cols, [:area, :subarea], variable_name=:filter_years,value_name=:ref_price) - - # stacked_row.filter_years .= "years=>" .* string.(stacked_row.filter_years) - - # tmp_retail_price = innerjoin(tmp_retail_price, stacked_row, on = [:filter_years]) - # println(tmp_retail_price) - - # end - # to do: the price may need to be energy demand/consumption - # CSV.write(get_out_path(config, string(m.name, ".csv")), table) -end -export modify_results! - -# function extract_results(m::WelfareTable, config, data) -# results = get_results(data) -# haskey(results, m.name) || modify_results!(m, config, data) -# return get_result(data, m.name) -# end - -# function combine_results(m::WelfareTable, post_config, post_data) - -# res = join_sim_tables(post_data, :value) - -# CSV.write(get_out_path(post_config, "$(m.name)_combined.csv"), res) -# end From 48aafee73baf5b9f6783c7b890461f16a8925abb Mon Sep 17 00:00:00 2001 From: Peplinski Date: Wed, 15 Oct 2025 23:29:05 -0400 Subject: [PATCH 19/35] Updates doc string, clean up and comment code --- src/results/retail_price.jl | 58 +++++---- src/types/modifications/RetailPrice.jl | 172 +++++++++++++++---------- 2 files changed, 134 insertions(+), 96 deletions(-) diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl index 8cf3a2cb..48900122 100644 --- a/src/results/retail_price.jl +++ b/src/results/retail_price.jl @@ -24,6 +24,8 @@ and the true retail rate to use as a calibrator value. If cal_mode is set to `ca # to do: update docstrings +# double check that get_cal_values will error if looking for national price and there isn't a ref +# double check that cal_mode calibration will error (or warn?) if looking for retail price that doesn't have a calibrator # adjust calibrator table to be read in with right format, or change function to read in correctly function setup_retail_price!(config, data) @@ -62,6 +64,7 @@ function get_retail_price(data) return data[:retail_price]::OrderedDict{Symbol, OrderedDict{Symbol,OrderedDict{Symbol,Function}}} end export get_retail_price + """ add_price_term!(data, price_type::Symbol, table_name::Symbol, result_name::Symbol, oper) """ @@ -80,10 +83,12 @@ function add_price_term!(data, price_type::Symbol, table_name::Symbol, result_na end export add_price_term! +# wrapper function that dispatches different methods based on cal_mod arg function compute_retail_price(m, data, price_type::Symbol, idxs, yr_idxs, hr_idxs) compute_retail_price((Val(Symbol(m.cal_mode))), m, data, price_type, idxs, yr_idxs, hr_idxs) end +# specialized method to calculate retail price for cal_mode none function compute_retail_price(::Val{:none}, m, data, price_type::Symbol, idxs...) value = 0.0 retail_price = get_retail_price(data) @@ -94,11 +99,13 @@ function compute_retail_price(::Val{:none}, m, data, price_type::Symbol, idxs... value += res end end + # divide by total generation to get dollars per MWh elserv_total = compute_result(data, :bus, :elserv_total, idxs...) return value/elserv_total end +# specialized method to calculate retail price for cal_mode get_cal_values function compute_retail_price(::Val{:get_cal_values}, m, data, price_type::Symbol, idxs, yr_idxs, hr_idxs) value = 0.0 retail_price = get_retail_price(data) @@ -113,12 +120,11 @@ function compute_retail_price(::Val{:get_cal_values}, m, data, price_type::Symbo elserv_total = compute_result(data, :bus, :elserv_total, idxs, yr_idxs, hr_idxs) retail_price = value/elserv_total - ref_price_table = read_table(m.calibrator_file) - - ref_value, area, subarea, year = get_ref_price(ref_price_table, idxs, yr_idxs, hr_idxs, retail_price) + ref_value, area, subarea, year = get_ref_price(m.calibrator_file, idxs, yr_idxs, hr_idxs, retail_price) subset = filter(row -> row.area == "" && row.subarea == "", ref_price_table) - + + # warn if there is no average reference price, error if more than 1 if nrow(subset) == 0 @warn "No full model reference price row. Outputting calibration values without a full adjustment." elserv_ratio = 0 @@ -154,9 +160,12 @@ end export compute_retail_price + # get corresponding price values function get_ref_price(ref_price_table, idxs, yr_idxs, hr_idxs, retail_price) - # get corresponding price values + # read in the reference prices + ref_price_table = read_table(m.calibrator_file) + # checks that there is only one filter, outside of hour and year filters if isempty(idxs) area = "" subarea = "" @@ -167,34 +176,34 @@ function get_ref_price(ref_price_table, idxs, yr_idxs, hr_idxs, retail_price) error("Retail price calibrator is not set up to handle multiple filters.") end - @assert yr_idxs != Any[] "Retail price calibrator is not set up to handle average retail rate across years." - year = yr_idxs - - @assert hr_idxs == Colon() "Retail price calibrator is not set up to handle hourly retail rates." - + # for each result row, get the corresponding reference price ref_values = [] for (i, row) in enumerate(eachrow(ref_price_table)) - if row.area == area && row.subarea ==subarea && row.year == year + if row.area == area && row.subarea == subarea && row.year == yr_idxs push!(ref_values, row["ref_price"]) elseif row.area == area && row.subarea ==subarea && isempty(row.year) # if no year provided, ref price is used for all years push!(ref_values, row["ref_price"]) end end + # error if there are multiple corresponding ref prices, and warn if there is none if length(ref_values) > 1 - error("Retail price calibator is not set up to handle multiple referenc prices") + error("Retail price calibator is not set up to handle multiple reference prices.") elseif isempty(ref_values) - @warn "There is no reference retail price for area $(area) and subarea $(subarea). This region will not be calibrated." + @warn "There is no reference retail price for area `$(area)` and subarea `$(subarea)`. This region will not get a calibration value." push!(ref_values, retail_price) end - - return sum(ref_values), area, subarea, year + + return sum(ref_values), area, subarea, yr_idxs end +# get the corresponding calibrator values function get_calibrator_value(calibrator_file, idxs, yr_idxs, hr_idxs) - # adjust retail price with calibrator values + + # read in table with cal values cal_table = read_table(calibrator_file) + # checks that there is only one filter, outside of hour and year filters if isempty(idxs) area = "" subarea = "" @@ -205,24 +214,19 @@ function get_calibrator_value(calibrator_file, idxs, yr_idxs, hr_idxs) error("Retail price calibrator is not set up to handle multiple filters.") end - @assert yr_idxs != Any[] "Retail price calibrator is not set up to handle average retail rate across years." - year = yr_idxs - - @assert hr_idxs == Colon() "Retail price calibrator is not set up to handle hourly retail rates." - + # for each result row, get the corresponding calibrator value cal_values =[] for (i, row) in enumerate(eachrow(cal_table)) - if row.area == "" && row.subarea == "" && !isempty(idxs) - push!(cal_values, row[year]) - elseif row.area == area && row.subarea ==subarea - push!(cal_values, row[year]) + if row.area == area && row.subarea == area + push!(cal_values, row[yr_idxs]) end end + # error if there are multiple corresponding calibrator values, and warn if there is none if length(cal_values) > 1 - error("Retail price calibator is not set up to handle multiple referenc prices") + error("Retail price calibator is not set up to handle multiple calibrator values for the same region.") elseif isempty(cal_values) - @warn "There is no calibrator value for area $(area) and subarea $(subarea). This region will not be calibrated." + @warn "There is no calibrator value for area `$(area)` and subarea `$(subarea)`. This region will not be calibrated." push!(cal_values, 0) end diff --git a/src/types/modifications/RetailPrice.jl b/src/types/modifications/RetailPrice.jl index 24aea177..0755c3dc 100644 --- a/src/types/modifications/RetailPrice.jl +++ b/src/types/modifications/RetailPrice.jl @@ -2,22 +2,28 @@ """ RetailPrice(;file, name, col_sort=:initial_order) <: Modification -This is a mod that outputs computed results, given a `file` representing the template of the things to be aggregated. `name` is simply the name of the modification, and will be used as the root for the filename that the aggregated information is saved to. This can be used for computing results or welfare. +This is a mod that outputs retail prices, given a `file` that indicates for which regions and years the retail price should be calculated. `name` is simply the name of the modification, and will be used as the root for the filename that the retail rates are saved to. +The mod pulls values across different tables to calculate one retail rate. The specific terms that go into this cross-table calculation can be found in retail_price.jl. + +The mod will also adjust the retail price values through calibration based on the `cal_mode` argument. If `cal_mode` is set to `none`, the retail prices will be unadjusted. If `cal_mode` is `get_val_values` the mod will use the reference price values to get calibration +values. If `cal_mod` is set to `calibrate`, the calibrator values will be read in from the `calibrator_file` and used to adjust the calculated retail rates. ## Keyword Arguments -* `file` - the file pointing to a table specifying which results to calculate +* `file` - the file pointing to a table specifying which retail prices to calculate * `name` - the name of the mod, do not need to specify in a config file +* `cal_mode` - a string that indicates the calibration mode. Options are `none`, `get_cal_values`, and `calibrate`. Defaults to `none`. +* `calibrator_file` - the file pointing to a table that contains reference price values or calibration values, depending on `cal_mode`. * `col_sort` - the column(s) to sort by. Defaults to the order in which they were originally specified. -* `cross_table` - indicates that the result is pulling results from multiple tables. Defaults to false. The `file` should represent a csv table with the following columns: * `table_name` - the name of the table being aggregated. i.e. `gen`, `bus`, etc. If you leave it empty, it will call `compute_welfare` instead of `compute_result` * `result_name` - the name of the column in the table being aggregated. Note that the column must have a Unit accessible via [`get_table_col_unit`](@ref). * `filter_` - the filtering conditions for the rows of the table. I.e. `filter1`. See [`parse_comparisons`](@ref) for information on what types of filters could be provided. * `filter_years` - the filtering conditions for the years to be aggregated. See [`parse_year_idxs`](@ref) for information on the year filters. -* `filter_hours` - the filtering conditions for the hours to be aggregated. See [`parse_hour_idxs`](@ref) for information on the hour filters. +* `filter_hours` - the filtering conditions for the hours to be aggregated. See [`parse_hour_idxs`](@ref) for information on the hour filters. the retail rate mod is not set up to calculate hourly values. Note that, for the `filter_` or `filter_hours` columns, if a column name of the data table (or hours table) is given, new rows will be created for each unique value of that column. I.e. if a value of `gentype` is given, there will be made a new row for `gentype=>coal`, `gentype=>ng`, etc. +The calibration feature can only handle one filter_ column beyond filter_hours and filter_years. """ struct RetailPrice <: Modification @@ -40,6 +46,7 @@ struct RetailPrice <: Modification hasproperty(table, col_name) || continue force_table_types!(table, name, col_name=>String) end + # errors if no calibrator file is provided if cal_mode != "none" isempty(calibrator_file) && error("Calibrator file required when cal_mode is set to $(cal_mode).") end @@ -54,7 +61,8 @@ mod_rank(::Type{<:RetailPrice}) = 5.0 fieldnames_for_yaml(::Type{RetailPrice}) = (:file,) - +# function takes the table from file and expands the filter columns so there is a row for each calculated result +# eg if file has a filter_ with the value "state", the table will be expanded so that there is a row for each state function modify_results!(m::RetailPrice, config, data) table = copy(m.table) table.initial_order = 1:nrow(table) @@ -98,52 +106,54 @@ function modify_results!(m::RetailPrice, config, data) not_pair_idx = findfirst(not_a_full_filter, eachrow(table)) end + # function that calculates retail price for each row in table get_retail_price(m, config, data, table) return end +function extract_results(m::RetailPrice, config, data) + results = get_results(data) + # haskey(results, m.name) || modify_results!(m, config, data) + modify_results!(m, config, data) + return get_result(data, m.name) +end -# function extract_results(m::RetailPrice, config, data) -# results = get_results(data) -# # haskey(results, m.name) || modify_results!(m, config, data) -# modify_results!(m, config, data) -# return get_result(data, m.name) -# end - -# function combine_results(m::RetailPrice, post_config, post_data) +function combine_results(m::RetailPrice, post_config, post_data) -# res = join_sim_tables(post_data, :value) + res = join_sim_tables(post_data, :value) -# CSV.write(get_out_path(post_config, "$(m.name)_combined.csv"), res) -# end + CSV.write(get_out_path(post_config, "$(m.name)_combined.csv"), res) +end +# wrapper function that will dispatch a different get_retail_price method based on cal_mode arg function get_retail_price(m::RetailPrice, config, data, table) @info "Calculating results for $(nrow(table)) rows in RetailPrice $(m.name)" get_retail_price((Val(Symbol(m.cal_mode))), m, config, data, table) end +# specialized method for retail rates with no cal_mode none function get_retail_price(::Val{:none}, m, config, data, table) if !hasproperty(table, :value) table.value = Vector{Union{Missing, Float64}}(missing, nrow(table)) end - for i in 1:nrow(table) - table_name = table[i, :table_name] - result_name = table[i,:result_name] - idxs = parse_comparisons(table[i,:]) - yr_idxs = parse_year_idxs(table[i,:filter_years]) - hr_idxs = parse_hour_idxs(table[i,:filter_hours]) - - if hr_idxs !== Colon() - @warn "Hourly retail price calculations are not set up." - return 0.0 - else - val = compute_retail_price(m, data, result_name, idxs, yr_idxs, hr_idxs) - end - table.value[i]= val - end + + for row in eachrow(table) + table_name = row[:table_name] + result_name = row[:result_name] + + idxs = parse_comparisons(row) + yr_idxs = parse_year_idxs(row[:filter_years]) + hr_idxs = parse_hour_idxs(row[:filter_hours]) + + @assert hr_idxs == Colon() "Retail price mod is not set up to handle hourly retail rates." + + val = compute_retail_price(m, data, result_name, idxs, yr_idxs, hr_idxs) + + row[:value] = val + end sort!(table, m.col_sort) select!(table, Not(:initial_order)) CSV.write(get_out_path(config, string(m.name, ".csv")), table) @@ -151,60 +161,79 @@ function get_retail_price(::Val{:none}, m, config, data, table) results[m.name] = table end +# specialized method for retail rates with no cal_mode get_cal_values function get_retail_price(::Val{:get_cal_values}, m::RetailPrice, config, data, table) - cal_table = DataFrame(area = String[], subarea = [], year=[], ref_price =[], retail_price = [], cal_value = [], elserv_total = [], elserv_ratio=[]) + # set up table that will contain calibrator values + cal_table = DataFrame( + area = String[], + subarea = String[], + year = String[], + ref_price = Float64[], + retail_price = Float64[], + cal_value = Float64[], + elserv_total = Float64[], + elserv_ratio = Float64[] + ) + + # add value column to results table if it doesn't exist if !hasproperty(table, :value) table.value = Vector{Union{Missing, Float64}}(missing, nrow(table)) end - for i in 1:nrow(table) - table_name = table[i, :table_name] - result_name = table[i,:result_name] - idxs = parse_comparisons(table[i,:]) - yr_idxs = parse_year_idxs(table[i,:filter_years]) - hr_idxs = parse_hour_idxs(table[i,:filter_hours]) - - if hr_idxs !== Colon() - @warn "Hourly retail price calculations are not set up." - return 0.0 - else - val, cal_row = compute_retail_price(m, data, result_name, idxs, yr_idxs, hr_idxs) - push!(cal_table, cal_row) - end - table.value[i]= val - end + for row in eachrow(table) + table_name = row[:table_name] + result_name = row[:result_name] + + idxs = parse_comparisons(row) + yr_idxs = parse_year_idxs(row[:filter_years]) + hr_idxs = parse_hour_idxs(row[:filter_hours]) + + @assert yr_idxs != Any[] "Retail price calibrator is not set up to handle average retail rate across years." + @assert hr_idxs == Colon() "Retail price mod is not set up to handle hourly retail rates." + + val, cal_row = compute_retail_price(m, data, result_name, idxs, yr_idxs, hr_idxs) + + push!(cal_table, cal_row) + row[:value] = val + end + sort!(table, m.col_sort) select!(table, Not(:initial_order)) CSV.write(get_out_path(config, string(m.name, ".csv")), table) results = get_results(data) results[m.name] = table + # second calibrator adjustment to calibrate with full region full_cal!(m, data, table, cal_table) CSV.write(get_out_path(config, string(m.name, "_cals.csv")), cal_table) end function get_retail_price(::Val{:calibrate}, m, config, data, table) + + # add value column to results table if it doesn't exist if !hasproperty(table, :value) table.value = Vector{Union{Missing, Float64}}(missing, nrow(table)) end - for i in 1:nrow(table) - table_name = table[i, :table_name] - result_name = table[i,:result_name] - idxs = parse_comparisons(table[i,:]) - yr_idxs = parse_year_idxs(table[i,:filter_years]) - hr_idxs = parse_hour_idxs(table[i,:filter_hours]) - - if hr_idxs !== Colon() - @warn "Hourly retail price calculations are not set up." - return 0.0 - else - val = compute_retail_price(m, data, result_name, idxs, yr_idxs, hr_idxs) - end - table.value[i]= val - end + + for row in eachrow(table) + table_name = row[:table_name] + result_name = row[:result_name] + + idxs = parse_comparisons(row) + yr_idxs = parse_year_idxs(row[:filter_years]) + hr_idxs = parse_hour_idxs(row[:filter_hours]) + + @assert yr_idxs != Any[] "Retail price calibrator is not set up to handle average retail rate across years." + @assert hr_idxs == Colon() "Retail price mod is not set up to handle hourly retail rates." + + val = compute_retail_price(m, data, result_name, idxs, yr_idxs, hr_idxs) + + row[:value] = val + end + sort!(table, m.col_sort) select!(table, Not(:initial_order)) CSV.write(get_out_path(config, string(m.name, ".csv")), table) @@ -212,20 +241,25 @@ function get_retail_price(::Val{:calibrate}, m, config, data, table) results[m.name] = table end +# final calibration value for full model +# example: for a state level model, ensure that the weighted average prices of all states are calibrated to the national average price function full_cal!(m, data, table, cal_table) + ref_price_table = read_table(m.calibrator_file) - subset = filter(row -> row.area == "" && row.subarea == "", ref_price_table) + # get the weighted average retail price acorss all areas + avg_price = sum(cal_table.ref_price .* cal_table.elserv_ratio) + + subset = filter(row -> row.area == "" && row.subarea == "", ref_price_table) + + # get the average reference price if nrow(subset) == 0 - @warn "No full model reference price row. Outputting calibration values without a full adjustment." - elseif nrow(subset) > 1 - error("Multiple full model reference price rows.") + avg_price_ref = avg_price # set refrence price to the calculated average price so that the calibrator value will be 0 else avg_price_ref = subset[1, :ref_price] end - - avg_price = sum(cal_table.ref_price .* cal_table.elserv_ratio) + # calculate and add the final cal value to existing cal values for row in eachrow(cal_table[(cal_table.area .!= "") .& (cal_table.subarea .!= ""), :]) println(row[:subarea]) cal = (avg_price_ref - avg_price) * row[:elserv_ratio] From 4b7033ba5c2c4e840b30d8742bbbcc63f282d56f Mon Sep 17 00:00:00 2001 From: Peplinski Date: Fri, 17 Oct 2025 12:15:17 -0400 Subject: [PATCH 20/35] Minor fixes for past invest calculation so that results doesn't error when left empty --- src/io/config.jl | 2 +- src/io/data.jl | 4 +++- src/results/formulas.jl | 10 ++++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/io/config.jl b/src/io/config.jl index 7356f02b..fa80b49a 100644 --- a/src/io/config.jl +++ b/src/io/config.jl @@ -79,7 +79,7 @@ function summarize_config() (:require_optimal, false, true, "Whether or not to require whether or not the model is solved to optimality. If set to true and the optimizer terminates with a suboptimal termination status, [`run_e4st`](@ref) returns after optimizing, without parsing results, etc."), (:model_string_names, false, false, "Whether or not to allow the model to have string names. Defaults to `false` for memory savings. Can be helpful to turn on for debugging, especially if you are encountering an infeasible model"), (:yearly_objective_scalars, false, 1, "The amount to scale the objective by for each year, defaults to 1 for each year."), - (:past_invest_file, false, 1, "Gen file used to calculate past investment costs. Only necessary when an aggregated gen table is used.") + (:past_invest_file, false, nothing, "File used to calculate past investment costs for existing generators. Only necessary when an aggregated gen table is used which would distort the past investment cost calculation.") ) diff --git a/src/io/data.jl b/src/io/data.jl index 6c7a3481..d6dcac52 100644 --- a/src/io/data.jl +++ b/src/io/data.jl @@ -585,9 +585,11 @@ Sets up the invest cost gen table. This is necessary when gens are aggregated so Creates age column which is a ByYear column. Unbuilt generators have a negative age before year_on. """ function setup_table!(config, data, ::Val{:past_invest}) - if !haskey(config, :past_invest) + + if !haskey(config, :past_invest_file) return end + bus = get_table(data, :bus) past_invest = get_table(data, :past_invest) years = get_years(data) diff --git a/src/results/formulas.jl b/src/results/formulas.jl index 000fd1be..9ea744a9 100644 --- a/src/results/formulas.jl +++ b/src/results/formulas.jl @@ -12,8 +12,10 @@ function setup_results_formulas!(config, data) results_formulas_table = read_table(data, results_formulas_file, :results_formulas) - for row in eachrow(results_formulas_table) - add_results_formula!(data, row.table_name, row.result_name, row.formula, row.unit, row.description) + foreach(eachrow(results_formulas_table)) do row + haskey(data, row.table_name) ? + add_results_formula!(data, row.table_name, row.result_name, row.formula, row.unit, row.description) : + @warn "There is no table $(row.table_name) in data. $(row.result_name) will not be added to formulas." end end export setup_results_formulas! @@ -809,8 +811,8 @@ function (f::CostOfServicePastCost)(data, table, idxs, yr_idxs, hr_idxs) res = 0.0 for i in idxs rf = reg_factor[i] - rev_prelim = compute_result(data, f.table_name, :past_invest_cost_total, i, yr_idxs, hr_idxs) - prod = rf * rev_prelim + past_invest = compute_result(data, f.table_name, :past_invest_cost_total, i, yr_idxs, hr_idxs) + prod = rf * past_invest res += prod end return res From d1c700f9059beefcf77352135b859b9a8a13ae8d Mon Sep 17 00:00:00 2001 From: Peplinski Date: Fri, 17 Oct 2025 17:37:48 -0400 Subject: [PATCH 21/35] Fixes bug in how cal value calculation handles reference values without a year attached --- src/io/data.jl | 12 ++- src/results/retail_price.jl | 115 +++++++++++++++---------- src/types/modifications/RetailPrice.jl | 14 ++- 3 files changed, 88 insertions(+), 53 deletions(-) diff --git a/src/io/data.jl b/src/io/data.jl index d6dcac52..ad2a23aa 100644 --- a/src/io/data.jl +++ b/src/io/data.jl @@ -1660,7 +1660,17 @@ Returns the number of years in this simulation function get_num_years(data) return length(get_years(data)) end -export get_num_years, get_years + +""" + get_first_sim_year(data) -> year + +Returns the first year as a string (i.e. "y2022") of the years being represented in the sim. +""" +function get_first_sim_year(data) + return data[:years][1]::String +end + +export get_num_years, get_years, get_first_sim_year """ get_bus_gens(data, bus_idx) diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl index 48900122..6127c4f6 100644 --- a/src/results/retail_price.jl +++ b/src/results/retail_price.jl @@ -22,10 +22,6 @@ there will be no calibration steps. If it is set to `get_cal_values`, the functi and the true retail rate to use as a calibrator value. If cal_mode is set to `calibrate`, the corresponding calibrator vaulue will be added to the retail price. """ - -# to do: update docstrings -# double check that get_cal_values will error if looking for national price and there isn't a ref -# double check that cal_mode calibration will error (or warn?) if looking for retail price that doesn't have a calibrator # adjust calibrator table to be read in with right format, or change function to read in correctly function setup_retail_price!(config, data) @@ -34,17 +30,20 @@ function setup_retail_price!(config, data) # price terms for average electricity rate add_price_term!(data, :avg_elec_rate, :bus, :electricity_cost, +) + # per MW cost adder for distribution costs add_price_term!(data, :avg_elec_rate, :bus, :distribution_cost_total, +) + # merchandising suplus is from selling electricity for higher price at one end of line than another add_price_term!(data, :avg_elec_rate, :bus, :merchandising_surplus_total, -) + # if the difference between revenue and total costs is positive, customers in COS regions get a rebate # total cost includes production costs, net policy costs, gs_rebate, and the net of past investment costs and subsidies add_price_term!(data, :avg_elec_rate, :gen, :cost_of_service_rebate, -) add_price_term!(data, :avg_elec_rate, :storage, :cost_of_service_rebate, -) - add_price_term!(data, :avg_elec_rate, :bus, :gs_payment, +) - # ToDo: include past subsidies, but these may be hard to track down + # add_price_term!(data, :avg_elec_rate, :bus, :gs_payment, +) + if haskey(config, :past_invest_file) add_price_term!(data, :avg_elec_rate, :past_invest, :cost_of_service_past_costs, +) end @@ -120,8 +119,21 @@ function compute_retail_price(::Val{:get_cal_values}, m, data, price_type::Symbo elserv_total = compute_result(data, :bus, :elserv_total, idxs, yr_idxs, hr_idxs) retail_price = value/elserv_total - ref_value, area, subarea, year = get_ref_price(m.calibrator_file, idxs, yr_idxs, hr_idxs, retail_price) + fsy = get_first_sim_year(data) + ref_price_table = read_table(m.calibrator_file) + + if !hasproperty(ref_price_table, :year) + if yr_idxs != fsy + return retail_price, [] + else + year = "" + ref_value, area, subarea = get_ref_price(ref_price_table, idxs, hr_idxs, retail_price) + end + else + ref_value, area, subarea, year = get_ref_price(ref_price_table, idxs, yr_idxs, hr_idxs, retail_price, first_sim_year) + end + subset = filter(row -> row.area == "" && row.subarea == "", ref_price_table) # warn if there is no average reference price, error if more than 1 @@ -134,8 +146,9 @@ function compute_retail_price(::Val{:get_cal_values}, m, data, price_type::Symbo elserv_total_all = compute_result(data, :bus, :elserv_total, :, yr_idxs, hr_idxs) elserv_ratio = elserv_total/elserv_total_all end - + return retail_price, [area, subarea, year, ref_value, retail_price, ref_value - retail_price, elserv_total, elserv_ratio] + end function compute_retail_price(::Val{:calibrate}, m, data, price_type::Symbol, idxs, yr_idxs, hr_idxs) @@ -160,43 +173,62 @@ end export compute_retail_price - # get corresponding price values +# get corresponding price values function get_ref_price(ref_price_table, idxs, yr_idxs, hr_idxs, retail_price) - # read in the reference prices - ref_price_table = read_table(m.calibrator_file) - + # checks that there is only one filter, outside of hour and year filters - if isempty(idxs) - area = "" - subarea = "" - elseif length(idxs)==1 && idxs[1] isa Pair - area = idxs[1].first - subarea = idxs[1].second - else - error("Retail price calibrator is not set up to handle multiple filters.") - end + area, subarea = + isempty(idxs) ? ("", "") : + length(idxs) == 1 && idxs[1] isa Pair ? (idxs[1].first, idxs[1].second) : + throw(ErrorException("Retail price calibrator is not set up to handle multiple filters.")) # for each result row, get the corresponding reference price ref_values = [] for (i, row) in enumerate(eachrow(ref_price_table)) if row.area == area && row.subarea == subarea && row.year == yr_idxs push!(ref_values, row["ref_price"]) - elseif row.area == area && row.subarea ==subarea && isempty(row.year) # if no year provided, ref price is used for all years - push!(ref_values, row["ref_price"]) end end # error if there are multiple corresponding ref prices, and warn if there is none - if length(ref_values) > 1 - error("Retail price calibator is not set up to handle multiple reference prices.") - elseif isempty(ref_values) + length(ref_values) > 1 && error("Retail price calibator is not set up to handle multiple reference prices for the same region and year.") + + isempty(ref_values) && begin @warn "There is no reference retail price for area `$(area)` and subarea `$(subarea)`. This region will not get a calibration value." - push!(ref_values, retail_price) + push!(cal_values, 0) end return sum(ref_values), area, subarea, yr_idxs end +# one ref price for all years +function get_ref_price(ref_price_table, idxs, hr_idxs, retail_price) + + # checks that there is only one filter, outside of hour and year filters + area, subarea = + isempty(idxs) ? ("", "") : + length(idxs) == 1 && idxs[1] isa Pair ? (idxs[1].first, idxs[1].second) : + throw(ErrorException("Retail price calibrator is not set up to handle multiple filters.")) + + # for each result row, get the corresponding reference price + ref_values = [] + for (i, row) in enumerate(eachrow(ref_price_table)) + if row.area == area && row.subarea == subarea + push!(ref_values, row["ref_price"]) + end + end + + # error if there are multiple corresponding ref prices, and warn if there is none + length(ref_values) > 1 && error("Retail price calibator is not set up to handle multiple reference prices for the same region and year.") + + isempty(ref_values) && begin + @warn "There is no reference retail price for area `$(area)` and subarea `$(subarea)`. This region will not get a calibration value." + push!(cal_values, 0) + end + + return sum(ref_values), area, subarea +end + # get the corresponding calibrator values function get_calibrator_value(calibrator_file, idxs, yr_idxs, hr_idxs) @@ -204,28 +236,23 @@ function get_calibrator_value(calibrator_file, idxs, yr_idxs, hr_idxs) cal_table = read_table(calibrator_file) # checks that there is only one filter, outside of hour and year filters - if isempty(idxs) - area = "" - subarea = "" - elseif length(idxs)==1 && idxs[1] isa Pair - area = idxs[1].first - subarea = idxs[1].second - else - error("Retail price calibrator is not set up to handle multiple filters.") - end + area, subarea = + isempty(idxs) ? ("", "") : + length(idxs) == 1 && idxs[1] isa Pair ? (idxs[1].first, idxs[1].second) : + throw(ErrorException("Retail price calibrator is not set up to handle multiple filters.")) - # for each result row, get the corresponding calibrator value + # for each result row, get the corresponding calibrator value for area, subarea, year cal_values =[] - for (i, row) in enumerate(eachrow(cal_table)) - if row.area == area && row.subarea == area - push!(cal_values, row[yr_idxs]) + for row in eachrow(cal_table) + if row.area == area && row.subarea == subarea && (!hasproperty(calibrator_file, :year) || row.year == yr_idxs) # if there is no year column only check that area, subarea match + push!(cal_values, row.cal_value) end - end + end # error if there are multiple corresponding calibrator values, and warn if there is none - if length(cal_values) > 1 - error("Retail price calibator is not set up to handle multiple calibrator values for the same region.") - elseif isempty(cal_values) + length(cal_values) > 1 && error("Retail price calibrator is not set up to handle multiple calibrator values for the same region.") + + isempty(cal_values) && begin @warn "There is no calibrator value for area `$(area)` and subarea `$(subarea)`. This region will not be calibrated." push!(cal_values, 0) end diff --git a/src/types/modifications/RetailPrice.jl b/src/types/modifications/RetailPrice.jl index 0755c3dc..e6dcba64 100644 --- a/src/types/modifications/RetailPrice.jl +++ b/src/types/modifications/RetailPrice.jl @@ -194,7 +194,7 @@ function get_retail_price(::Val{:get_cal_values}, m::RetailPrice, config, data, val, cal_row = compute_retail_price(m, data, result_name, idxs, yr_idxs, hr_idxs) - push!(cal_table, cal_row) + !isempty(cal_row) && push!(cal_table, cal_row) row[:value] = val end @@ -206,6 +206,8 @@ function get_retail_price(::Val{:get_cal_values}, m::RetailPrice, config, data, # second calibrator adjustment to calibrate with full region full_cal!(m, data, table, cal_table) + select!(cal_table, + [c for c in (:area, :subarea, :year, :cal_value) if any(!ismissing, cal_table[!, c]) && any(x -> x != "" && !ismissing(x), cal_table[!, c])]) CSV.write(get_out_path(config, string(m.name, "_cals.csv")), cal_table) end @@ -217,8 +219,7 @@ function get_retail_price(::Val{:calibrate}, m, config, data, table) table.value = Vector{Union{Missing, Float64}}(missing, nrow(table)) end - - for row in eachrow(table) + for row in eachrow(table) table_name = row[:table_name] result_name = row[:result_name] @@ -251,7 +252,7 @@ function full_cal!(m, data, table, cal_table) avg_price = sum(cal_table.ref_price .* cal_table.elserv_ratio) subset = filter(row -> row.area == "" && row.subarea == "", ref_price_table) - + # get the average reference price if nrow(subset) == 0 avg_price_ref = avg_price # set refrence price to the calculated average price so that the calibrator value will be 0 @@ -261,11 +262,8 @@ function full_cal!(m, data, table, cal_table) # calculate and add the final cal value to existing cal values for row in eachrow(cal_table[(cal_table.area .!= "") .& (cal_table.subarea .!= ""), :]) - println(row[:subarea]) + # average calibrator value is the differnce between the reference price minus the calculated price weighted by load cal = (avg_price_ref - avg_price) * row[:elserv_ratio] - println(row[:cal_value]) - println(row[:retail_price]) - println(row[:ref_price]) row[:cal_value] += cal end From d9966bcf0b6b141378f6cabf336ab7faf95f92fa Mon Sep 17 00:00:00 2001 From: Peplinski Date: Tue, 21 Oct 2025 15:11:51 -0400 Subject: [PATCH 22/35] Removes incorrect calibration mod from e4st.jl --- src/E4ST.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/E4ST.jl b/src/E4ST.jl index 5ecb78cd..9ba3cc16 100644 --- a/src/E4ST.jl +++ b/src/E4ST.jl @@ -73,7 +73,7 @@ include("types/modifications/GenHashID.jl") include("types/modifications/LeftJoinCols.jl") include("types/modifications/CapacityConstraint.jl") include("types/modifications/PerfectForesight.jl") -include("types/modifications/RetailPriceCalibration.jl") +# include("types/modifications/RetailPriceCalibration.jl") include("types/modifications/RetailPrice.jl") # Include Policies From fb8cac61693d63760816680d47ed27335b26052c Mon Sep 17 00:00:00 2001 From: Peplinski Date: Mon, 3 Nov 2025 16:51:43 -0500 Subject: [PATCH 23/35] Adjusts how mode and input files are handled in mod --- src/results/retail_price.jl | 4 +-- src/types/modifications/RetailPrice.jl | 36 +++++++++++++++++++++++--- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl index 6127c4f6..6b80c1f3 100644 --- a/src/results/retail_price.jl +++ b/src/results/retail_price.jl @@ -42,7 +42,7 @@ function setup_retail_price!(config, data) add_price_term!(data, :avg_elec_rate, :gen, :cost_of_service_rebate, -) add_price_term!(data, :avg_elec_rate, :storage, :cost_of_service_rebate, -) - # add_price_term!(data, :avg_elec_rate, :bus, :gs_payment, +) + add_price_term!(data, :avg_elec_rate, :bus, :gs_payment, +) if haskey(config, :past_invest_file) add_price_term!(data, :avg_elec_rate, :past_invest, :cost_of_service_past_costs, +) @@ -121,7 +121,7 @@ function compute_retail_price(::Val{:get_cal_values}, m, data, price_type::Symbo fsy = get_first_sim_year(data) - ref_price_table = read_table(m.calibrator_file) + ref_price_table = read_table(m.ref_price_file) if !hasproperty(ref_price_table, :year) if yr_idxs != fsy diff --git a/src/types/modifications/RetailPrice.jl b/src/types/modifications/RetailPrice.jl index e6dcba64..5298e2a8 100644 --- a/src/types/modifications/RetailPrice.jl +++ b/src/types/modifications/RetailPrice.jl @@ -31,9 +31,10 @@ struct RetailPrice <: Modification name::Symbol table::DataFrame cal_mode:: String + ref_price_file::String calibrator_file::String col_sort - function RetailPrice(;file, name, calibrator_file="", cal_mode="none", col_sort=:initial_order) + function RetailPrice(;file, name, ref_price_file="", calibrator_file="", cal_mode="none", col_sort=:initial_order) table = read_table(file) force_table_types!(table, name, :table_name=>Symbol, @@ -47,10 +48,12 @@ struct RetailPrice <: Modification force_table_types!(table, name, col_name=>String) end # errors if no calibrator file is provided - if cal_mode != "none" + if cal_mode == "get_cal_values" + isempty(ref_price_file) && error("Ref price file required when cal_mode is set to $(cal_mode).") + elseif cal_mode == "calibrate" isempty(calibrator_file) && error("Calibrator file required when cal_mode is set to $(cal_mode).") end - return new(file, name, table, cal_mode, calibrator_file, col_sort) + return new(file, name, table, cal_mode, ref_price_file, calibrator_file, col_sort) end end @@ -61,6 +64,31 @@ mod_rank(::Type{<:RetailPrice}) = 5.0 fieldnames_for_yaml(::Type{RetailPrice}) = (:file,) + +function summarize_table(::Val{:ref_price}) + df = TableSummary() + push!(df, + (:area, String, NA, true, "The area that the price applies for i.e. `nation`. Leave blank if grid-wide"), + (:subarea, String, NA, true, "The subarea that the price applies for i.e. `narnia`. Leave blank if grid-wide"), + (:year, String, NA, false, "Year of corresponding reference price. If no column, them same calibrator value will be applied in each year."), + (:ref_price, Float64, DollarsPerMWhServed, true, "Reference price for retail rate in \$/MWh."), + ) + return df +end + + + function summarize_table(::Val{:retail_calibrator}) + df = TableSummary() + push!(df, + (:area, String, NA, true, "The area that the price applies for i.e. `nation`. Leave blank if grid-wide"), + (:subarea, String, NA, true, "The subarea that the price applies for i.e. `narnia`. Leave blank if grid-wide"), + (:year, String, NA, false, "Year of corresponding reference price. If no column, the same calibrator value is used in every year."), + (:cal_value, Float64, DollarsPerMWhServed, true, "Calibrator value for retail rate in \$/MWh."), + ) + return df +end + + # function takes the table from file and expands the filter columns so there is a row for each calculated result # eg if file has a filter_ with the value "state", the table will be expanded so that there is a row for each state function modify_results!(m::RetailPrice, config, data) @@ -246,7 +274,7 @@ end # example: for a state level model, ensure that the weighted average prices of all states are calibrated to the national average price function full_cal!(m, data, table, cal_table) - ref_price_table = read_table(m.calibrator_file) + ref_price_table = read_table(m.ref_price_file) # get the weighted average retail price acorss all areas avg_price = sum(cal_table.ref_price .* cal_table.elserv_ratio) From 63fcce42ab2634a3c8debe689999033f79c3e13c Mon Sep 17 00:00:00 2001 From: Peplinski Date: Mon, 3 Nov 2025 16:53:21 -0500 Subject: [PATCH 24/35] Updates calculation of generation standard payments --- src/types/modifications/GenerationStandard.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/types/modifications/GenerationStandard.jl b/src/types/modifications/GenerationStandard.jl index 18d2fcc6..45c6a861 100644 --- a/src/types/modifications/GenerationStandard.jl +++ b/src/types/modifications/GenerationStandard.jl @@ -173,6 +173,7 @@ Modifies the results by adding the following columns to the bus table: function modify_results!(pol::GenerationStandard, config, data) bus = get_table(data, :bus) gen = get_table(data, :gen) + nyr = get_num_years(data) prc_name = Symbol("$(pol.name)_prc") cost_name = Symbol("$(pol.name)_cost") @@ -201,19 +202,19 @@ function modify_results!(pol::GenerationStandard, config, data) end for (k,d) in pol.load_targets - targets = d[:targets] + targets = collect(values(d[:targets]))[1:nyr] filters = d[:filters] bus_idxs = get_row_idxs(bus, parse_comparisons(d[:filters])) # set to shadow_prc for bus for i in bus_idxs - bus[i, prc_name] = -(shadow_prc) + bus[i, prc_name] = -(shadow_prc) .* targets end end # policy cost, price * credit * generation add_results_formula!(data, :gen, cost_name, "SumHourlyWeighted($(prc_name), pgen)", Dollars, "Cost of $(pol.name) based on the shadow price on the constraint and the generator credit level.") add_to_results_formula!(data, :gen, :gs_rebate, cost_name) - add_results_formula!(data, :bus, cost_name, "SumHourlyWeighted($(prc_name), plserv)", Dollars, "Cost of $(pol.name) based on the shadow price on the constraint and the generator credit level.") + add_results_formula!(data, :bus, cost_name, "SumHourlyWeighted($(prc_name), pl_gs)", Dollars, "Cost of $(pol.name) based on the shadow price on the constraint and the generator credit level.") add_to_results_formula!(data, :bus, :gs_payment, cost_name) end export modify_results! From 3f15a0ca0cdb8644f0f8e60952194462b354a688 Mon Sep 17 00:00:00 2001 From: Peplinski Date: Tue, 11 Nov 2025 18:18:44 -0500 Subject: [PATCH 25/35] Fix bugs that were found in tests --- src/results/retail_price.jl | 86 +++++++++++++++++++------- src/types/modifications/RetailPrice.jl | 6 +- 2 files changed, 66 insertions(+), 26 deletions(-) diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl index 6b80c1f3..955c47f9 100644 --- a/src/results/retail_price.jl +++ b/src/results/retail_price.jl @@ -115,23 +115,23 @@ function compute_retail_price(::Val{:get_cal_values}, m, data, price_type::Symbo value += res end end + # divide by total generation to get dollars per MWh elserv_total = compute_result(data, :bus, :elserv_total, idxs, yr_idxs, hr_idxs) retail_price = value/elserv_total - fsy = get_first_sim_year(data) - + _yr_idxs = get_year_idxs(data, yr_idxs) ref_price_table = read_table(m.ref_price_file) if !hasproperty(ref_price_table, :year) - if yr_idxs != fsy + if _yr_idxs != 1 return retail_price, [] else year = "" ref_value, area, subarea = get_ref_price(ref_price_table, idxs, hr_idxs, retail_price) end else - ref_value, area, subarea, year = get_ref_price(ref_price_table, idxs, yr_idxs, hr_idxs, retail_price, first_sim_year) + ref_value, area, subarea, year = get_ref_price(ref_price_table, idxs, yr_idxs, hr_idxs, retail_price) end subset = filter(row -> row.area == "" && row.subarea == "", ref_price_table) @@ -164,8 +164,15 @@ function compute_retail_price(::Val{:calibrate}, m, data, price_type::Symbol, i # divide by total generation to get dollars per MWh elserv_total = compute_result(data, :bus, :elserv_total, idxs, yr_idxs, hr_idxs) retail_price = value/elserv_total - - cal = get_calibrator_value(m.calibrator_file, idxs, yr_idxs, hr_idxs) + + cal_table = read_table(m.calibrator_file) + cal = get_calibrator_value(data, + cal_table, + idxs, + (hasproperty(cal_table, :year) ? (yr_idxs,) : ())..., # include yr_idxs only if present + hr_idxs + ) + retail_price = retail_price + cal return retail_price @@ -175,17 +182,18 @@ export compute_retail_price # get corresponding price values function get_ref_price(ref_price_table, idxs, yr_idxs, hr_idxs, retail_price) - # checks that there is only one filter, outside of hour and year filters - area, subarea = + area, subarea = isempty(idxs) ? ("", "") : - length(idxs) == 1 && idxs[1] isa Pair ? (idxs[1].first, idxs[1].second) : + isa(idxs, Pair) ? (string(first(idxs)), string(last(idxs))) : + length(idxs) == 1 && isa(idxs[1], Pair) ? + (string(first(idxs[1])), string(last(idxs[1]))) : throw(ErrorException("Retail price calibrator is not set up to handle multiple filters.")) # for each result row, get the corresponding reference price ref_values = [] for (i, row) in enumerate(eachrow(ref_price_table)) - if row.area == area && row.subarea == subarea && row.year == yr_idxs + if string(row.area) == area && string(row.subarea) == subarea && row.year == yr_idxs push!(ref_values, row["ref_price"]) end end @@ -195,7 +203,7 @@ function get_ref_price(ref_price_table, idxs, yr_idxs, hr_idxs, retail_price) isempty(ref_values) && begin @warn "There is no reference retail price for area `$(area)` and subarea `$(subarea)`. This region will not get a calibration value." - push!(cal_values, 0) + push!(ref_values, 0) end return sum(ref_values), area, subarea, yr_idxs @@ -203,17 +211,18 @@ end # one ref price for all years function get_ref_price(ref_price_table, idxs, hr_idxs, retail_price) - # checks that there is only one filter, outside of hour and year filters - area, subarea = + area, subarea = isempty(idxs) ? ("", "") : - length(idxs) == 1 && idxs[1] isa Pair ? (idxs[1].first, idxs[1].second) : + isa(idxs, Pair) ? (string(first(idxs)), string(last(idxs))) : + length(idxs) == 1 && isa(idxs[1], Pair) ? + (string(first(idxs[1])), string(last(idxs[1]))) : throw(ErrorException("Retail price calibrator is not set up to handle multiple filters.")) # for each result row, get the corresponding reference price ref_values = [] for (i, row) in enumerate(eachrow(ref_price_table)) - if row.area == area && row.subarea == subarea + if string(row.area) == area && string(row.subarea) == subarea push!(ref_values, row["ref_price"]) end end @@ -223,28 +232,61 @@ function get_ref_price(ref_price_table, idxs, hr_idxs, retail_price) isempty(ref_values) && begin @warn "There is no reference retail price for area `$(area)` and subarea `$(subarea)`. This region will not get a calibration value." - push!(cal_values, 0) + push!(ref_values, 0) end return sum(ref_values), area, subarea end # get the corresponding calibrator values -function get_calibrator_value(calibrator_file, idxs, yr_idxs, hr_idxs) - +function get_calibrator_value(data, cal_table, idxs, yr_idxs, hr_idxs) + # read in table with cal values - cal_table = read_table(calibrator_file) + yr_idxs = get_year_idxs(data, yr_idxs) + cal_table.year .= get_year_idxs(data, cal_table.year) # checks that there is only one filter, outside of hour and year filters - area, subarea = + area, subarea = isempty(idxs) ? ("", "") : - length(idxs) == 1 && idxs[1] isa Pair ? (idxs[1].first, idxs[1].second) : + isa(idxs, Pair) ? (string(first(idxs)), string(last(idxs))) : + length(idxs) == 1 && isa(idxs[1], Pair) ? + (string(first(idxs[1])), string(last(idxs[1]))) : throw(ErrorException("Retail price calibrator is not set up to handle multiple filters.")) + + # for each result row, get the corresponding calibrator value for area, subarea, year + cal_values =[] + for row in eachrow(cal_table) + if string(row.area) == area && string(row.subarea) == subarea && row.year == yr_idxs # if there is no year column only check that area, subarea match + push!(cal_values, row.cal_value) + end + end + # error if there are multiple corresponding calibrator values, and warn if there is none + length(cal_values) > 1 && error("Retail price calibrator is not set up to handle multiple calibrator values for the same region.") + + isempty(cal_values) && begin + @warn "There is no calibrator value for area `$(area)` and subarea `$(subarea)`. This region will not be calibrated." + push!(cal_values, 0) + end + + return sum(cal_values) +end + + +function get_calibrator_value(data, cal_table, idxs, hr_idxs) + + # checks that there is only one filter, outside of hour and year filters + area, subarea = + isempty(idxs) ? ("", "") : + isa(idxs, Pair) ? (string(first(idxs)), string(last(idxs))) : + length(idxs) == 1 && isa(idxs[1], Pair) ? + (string(first(idxs[1])), string(last(idxs[1]))) : + throw(ErrorException("Retail price calibrator is not set up to handle multiple filters.")) + # for each result row, get the corresponding calibrator value for area, subarea, year cal_values =[] for row in eachrow(cal_table) - if row.area == area && row.subarea == subarea && (!hasproperty(calibrator_file, :year) || row.year == yr_idxs) # if there is no year column only check that area, subarea match + if string(row.area) == area && string(row.subarea) == subarea push!(cal_values, row.cal_value) end end diff --git a/src/types/modifications/RetailPrice.jl b/src/types/modifications/RetailPrice.jl index 5298e2a8..1958ab15 100644 --- a/src/types/modifications/RetailPrice.jl +++ b/src/types/modifications/RetailPrice.jl @@ -148,9 +148,7 @@ function extract_results(m::RetailPrice, config, data) end function combine_results(m::RetailPrice, post_config, post_data) - res = join_sim_tables(post_data, :value) - CSV.write(get_out_path(post_config, "$(m.name)_combined.csv"), res) end @@ -191,11 +189,10 @@ end # specialized method for retail rates with no cal_mode get_cal_values function get_retail_price(::Val{:get_cal_values}, m::RetailPrice, config, data, table) - # set up table that will contain calibrator values cal_table = DataFrame( area = String[], - subarea = String[], + subarea = Union{String, Int}[], year = String[], ref_price = Float64[], retail_price = Float64[], @@ -237,6 +234,7 @@ function get_retail_price(::Val{:get_cal_values}, m::RetailPrice, config, data, select!(cal_table, [c for c in (:area, :subarea, :year, :cal_value) if any(!ismissing, cal_table[!, c]) && any(x -> x != "" && !ismissing(x), cal_table[!, c])]) CSV.write(get_out_path(config, string(m.name, "_cals.csv")), cal_table) + results[:calibrator_values] = cal_table end From 3aed8a5c807b3e46805010d504e599a58390a22e Mon Sep 17 00:00:00 2001 From: Peplinski Date: Tue, 11 Nov 2025 18:19:03 -0500 Subject: [PATCH 26/35] Set up tests for retail price mode --- test/data/3bus/past_invest_costs.csv | 7 + test/data/3bus/ref_retail_price.csv | 5 + test/data/3bus/ref_retail_price_yearly.csv | 10 + test/data/3bus/results_retail_price.csv | 2 + test/data/3bus/retail_price_calibrator.csv | 4 + .../3bus/retail_price_calibrator_yearly.csv | 10 + test/runtests.jl | 4 + test/testretailprice.jl | 285 ++++++++++++++++++ 8 files changed, 327 insertions(+) create mode 100644 test/data/3bus/past_invest_costs.csv create mode 100644 test/data/3bus/ref_retail_price.csv create mode 100644 test/data/3bus/ref_retail_price_yearly.csv create mode 100644 test/data/3bus/results_retail_price.csv create mode 100644 test/data/3bus/retail_price_calibrator.csv create mode 100644 test/data/3bus/retail_price_calibrator_yearly.csv create mode 100644 test/testretailprice.jl diff --git a/test/data/3bus/past_invest_costs.csv b/test/data/3bus/past_invest_costs.csv new file mode 100644 index 00000000..8720ba87 --- /dev/null +++ b/test/data/3bus/past_invest_costs.csv @@ -0,0 +1,7 @@ +bus_idx,status,reg_factor,build_status,build_type,build_id,genfuel,gentype,econ_life,pcap_inv,pcap0,pcap_min,pcap_max,cf_min,cf_hist,vom,fuel_price,fom,capex,transmission_capex,routine_capex,year_on,year_off,year_shutdown,emis_co2,capt_co2_percent,heat_rate,chp_co2_multi,chp,pcap_plant_avg,past_invest_cost +3,1,0.75,built,exog,,coal,coal,30,2,2,0,2,0.6,0.68,2,0.555555556,20,7,0.1,0.2,y2020,y9999,y2045,1,0,9,1,0,2,7 +1,1,0,built,exog,,solar,solar,30,0.5,0.5,0,0.5,0,0.25,2,0,5,5,0.5,0.1,y2027,y9999,y2050,0,0,0,1,0,0.5,5 +3,1,0.75,built,exog,,ng,ngccccs,30,0.5,0.5,0,0.5,0.5,0.55,1.5,0.142857143,6,6,0.1,0.2,y2025,y9999,y2055,0.6,0.9,7,1,0,0.5,6 +1,1,0,built,exog,,wind,wind,30,0.05,0.05,0,0.05,0,0.4,0.1,0,2,3,0.5,0.3,y2015,y9999,y2035,0,0,0,1,0,0.05,3 +1,1,0.1,built,exog,,ng,ngt,30,0.01,0.01,0,0.01,0.6,0.23,2,0.25,7,7,0.1,0.1,y2030,y9999,y2050,0.6,0,8,0.67,1,0.01,7 +2,1,0.1,built,exog,,ng,ngt,30,2,2,0,2,0,0.23,100,0.25,5,7,0.1,0,y2020,y9999,y2050,0.6,0,8,0.67,1,0.01,7 diff --git a/test/data/3bus/ref_retail_price.csv b/test/data/3bus/ref_retail_price.csv new file mode 100644 index 00000000..ddd675c3 --- /dev/null +++ b/test/data/3bus/ref_retail_price.csv @@ -0,0 +1,5 @@ +area,subarea,ref_price +bus_idx,1,87.172 +bus_idx,2,72.716 +bus_idx,3,93.828 +,,78 diff --git a/test/data/3bus/ref_retail_price_yearly.csv b/test/data/3bus/ref_retail_price_yearly.csv new file mode 100644 index 00000000..c0be3a29 --- /dev/null +++ b/test/data/3bus/ref_retail_price_yearly.csv @@ -0,0 +1,10 @@ +area,subarea,year,ref_price +bus_idx,1,y2030,87.172 +bus_idx,2,y2030,72.716 +bus_idx,3,y2030,93.828 +bus_idx,1,y2035,88.172 +bus_idx,2,y2035,73.716 +bus_idx,3,y2035,94.828 +bus_idx,1,y2040,89.172 +bus_idx,2,y2040,75.716 +bus_idx,3,y2040,97.828 \ No newline at end of file diff --git a/test/data/3bus/results_retail_price.csv b/test/data/3bus/results_retail_price.csv new file mode 100644 index 00000000..13560f7b --- /dev/null +++ b/test/data/3bus/results_retail_price.csv @@ -0,0 +1,2 @@ +table_name,result_name,filter1,filter2,filter3,filter_years,filter_hours +retail_price,avg_elec_rate,bus_idx,,,:, diff --git a/test/data/3bus/retail_price_calibrator.csv b/test/data/3bus/retail_price_calibrator.csv new file mode 100644 index 00000000..4833eaab --- /dev/null +++ b/test/data/3bus/retail_price_calibrator.csv @@ -0,0 +1,4 @@ +area,subarea,cal_value +bus_idx,1,10 +bus_idx,2,5 +bus_idx,3,3 diff --git a/test/data/3bus/retail_price_calibrator_yearly.csv b/test/data/3bus/retail_price_calibrator_yearly.csv new file mode 100644 index 00000000..a88270ca --- /dev/null +++ b/test/data/3bus/retail_price_calibrator_yearly.csv @@ -0,0 +1,10 @@ +area,subarea,year,cal_value +bus_idx,1,y2030,10 +bus_idx,2,y2030,5 +bus_idx,3,y2030,3 +bus_idx,1,y2035,12 +bus_idx,2,y2035,6 +bus_idx,3,y2035,4 +bus_idx,1,y2040,12 +bus_idx,2,y2040,6 +bus_idx,3,y2040,4 \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index 16208465..d3d8d615 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,6 +5,8 @@ using JuMP using DataFrames using Logging using BasicInterpolators +using Statistics +using CSV import OrderedCollections: OrderedDict import YAML @@ -27,6 +29,7 @@ rm(joinpath(@__DIR__, "out"), force=true, recursive=true) include("testpoltypes.jl") include("testoptimizemodel.jl") include("testwelfare.jl") + include("testretailprice.jl") include("testresultprocessing.jl") include("testiteration.jl") include("teststorage.jl") @@ -35,6 +38,7 @@ rm(joinpath(@__DIR__, "out"), force=true, recursive=true) include("testadjust.jl") include("testutil.jl") include("testpost.jl") + include("testretailprice.jl") end global_logger(original_logger) diff --git a/test/testretailprice.jl b/test/testretailprice.jl new file mode 100644 index 00000000..cf5dba79 --- /dev/null +++ b/test/testretailprice.jl @@ -0,0 +1,285 @@ +@testset "Test Retail Price" begin + @testset "Retail Price Set Up" begin + config_file_ref = joinpath(@__DIR__, "config", "config_3bus.yml") + config_file = joinpath(@__DIR__, "config", "config_3bus_reserve_req.yml") + config = read_config(config_file_ref, config_file, log_model_summary=true) + config[:past_invest_file] = "data/3bus/past_invest_costs.csv" + + data = read_data(config) + model = setup_model(config, data) + + optimize!(model) + # solution_summary(model) + + @test check(config, data, model) + + parse_results!(config, data, model) + process_results!(config, data) + + setup_retail_price!(config, data) + + @test haskey(data, :retail_price) + + @testset "Check Retail Price Terms" begin + retail_price = data[:retail_price][:avg_elec_rate] + + #check retail price terms + @test all(k -> haskey(retail_price, k), [:bus, :gen, :storage, :past_invest]) + bus_terms = retail_price[:bus] + @test all(k -> haskey(bus_terms, k), [:electricity_cost, :distribution_cost_total, :merchandising_surplus_total, :gs_payment]) + @test !haskey(bus_terms, :baa_reserve_requirement_cost) + gen_terms = retail_price[:gen] + @test haskey(gen_terms, :cost_of_service_rebate) + storage_terms = retail_price[:storage] + @test haskey(storage_terms, :cost_of_service_rebate) + past_invest_terms = retail_price[:past_invest] + @test haskey(past_invest_terms, :cost_of_service_past_costs) + end + + @testset "Without Past Invest" begin + delete!(config, :past_invest_file) + + setup_retail_price!(config, data) + retail_price = data[:retail_price][:avg_elec_rate] + @test !haskey(retail_price, :past_invest) + + end + + @testset "Without Reserve Requirements" begin + delete!(config[:mods], :state_reserve) + + setup_retail_price!(config, data) + + retail_price = data[:retail_price][:avg_elec_rate] + + bus_terms = retail_price[:bus] + @test !haskey(bus_terms, :state_reserve) + end + + end + + + @testset "Test RetailPrice Mod" begin + + @testset "Without calibration" begin + config_file = joinpath(@__DIR__, "config", "config_3bus.yml") + storage_config_file = joinpath(@__DIR__, "config", "config_stor.yml") + config = read_config(config_file, storage_config_file) + data = read_data(config) + model = setup_model(config, data) + optimize!(model) + parse_results!(config, data, model) + # Make new mod + rtlprc_file = joinpath(@__DIR__, "data/3bus/results_retail_price.csv") + name = :retail_price + mod = RetailPrice(;file=rtlprc_file, name) + + mods = get_mods(config) + mods[name] = mod + + process_results!(config, data) + + results = get_results(data) + @test haskey(results, name) + table = get_result(data, name) + @test table[end, :filter1] |> contains("=>") + + @test table.value[1] == compute_retail_price(mod, data, :avg_elec_rate, :bus_idx=>3, 3,:) + + # test when filters that arent in all tables are provided + ref_price_file = joinpath(@__DIR__, "data/3bus/ref_retail_price.csv") + rtlprc = read_table(rtlprc_file) + rtlprc.filter2 .= "genfuel" + CSV.write(get_out_path(config, "results_retail_price.csv"), rtlprc) + + rtlprc_file = get_out_path(config, "results_retail_price.csv") + mod = RetailPrice(;file=rtlprc_file, name) + + mods = get_mods(config) + mods[name] = mod + + @test_throws Exception process_results!(config, data) + + + end + + @testset "With yearly reference price" begin + config_file = joinpath(@__DIR__, "config", "config_3bus.yml") + storage_config_file = joinpath(@__DIR__, "config", "config_stor.yml") + config = read_config(config_file, storage_config_file) + data = read_data(config) + model = setup_model(config, data) + optimize!(model) + parse_results!(config, data, model) + + # Make new mod + rtlprc_file = joinpath(@__DIR__, "data/3bus/results_retail_price.csv") + ref_price_file = joinpath(@__DIR__, "data/3bus/ref_retail_price_yearly.csv") + name = :retail_price + mod = RetailPrice(;file=rtlprc_file, name, ref_price_file=ref_price_file, cal_mode = "get_cal_values") + + mods = get_mods(config) + mods[name] = mod + + process_results!(config, data) + + results = get_results(data) + table = get_result(data, name) + + @test isfile(get_out_path(config, "$(name)_cals.csv")) + + @test table.value[2] == compute_retail_price(mod, data, :avg_elec_rate, :bus_idx=>3, 2,:)[1] + + cal_table = get_result(data, :calibrator_values) + ref_price = read_table(ref_price_file) + @test filter(row->row.subarea==3 && row.year=="y2035", ref_price)[1,"ref_price"] - compute_retail_price(mod, data, :avg_elec_rate, :bus_idx=>3, 2,:)[1] == filter(row->row.subarea=="3" && row.year=="y2035", cal_table)[1,"cal_value"] + + # test for error when multiple filters are provided + ref_price = read_table(ref_price_file) + insertcols!(ref_price, 3, :filter1 => "genfuel"=>"ng") + CSV.write(get_out_path(config, "ref_retail_price_yearly.csv"), ref_price) + + ref_price_file = get_out_path(config, "ref_retail_price_yearly.csv") + mod = RetailPrice(;file=rtlprc_file, name, ref_price_file=ref_price_file, cal_mode = "get_cal_values") + + # test for error when multiple filters are provided + @test_throws Exception compute_retail_price(mod, data, :avg_elec_rate, [:bus_idx=>3, :nation=>"narnia"], 1,:) + + end + + @testset "With single reference price" begin + config_file = joinpath(@__DIR__, "config", "config_3bus.yml") + storage_config_file = joinpath(@__DIR__, "config", "config_stor.yml") + config = read_config(config_file, storage_config_file) + data = read_data(config) + model = setup_model(config, data) + optimize!(model) + parse_results!(config, data, model) + + # Make new mod + rtlprc_file = joinpath(@__DIR__, "data/3bus/results_retail_price.csv") + ref_price_file = joinpath(@__DIR__, "data/3bus/ref_retail_price.csv") + name = :retail_price + mod = RetailPrice(;file=rtlprc_file, name, ref_price_file=ref_price_file, cal_mode = "get_cal_values") + + mods = get_mods(config) + mods[name] = mod + + process_results!(config, data) + + results = get_results(data) + table = get_result(data, name) + + @test isfile(get_out_path(config, "$(name)_cals.csv")) + @test table.value[3] == compute_retail_price(mod, data, :avg_elec_rate, :bus_idx=>3, 1,:)[1] + + cal_table = get_result(data, :calibrator_values) + ref_price = read_table(ref_price_file) + + @test filter(row->row.subarea=="3", ref_price)[1,"ref_price"] - compute_retail_price(mod, data, :avg_elec_rate, :bus_idx=>3, 1,:)[1] == filter(row->row.subarea=="3", cal_table)[1,"cal_value"] + + # test for error when multiple grid-wide prices are provided + ref_price = read_table(ref_price_file) + push!(ref_price, ["","",95]) + CSV.write(get_out_path(config, "ref_retail_price.csv"), ref_price) + + ref_price_file = get_out_path(config, "ref_retail_price.csv") + mod = RetailPrice(;file=rtlprc_file, name, ref_price_file=ref_price_file, cal_mode = "get_cal_values") + + @test_throws Exception compute_retail_price(mod, data, :avg_elec_rate, :bus_idx=>3, 1,:) + + # test for error when multiple prices are provided for a regoin + ref_price_file = joinpath(@__DIR__, "data/3bus/ref_retail_price.csv") + ref_price = read_table(ref_price_file) + push!(ref_price, ["bus_idx","3",89]) + CSV.write(get_out_path(config, "ref_retail_price.csv"), ref_price) + + ref_price_file = get_out_path(config, "ref_retail_price.csv") + mod = RetailPrice(;file=rtlprc_file, name, ref_price_file=ref_price_file, cal_mode = "get_cal_values") + + @test_throws Exception compute_retail_price(mod, data, :avg_elec_rate, :bus_idx=>3, 1,:) + + # test for error when multiple filters are provided + @test_throws Exception compute_retail_price(mod, data, :avg_elec_rate, [:bus_idx=>3, :nation=>"narnia"], 1,:) + + end + + @testset "With yearly calibrator values" begin + config_file = joinpath(@__DIR__, "config", "config_3bus.yml") + storage_config_file = joinpath(@__DIR__, "config", "config_stor.yml") + config = read_config(config_file, storage_config_file) + data = read_data(config) + model = setup_model(config, data) + optimize!(model) + parse_results!(config, data, model) + + # Make new mod + rtlprc_file = joinpath(@__DIR__, "data/3bus/results_retail_price.csv") + cal_file = joinpath(@__DIR__, "data/3bus/retail_price_calibrator_yearly.csv") + name = :retail_price + mod = RetailPrice(;file=rtlprc_file, name, calibrator_file=cal_file, cal_mode = "calibrate") + + mods = get_mods(config) + mods[name] = mod + + process_results!(config, data) + + results = get_results(data) + table = get_result(data, name) + + cal_table = read_table(cal_file) + @test table.value[1] == compute_retail_price(mod, data, :avg_elec_rate, :bus_idx=>3, 3,:)[1] + + # check that calibrator values were added + mod_no_calibrate = RetailPrice(;file=rtlprc_file, name) + @test table.value[1] == compute_retail_price(mod_no_calibrate, data, :avg_elec_rate, :bus_idx=>3, 3,:)[1] + filter(row->row.subarea==3 && row.year=="y2040", cal_table)[1,:"cal_value"] + @test table.value[2] == compute_retail_price(mod_no_calibrate, data, :avg_elec_rate, :bus_idx=>3, 2,:)[1] + filter(row->row.subarea==3 && row.year=="y2035", cal_table)[1,:"cal_value"] + + # test for error when multiple filters are provided + @test_throws Exception compute_retail_price(mod, data, :avg_elec_rate, [:bus_idx=>3, :nation=>"narnia"], 1,:) + end + + + @testset "With single calibrator value" begin + config_file = joinpath(@__DIR__, "config", "config_3bus.yml") + storage_config_file = joinpath(@__DIR__, "config", "config_stor.yml") + config = read_config(config_file, storage_config_file) + data = read_data(config) + model = setup_model(config, data) + optimize!(model) + parse_results!(config, data, model) + + # Make new mod + rtlprc_file = joinpath(@__DIR__, "data/3bus/results_retail_price.csv") + cal_file = joinpath(@__DIR__, "data/3bus/retail_price_calibrator.csv") + name = :retail_price + mod = RetailPrice(;file=rtlprc_file, name, calibrator_file=cal_file, cal_mode = "calibrate") + + mods = get_mods(config) + mods[name] = mod + + process_results!(config, data) + + results = get_results(data) + table = get_result(data, name) + + cal_table = read_table(cal_file) + @test table.value[1] == compute_retail_price(mod, data, :avg_elec_rate, :bus_idx=>3, 3,:)[1] #+ filter(row->row.subarea==3, cal_table)[1,:"cal_value"] + + # check that calibrator values were added + mod_no_calibrate = RetailPrice(;file=rtlprc_file, name) + @test table.value[1] == compute_retail_price(mod_no_calibrate, data, :avg_elec_rate, :bus_idx=>3, 3,:)[1] + filter(row->row.subarea==3, cal_table)[1,:"cal_value"] + @test table.value[2] == compute_retail_price(mod_no_calibrate, data, :avg_elec_rate, :bus_idx=>3, 2,:)[1] + filter(row->row.subarea==3, cal_table)[1,:"cal_value"] + + # test for error when multiple filters are provided + @test_throws Exception compute_retail_price(mod, data, :avg_elec_rate, [:bus_idx=>3, :nation=>"narnia"], 1,:) + end + end +end + +# results_template_rtl_price: +# type: RetailPrice +# file: "../res_templates/results_template_tests_rtl_prc.csv" +# cal_mode: calibrate +# # ref_price_file: "../../Data/config/mods/retail_price_calibrator/retail_price_ref_values.csv" +# calibrator_file: "L:/Project-Gurobi/Workspace3/E4ST_Output/haiku_merge/pipeflow_runs/rr_cal_251028/results_template_rtl_price_cals.csv" \ No newline at end of file From 31e185d062b94236b0238ecd8c3d5d6de9541985 Mon Sep 17 00:00:00 2001 From: Peplinski Date: Tue, 11 Nov 2025 19:03:35 -0500 Subject: [PATCH 27/35] Adjusts payment calculation for when a year doesn't have a target --- src/types/modifications/GenerationStandard.jl | 3 +- src/types/modifications/ResultsTemplate.jl | 168 ++++-------------- 2 files changed, 36 insertions(+), 135 deletions(-) diff --git a/src/types/modifications/GenerationStandard.jl b/src/types/modifications/GenerationStandard.jl index 45c6a861..aae06040 100644 --- a/src/types/modifications/GenerationStandard.jl +++ b/src/types/modifications/GenerationStandard.jl @@ -174,6 +174,7 @@ function modify_results!(pol::GenerationStandard, config, data) bus = get_table(data, :bus) gen = get_table(data, :gen) nyr = get_num_years(data) + years = Symbol.(get_years(data)) prc_name = Symbol("$(pol.name)_prc") cost_name = Symbol("$(pol.name)_cost") @@ -202,7 +203,7 @@ function modify_results!(pol::GenerationStandard, config, data) end for (k,d) in pol.load_targets - targets = collect(values(d[:targets]))[1:nyr] + targets = collect(values(OrderedDict(y => get(d[:targets], y, 0.0) for y in years)))[1:nyr] # target set to 0 if missing filters = d[:filters] bus_idxs = get_row_idxs(bus, parse_comparisons(d[:filters])) # set to shadow_prc for bus diff --git a/src/types/modifications/ResultsTemplate.jl b/src/types/modifications/ResultsTemplate.jl index 15ddedec..2e97026b 100644 --- a/src/types/modifications/ResultsTemplate.jl +++ b/src/types/modifications/ResultsTemplate.jl @@ -8,7 +8,6 @@ This is a mod that outputs computed results, given a `file` representing the tem * `file` - the file pointing to a table specifying which results to calculate * `name` - the name of the mod, do not need to specify in a config file * `col_sort` - the column(s) to sort by. Defaults to the order in which they were originally specified. -* `cross_table` - indicates that the result is pulling results from multiple tables. Defaults to false. The `file` should represent a csv table with the following columns: * `table_name` - the name of the table being aggregated. i.e. `gen`, `bus`, etc. If you leave it empty, it will call `compute_welfare` instead of `compute_result` @@ -23,10 +22,8 @@ struct ResultsTemplate <: Modification file::String name::Symbol table::DataFrame - cross_table::Bool - calibrator_file::String col_sort - function ResultsTemplate(;file, name, cross_table=false, calibrator_file="", col_sort=:initial_order) + function ResultsTemplate(;file, name, col_sort=:initial_order) table = read_table(file) force_table_types!(table, name, :table_name=>Symbol, @@ -39,16 +36,12 @@ struct ResultsTemplate <: Modification hasproperty(table, col_name) || continue force_table_types!(table, name, col_name=>String) end - return new(file, name, table, cross_table, calibrator_file, col_sort) + return new(file, name, table, col_sort) end end export ResultsTemplate -# Outer constructor to allow positional arguments if needed -ResultsTemplate(file::String, name::Symbol, table::DataFrame, cross_table::Bool, calibrator_file::String, col_sort) = - ResultsTemplate(file=file, name=name, cross_table=cross_table, calibrator_file=calibrator_file, col_sort=col_sort) - # Deal with backwards compatibility const AggregationTemplate = ResultsTemplate SYM2TYPE[:AggregationTemplate] = ResultsTemplate @@ -60,17 +53,7 @@ export AggregationTemplate mod_rank(::Type{<:ResultsTemplate}) = 5.0 fieldnames_for_yaml(::Type{ResultsTemplate}) = (:file,) -# dispatches the single or cross-table results method based on cross_table argument function modify_results!(m::ResultsTemplate, config, data) - if m.cross_table == true - modify_results!(m::ResultsTemplate, Val(:true), config, data) - elseif m.cross_table == false - modify_results!(m::ResultsTemplate, Val(:false), config, data) - end -end - -# method for single-table result -function modify_results!(m::ResultsTemplate, ::Val{:false}, config, data) table = copy(m.table) table.initial_order = 1:nrow(table) @@ -78,7 +61,6 @@ function modify_results!(m::ResultsTemplate, ::Val{:false}, config, data) # for any rows that are not a pair, separate into multiple rows not_pair_idx = findfirst(not_a_full_filter, eachrow(table)) - while not_pair_idx !== nothing row = table[not_pair_idx, :] filter_col_idx = findfirst(filter_col->not_a_full_filter(row[filter_col]), filter_cols) @@ -119,97 +101,15 @@ function modify_results!(m::ResultsTemplate, ::Val{:false}, config, data) idxs = parse_comparisons(row) yr_idxs = parse_year_idxs(row.filter_years) hr_idxs = parse_hour_idxs(row.filter_hours) - - try - return compute_result(data, table_name, result_name, idxs, yr_idxs, hr_idxs) - catch e - @warn "No single-table results formula found for table $table_name and result $result_name" - return 0.0 - end - - end - sort!(table, m.col_sort) - select!(table, Not(:initial_order)) - CSV.write(get_out_path(config, string(m.name, ".csv")), table) - results = get_results(data) - results[m.name] = table - return -end - -# cross-table results -function modify_results!(m::ResultsTemplate, ::Val{:true}, config, data) - table = copy(m.table) - table.initial_order = 1:nrow(table) - - filter_cols = setdiff(propertynames(table), [:table_name, :result_name]) - - # for any rows that are not a pair, separate into multiple rows - not_pair_idx = findfirst(not_a_full_filter, eachrow(table)) - while not_pair_idx !== nothing - row = table[not_pair_idx, :] - filter_col_idx = findfirst(filter_col->not_a_full_filter(row[filter_col]), filter_cols) - col_to_expand = filter_cols[filter_col_idx] - table_name = row[:table_name] - result_name = row[:result_name] - - if col_to_expand == :filter_hours - area = row.filter_hours - hours_table_col = get_table_col(data, :hours, area) - subareas = Base.sort!(String.(string.(unique(hours_table_col))), by=hours_sortby) - elseif col_to_expand == :filter_years && row[col_to_expand] == ":" - area = :years - subareas = data[area] + if table_name == Symbol("") + return compute_welfare(data, result_name, idxs, yr_idxs, hr_idxs) else - area = row[col_to_expand] - table_names = get_cross_table(data, table_name)[result_name] - all(hasproperty(get_table(data, t), area) for (t, _) in table_names) || error("Some tables are missing property $(area)") - data_table_col = get_table_col(data, first(keys(table_names)), area) - subareas = sort!(unique(data_table_col)) - end - - row_dict = Dict(pairs(row)) - for subarea in subareas - # Add a row right after the original row - row_dict[col_to_expand] = "$area=>$subarea" - insert!(table, not_pair_idx+1, row_dict) - end - - deleteat!(table, not_pair_idx) - - # Find the next index that is not a pair, to be expanded - not_pair_idx = findfirst(not_a_full_filter, eachrow(table)) - end - - @info "Calculating results for $(nrow(table)) rows in ResultsTemplate $(m.name)" - results_formulas = get_results_formulas(data) - table.value = map(eachrow(table)) do row - table_name = row.table_name - result_name = row.result_name - idxs = parse_comparisons(row) - yr_idxs = parse_year_idxs(row.filter_years) - hr_idxs = parse_hour_idxs(row.filter_hours) - - if table_name == Symbol("welfare") - if hr_idxs !== Colon() - @warn "Hourly welfare calculations are not set up." - return 0.0 - else - return compute_welfare(data, result_name, idxs, yr_idxs, hr_idxs) - end - elseif table_name == Symbol("retail_price") - if hr_idxs !== Colon() - @warn "Hourly retail price calculations are not set up." + try + return compute_result(data, table_name, result_name, idxs, yr_idxs, hr_idxs) + catch e + @warn "No results formula found for table $table_name and result $result_name" return 0.0 - else - if isempty(m.calibrator_file) - return compute_retail_price(data, result_name, idxs, yr_idxs, hr_idxs) - else - return compute_retail_price(data, result_name, m.calibrator_file, idxs, yr_idxs, hr_idxs) - end end - else - @warn "No cross-table results formula found for table $table_name and result $result_name" - return 0.0 end end sort!(table, m.col_sort) @@ -220,14 +120,14 @@ function modify_results!(m::ResultsTemplate, ::Val{:true}, config, data) return end -# function hours_sortby(s::T) where T -# if endswith(s, r"h\d+") -# m = match(r"h(\d+)", s) -# return lpad(m.captures[1], 4, '0') |> T -# else -# return s |> T -# end -# end +function hours_sortby(s::T) where T + if endswith(s, r"h\d+") + m = match(r"h(\d+)", s) + return lpad(m.captures[1], 4, '0') |> T + else + return s |> T + end +end function extract_results(m::ResultsTemplate, config, data) results = get_results(data) @@ -243,22 +143,22 @@ function combine_results(m::ResultsTemplate, post_config, post_data) CSV.write(get_out_path(post_config, "$(m.name)_combined.csv"), res) end -# function not_a_full_filter(row::DataFrameRow) -# not_a_full_filter(row.filter_years) && return true -# not_a_full_filter(row.filter_hours) && return true -# for i in 1:1000 -# col_name = "filter$i" -# hasproperty(row, col_name) || break -# not_a_full_filter(row[col_name]) && return true -# end -# return false -# end +function not_a_full_filter(row::DataFrameRow) + not_a_full_filter(row.filter_years) && return true + not_a_full_filter(row.filter_hours) && return true + for i in 1:1000 + col_name = "filter$i" + hasproperty(row, col_name) || break + not_a_full_filter(row[col_name]) && return true + end + return false +end -# function not_a_full_filter(s::AbstractString) -# isempty(s) && return false -# all(isnumeric, s) && return false -# contains(s, "=>") && return false -# startswith(s, "[") && return false -# startswith(s, "y2") && return false -# return true -# end \ No newline at end of file +function not_a_full_filter(s::AbstractString) + isempty(s) && return false + all(isnumeric, s) && return false + contains(s, "=>") && return false + startswith(s, "[") && return false + startswith(s, "y2") && return false + return true +end \ No newline at end of file From 7d7fd61b1b5e52c5aed1cf326a15ac44d62ee6e0 Mon Sep 17 00:00:00 2001 From: Peplinski Date: Tue, 11 Nov 2025 19:43:01 -0500 Subject: [PATCH 28/35] Delete one of two retail price includes in runtests --- test/runtests.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index d3d8d615..c849943b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -29,7 +29,6 @@ rm(joinpath(@__DIR__, "out"), force=true, recursive=true) include("testpoltypes.jl") include("testoptimizemodel.jl") include("testwelfare.jl") - include("testretailprice.jl") include("testresultprocessing.jl") include("testiteration.jl") include("teststorage.jl") From ea6a0ad667f2a69a99619159c838339d5161f586 Mon Sep 17 00:00:00 2001 From: Peplinski Date: Wed, 12 Nov 2025 12:29:56 -0500 Subject: [PATCH 29/35] Fixes so that resreq is added with any name and code cov improves --- src/results/retail_price.jl | 11 ++++++++--- test/testretailprice.jl | 6 +++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl index 955c47f9..a5c70e6c 100644 --- a/src/results/retail_price.jl +++ b/src/results/retail_price.jl @@ -48,9 +48,14 @@ function setup_retail_price!(config, data) add_price_term!(data, :avg_elec_rate, :past_invest, :cost_of_service_past_costs, +) end - if haskey(config, :mods) && haskey(config[:mods], :baa_reserve_requirement) - add_price_term!(data, :avg_elec_rate, :bus, :baa_reserve_requirement_cost, +) - add_price_term!(data, :avg_elec_rate, :bus, :baa_reserve_requirement_merchandising_surplus_total, -) + if haskey(config, :mods) && any(v -> v isa ReserveRequirement, values(config[:mods])) + reserve_mods = collect(k for (k, v) in config[:mods] if v isa ReserveRequirement) + length(reserve_mods) > 1 && @warn "Multiple ReserveRequirement mods found; using the first." reserve_mods + reserve_name = isempty(reserve_mods) ? nothing : first(reserve_mods) + reserve_cost = Symbol(string(reserve_name, "_cost")) + reserve_ms = Symbol(string(reserve_name, "_merchandising_surplus_total")) + add_price_term!(data, :avg_elec_rate, :bus, reserve_cost, +) + add_price_term!(data, :avg_elec_rate, :bus, reserve_ms, -) end end diff --git a/test/testretailprice.jl b/test/testretailprice.jl index cf5dba79..fc9a05e2 100644 --- a/test/testretailprice.jl +++ b/test/testretailprice.jl @@ -26,8 +26,7 @@ #check retail price terms @test all(k -> haskey(retail_price, k), [:bus, :gen, :storage, :past_invest]) bus_terms = retail_price[:bus] - @test all(k -> haskey(bus_terms, k), [:electricity_cost, :distribution_cost_total, :merchandising_surplus_total, :gs_payment]) - @test !haskey(bus_terms, :baa_reserve_requirement_cost) + @test all(k -> haskey(bus_terms, k), [:electricity_cost, :distribution_cost_total, :merchandising_surplus_total, :gs_payment, :state_reserve_cost]) gen_terms = retail_price[:gen] @test haskey(gen_terms, :cost_of_service_rebate) storage_terms = retail_price[:storage] @@ -53,7 +52,7 @@ retail_price = data[:retail_price][:avg_elec_rate] bus_terms = retail_price[:bus] - @test !haskey(bus_terms, :state_reserve) + @test !haskey(bus_terms, :state_reserve_cost) end end @@ -65,6 +64,7 @@ config_file = joinpath(@__DIR__, "config", "config_3bus.yml") storage_config_file = joinpath(@__DIR__, "config", "config_stor.yml") config = read_config(config_file, storage_config_file) + config[:past_invest_file] = "data/3bus/past_invest_costs.csv" data = read_data(config) model = setup_model(config, data) optimize!(model) From 73cd20a11984af215292647b1642a8829ce653f7 Mon Sep 17 00:00:00 2001 From: Peplinski Date: Wed, 12 Nov 2025 16:25:30 -0500 Subject: [PATCH 30/35] Retail price calc updated so that merchandising surplus is not distributed in cos regions --- src/results/parse.jl | 14 ++++++++++---- src/results/results_formulas.csv | 1 + 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/results/parse.jl b/src/results/parse.jl index 5c9bde41..279bbda1 100644 --- a/src/results/parse.jl +++ b/src/results/parse.jl @@ -375,6 +375,7 @@ function parse_lmp_results!(config, data) res_raw = get_raw_results(data) branch = get_table(data, :branch) + bus = get_table(data, :bus) f_bus_idxs = branch.f_bus_idx::Vector{Int64} t_bus_idxs = branch.t_bus_idx::Vector{Int64} @@ -420,6 +421,7 @@ function parse_lmp_results!(config, data) hour_weights_mat = [hour_weights[hr_idx] for yr_idx in 1:nyr, hr_idx in 1:nhr] # Loop through each branch and add the hourly merchandising surplus, in dollars, to the appropriate bus + ms_all = zeros(size(lmp_elserv)) # nbus x nyr x nhr ms = zeros(size(lmp_elserv)) # nbus x nyr x nhr ms_branch = zeros(size(pflow_branch)) @@ -430,13 +432,17 @@ function parse_lmp_results!(config, data) t_bus_lmp = view(lmp_elserv, t_bus_idx, :, :) # nyr x nhr pflow = view(pflow_branch, branch_idx, :, :) # nyr x nhr ms_per_bus = ((t_bus_lmp .- f_bus_lmp) .* pflow) .* hour_weights_mat .* 0.5 - ms[f_bus_idx, :, :] .+= ms_per_bus - ms[t_bus_idx, :, :] .+= ms_per_bus + ms_all[f_bus_idx, :, :] .+= ms_per_bus + ms_all[t_bus_idx, :, :] .+= ms_per_bus + f_bus_reg_factor = first(filter(row -> row.bus_idx == f_bus_idx, bus))[:reg_factor] + t_bus_reg_factor = first(filter(row -> row.bus_idx == t_bus_idx, bus))[:reg_factor] + ms[f_bus_idx, :, :] .+= ms_per_bus .* (1 - f_bus_reg_factor) + ms[t_bus_idx, :, :] .+= ms_per_bus .* (1 - t_bus_reg_factor) ms_branch[branch_idx, :, :] = ms_per_bus .* 2 end - add_table_col!(data, :bus, :merchandising_surplus, ms, Dollars, "Merchandising surplus, in dollars, from selling electricity for a higher price at one end of a line than another.") - + add_table_col!(data, :bus, :merchandising_surplus, ms_all, Dollars, "Merchandising surplus, in dollars, from selling electricity for a higher price at one end of a line than another, calculated for every node.") + add_table_col!(data, :bus, :merchandising_surplus_comp, ms, Dollars, "Merchandising surplus, in dollars, from selling electricity for a higher price at one end of a line than another, calculated for nodes in competitive regions only.") add_table_col!(data, :branch, :merchandising_surplus, ms_branch, Dollars, "Merchandising surplus, in dollars, from selling electricity for a higher price at one end of a line than another.") # # Add the LMP's to the results and to the branch table diff --git a/src/results/results_formulas.csv b/src/results/results_formulas.csv index 01d9d0a3..05b4c472 100644 --- a/src/results/results_formulas.csv +++ b/src/results/results_formulas.csv @@ -101,6 +101,7 @@ bus,plcurt_min,MinHourly(plcurt),MWCurtailed,Minimum hourly load power curtailed bus,electricity_cost,"SumHourlyWeighted(plserv,lmp_elserv)",Dollars,Total cost of electricity served bus,electricity_price,electricity_cost / elserv_total,DollarsPerMWhServed,Average cost of electricity served bus,merchandising_surplus_total,SumHourly(merchandising_surplus),Dollars,Total merchandising surplus from selling electricity for a higher price at one end of a line than another. Lines that are split across the region add half of their merchandising surplus to each region. +bus,merchandising_surplus_comp_total,SumHourly(merchandising_surplus),Dollars,Total merchandising surplus from selling electricity for a higher price at one end of a line than another. Lines that are split across the region add half of their merchandising surplus to each region. Calculated only for competitive regions. bus,distribution_cost_total,"SumHourlyWeighted(plserv, distribution_cost)",Dollars,Total cost to consumers for the transmission and distribution of power. bus,unserved_load_cost_total, "SumHourlyWeighted(plcurt, voll)",Dollars,Total cost of unserved load. bus,gs_payment,0,Dollars,Cost of required credits for clean/renewable generation for all generation standards (RPS's and CES's) for the qualifying demand at each given bus. From 8a6e5ebd1d08345aa42fad449a4e16b09f131e2d Mon Sep 17 00:00:00 2001 From: McKenna Peplinski <56940653+mckennapep@users.noreply.github.com> Date: Wed, 12 Nov 2025 17:02:59 -0500 Subject: [PATCH 31/35] Includes comments in retail price --- src/results/retail_price.jl | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl index a5c70e6c..7c83168f 100644 --- a/src/results/retail_price.jl +++ b/src/results/retail_price.jl @@ -7,9 +7,10 @@ Add in retail price terms to calculate retail electricity rates in \$/MWh. The relevant price terms are: * `electricity_cost` * `distribution_cost_total` -* `merchandising_surplus_total` +* `merchandising_surplus_comp_total` * `cost_of_service_rebate` * `net_production_cost` +* `cost_of_service_past_costs` * `baa_reserve_requirement_cost` * `baa_reserve_requiremetn_merchandising_surplus_total` Reference the results formulas for more detailed descriptions of each of these terms. @@ -35,19 +36,23 @@ function setup_retail_price!(config, data) add_price_term!(data, :avg_elec_rate, :bus, :distribution_cost_total, +) # merchandising suplus is from selling electricity for higher price at one end of line than another - add_price_term!(data, :avg_elec_rate, :bus, :merchandising_surplus_total, -) + add_price_term!(data, :avg_elec_rate, :bus, :merchandising_surplus_comp_total, -) # if the difference between revenue and total costs is positive, customers in COS regions get a rebate - # total cost includes production costs, net policy costs, gs_rebate, and the net of past investment costs and subsidies + # total cost includes production costs, net policy costs, gs_rebate, and the net of past investment costs and subsidies + # past investment costs may be handled separately when not included in the generator file add_price_term!(data, :avg_elec_rate, :gen, :cost_of_service_rebate, -) add_price_term!(data, :avg_elec_rate, :storage, :cost_of_service_rebate, -) + # payments for RPS and CES policies add_price_term!(data, :avg_elec_rate, :bus, :gs_payment, +) + # past invest file will overwrite the past invest column of the gen table if haskey(config, :past_invest_file) add_price_term!(data, :avg_elec_rate, :past_invest, :cost_of_service_past_costs, +) end + # capacity market costs in competitive regions (cost of service rebate subtracts out the capacity costs) if haskey(config, :mods) && any(v -> v isa ReserveRequirement, values(config[:mods])) reserve_mods = collect(k for (k, v) in config[:mods] if v isa ReserveRequirement) length(reserve_mods) > 1 && @warn "Multiple ReserveRequirement mods found; using the first." reserve_mods @@ -305,4 +310,4 @@ function get_calibrator_value(data, cal_table, idxs, hr_idxs) end return sum(cal_values) -end \ No newline at end of file +end From cdd3941ee524520208ddc5676b2e7ddb9b55f1a6 Mon Sep 17 00:00:00 2001 From: McKenna Peplinski <56940653+mckennapep@users.noreply.github.com> Date: Wed, 12 Nov 2025 17:06:32 -0500 Subject: [PATCH 32/35] Add to docs of RetailPrice mod --- src/types/modifications/RetailPrice.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/types/modifications/RetailPrice.jl b/src/types/modifications/RetailPrice.jl index 1958ab15..789bc091 100644 --- a/src/types/modifications/RetailPrice.jl +++ b/src/types/modifications/RetailPrice.jl @@ -23,7 +23,7 @@ The `file` should represent a csv table with the following columns: * `filter_hours` - the filtering conditions for the hours to be aggregated. See [`parse_hour_idxs`](@ref) for information on the hour filters. the retail rate mod is not set up to calculate hourly values. Note that, for the `filter_` or `filter_hours` columns, if a column name of the data table (or hours table) is given, new rows will be created for each unique value of that column. I.e. if a value of `gentype` is given, there will be made a new row for `gentype=>coal`, `gentype=>ng`, etc. -The calibration feature can only handle one filter_ column beyond filter_hours and filter_years. +However, the filter must be present in each of tables in the cross-table calculation for retail price or it will error. Further, the calibration feature can only handle one filter_ column beyond filter_hours and filter_years (which will most likely designate the region of interest for the retail price calcualation). """ struct RetailPrice <: Modification @@ -294,4 +294,4 @@ function full_cal!(m, data, table, cal_table) end return cal_table -end \ No newline at end of file +end From 243ce17ff1b8884508ccb5c631f8a7dbf584bd59 Mon Sep 17 00:00:00 2001 From: McKenna Peplinski <56940653+mckennapep@users.noreply.github.com> Date: Wed, 12 Nov 2025 17:37:54 -0500 Subject: [PATCH 33/35] Fix bug in test --- test/testretailprice.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/testretailprice.jl b/test/testretailprice.jl index fc9a05e2..f26ad996 100644 --- a/test/testretailprice.jl +++ b/test/testretailprice.jl @@ -26,7 +26,7 @@ #check retail price terms @test all(k -> haskey(retail_price, k), [:bus, :gen, :storage, :past_invest]) bus_terms = retail_price[:bus] - @test all(k -> haskey(bus_terms, k), [:electricity_cost, :distribution_cost_total, :merchandising_surplus_total, :gs_payment, :state_reserve_cost]) + @test all(k -> haskey(bus_terms, k), [:electricity_cost, :distribution_cost_total, :merchandising_surplus_comp_total, :gs_payment, :state_reserve_cost]) gen_terms = retail_price[:gen] @test haskey(gen_terms, :cost_of_service_rebate) storage_terms = retail_price[:storage] @@ -282,4 +282,4 @@ end # file: "../res_templates/results_template_tests_rtl_prc.csv" # cal_mode: calibrate # # ref_price_file: "../../Data/config/mods/retail_price_calibrator/retail_price_ref_values.csv" -# calibrator_file: "L:/Project-Gurobi/Workspace3/E4ST_Output/haiku_merge/pipeflow_runs/rr_cal_251028/results_template_rtl_price_cals.csv" \ No newline at end of file +# calibrator_file: "L:/Project-Gurobi/Workspace3/E4ST_Output/haiku_merge/pipeflow_runs/rr_cal_251028/results_template_rtl_price_cals.csv" From d2dd0e5bb20bceb3792991606979f440f0bf8302 Mon Sep 17 00:00:00 2001 From: Peplinski Date: Mon, 17 Nov 2025 12:23:45 -0500 Subject: [PATCH 34/35] Fixes for get_cal_values mode with multi-year reference prices --- src/results/retail_price.jl | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl index 7c83168f..1e87b391 100644 --- a/src/results/retail_price.jl +++ b/src/results/retail_price.jl @@ -86,7 +86,6 @@ function add_price_term!(data, price_type::Symbol, table_name::Symbol, result_na OrderedDict{Symbol, Function}() end - get(subretail_price, result_name, oper) == oper || @warn "Changing price sign for price[$price_type][$table_name][$result_name] to $oper" subretail_price[result_name] = oper end @@ -145,13 +144,16 @@ function compute_retail_price(::Val{:get_cal_values}, m, data, price_type::Symbo end subset = filter(row -> row.area == "" && row.subarea == "", ref_price_table) + n = nrow(subset) - # warn if there is no average reference price, error if more than 1 - if nrow(subset) == 0 + # ratio between load for region and full grid, used to adjust the calibration value + if n == 0 @warn "No full model reference price row. Outputting calibration values without a full adjustment." elserv_ratio = 0 - elseif nrow(subset) > 1 - error("Multiple full model reference price rows.") + elseif !hasproperty(ref_price_table, :year) && n > 1 + error("Multiple full model reference price rows for single reference price file.") + elseif hasproperty(ref_price_table, :year)&& n > length(unique((ref_price_table.year))) + error("Number of full model reference price rows is greater than number of years.") else elserv_total_all = compute_result(data, :bus, :elserv_total, :, yr_idxs, hr_idxs) elserv_ratio = elserv_total/elserv_total_all @@ -161,6 +163,7 @@ function compute_retail_price(::Val{:get_cal_values}, m, data, price_type::Symbo end +# specialized method to calculate retail price for cal_mode calibrate function compute_retail_price(::Val{:calibrate}, m, data, price_type::Symbol, idxs, yr_idxs, hr_idxs) value = 0.0 retail_price = get_retail_price(data) @@ -183,6 +186,7 @@ function compute_retail_price(::Val{:calibrate}, m, data, price_type::Symbol, i hr_idxs ) + # add corresponding cal value to retail price calculation retail_price = retail_price + cal return retail_price @@ -190,7 +194,7 @@ end export compute_retail_price -# get corresponding price values +# get corresponding ref price values by year function get_ref_price(ref_price_table, idxs, yr_idxs, hr_idxs, retail_price) # checks that there is only one filter, outside of hour and year filters area, subarea = @@ -219,7 +223,7 @@ function get_ref_price(ref_price_table, idxs, yr_idxs, hr_idxs, retail_price) return sum(ref_values), area, subarea, yr_idxs end -# one ref price for all years +# get corresponding ref price value for region (single value for all years) function get_ref_price(ref_price_table, idxs, hr_idxs, retail_price) # checks that there is only one filter, outside of hour and year filters area, subarea = From 6ac56f9c1030c6b3b9bf697150fe2e00e8a942a2 Mon Sep 17 00:00:00 2001 From: Peplinski Date: Mon, 5 Jan 2026 11:54:30 -0500 Subject: [PATCH 35/35] Fixes calibrator output table when area is empty, and improves comments --- src/types/modifications/RetailPrice.jl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/types/modifications/RetailPrice.jl b/src/types/modifications/RetailPrice.jl index 789bc091..3f34724b 100644 --- a/src/types/modifications/RetailPrice.jl +++ b/src/types/modifications/RetailPrice.jl @@ -5,14 +5,15 @@ This is a mod that outputs retail prices, given a `file` that indicates for which regions and years the retail price should be calculated. `name` is simply the name of the modification, and will be used as the root for the filename that the retail rates are saved to. The mod pulls values across different tables to calculate one retail rate. The specific terms that go into this cross-table calculation can be found in retail_price.jl. -The mod will also adjust the retail price values through calibration based on the `cal_mode` argument. If `cal_mode` is set to `none`, the retail prices will be unadjusted. If `cal_mode` is `get_val_values` the mod will use the reference price values to get calibration -values. If `cal_mod` is set to `calibrate`, the calibrator values will be read in from the `calibrator_file` and used to adjust the calculated retail rates. +The mod will also adjust the retail price values through calibration based on the `cal_mode` argument. If `cal_mode` is set to `none`, the retail prices will be unadjusted. If `cal_mode` is `get_val_values` the mod will use the reference price values in `ref_price_file` to +get calibration values. If `cal_mode` is set to `calibrate`, the calibrator values will be read in from the `calibrator_file` and used to adjust the calculated retail rates. ## Keyword Arguments * `file` - the file pointing to a table specifying which retail prices to calculate * `name` - the name of the mod, do not need to specify in a config file * `cal_mode` - a string that indicates the calibration mode. Options are `none`, `get_cal_values`, and `calibrate`. Defaults to `none`. -* `calibrator_file` - the file pointing to a table that contains reference price values or calibration values, depending on `cal_mode`. +* `ref_price_file` - the file pointing to a table that contains the reference price values, only necessary when `cal_mode` is `get_cal_values`. +* `calibrator_file` - the file pointing to a table that contains calibration values, only necessary when `cal_mode` is `calibrate`. * `col_sort` - the column(s) to sort by. Defaults to the order in which they were originally specified. The `file` should represent a csv table with the following columns: @@ -22,7 +23,7 @@ The `file` should represent a csv table with the following columns: * `filter_years` - the filtering conditions for the years to be aggregated. See [`parse_year_idxs`](@ref) for information on the year filters. * `filter_hours` - the filtering conditions for the hours to be aggregated. See [`parse_hour_idxs`](@ref) for information on the hour filters. the retail rate mod is not set up to calculate hourly values. -Note that, for the `filter_` or `filter_hours` columns, if a column name of the data table (or hours table) is given, new rows will be created for each unique value of that column. I.e. if a value of `gentype` is given, there will be made a new row for `gentype=>coal`, `gentype=>ng`, etc. +Note that, for the `filter_` or `filter_hours` columns, if a column name of the data table (or hours table) is given, new rows will be created for each unique value of that column. I.e. if a value of `state` is given, there will be made a new row for `state=>california`, `state=>colorado`, etc. However, the filter must be present in each of tables in the cross-table calculation for retail price or it will error. Further, the calibration feature can only handle one filter_ column beyond filter_hours and filter_years (which will most likely designate the region of interest for the retail price calcualation). """ @@ -231,13 +232,13 @@ function get_retail_price(::Val{:get_cal_values}, m::RetailPrice, config, data, # second calibrator adjustment to calibrate with full region full_cal!(m, data, table, cal_table) - select!(cal_table, - [c for c in (:area, :subarea, :year, :cal_value) if any(!ismissing, cal_table[!, c]) && any(x -> x != "" && !ismissing(x), cal_table[!, c])]) + select!(cal_table, [c for c in (:area, :subarea, :year, :cal_value) if c != :year ||any(x -> !ismissing(x) && x != "", cal_table.year)]) CSV.write(get_out_path(config, string(m.name, "_cals.csv")), cal_table) results[:calibrator_values] = cal_table end +# specialized method for retail rates with no cal_mode calibrate function get_retail_price(::Val{:calibrate}, m, config, data, table) # add value column to results table if it doesn't exist @@ -268,7 +269,7 @@ function get_retail_price(::Val{:calibrate}, m, config, data, table) results[m.name] = table end -# final calibration value for full model +# final calibration step for full model # example: for a state level model, ensure that the weighted average prices of all states are calibrated to the national average price function full_cal!(m, data, table, cal_table)