diff --git a/src/E4ST.jl b/src/E4ST.jl index af804b6a..9ba3cc16 100644 --- a/src/E4ST.jl +++ b/src/E4ST.jl @@ -73,6 +73,8 @@ include("types/modifications/GenHashID.jl") include("types/modifications/LeftJoinCols.jl") include("types/modifications/CapacityConstraint.jl") include("types/modifications/PerfectForesight.jl") +# include("types/modifications/RetailPriceCalibration.jl") +include("types/modifications/RetailPrice.jl") # Include Policies include("types/policies/ITC.jl") @@ -106,6 +108,7 @@ include("results/process.jl") include("results/aggregate.jl") include("results/welfare.jl") include("results/util.jl") +include("results/retail_price.jl") # Include postprocessing include("post/post.jl") diff --git a/src/io/config.jl b/src/io/config.jl index 3ad13941..fa80b49a 100644 --- a/src/io/config.jl +++ b/src/io/config.jl @@ -79,6 +79,7 @@ function summarize_config() (:require_optimal, false, true, "Whether or not to require whether or not the model is solved to optimality. If set to true and the optimizer terminates with a suboptimal termination status, [`run_e4st`](@ref) returns after optimizing, without parsing results, etc."), (:model_string_names, false, false, "Whether or not to allow the model to have string names. Defaults to `false` for memory savings. Can be helpful to turn on for debugging, especially if you are encountering an infeasible model"), (:yearly_objective_scalars, false, 1, "The amount to scale the objective by for each year, defaults to 1 for each year."), + (:past_invest_file, false, nothing, "File used to calculate past investment costs for existing generators. Only necessary when an aggregated gen table is used which would distort the past investment cost calculation.") ) diff --git a/src/io/data.jl b/src/io/data.jl index 86f45f22..ad2a23aa 100644 --- a/src/io/data.jl +++ b/src/io/data.jl @@ -49,6 +49,7 @@ function read_data!(config, data) promote_cols!(data) setup_results_formulas!(config, data) setup_welfare!(config, data) + setup_retail_price!(config, data) # Save the data to file as specified. if get(config, :save_data, true) @@ -84,6 +85,7 @@ function read_data_files!(config, data) read_table!(config, data, :load_add_file=>:load_add, optional=true) read_table!(config, data, :build_gen_file => :build_gen, optional=true) read_table!(config, data, :gentype_genfuel_file => :genfuel, optional=true) + read_table!(config, data, :past_invest_file => :past_invest, optional=true) end export read_data_files! @@ -179,6 +181,7 @@ function setup_data!(config, data) setup_table!(config, data, :nominal_load) setup_table!(config, data, :gen) # needs to come after build_gen setup for newgens setup_table!(config, data, :af_table) + setup_table!(config, data, :past_invest) end export setup_data! @@ -518,17 +521,19 @@ function setup_table!(config, data, ::Val{:gen}) z = Container(0.0) to_container!(gen, :past_invest_cost) to_container!(gen, :past_invest_subsidy) - for (idx_g, g) in enumerate(eachrow(gen)) - if g.build_status == "unbuilt" - if any(!=(0), g.past_invest_cost) || any(!=(0), g.past_invest_subsidy) - @warn "Generator $idx_g is unbuilt yet has past capex cost/subsidy, setting to zero" - g.past_invest_cost = z - g.past_invest_subsidy = z + if !haskey(config,:past_invest_file) + for (idx_g, g) in enumerate(eachrow(gen)) + if g.build_status == "unbuilt" + if any(!=(0), g.past_invest_cost) || any(!=(0), g.past_invest_subsidy) + @warn "Generator $idx_g is unbuilt yet has past capex cost/subsidy, setting to zero" + g.past_invest_cost = z + g.past_invest_subsidy = z + end + else + past_invest_percentages = get_past_invest_percentages(g, years) + g.past_invest_cost = g.past_invest_cost .* past_invest_percentages + g.past_invest_subsidy = g.past_invest_subsidy .* past_invest_percentages end - else - past_invest_percentages = get_past_invest_percentages(g, years) - g.past_invest_cost = g.past_invest_cost .* past_invest_percentages - g.past_invest_subsidy = g.past_invest_subsidy .* past_invest_percentages end end @@ -573,6 +578,76 @@ function setup_table!(config, data, ::Val{:gen}) return gen end export setup_table! +""" + setup_table!(config, data, ::Val{:past_invest}) + +Sets up the invest cost gen table. This is necessary when gens are aggregated so that capex for built gens can be found. +Creates age column which is a ByYear column. Unbuilt generators have a negative age before year_on. +""" +function setup_table!(config, data, ::Val{:past_invest}) + + if !haskey(config, :past_invest_file) + return + end + + bus = get_table(data, :bus) + past_invest = get_table(data, :past_invest) + years = get_years(data) + + # Set up year_unbuilt before setting up new gens. Plus we will want to save the column + hasproperty(past_invest, :year_unbuilt) || (past_invest.year_unbuilt = map(y->add_to_year(y, -1), past_invest.year_on)) + + # Set up past capex cost and subsidy to be for built generators only + # Make columns as needed + hasproperty(past_invest, :past_invest_cost) || (past_invest.past_invest_cost = zeros(nrow(past_invest))) + hasproperty(past_invest, :past_invest_subsidy) || (past_invest.past_invest_subsidy = zeros(nrow(past_invest))) + z = Container(0.0) + to_container!(past_invest, :past_invest_cost) + to_container!(past_invest, :past_invest_subsidy) + for (idx_g, g) in enumerate(eachrow(past_invest)) + if g.build_status == "unbuilt" + if any(!=(0), g.past_invest_cost) || any(!=(0), g.past_invest_subsidy) + @warn "Generator $idx_g is unbuilt yet has past capex cost/subsidy, setting to zero" + g.past_invest_cost = z + g.past_invest_subsidy = z + end + else + past_invest_percentages = get_past_invest_percentages(g, years) + g.past_invest_cost = g.past_invest_cost .* past_invest_percentages + g.past_invest_subsidy = g.past_invest_subsidy .* past_invest_percentages + end + end + + + original_cols = propertynames(past_invest) + data[:past_invest_table_original_cols] = original_cols + + #removes capex_obj if read in from previous sim + :capex_obj in propertynames(data[:past_invest]) && select!(data[:past_invest], Not(:capex_obj)) + + #set build_status to 'built' for all gens marked 'new'. This marks gens built in a previous sim as 'built'. + b = "built" # pre-allocate + transform!(past_invest, :build_status => ByRow(s->isnew(s) ? b : s) => :build_status) # transform in-place + + # Set the pcap_max to be equal to pcap0 for built generators + past_invest.pcap_max = map(row->isbuilt(row) ? row.pcap0 : row.pcap_max, eachrow(past_invest)) + past_invest.pcap0 = map(row->isbuilt(row) ? row.pcap0 : 0.0, eachrow(past_invest)) + + + ### Add age column as by ByYear based on year_on + years = year2float.(get_years(data)) + gen_age = Container[ByNothing(0.0) for i in 1:nrow(past_invest)] + for idx_g in 1:nrow(past_invest) + year_on = year2float(past_invest[idx_g, :year_on]) + g_age = [year - year_on for year in years] + gen_age[idx_g] = ByYear(g_age) + end + + add_table_col!(data, :past_invest, :age, gen_age, NumYears, "The age of the generator in each simulation year, given as a byYear container. Negative age is given for gens before their year_on.") + + return past_invest +end + """ join_bus_columns!(data, table_name) @@ -977,6 +1052,51 @@ function summarize_table(::Val{:gen}) return df end +@doc """ + summarize_table(::Val{:past_invest}) + +$(table2markdown(summarize_table(Val(:past_invest)))) +""" +function summarize_table(::Val{:past_invest}) + df = TableSummary() + push!(df, + (:bus_idx, Int64, NA, true, "The index of the `bus` table that the generator corresponds to"), + (:status, Bool, NA, false, "Whether or not the generator is in service"), + (:build_status, String15, NA, true, "Whether the generator is `built`, `new`, `unbuilt`, or `unretrofitted`. All generators marked `new` when the gen file is read in will be changed to `built`. Can also be changed to `retired_exog` or `retired_endog` after the simulation is run. See [`update_build_status!`](@ref). Note that `unretrofitted` means it is a [`Retrofit`](@ref) option based on a `built` generator."), + (:build_type, AbstractString, NA, true, "Whether the generator is 'real', 'exog' (exogenously built), or 'endog' (endogenously built)"), + (:build_id, AbstractString, NA, true, "Identifier of the build row. For pre-existing generators not specified in the build file, this is usually left empty"), + (:year_on, YearString, Year, true, "The first year of operation for the generator. (For new gens this is also the year it was built)"), + (:year_unbuilt,YearString, Year, false, "The latest year the generator was known not to be built. Defaults to year_on - 1. Used for past capex accounting."), + (:econ_life, Float64, NumYears, true, "The number of years in the economic lifetime of the generator."), + (:year_off, YearString, Year, true, "The first year that the generator is no longer operating in the simulation, computed from the simulation. Leave as y9999 if an existing generator that has not been retired in the simulation yet."), + (:year_shutdown, YearString, Year, true, "The forced (exogenous) shutdown year for the generator. Often equal to the year_on plus the econ_life"), + (:genfuel, AbstractString, NA, true, "The fuel type that the generator uses"), + (:gentype, String, NA, true, "The generation technology type that the generator uses"), + (:pcap_inv, Float64, MWCapacity, true, "Original invested nameplate power generation capacity for the generator. This is the original invested capacity of exogenously built generators (even if there have been retirements ), and the original invested capacity in year_on for endogenously built generators."), + (:pcap0, Float64, MWCapacity, true, "Nameplate power generation capacity for the generator at the start of the simulation"), + (:pcap_min, Float64, MWCapacity, true, "Minimum nameplate power generation capacity of the generator (normally set to zero to allow for retirement)"), + (:pcap_max, Float64, MWCapacity, true, "Maximum nameplate power generation capacity of the generator"), + (:vom, Float64, DollarsPerMWhGenerated, true, "Variable operation and maintenance cost per MWh of generation"), + (:fuel_price, Float64, DollarsPerMMBtu, false, "Fuel cost per MMBtu of fuel used. `heat_rate` column also necessary when supplying `fuel_price`"), + (:heat_rate, Float64, MMBtuPerMWhGenerated, false, "Heat rate, or MMBtu of fuel consumed per MWh electricity generated (0 for generators that don't use combustion)"), + (:fom, Float64, DollarsPerMWCapacityPerHour, true, "Hourly fixed operation and maintenance cost for a MW of generation capacity"), + (:capex, Float64, DollarsPerMWBuiltCapacityPerHour, true, "Hourly capital expenditures for a MW of generation capacity. For already-built generators, this is not accounted for in the optimization or accounting. For accounting for investment costs and subsidies in built generators, use `past_invest_cost` and `past_invest_subsidy`"), + (:transmission_capex, Float64, DollarsPerMWBuiltCapacityPerHour, true, "Hourly capital expenditures for the transmission supporting a MW of generation capacity"), + (:routine_capex, Float64, DollarsPerMWCapacityPerHour, true, "Routine capital expenditures for a MW of discharge capacity"), + (:past_invest_cost, Float64, DollarsPerMWCapacityPerHour, false, "Investment costs per MW of initial capacity per hour, for past investments"), + (:past_invest_subsidy, Float64, DollarsPerMWCapacityPerHour, false, "Investment subsidies from govt. per MW of initial capacity per hour, for past investments"), + (:cf_min, Float64, MWhGeneratedPerMWhCapacity, false, "The minimum capacity factor, or operable ratio of power generation to capacity for the generator to operate. Take care to ensure this is not above the hourly availability factor in any of the hours, or else the model may be infeasible. Set to zero by default."), + (:cf_max, Float64, MWhGeneratedPerMWhCapacity, false, "The maximum capacity factor, or operable ratio of power generation to capacity for the generator to operate"), + (:cf_hist, Float64, MWhGeneratedPerMWhCapacity, false, "The historical capacity factor for the generator, or the gentype if no previous data is available. Primarily used to calculate estimate policy value (PTC and EmissionPrice capex_adj)"), + (:af, Float64, MWhGeneratedPerMWhCapacity, false, "The availability factor, or maximum available ratio of pewer generation to nameplate capacity for the generator."), + (:emis_co2, Float64, ShortTonsPerMWhGenerated, false, "The emission rate per MWh of CO2"), + (:capt_co2_percent, Float64, NA, false, "The percentage of co2 emissions captured, to be sequestered."), + (:chp_co2_multi,Float64,NA,false,"The percentage of CO2 emissions from CHP attributed to the power generation. Used to calculate CO2e"), + (:reg_factor, Float64, NA, true, "The percentage of generation that dispatches to a cost-of-service regulated market"), + ) + return df +end + @doc """ summarize_table(::Val{:bus}) @@ -1128,6 +1248,11 @@ end export get_table export get_table +function get_cross_table(data, table_name) + return data[table_name]::OrderedDict{Symbol, OrderedDict{Symbol,OrderedDict{Symbol,Function}}} +end +export get_cross_table + """ get_subtable(table::DataFrame, conditions...) @@ -1535,7 +1660,17 @@ Returns the number of years in this simulation function get_num_years(data) return length(get_years(data)) end -export get_num_years, get_years + +""" + get_first_sim_year(data) -> year + +Returns the first year as a string (i.e. "y2022") of the years being represented in the sim. +""" +function get_first_sim_year(data) + return data[:years][1]::String +end + +export get_num_years, get_years, get_first_sim_year """ get_bus_gens(data, bus_idx) diff --git a/src/results/formulas.jl b/src/results/formulas.jl index e83ad8a4..9ea744a9 100644 --- a/src/results/formulas.jl +++ b/src/results/formulas.jl @@ -12,8 +12,10 @@ function setup_results_formulas!(config, data) results_formulas_table = read_table(data, results_formulas_file, :results_formulas) - for row in eachrow(results_formulas_table) - add_results_formula!(data, row.table_name, row.result_name, row.formula, row.unit, row.description) + foreach(eachrow(results_formulas_table)) do row + haskey(data, row.table_name) ? + add_results_formula!(data, row.table_name, row.result_name, row.formula, row.unit, row.description) : + @warn "There is no table $(row.table_name) in data. $(row.result_name) will not be added to formulas." end end export setup_results_formulas! @@ -801,6 +803,22 @@ function (f::CostOfServiceRebate)(data, table, idxs, yr_idxs, hr_idxs) end export CostOfServiceRebate +struct CostOfServicePastCost <: Function + table_name::Symbol +end +function (f::CostOfServicePastCost)(data, table, idxs, yr_idxs, hr_idxs) + reg_factor = table.reg_factor::Vector{Float64} + res = 0.0 + for i in idxs + rf = reg_factor[i] + past_invest = compute_result(data, f.table_name, :past_invest_cost_total, i, yr_idxs, hr_idxs) + prod = rf * past_invest + res += prod + end + return res + # return sum0(reg_factor[i] * compute_result(data, f.table_name, :net_total_revenue_prelim, i, yr_idxs, hr_idxs) for i in idxs) +end +export CostOfServicePastCost function _sum(v1, idxs) res = 0.0 diff --git a/src/results/parse.jl b/src/results/parse.jl index c9aeb20f..bf721ecb 100644 --- a/src/results/parse.jl +++ b/src/results/parse.jl @@ -375,6 +375,7 @@ function parse_lmp_results!(config, data) res_raw = get_raw_results(data) branch = get_table(data, :branch) + bus = get_table(data, :bus) f_bus_idxs = branch.f_bus_idx::Vector{Int64} t_bus_idxs = branch.t_bus_idx::Vector{Int64} @@ -420,6 +421,7 @@ function parse_lmp_results!(config, data) hour_weights_mat = [hour_weights[hr_idx] for yr_idx in 1:nyr, hr_idx in 1:nhr] # Loop through each branch and add the hourly merchandising surplus, in dollars, to the appropriate bus + ms_all = zeros(size(lmp_elserv)) # nbus x nyr x nhr ms = zeros(size(lmp_elserv)) # nbus x nyr x nhr ms_branch = zeros(size(pflow_branch)) @@ -430,13 +432,17 @@ function parse_lmp_results!(config, data) t_bus_lmp = view(lmp_elserv, t_bus_idx, :, :) # nyr x nhr pflow = view(pflow_branch, branch_idx, :, :) # nyr x nhr ms_per_bus = ((t_bus_lmp .- f_bus_lmp) .* pflow) .* hour_weights_mat .* 0.5 - ms[f_bus_idx, :, :] .+= ms_per_bus - ms[t_bus_idx, :, :] .+= ms_per_bus + ms_all[f_bus_idx, :, :] .+= ms_per_bus + ms_all[t_bus_idx, :, :] .+= ms_per_bus + f_bus_reg_factor = first(filter(row -> row.bus_idx == f_bus_idx, bus))[:reg_factor] + t_bus_reg_factor = first(filter(row -> row.bus_idx == t_bus_idx, bus))[:reg_factor] + ms[f_bus_idx, :, :] .+= ms_per_bus .* (1 - f_bus_reg_factor) + ms[t_bus_idx, :, :] .+= ms_per_bus .* (1 - t_bus_reg_factor) ms_branch[branch_idx, :, :] = ms_per_bus .* 2 end - add_table_col!(data, :bus, :merchandising_surplus, ms, Dollars, "Merchandising surplus, in dollars, from selling electricity for a higher price at one end of a line than another.") - + add_table_col!(data, :bus, :merchandising_surplus, ms_all, Dollars, "Merchandising surplus, in dollars, from selling electricity for a higher price at one end of a line than another, calculated for every node.") + add_table_col!(data, :bus, :merchandising_surplus_comp, ms, Dollars, "Merchandising surplus, in dollars, from selling electricity for a higher price at one end of a line than another, calculated for nodes in competitive regions only.") add_table_col!(data, :branch, :merchandising_surplus, ms_branch, Dollars, "Merchandising surplus, in dollars, from selling electricity for a higher price at one end of a line than another.") # # Add the LMP's to the results and to the branch table diff --git a/src/results/results_formulas.csv b/src/results/results_formulas.csv index dc4201f2..0e5b3567 100644 --- a/src/results/results_formulas.csv +++ b/src/results/results_formulas.csv @@ -71,6 +71,7 @@ gen,emission_cost,0,Dollars,Cost for paying all emissions prices. gen,emission_cost_per_mwh,emission_cost / egen_total,DollarsPerMWhGenerated,"Average cost, per MWh, for paying emission prices" gen,net_pol_cost_for_egus,emission_cap_cost + emission_cost - gs_rebate - invest_subsidy - ptc_subsidy,Dollars,Net cost for all generators from all policy types. gen,net_pol_cost_for_egus_per_mwh,net_pol_cost_for_egus / egen_total,DollarsPerMWhGenerated,"Average cost for all generators from all policy types, per MWh." +gen,net_pol_prod_cost_for_egus,emission_cost - invest_subsidy - ptc_subsidy,Dollars,Net cost for all generators from policy types that impact the production cost. gen,net_government_revenue,emission_cap_cost + emission_cost - invest_subsidy - ptc_subsidy - past_invest_subsidy_total,Dollars,Net government revenue earned from generators gen,going_forward_cost,production_cost + net_pol_cost_for_egus,Dollars,Total going forward cost gen,total_cost_prelim,going_forward_cost + past_invest_cost_total - past_invest_subsidy_total,Dollars,"Total cost of production, including past investment costs and subsidies for investments still within their economic lifetimes, before adjusting for cost-of-service rebates" @@ -100,8 +101,14 @@ bus,plcurt_min,MinHourly(plcurt),MWCurtailed,Minimum hourly load power curtailed bus,electricity_cost,"SumHourlyWeighted(plserv,lmp_elserv)",Dollars,Total cost of electricity served bus,electricity_price,electricity_cost / elserv_total,DollarsPerMWhServed,Average cost of electricity served bus,merchandising_surplus_total,SumHourly(merchandising_surplus),Dollars,Total merchandising surplus from selling electricity for a higher price at one end of a line than another. Lines that are split across the region add half of their merchandising surplus to each region. -bus,distribution_cost_total,"SumHourlyWeighted(plserv, distribution_cost)",Dollars,Total cost to consumers per MWh of served power for the transmission and distribution of the power. +bus,merchandising_surplus_comp_total,SumHourly(merchandising_surplus),Dollars,Total merchandising surplus from selling electricity for a higher price at one end of a line than another. Lines that are split across the region add half of their merchandising surplus to each region. Calculated only for competitive regions. +bus,distribution_cost_total,"SumHourlyWeighted(plserv, distribution_cost)",Dollars,Total cost to consumers for the transmission and distribution of power. bus,unserved_load_cost_total, "SumHourlyWeighted(plcurt, voll)",Dollars,Total cost of unserved load. +bus,gs_payment,0,Dollars,Cost of required credits for clean/renewable generation for all generation standards (RPS's and CES's) for the qualifying demand at each given bus. +bus,emission_cost,0,Dollars,Cost for paying all emissions prices for imported energy. branch,eflow_total,SumHourlyWeighted(pflow),MWhFlow,Total energy flowing in this branch branch,pflow_hourly_min,MinHourly(pflow),MWFlow,Minimum sum of power flowing in these branches branch,pflow_hourly_max,MaxHourly(pflow),MWFlow,Maximum sum of power flowing in these branches +past_invest,past_invest_cost_total,"SumHourlyWeighted(past_invest_cost, pcap_inv)",Dollars,"Investment costs from past investments. This only applies to generators built prior to the simulation. This includes the full annualized investment cost (""invest_cost""), times the percentage likelihood that the generator would still be within its the economic lifetime for the year calculated, given that endogenously built generators can be built in a range of years" +past_invest,past_invest_subsidy_total,"SumHourlyWeighted(past_invest_subsidy, pcap_inv)",Dollars,"Investment subsidies from past investments. This only applies to generators built prior to the simulation. This includes the full annualized investment subsidy (""invest_subsidy""), times the percentage likelihood that the generator would still be within its the economic lifetime for the year calculated, given that endogenously built generators can be built in a range of years" +past_invest,cost_of_service_past_costs,CostOfServicePastCost(past_invest),Dollars,"This is a specially calculated result, which is the sum of past_invest_cost_total * reg_factor for each generator" \ No newline at end of file diff --git a/src/results/retail_price.jl b/src/results/retail_price.jl new file mode 100644 index 00000000..1e87b391 --- /dev/null +++ b/src/results/retail_price.jl @@ -0,0 +1,317 @@ +""" + setup_retail_price!(config, data) + +Sets up the retail price structure. +Add in retail price terms to calculate retail electricity rates in \$/MWh. + +The relevant price terms are: +* `electricity_cost` +* `distribution_cost_total` +* `merchandising_surplus_comp_total` +* `cost_of_service_rebate` +* `net_production_cost` +* `cost_of_service_past_costs` +* `baa_reserve_requirement_cost` +* `baa_reserve_requiremetn_merchandising_surplus_total` +Reference the results formulas for more detailed descriptions of each of these terms. + +The results template can calculate annual rates by specified region, but is not set up for hourly rates. + + +There are three specialized methods for calculating the retail rate that depend on the cal_mode argument in the RetailPrice mod. If cal_mode is set to `none` +there will be no calibration steps. If it is set to `get_cal_values`, the function will find the difference between the estimated retail rates +and the true retail rate to use as a calibrator value. If cal_mode is set to `calibrate`, the corresponding calibrator vaulue will be added to the retail price. +""" + +# adjust calibrator table to be read in with right format, or change function to read in correctly + +function setup_retail_price!(config, data) + retail_price = OrderedDict{Symbol, OrderedDict{Symbol,OrderedDict{Symbol,Function}}}() + data[:retail_price] = retail_price + + # price terms for average electricity rate + add_price_term!(data, :avg_elec_rate, :bus, :electricity_cost, +) + + # per MW cost adder for distribution costs + add_price_term!(data, :avg_elec_rate, :bus, :distribution_cost_total, +) + + # merchandising suplus is from selling electricity for higher price at one end of line than another + add_price_term!(data, :avg_elec_rate, :bus, :merchandising_surplus_comp_total, -) + + # if the difference between revenue and total costs is positive, customers in COS regions get a rebate + # total cost includes production costs, net policy costs, gs_rebate, and the net of past investment costs and subsidies + # past investment costs may be handled separately when not included in the generator file + add_price_term!(data, :avg_elec_rate, :gen, :cost_of_service_rebate, -) + add_price_term!(data, :avg_elec_rate, :storage, :cost_of_service_rebate, -) + + # payments for RPS and CES policies + add_price_term!(data, :avg_elec_rate, :bus, :gs_payment, +) + + # past invest file will overwrite the past invest column of the gen table + if haskey(config, :past_invest_file) + add_price_term!(data, :avg_elec_rate, :past_invest, :cost_of_service_past_costs, +) + end + + # capacity market costs in competitive regions (cost of service rebate subtracts out the capacity costs) + if haskey(config, :mods) && any(v -> v isa ReserveRequirement, values(config[:mods])) + reserve_mods = collect(k for (k, v) in config[:mods] if v isa ReserveRequirement) + length(reserve_mods) > 1 && @warn "Multiple ReserveRequirement mods found; using the first." reserve_mods + reserve_name = isempty(reserve_mods) ? nothing : first(reserve_mods) + reserve_cost = Symbol(string(reserve_name, "_cost")) + reserve_ms = Symbol(string(reserve_name, "_merchandising_surplus_total")) + add_price_term!(data, :avg_elec_rate, :bus, reserve_cost, +) + add_price_term!(data, :avg_elec_rate, :bus, reserve_ms, -) + end + +end +export setup_retail_price! + +""" + get_retail_price(data) -> retail_price::OrderedDict{Symbol, OrderedDict{Symbol,OrderedDict{Symbol,Function}}} +""" +function get_retail_price(data) + return data[:retail_price]::OrderedDict{Symbol, OrderedDict{Symbol,OrderedDict{Symbol,Function}}} +end +export get_retail_price + +""" + add_price_term!(data, price_type::Symbol, table_name::Symbol, result_name::Symbol, oper) +""" +function add_price_term!(data, price_type::Symbol, table_name::Symbol, result_name::Symbol, oper::Function) + retail_price = get_retail_price(data) + subretail_price = get!(retail_price, price_type) do + OrderedDict{Symbol, OrderedDict{Symbol, Function}}() + end + subretail_price = get!(subretail_price, table_name) do + OrderedDict{Symbol, Function}() + end + + get(subretail_price, result_name, oper) == oper || @warn "Changing price sign for price[$price_type][$table_name][$result_name] to $oper" + subretail_price[result_name] = oper +end +export add_price_term! + +# wrapper function that dispatches different methods based on cal_mod arg +function compute_retail_price(m, data, price_type::Symbol, idxs, yr_idxs, hr_idxs) + compute_retail_price((Val(Symbol(m.cal_mode))), m, data, price_type, idxs, yr_idxs, hr_idxs) +end + +# specialized method to calculate retail price for cal_mode none +function compute_retail_price(::Val{:none}, m, data, price_type::Symbol, idxs...) + value = 0.0 + retail_price = get_retail_price(data) + table_names = retail_price[price_type] + for (table_name, result_names) in table_names + for (result_name, result_sign) in result_names + res = compute_result(data, table_name, result_name, idxs...) |> result_sign + value += res + end + end + + # divide by total generation to get dollars per MWh + elserv_total = compute_result(data, :bus, :elserv_total, idxs...) + return value/elserv_total +end + +# specialized method to calculate retail price for cal_mode get_cal_values +function compute_retail_price(::Val{:get_cal_values}, m, data, price_type::Symbol, idxs, yr_idxs, hr_idxs) + value = 0.0 + retail_price = get_retail_price(data) + table_names = retail_price[price_type] + for (table_name, result_names) in table_names + for (result_name, result_sign) in result_names + res = compute_result(data, table_name, result_name, idxs, yr_idxs, hr_idxs) |> result_sign + value += res + end + end + + # divide by total generation to get dollars per MWh + elserv_total = compute_result(data, :bus, :elserv_total, idxs, yr_idxs, hr_idxs) + retail_price = value/elserv_total + + _yr_idxs = get_year_idxs(data, yr_idxs) + ref_price_table = read_table(m.ref_price_file) + + if !hasproperty(ref_price_table, :year) + if _yr_idxs != 1 + return retail_price, [] + else + year = "" + ref_value, area, subarea = get_ref_price(ref_price_table, idxs, hr_idxs, retail_price) + end + else + ref_value, area, subarea, year = get_ref_price(ref_price_table, idxs, yr_idxs, hr_idxs, retail_price) + end + + subset = filter(row -> row.area == "" && row.subarea == "", ref_price_table) + n = nrow(subset) + + # ratio between load for region and full grid, used to adjust the calibration value + if n == 0 + @warn "No full model reference price row. Outputting calibration values without a full adjustment." + elserv_ratio = 0 + elseif !hasproperty(ref_price_table, :year) && n > 1 + error("Multiple full model reference price rows for single reference price file.") + elseif hasproperty(ref_price_table, :year)&& n > length(unique((ref_price_table.year))) + error("Number of full model reference price rows is greater than number of years.") + else + elserv_total_all = compute_result(data, :bus, :elserv_total, :, yr_idxs, hr_idxs) + elserv_ratio = elserv_total/elserv_total_all + end + + return retail_price, [area, subarea, year, ref_value, retail_price, ref_value - retail_price, elserv_total, elserv_ratio] + +end + +# specialized method to calculate retail price for cal_mode calibrate +function compute_retail_price(::Val{:calibrate}, m, data, price_type::Symbol, idxs, yr_idxs, hr_idxs) + value = 0.0 + retail_price = get_retail_price(data) + table_names = retail_price[price_type] + for (table_name, result_names) in table_names + for (result_name, result_sign) in result_names + res = compute_result(data, table_name, result_name, idxs, yr_idxs, hr_idxs) |> result_sign + value += res + end + end + # divide by total generation to get dollars per MWh + elserv_total = compute_result(data, :bus, :elserv_total, idxs, yr_idxs, hr_idxs) + retail_price = value/elserv_total + + cal_table = read_table(m.calibrator_file) + cal = get_calibrator_value(data, + cal_table, + idxs, + (hasproperty(cal_table, :year) ? (yr_idxs,) : ())..., # include yr_idxs only if present + hr_idxs + ) + + # add corresponding cal value to retail price calculation + retail_price = retail_price + cal + return retail_price + +end + +export compute_retail_price + +# get corresponding ref price values by year +function get_ref_price(ref_price_table, idxs, yr_idxs, hr_idxs, retail_price) + # checks that there is only one filter, outside of hour and year filters + area, subarea = + isempty(idxs) ? ("", "") : + isa(idxs, Pair) ? (string(first(idxs)), string(last(idxs))) : + length(idxs) == 1 && isa(idxs[1], Pair) ? + (string(first(idxs[1])), string(last(idxs[1]))) : + throw(ErrorException("Retail price calibrator is not set up to handle multiple filters.")) + + # for each result row, get the corresponding reference price + ref_values = [] + for (i, row) in enumerate(eachrow(ref_price_table)) + if string(row.area) == area && string(row.subarea) == subarea && row.year == yr_idxs + push!(ref_values, row["ref_price"]) + end + end + + # error if there are multiple corresponding ref prices, and warn if there is none + length(ref_values) > 1 && error("Retail price calibator is not set up to handle multiple reference prices for the same region and year.") + + isempty(ref_values) && begin + @warn "There is no reference retail price for area `$(area)` and subarea `$(subarea)`. This region will not get a calibration value." + push!(ref_values, 0) + end + + return sum(ref_values), area, subarea, yr_idxs +end + +# get corresponding ref price value for region (single value for all years) +function get_ref_price(ref_price_table, idxs, hr_idxs, retail_price) + # checks that there is only one filter, outside of hour and year filters + area, subarea = + isempty(idxs) ? ("", "") : + isa(idxs, Pair) ? (string(first(idxs)), string(last(idxs))) : + length(idxs) == 1 && isa(idxs[1], Pair) ? + (string(first(idxs[1])), string(last(idxs[1]))) : + throw(ErrorException("Retail price calibrator is not set up to handle multiple filters.")) + + # for each result row, get the corresponding reference price + ref_values = [] + for (i, row) in enumerate(eachrow(ref_price_table)) + if string(row.area) == area && string(row.subarea) == subarea + push!(ref_values, row["ref_price"]) + end + end + + # error if there are multiple corresponding ref prices, and warn if there is none + length(ref_values) > 1 && error("Retail price calibator is not set up to handle multiple reference prices for the same region and year.") + + isempty(ref_values) && begin + @warn "There is no reference retail price for area `$(area)` and subarea `$(subarea)`. This region will not get a calibration value." + push!(ref_values, 0) + end + + return sum(ref_values), area, subarea +end + +# get the corresponding calibrator values +function get_calibrator_value(data, cal_table, idxs, yr_idxs, hr_idxs) + + # read in table with cal values + yr_idxs = get_year_idxs(data, yr_idxs) + cal_table.year .= get_year_idxs(data, cal_table.year) + + # checks that there is only one filter, outside of hour and year filters + area, subarea = + isempty(idxs) ? ("", "") : + isa(idxs, Pair) ? (string(first(idxs)), string(last(idxs))) : + length(idxs) == 1 && isa(idxs[1], Pair) ? + (string(first(idxs[1])), string(last(idxs[1]))) : + throw(ErrorException("Retail price calibrator is not set up to handle multiple filters.")) + + # for each result row, get the corresponding calibrator value for area, subarea, year + cal_values =[] + for row in eachrow(cal_table) + if string(row.area) == area && string(row.subarea) == subarea && row.year == yr_idxs # if there is no year column only check that area, subarea match + push!(cal_values, row.cal_value) + end + end + + # error if there are multiple corresponding calibrator values, and warn if there is none + length(cal_values) > 1 && error("Retail price calibrator is not set up to handle multiple calibrator values for the same region.") + + isempty(cal_values) && begin + @warn "There is no calibrator value for area `$(area)` and subarea `$(subarea)`. This region will not be calibrated." + push!(cal_values, 0) + end + + return sum(cal_values) +end + + +function get_calibrator_value(data, cal_table, idxs, hr_idxs) + + # checks that there is only one filter, outside of hour and year filters + area, subarea = + isempty(idxs) ? ("", "") : + isa(idxs, Pair) ? (string(first(idxs)), string(last(idxs))) : + length(idxs) == 1 && isa(idxs[1], Pair) ? + (string(first(idxs[1])), string(last(idxs[1]))) : + throw(ErrorException("Retail price calibrator is not set up to handle multiple filters.")) + + # for each result row, get the corresponding calibrator value for area, subarea, year + cal_values =[] + for row in eachrow(cal_table) + if string(row.area) == area && string(row.subarea) == subarea + push!(cal_values, row.cal_value) + end + end + + # error if there are multiple corresponding calibrator values, and warn if there is none + length(cal_values) > 1 && error("Retail price calibrator is not set up to handle multiple calibrator values for the same region.") + + isempty(cal_values) && begin + @warn "There is no calibrator value for area `$(area)` and subarea `$(subarea)`. This region will not be calibrated." + push!(cal_values, 0) + end + + return sum(cal_values) +end diff --git a/src/results/util.jl b/src/results/util.jl index 8e8f1145..39baf337 100644 --- a/src/results/util.jl +++ b/src/results/util.jl @@ -155,3 +155,34 @@ function unweight_hourly(data, v::Vector{<:Container}, s=+) return [s(v[i][y,h]) / w[h] for i in 1:length(v), y in 1:ny, h in 1:nh] end export unweight_hourly + +function hours_sortby(s::T) where T + if endswith(s, r"h\d+") + m = match(r"h(\d+)", s) + return lpad(m.captures[1], 4, '0') |> T + else + return s |> T + end +end +export hours_sortby + +function not_a_full_filter(row::DataFrameRow) + not_a_full_filter(row.filter_years) && return true + not_a_full_filter(row.filter_hours) && return true + for i in 1:1000 + col_name = "filter$i" + hasproperty(row, col_name) || break + not_a_full_filter(row[col_name]) && return true + end + return false +end + +function not_a_full_filter(s::AbstractString) + isempty(s) && return false + all(isnumeric, s) && return false + contains(s, "=>") && return false + startswith(s, "[") && return false + startswith(s, "y2") && return false + return true +end +export not_a_full_filter \ No newline at end of file diff --git a/src/types/modifications/GenerationStandard.jl b/src/types/modifications/GenerationStandard.jl index 756faeb2..aae06040 100644 --- a/src/types/modifications/GenerationStandard.jl +++ b/src/types/modifications/GenerationStandard.jl @@ -2,8 +2,10 @@ struct GenerationStandard{T} <: Policy A generation standard (also refered to as a portfolio standard) is a constraint on generation where a portion of generation from certain generators must meet the a portion of the load in a specified region. -This encompasses RPSs, CESs, and technology carveouts. +This encompasses RPSs, CESs, and technology carveouts. To assign the credit (the portion of generation that can contribute) to generators, the [`Crediting`](@ref) type is used. +In certain instances, there can be a mismatch between the location of the qualifying load and the location of the generation that is used to meet the load. To handle this, gs_rebate is calculated as the payment +that a generators get for providing generation for qualifying load while gs_payment is calculated as the payment that would be associated with the qualifying load. ### Keyword Arguments * `name` - Name of the policy @@ -171,6 +173,8 @@ Modifies the results by adding the following columns to the bus table: function modify_results!(pol::GenerationStandard, config, data) bus = get_table(data, :bus) gen = get_table(data, :gen) + nyr = get_num_years(data) + years = Symbol.(get_years(data)) prc_name = Symbol("$(pol.name)_prc") cost_name = Symbol("$(pol.name)_cost") @@ -179,7 +183,7 @@ function modify_results!(pol::GenerationStandard, config, data) if !hasproperty(bus, :pl_gs) pl_gs_bus = get_raw_result(data, :pl_gs_bus) el_gs_bus = weight_hourly(data, pl_gs_bus) - + add_table_col!(data, :bus, :pl_gs, pl_gs_bus, MWServed, "Served Load Power that qualifies for generation standards") add_table_col!(data, :bus, :el_gs, el_gs_bus, MWhServed, "Served Load Energy that qualifies for generation standards") @@ -191,16 +195,28 @@ function modify_results!(pol::GenerationStandard, config, data) gen_idxs = get_row_idxs(gen, parse_comparisons(pol.gen_filters)) add_table_col!(data, :gen, prc_name, Container[ByNothing(0.0) for i in 1:nrow(gen)], DollarsPerMWhGenerated, "Policy price based on shadow price of $(pol.name) (converted to DollarsPerMWhGenerated) multiplied by the credit.") + add_table_col!(data, :bus, prc_name, Container[ByNothing(0.0) for i in 1:nrow(bus)], DollarsPerMWhGenerated, "Policy cost based on shadow price of $(pol.name) (converted to DollarsPerMWhGenerated).") # set to shadow_prc * crediting for i in gen_idxs gen[i, prc_name] = -(shadow_prc) .* gen[i, pol.name] end + for (k,d) in pol.load_targets + targets = collect(values(OrderedDict(y => get(d[:targets], y, 0.0) for y in years)))[1:nyr] # target set to 0 if missing + filters = d[:filters] + bus_idxs = get_row_idxs(bus, parse_comparisons(d[:filters])) + # set to shadow_prc for bus + for i in bus_idxs + bus[i, prc_name] = -(shadow_prc) .* targets + end + end # policy cost, price * credit * generation add_results_formula!(data, :gen, cost_name, "SumHourlyWeighted($(prc_name), pgen)", Dollars, "Cost of $(pol.name) based on the shadow price on the constraint and the generator credit level.") add_to_results_formula!(data, :gen, :gs_rebate, cost_name) + add_results_formula!(data, :bus, cost_name, "SumHourlyWeighted($(prc_name), pl_gs)", Dollars, "Cost of $(pol.name) based on the shadow price on the constraint and the generator credit level.") + add_to_results_formula!(data, :bus, :gs_payment, cost_name) end export modify_results! diff --git a/src/types/modifications/ReserveRequirement.jl b/src/types/modifications/ReserveRequirement.jl index 15182751..0666df54 100644 --- a/src/types/modifications/ReserveRequirement.jl +++ b/src/types/modifications/ReserveRequirement.jl @@ -454,7 +454,7 @@ function modify_results!(mod::ReserveRequirement, config, data) add_results_formula!(data, :gen, rebate_price_result_name, "$(rebate_result_name)/pcap_qual_$(mod.name)",DollarsPerMWCapacity, "The per MW of qualifying capacity price of the rebate receive by EGU's from the $(mod.name) reserve requirement.") # Add it to net_total_revenue_prelim - add_to_results_formula!(data, :gen, :net_total_revenue_prelim, "+ $rebate_result_name") + add_to_results_formula!(data, :gen, :net_total_revenue_prelim, "$rebate_result_name") # Subtract the cost from user surplus add_welfare_term!(data, :user, :bus, cost_result_name, -) @@ -490,7 +490,7 @@ function modify_results!(mod::ReserveRequirement, config, data) add_results_formula!(data, :storage, rebate_price_result_name, "$(rebate_result_name)/pcap_qual_$(mod.name)",DollarsPerMWCapacity, "The per MW price of the rebate receive by EGU's from the $(mod.name) reserve requirement.") # Add it to net_total_revenue_prelim - add_to_results_formula!(data, :storage, :net_total_revenue_prelim, "+ $rebate_result_name") + add_to_results_formula!(data, :storage, :net_total_revenue_prelim, "$rebate_result_name") end # Add merchandising surplus if applicable diff --git a/src/types/modifications/RetailPrice.jl b/src/types/modifications/RetailPrice.jl new file mode 100644 index 00000000..3f34724b --- /dev/null +++ b/src/types/modifications/RetailPrice.jl @@ -0,0 +1,298 @@ + +""" + RetailPrice(;file, name, col_sort=:initial_order) <: Modification + +This is a mod that outputs retail prices, given a `file` that indicates for which regions and years the retail price should be calculated. `name` is simply the name of the modification, and will be used as the root for the filename that the retail rates are saved to. +The mod pulls values across different tables to calculate one retail rate. The specific terms that go into this cross-table calculation can be found in retail_price.jl. + +The mod will also adjust the retail price values through calibration based on the `cal_mode` argument. If `cal_mode` is set to `none`, the retail prices will be unadjusted. If `cal_mode` is `get_val_values` the mod will use the reference price values in `ref_price_file` to +get calibration values. If `cal_mode` is set to `calibrate`, the calibrator values will be read in from the `calibrator_file` and used to adjust the calculated retail rates. + +## Keyword Arguments +* `file` - the file pointing to a table specifying which retail prices to calculate +* `name` - the name of the mod, do not need to specify in a config file +* `cal_mode` - a string that indicates the calibration mode. Options are `none`, `get_cal_values`, and `calibrate`. Defaults to `none`. +* `ref_price_file` - the file pointing to a table that contains the reference price values, only necessary when `cal_mode` is `get_cal_values`. +* `calibrator_file` - the file pointing to a table that contains calibration values, only necessary when `cal_mode` is `calibrate`. +* `col_sort` - the column(s) to sort by. Defaults to the order in which they were originally specified. + +The `file` should represent a csv table with the following columns: +* `table_name` - the name of the table being aggregated. i.e. `gen`, `bus`, etc. If you leave it empty, it will call `compute_welfare` instead of `compute_result` +* `result_name` - the name of the column in the table being aggregated. Note that the column must have a Unit accessible via [`get_table_col_unit`](@ref). +* `filter_` - the filtering conditions for the rows of the table. I.e. `filter1`. See [`parse_comparisons`](@ref) for information on what types of filters could be provided. +* `filter_years` - the filtering conditions for the years to be aggregated. See [`parse_year_idxs`](@ref) for information on the year filters. +* `filter_hours` - the filtering conditions for the hours to be aggregated. See [`parse_hour_idxs`](@ref) for information on the hour filters. the retail rate mod is not set up to calculate hourly values. + +Note that, for the `filter_` or `filter_hours` columns, if a column name of the data table (or hours table) is given, new rows will be created for each unique value of that column. I.e. if a value of `state` is given, there will be made a new row for `state=>california`, `state=>colorado`, etc. +However, the filter must be present in each of tables in the cross-table calculation for retail price or it will error. Further, the calibration feature can only handle one filter_ column beyond filter_hours and filter_years (which will most likely designate the region of interest for the retail price calcualation). +""" + +struct RetailPrice <: Modification + file::String + name::Symbol + table::DataFrame + cal_mode:: String + ref_price_file::String + calibrator_file::String + col_sort + function RetailPrice(;file, name, ref_price_file="", calibrator_file="", cal_mode="none", col_sort=:initial_order) + table = read_table(file) + force_table_types!(table, name, + :table_name=>Symbol, + :result_name=>Symbol, + :filter_years=>String, + :filter_hours=>String, + ) + for i in 1:1000 + col_name = "filter$i" + hasproperty(table, col_name) || continue + force_table_types!(table, name, col_name=>String) + end + # errors if no calibrator file is provided + if cal_mode == "get_cal_values" + isempty(ref_price_file) && error("Ref price file required when cal_mode is set to $(cal_mode).") + elseif cal_mode == "calibrate" + isempty(calibrator_file) && error("Calibrator file required when cal_mode is set to $(cal_mode).") + end + return new(file, name, table, cal_mode, ref_price_file, calibrator_file, col_sort) + end +end + +export RetailPrice + + +mod_rank(::Type{<:RetailPrice}) = 5.0 + +fieldnames_for_yaml(::Type{RetailPrice}) = (:file,) + + +function summarize_table(::Val{:ref_price}) + df = TableSummary() + push!(df, + (:area, String, NA, true, "The area that the price applies for i.e. `nation`. Leave blank if grid-wide"), + (:subarea, String, NA, true, "The subarea that the price applies for i.e. `narnia`. Leave blank if grid-wide"), + (:year, String, NA, false, "Year of corresponding reference price. If no column, them same calibrator value will be applied in each year."), + (:ref_price, Float64, DollarsPerMWhServed, true, "Reference price for retail rate in \$/MWh."), + ) + return df +end + + + function summarize_table(::Val{:retail_calibrator}) + df = TableSummary() + push!(df, + (:area, String, NA, true, "The area that the price applies for i.e. `nation`. Leave blank if grid-wide"), + (:subarea, String, NA, true, "The subarea that the price applies for i.e. `narnia`. Leave blank if grid-wide"), + (:year, String, NA, false, "Year of corresponding reference price. If no column, the same calibrator value is used in every year."), + (:cal_value, Float64, DollarsPerMWhServed, true, "Calibrator value for retail rate in \$/MWh."), + ) + return df +end + + +# function takes the table from file and expands the filter columns so there is a row for each calculated result +# eg if file has a filter_ with the value "state", the table will be expanded so that there is a row for each state +function modify_results!(m::RetailPrice, config, data) + table = copy(m.table) + table.initial_order = 1:nrow(table) + + filter_cols = setdiff(propertynames(table), [:table_name, :result_name]) + + # for any rows that are not a pair, separate into multiple rows + not_pair_idx = findfirst(not_a_full_filter, eachrow(table)) + while not_pair_idx !== nothing + row = table[not_pair_idx, :] + filter_col_idx = findfirst(filter_col->not_a_full_filter(row[filter_col]), filter_cols) + col_to_expand = filter_cols[filter_col_idx] + table_name = row[:table_name] + result_name = row[:result_name] + + if col_to_expand == :filter_hours + area = row.filter_hours + hours_table_col = get_table_col(data, :hours, area) + subareas = Base.sort!(String.(string.(unique(hours_table_col))), by=hours_sortby) + elseif col_to_expand == :filter_years && row[col_to_expand] == ":" + area = :years + subareas = data[area] + else + area = row[col_to_expand] + table_names = get_cross_table(data, table_name)[result_name] + all(hasproperty(get_table(data, t), area) for (t, _) in table_names) || error("Some tables are missing property $(area)") + data_table_col = get_table_col(data, first(keys(table_names)), area) + subareas = sort!(unique(data_table_col)) + end + + row_dict = Dict(pairs(row)) + for subarea in subareas + # Add a row right after the original row + row_dict[col_to_expand] = "$area=>$subarea" + insert!(table, not_pair_idx+1, row_dict) + end + + deleteat!(table, not_pair_idx) + + # Find the next index that is not a pair, to be expanded + not_pair_idx = findfirst(not_a_full_filter, eachrow(table)) + end + + # function that calculates retail price for each row in table + get_retail_price(m, config, data, table) + + return +end + +function extract_results(m::RetailPrice, config, data) + results = get_results(data) + # haskey(results, m.name) || modify_results!(m, config, data) + modify_results!(m, config, data) + return get_result(data, m.name) +end + +function combine_results(m::RetailPrice, post_config, post_data) + res = join_sim_tables(post_data, :value) + CSV.write(get_out_path(post_config, "$(m.name)_combined.csv"), res) +end + +# wrapper function that will dispatch a different get_retail_price method based on cal_mode arg +function get_retail_price(m::RetailPrice, config, data, table) + @info "Calculating results for $(nrow(table)) rows in RetailPrice $(m.name)" + get_retail_price((Val(Symbol(m.cal_mode))), m, config, data, table) +end + +# specialized method for retail rates with no cal_mode none +function get_retail_price(::Val{:none}, m, config, data, table) + + if !hasproperty(table, :value) + table.value = Vector{Union{Missing, Float64}}(missing, nrow(table)) + end + + + for row in eachrow(table) + table_name = row[:table_name] + result_name = row[:result_name] + + idxs = parse_comparisons(row) + yr_idxs = parse_year_idxs(row[:filter_years]) + hr_idxs = parse_hour_idxs(row[:filter_hours]) + + @assert hr_idxs == Colon() "Retail price mod is not set up to handle hourly retail rates." + + val = compute_retail_price(m, data, result_name, idxs, yr_idxs, hr_idxs) + + row[:value] = val + end + sort!(table, m.col_sort) + select!(table, Not(:initial_order)) + CSV.write(get_out_path(config, string(m.name, ".csv")), table) + results = get_results(data) + results[m.name] = table +end + +# specialized method for retail rates with no cal_mode get_cal_values +function get_retail_price(::Val{:get_cal_values}, m::RetailPrice, config, data, table) + # set up table that will contain calibrator values + cal_table = DataFrame( + area = String[], + subarea = Union{String, Int}[], + year = String[], + ref_price = Float64[], + retail_price = Float64[], + cal_value = Float64[], + elserv_total = Float64[], + elserv_ratio = Float64[] + ) + + # add value column to results table if it doesn't exist + if !hasproperty(table, :value) + table.value = Vector{Union{Missing, Float64}}(missing, nrow(table)) + end + + for row in eachrow(table) + table_name = row[:table_name] + result_name = row[:result_name] + + idxs = parse_comparisons(row) + yr_idxs = parse_year_idxs(row[:filter_years]) + hr_idxs = parse_hour_idxs(row[:filter_hours]) + + @assert yr_idxs != Any[] "Retail price calibrator is not set up to handle average retail rate across years." + @assert hr_idxs == Colon() "Retail price mod is not set up to handle hourly retail rates." + + val, cal_row = compute_retail_price(m, data, result_name, idxs, yr_idxs, hr_idxs) + + !isempty(cal_row) && push!(cal_table, cal_row) + row[:value] = val + end + + sort!(table, m.col_sort) + select!(table, Not(:initial_order)) + CSV.write(get_out_path(config, string(m.name, ".csv")), table) + results = get_results(data) + results[m.name] = table + + # second calibrator adjustment to calibrate with full region + full_cal!(m, data, table, cal_table) + select!(cal_table, [c for c in (:area, :subarea, :year, :cal_value) if c != :year ||any(x -> !ismissing(x) && x != "", cal_table.year)]) + CSV.write(get_out_path(config, string(m.name, "_cals.csv")), cal_table) + results[:calibrator_values] = cal_table + +end + +# specialized method for retail rates with no cal_mode calibrate +function get_retail_price(::Val{:calibrate}, m, config, data, table) + + # add value column to results table if it doesn't exist + if !hasproperty(table, :value) + table.value = Vector{Union{Missing, Float64}}(missing, nrow(table)) + end + + for row in eachrow(table) + table_name = row[:table_name] + result_name = row[:result_name] + + idxs = parse_comparisons(row) + yr_idxs = parse_year_idxs(row[:filter_years]) + hr_idxs = parse_hour_idxs(row[:filter_hours]) + + @assert yr_idxs != Any[] "Retail price calibrator is not set up to handle average retail rate across years." + @assert hr_idxs == Colon() "Retail price mod is not set up to handle hourly retail rates." + + val = compute_retail_price(m, data, result_name, idxs, yr_idxs, hr_idxs) + + row[:value] = val + end + + sort!(table, m.col_sort) + select!(table, Not(:initial_order)) + CSV.write(get_out_path(config, string(m.name, ".csv")), table) + results = get_results(data) + results[m.name] = table +end + +# final calibration step for full model +# example: for a state level model, ensure that the weighted average prices of all states are calibrated to the national average price +function full_cal!(m, data, table, cal_table) + + ref_price_table = read_table(m.ref_price_file) + + # get the weighted average retail price acorss all areas + avg_price = sum(cal_table.ref_price .* cal_table.elserv_ratio) + + subset = filter(row -> row.area == "" && row.subarea == "", ref_price_table) + + # get the average reference price + if nrow(subset) == 0 + avg_price_ref = avg_price # set refrence price to the calculated average price so that the calibrator value will be 0 + else + avg_price_ref = subset[1, :ref_price] + end + + # calculate and add the final cal value to existing cal values + for row in eachrow(cal_table[(cal_table.area .!= "") .& (cal_table.subarea .!= ""), :]) + # average calibrator value is the differnce between the reference price minus the calculated price weighted by load + cal = (avg_price_ref - avg_price) * row[:elserv_ratio] + row[:cal_value] += cal + end + + return cal_table +end diff --git a/test/data/3bus/past_invest_costs.csv b/test/data/3bus/past_invest_costs.csv new file mode 100644 index 00000000..8720ba87 --- /dev/null +++ b/test/data/3bus/past_invest_costs.csv @@ -0,0 +1,7 @@ +bus_idx,status,reg_factor,build_status,build_type,build_id,genfuel,gentype,econ_life,pcap_inv,pcap0,pcap_min,pcap_max,cf_min,cf_hist,vom,fuel_price,fom,capex,transmission_capex,routine_capex,year_on,year_off,year_shutdown,emis_co2,capt_co2_percent,heat_rate,chp_co2_multi,chp,pcap_plant_avg,past_invest_cost +3,1,0.75,built,exog,,coal,coal,30,2,2,0,2,0.6,0.68,2,0.555555556,20,7,0.1,0.2,y2020,y9999,y2045,1,0,9,1,0,2,7 +1,1,0,built,exog,,solar,solar,30,0.5,0.5,0,0.5,0,0.25,2,0,5,5,0.5,0.1,y2027,y9999,y2050,0,0,0,1,0,0.5,5 +3,1,0.75,built,exog,,ng,ngccccs,30,0.5,0.5,0,0.5,0.5,0.55,1.5,0.142857143,6,6,0.1,0.2,y2025,y9999,y2055,0.6,0.9,7,1,0,0.5,6 +1,1,0,built,exog,,wind,wind,30,0.05,0.05,0,0.05,0,0.4,0.1,0,2,3,0.5,0.3,y2015,y9999,y2035,0,0,0,1,0,0.05,3 +1,1,0.1,built,exog,,ng,ngt,30,0.01,0.01,0,0.01,0.6,0.23,2,0.25,7,7,0.1,0.1,y2030,y9999,y2050,0.6,0,8,0.67,1,0.01,7 +2,1,0.1,built,exog,,ng,ngt,30,2,2,0,2,0,0.23,100,0.25,5,7,0.1,0,y2020,y9999,y2050,0.6,0,8,0.67,1,0.01,7 diff --git a/test/data/3bus/ref_retail_price.csv b/test/data/3bus/ref_retail_price.csv new file mode 100644 index 00000000..ddd675c3 --- /dev/null +++ b/test/data/3bus/ref_retail_price.csv @@ -0,0 +1,5 @@ +area,subarea,ref_price +bus_idx,1,87.172 +bus_idx,2,72.716 +bus_idx,3,93.828 +,,78 diff --git a/test/data/3bus/ref_retail_price_yearly.csv b/test/data/3bus/ref_retail_price_yearly.csv new file mode 100644 index 00000000..c0be3a29 --- /dev/null +++ b/test/data/3bus/ref_retail_price_yearly.csv @@ -0,0 +1,10 @@ +area,subarea,year,ref_price +bus_idx,1,y2030,87.172 +bus_idx,2,y2030,72.716 +bus_idx,3,y2030,93.828 +bus_idx,1,y2035,88.172 +bus_idx,2,y2035,73.716 +bus_idx,3,y2035,94.828 +bus_idx,1,y2040,89.172 +bus_idx,2,y2040,75.716 +bus_idx,3,y2040,97.828 \ No newline at end of file diff --git a/test/data/3bus/results_retail_price.csv b/test/data/3bus/results_retail_price.csv new file mode 100644 index 00000000..13560f7b --- /dev/null +++ b/test/data/3bus/results_retail_price.csv @@ -0,0 +1,2 @@ +table_name,result_name,filter1,filter2,filter3,filter_years,filter_hours +retail_price,avg_elec_rate,bus_idx,,,:, diff --git a/test/data/3bus/retail_price_calibrator.csv b/test/data/3bus/retail_price_calibrator.csv new file mode 100644 index 00000000..4833eaab --- /dev/null +++ b/test/data/3bus/retail_price_calibrator.csv @@ -0,0 +1,4 @@ +area,subarea,cal_value +bus_idx,1,10 +bus_idx,2,5 +bus_idx,3,3 diff --git a/test/data/3bus/retail_price_calibrator_yearly.csv b/test/data/3bus/retail_price_calibrator_yearly.csv new file mode 100644 index 00000000..a88270ca --- /dev/null +++ b/test/data/3bus/retail_price_calibrator_yearly.csv @@ -0,0 +1,10 @@ +area,subarea,year,cal_value +bus_idx,1,y2030,10 +bus_idx,2,y2030,5 +bus_idx,3,y2030,3 +bus_idx,1,y2035,12 +bus_idx,2,y2035,6 +bus_idx,3,y2035,4 +bus_idx,1,y2040,12 +bus_idx,2,y2040,6 +bus_idx,3,y2040,4 \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index 16208465..c849943b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,6 +5,8 @@ using JuMP using DataFrames using Logging using BasicInterpolators +using Statistics +using CSV import OrderedCollections: OrderedDict import YAML @@ -35,6 +37,7 @@ rm(joinpath(@__DIR__, "out"), force=true, recursive=true) include("testadjust.jl") include("testutil.jl") include("testpost.jl") + include("testretailprice.jl") end global_logger(original_logger) diff --git a/test/testretailprice.jl b/test/testretailprice.jl new file mode 100644 index 00000000..f26ad996 --- /dev/null +++ b/test/testretailprice.jl @@ -0,0 +1,285 @@ +@testset "Test Retail Price" begin + @testset "Retail Price Set Up" begin + config_file_ref = joinpath(@__DIR__, "config", "config_3bus.yml") + config_file = joinpath(@__DIR__, "config", "config_3bus_reserve_req.yml") + config = read_config(config_file_ref, config_file, log_model_summary=true) + config[:past_invest_file] = "data/3bus/past_invest_costs.csv" + + data = read_data(config) + model = setup_model(config, data) + + optimize!(model) + # solution_summary(model) + + @test check(config, data, model) + + parse_results!(config, data, model) + process_results!(config, data) + + setup_retail_price!(config, data) + + @test haskey(data, :retail_price) + + @testset "Check Retail Price Terms" begin + retail_price = data[:retail_price][:avg_elec_rate] + + #check retail price terms + @test all(k -> haskey(retail_price, k), [:bus, :gen, :storage, :past_invest]) + bus_terms = retail_price[:bus] + @test all(k -> haskey(bus_terms, k), [:electricity_cost, :distribution_cost_total, :merchandising_surplus_comp_total, :gs_payment, :state_reserve_cost]) + gen_terms = retail_price[:gen] + @test haskey(gen_terms, :cost_of_service_rebate) + storage_terms = retail_price[:storage] + @test haskey(storage_terms, :cost_of_service_rebate) + past_invest_terms = retail_price[:past_invest] + @test haskey(past_invest_terms, :cost_of_service_past_costs) + end + + @testset "Without Past Invest" begin + delete!(config, :past_invest_file) + + setup_retail_price!(config, data) + retail_price = data[:retail_price][:avg_elec_rate] + @test !haskey(retail_price, :past_invest) + + end + + @testset "Without Reserve Requirements" begin + delete!(config[:mods], :state_reserve) + + setup_retail_price!(config, data) + + retail_price = data[:retail_price][:avg_elec_rate] + + bus_terms = retail_price[:bus] + @test !haskey(bus_terms, :state_reserve_cost) + end + + end + + + @testset "Test RetailPrice Mod" begin + + @testset "Without calibration" begin + config_file = joinpath(@__DIR__, "config", "config_3bus.yml") + storage_config_file = joinpath(@__DIR__, "config", "config_stor.yml") + config = read_config(config_file, storage_config_file) + config[:past_invest_file] = "data/3bus/past_invest_costs.csv" + data = read_data(config) + model = setup_model(config, data) + optimize!(model) + parse_results!(config, data, model) + # Make new mod + rtlprc_file = joinpath(@__DIR__, "data/3bus/results_retail_price.csv") + name = :retail_price + mod = RetailPrice(;file=rtlprc_file, name) + + mods = get_mods(config) + mods[name] = mod + + process_results!(config, data) + + results = get_results(data) + @test haskey(results, name) + table = get_result(data, name) + @test table[end, :filter1] |> contains("=>") + + @test table.value[1] == compute_retail_price(mod, data, :avg_elec_rate, :bus_idx=>3, 3,:) + + # test when filters that arent in all tables are provided + ref_price_file = joinpath(@__DIR__, "data/3bus/ref_retail_price.csv") + rtlprc = read_table(rtlprc_file) + rtlprc.filter2 .= "genfuel" + CSV.write(get_out_path(config, "results_retail_price.csv"), rtlprc) + + rtlprc_file = get_out_path(config, "results_retail_price.csv") + mod = RetailPrice(;file=rtlprc_file, name) + + mods = get_mods(config) + mods[name] = mod + + @test_throws Exception process_results!(config, data) + + + end + + @testset "With yearly reference price" begin + config_file = joinpath(@__DIR__, "config", "config_3bus.yml") + storage_config_file = joinpath(@__DIR__, "config", "config_stor.yml") + config = read_config(config_file, storage_config_file) + data = read_data(config) + model = setup_model(config, data) + optimize!(model) + parse_results!(config, data, model) + + # Make new mod + rtlprc_file = joinpath(@__DIR__, "data/3bus/results_retail_price.csv") + ref_price_file = joinpath(@__DIR__, "data/3bus/ref_retail_price_yearly.csv") + name = :retail_price + mod = RetailPrice(;file=rtlprc_file, name, ref_price_file=ref_price_file, cal_mode = "get_cal_values") + + mods = get_mods(config) + mods[name] = mod + + process_results!(config, data) + + results = get_results(data) + table = get_result(data, name) + + @test isfile(get_out_path(config, "$(name)_cals.csv")) + + @test table.value[2] == compute_retail_price(mod, data, :avg_elec_rate, :bus_idx=>3, 2,:)[1] + + cal_table = get_result(data, :calibrator_values) + ref_price = read_table(ref_price_file) + @test filter(row->row.subarea==3 && row.year=="y2035", ref_price)[1,"ref_price"] - compute_retail_price(mod, data, :avg_elec_rate, :bus_idx=>3, 2,:)[1] == filter(row->row.subarea=="3" && row.year=="y2035", cal_table)[1,"cal_value"] + + # test for error when multiple filters are provided + ref_price = read_table(ref_price_file) + insertcols!(ref_price, 3, :filter1 => "genfuel"=>"ng") + CSV.write(get_out_path(config, "ref_retail_price_yearly.csv"), ref_price) + + ref_price_file = get_out_path(config, "ref_retail_price_yearly.csv") + mod = RetailPrice(;file=rtlprc_file, name, ref_price_file=ref_price_file, cal_mode = "get_cal_values") + + # test for error when multiple filters are provided + @test_throws Exception compute_retail_price(mod, data, :avg_elec_rate, [:bus_idx=>3, :nation=>"narnia"], 1,:) + + end + + @testset "With single reference price" begin + config_file = joinpath(@__DIR__, "config", "config_3bus.yml") + storage_config_file = joinpath(@__DIR__, "config", "config_stor.yml") + config = read_config(config_file, storage_config_file) + data = read_data(config) + model = setup_model(config, data) + optimize!(model) + parse_results!(config, data, model) + + # Make new mod + rtlprc_file = joinpath(@__DIR__, "data/3bus/results_retail_price.csv") + ref_price_file = joinpath(@__DIR__, "data/3bus/ref_retail_price.csv") + name = :retail_price + mod = RetailPrice(;file=rtlprc_file, name, ref_price_file=ref_price_file, cal_mode = "get_cal_values") + + mods = get_mods(config) + mods[name] = mod + + process_results!(config, data) + + results = get_results(data) + table = get_result(data, name) + + @test isfile(get_out_path(config, "$(name)_cals.csv")) + @test table.value[3] == compute_retail_price(mod, data, :avg_elec_rate, :bus_idx=>3, 1,:)[1] + + cal_table = get_result(data, :calibrator_values) + ref_price = read_table(ref_price_file) + + @test filter(row->row.subarea=="3", ref_price)[1,"ref_price"] - compute_retail_price(mod, data, :avg_elec_rate, :bus_idx=>3, 1,:)[1] == filter(row->row.subarea=="3", cal_table)[1,"cal_value"] + + # test for error when multiple grid-wide prices are provided + ref_price = read_table(ref_price_file) + push!(ref_price, ["","",95]) + CSV.write(get_out_path(config, "ref_retail_price.csv"), ref_price) + + ref_price_file = get_out_path(config, "ref_retail_price.csv") + mod = RetailPrice(;file=rtlprc_file, name, ref_price_file=ref_price_file, cal_mode = "get_cal_values") + + @test_throws Exception compute_retail_price(mod, data, :avg_elec_rate, :bus_idx=>3, 1,:) + + # test for error when multiple prices are provided for a regoin + ref_price_file = joinpath(@__DIR__, "data/3bus/ref_retail_price.csv") + ref_price = read_table(ref_price_file) + push!(ref_price, ["bus_idx","3",89]) + CSV.write(get_out_path(config, "ref_retail_price.csv"), ref_price) + + ref_price_file = get_out_path(config, "ref_retail_price.csv") + mod = RetailPrice(;file=rtlprc_file, name, ref_price_file=ref_price_file, cal_mode = "get_cal_values") + + @test_throws Exception compute_retail_price(mod, data, :avg_elec_rate, :bus_idx=>3, 1,:) + + # test for error when multiple filters are provided + @test_throws Exception compute_retail_price(mod, data, :avg_elec_rate, [:bus_idx=>3, :nation=>"narnia"], 1,:) + + end + + @testset "With yearly calibrator values" begin + config_file = joinpath(@__DIR__, "config", "config_3bus.yml") + storage_config_file = joinpath(@__DIR__, "config", "config_stor.yml") + config = read_config(config_file, storage_config_file) + data = read_data(config) + model = setup_model(config, data) + optimize!(model) + parse_results!(config, data, model) + + # Make new mod + rtlprc_file = joinpath(@__DIR__, "data/3bus/results_retail_price.csv") + cal_file = joinpath(@__DIR__, "data/3bus/retail_price_calibrator_yearly.csv") + name = :retail_price + mod = RetailPrice(;file=rtlprc_file, name, calibrator_file=cal_file, cal_mode = "calibrate") + + mods = get_mods(config) + mods[name] = mod + + process_results!(config, data) + + results = get_results(data) + table = get_result(data, name) + + cal_table = read_table(cal_file) + @test table.value[1] == compute_retail_price(mod, data, :avg_elec_rate, :bus_idx=>3, 3,:)[1] + + # check that calibrator values were added + mod_no_calibrate = RetailPrice(;file=rtlprc_file, name) + @test table.value[1] == compute_retail_price(mod_no_calibrate, data, :avg_elec_rate, :bus_idx=>3, 3,:)[1] + filter(row->row.subarea==3 && row.year=="y2040", cal_table)[1,:"cal_value"] + @test table.value[2] == compute_retail_price(mod_no_calibrate, data, :avg_elec_rate, :bus_idx=>3, 2,:)[1] + filter(row->row.subarea==3 && row.year=="y2035", cal_table)[1,:"cal_value"] + + # test for error when multiple filters are provided + @test_throws Exception compute_retail_price(mod, data, :avg_elec_rate, [:bus_idx=>3, :nation=>"narnia"], 1,:) + end + + + @testset "With single calibrator value" begin + config_file = joinpath(@__DIR__, "config", "config_3bus.yml") + storage_config_file = joinpath(@__DIR__, "config", "config_stor.yml") + config = read_config(config_file, storage_config_file) + data = read_data(config) + model = setup_model(config, data) + optimize!(model) + parse_results!(config, data, model) + + # Make new mod + rtlprc_file = joinpath(@__DIR__, "data/3bus/results_retail_price.csv") + cal_file = joinpath(@__DIR__, "data/3bus/retail_price_calibrator.csv") + name = :retail_price + mod = RetailPrice(;file=rtlprc_file, name, calibrator_file=cal_file, cal_mode = "calibrate") + + mods = get_mods(config) + mods[name] = mod + + process_results!(config, data) + + results = get_results(data) + table = get_result(data, name) + + cal_table = read_table(cal_file) + @test table.value[1] == compute_retail_price(mod, data, :avg_elec_rate, :bus_idx=>3, 3,:)[1] #+ filter(row->row.subarea==3, cal_table)[1,:"cal_value"] + + # check that calibrator values were added + mod_no_calibrate = RetailPrice(;file=rtlprc_file, name) + @test table.value[1] == compute_retail_price(mod_no_calibrate, data, :avg_elec_rate, :bus_idx=>3, 3,:)[1] + filter(row->row.subarea==3, cal_table)[1,:"cal_value"] + @test table.value[2] == compute_retail_price(mod_no_calibrate, data, :avg_elec_rate, :bus_idx=>3, 2,:)[1] + filter(row->row.subarea==3, cal_table)[1,:"cal_value"] + + # test for error when multiple filters are provided + @test_throws Exception compute_retail_price(mod, data, :avg_elec_rate, [:bus_idx=>3, :nation=>"narnia"], 1,:) + end + end +end + +# results_template_rtl_price: +# type: RetailPrice +# file: "../res_templates/results_template_tests_rtl_prc.csv" +# cal_mode: calibrate +# # ref_price_file: "../../Data/config/mods/retail_price_calibrator/retail_price_ref_values.csv" +# calibrator_file: "L:/Project-Gurobi/Workspace3/E4ST_Output/haiku_merge/pipeflow_runs/rr_cal_251028/results_template_rtl_price_cals.csv"