Examples

Bid stack equivalent market

This example implements a one region market that mirrors the ‘bid stack’ model of an electricity market. Under the bid stack model, generators are dispatched according to their bid prices, from cheapest to most expensive, until all demand is satisfied. No loss factors, ramping constraints or other factors are considered.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
import pandas as pd
from nempy import markets

# Volume of each bid, number of bands must equal number of bands in price_bids.
volume_bids = pd.DataFrame({
    'unit': ['A', 'B'],
    '1': [20.0, 50.0],  # MW
    '2': [20.0, 30.0],  # MW
    '3': [5.0, 10.0]  # More bid bands could be added.
})

# Price of each bid, bids must be monotonically increasing.
price_bids = pd.DataFrame({
    'unit': ['A', 'B'],
    '1': [50.0, 50.0],  # $/MW
    '2': [60.0, 55.0],  # $/MW
    '3': [100.0, 80.0]  # . . .
})

# Other unit properties
unit_info = pd.DataFrame({
    'unit': ['A', 'B'],
    'region': ['NSW', 'NSW'],  # MW
})

# The demand in the region\s being dispatched
demand = pd.DataFrame({
    'region': ['NSW'],
    'demand': [120.0]  # MW
})

# Create the market model
market = markets.SpotMarket(unit_info=unit_info, market_regions=['NSW'])
market.set_unit_volume_bids(volume_bids)
market.set_unit_price_bids(price_bids)
market.set_demand_constraints(demand)

# Calculate dispatch and pricing
market.dispatch()

# Return the total dispatch of each unit in MW.
print(market.get_unit_dispatch())
#   unit service  dispatch
# 0    A  energy      40.0
# 1    B  energy      80.0

# Return the price of energy in each region.
print(market.get_energy_prices())
#   region  price
# 0    NSW   60.0

Unit loss factors, capacities and ramp rates

In this example units are given loss factors, capacity values and ramp rates.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
import pandas as pd
from nempy import markets

# Volume of each bid, number of bands must equal number of bands in price_bids.
volume_bids = pd.DataFrame({
    'unit': ['A', 'B'],
    '1': [20.0, 50.0],  # MW
    '2': [20.0, 30.0],  # MW
    '3': [5.0, 10.0]  # More bid bands could be added.
})

# Price of each bid, bids must be monotonically increasing.
price_bids = pd.DataFrame({
    'unit': ['A', 'B'],
    '1': [50.0, 50.0],  # $/MW
    '2': [60.0, 55.0],  # $/MW
    '3': [100.0, 80.0]  # . . .
})

# Factors limiting unit output.
unit_limits = pd.DataFrame({
    'unit': ['A', 'B'],
    'initial_output': [55.0, 90.0],  # MW
    'capacity': [55.0, 90.0],  # MW
    'ramp_up_rate': [1000.0, 1500.0],  # MW/h
    'ramp_down_rate': [1000.0, 1500.0]  # MW/h
})

# Other unit properties including loss factors.
unit_info = pd.DataFrame({
    'unit': ['A', 'B'],
    'region': ['NSW', 'NSW'],  # MW
    'loss_factor': [0.9, 0.95]
})

# The demand in the region\s being dispatched.
demand = pd.DataFrame({
    'region': ['NSW'],
    'demand': [100.0]  # MW
})

# Create the market model
market = markets.SpotMarket(unit_info=unit_info,
                            market_regions=['NSW'])
market.set_unit_volume_bids(volume_bids)
market.set_unit_price_bids(price_bids)
market.set_unit_bid_capacity_constraints(
    unit_limits.loc[:, ['unit', 'capacity']])
market.set_unit_ramp_up_constraints(
    unit_limits.loc[:, ['unit', 'initial_output', 'ramp_up_rate']])
market.set_unit_ramp_down_constraints(
    unit_limits.loc[:, ['unit', 'initial_output', 'ramp_down_rate']])
market.set_demand_constraints(demand)

# Calculate dispatch and pricing
market.dispatch()

# Return the total dispatch of each unit in MW.
print(market.get_unit_dispatch())
#   unit service  dispatch
# 0    A  energy      20.0
# 1    B  energy      80.0

# Return the price of energy in each region.
print(market.get_energy_prices())
#   region  price
# 0    NSW   57.89

Interconnector with losses

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
import pandas as pd
from nempy import markets

# The only generator is located in NSW.
unit_info = pd.DataFrame({
    'unit': ['A'],
    'region': ['NSW']  # MW
})

# Create a market instance.
market = markets.SpotMarket(unit_info=unit_info, market_regions=['NSW', 'VIC'])

# Volume of each bids.
volume_bids = pd.DataFrame({
    'unit': ['A'],
    '1': [100.0]  # MW
})

market.set_unit_volume_bids(volume_bids)

# Price of each bid.
price_bids = pd.DataFrame({
    'unit': ['A'],
    '1': [50.0]  # $/MW
})

market.set_unit_price_bids(price_bids)

# NSW has no demand but VIC has 90 MW.
demand = pd.DataFrame({
    'region': ['NSW', 'VIC'],
    'demand': [0.0, 90.0]  # MW
})

market.set_demand_constraints(demand)

# There is one interconnector between NSW and VIC. Its nominal direction is towards VIC.
interconnectors = pd.DataFrame({
    'interconnector': ['little_link'],
    'to_region': ['VIC'],
    'from_region': ['NSW'],
    'max': [100.0],
    'min': [-120.0]
})

market.set_interconnectors(interconnectors)


# The interconnector loss function. In this case losses are always 5 % of line flow.
def constant_losses(flow):
    return abs(flow) * 0.05


# The loss function on a per interconnector basis. Also details how the losses should be proportioned to the
# connected regions.
loss_functions = pd.DataFrame({
    'interconnector': ['little_link'],
    'from_region_loss_share': [0.5],  # losses are shared equally.
    'loss_function': [constant_losses]
})

# The points to linearly interpolate the loss function between. In this example the loss function is linear so only
# three points are needed, but if a non linear loss function was used then more points would be better.
interpolation_break_points = pd.DataFrame({
    'interconnector': ['little_link', 'little_link', 'little_link'],
    'loss_segment': [1, 2, 3],
    'break_point': [-120.0, 0.0, 100]
})

market.set_interconnector_losses(loss_functions, interpolation_break_points)

# Calculate dispatch.
market.dispatch()

# Return the total dispatch of each unit in MW.
print(market.get_unit_dispatch())
#   unit service   dispatch
# 0    A  energy  94.615385

# Return interconnector flow and losses.
print(market.get_interconnector_flows())
#   interconnector       flow    losses
# 0    little_link  92.307692  4.615385

# Return the price of energy in each region.
print(market.get_energy_prices())
#   region      price
# 0    NSW  50.000000
# 1    VIC  52.564103

Dynamic non-linear interconnector losses

Implements creating loss functions as described in Marginal Loss Factors documentation section 3 to 5.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
import pandas as pd
from nempy import markets
from nempy.historical_inputs import interconnectors as interconnector_inputs


# The only generator is located in NSW.
unit_info = pd.DataFrame({
    'unit': ['A'],
    'region': ['NSW']  # MW
})

# Create a market instance.
market = markets.SpotMarket(unit_info=unit_info,
                            market_regions=['NSW', 'VIC'])

# Volume of each bids.
volume_bids = pd.DataFrame({
    'unit': ['A'],
    '1': [1000.0]  # MW
})

market.set_unit_volume_bids(volume_bids)

# Price of each bid.
price_bids = pd.DataFrame({
    'unit': ['A'],
    '1': [50.0]  # $/MW
})

market.set_unit_price_bids(price_bids)

# NSW has no demand but VIC has 800 MW.
demand = pd.DataFrame({
    'region': ['NSW', 'VIC'],
    'demand': [0.0, 800.0],  # MW
    'loss_function_demand': [0.0, 800.0]  # MW
})

market.set_demand_constraints(demand.loc[:, ['region', 'demand']])

# There is one interconnector between NSW and VIC.
# Its nominal direction is towards VIC.
interconnectors = pd.DataFrame({
    'interconnector': ['VIC1-NSW1'],
    'to_region': ['VIC'],
    'from_region': ['NSW'],
    'max': [1000.0],
    'min': [-1200.0]
})

market.set_interconnectors(interconnectors)

# Create a demand dependent loss function.
# Specify the demand dependency
demand_coefficients = pd.DataFrame({
    'interconnector': ['VIC1-NSW1', 'VIC1-NSW1'],
    'region': ['NSW1', 'VIC1'],
    'demand_coefficient': [0.000021734, -0.000031523]})

# Specify the loss function constant and flow coefficient.
interconnector_coefficients = pd.DataFrame({
    'interconnector': ['VIC1-NSW1'],
    'loss_constant': [1.0657],
    'flow_coefficient': [0.00017027],
    'from_region_loss_share': [0.5]})

# Create loss functions on per interconnector basis.
loss_functions = interconnector_inputs.create_loss_functions(
    interconnector_coefficients, demand_coefficients,
    demand.loc[:, ['region', 'loss_function_demand']])

# The points to linearly interpolate the loss function between.
interpolation_break_points = pd.DataFrame({
    'interconnector': 'VIC1-NSW1',
    'loss_segment': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],
    'break_point': [-1200.0, -1000.0, -800.0, -600.0, -400.0, -200.0,
                    0.0, 200.0, 400.0, 600.0, 800.0, 1000]
})

market.set_interconnector_losses(loss_functions,
                                 interpolation_break_points)

# Calculate dispatch.
market.dispatch()

# Return the total dispatch of each unit in MW.
print(market.get_unit_dispatch())
#   unit service    dispatch
# 0    A  energy  920.205473

# Return interconnector flow and losses.
print(market.get_interconnector_flows())
#   interconnector        flow      losses
# 0      VIC1-NSW1  860.102737  120.205473

# Return the price of energy in each region.
print(market.get_energy_prices())
#   region      price
# 0    NSW  50.000000
# 1    VIC  62.292869

Simple FCAS markets

Implements a market for energy, regulation raise and contingency 6 sec raise, with co-optimisation constraints as described in section 6.2 and 6.3 of FCAS Model in NEMDE.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
import pandas as pd
from nempy import markets


# Set options so you see all DataFrame columns in print outs.
pd.options.display.width = 0

# Volume of each bid.
volume_bids = pd.DataFrame({
    'unit': ['A', 'A', 'B', 'B', 'B'],
    'service': ['energy', 'raise_6s', 'energy',
                'raise_6s', 'raise_reg'],
    '1': [100.0, 10.0, 110.0, 15.0, 15.0],  # MW
})

print(volume_bids)
#   unit    service      1
# 0    A     energy  100.0
# 1    A   raise_6s   10.0
# 2    B     energy  110.0
# 3    B   raise_6s   15.0
# 4    B  raise_reg   15.0

# Price of each bid.
price_bids = pd.DataFrame({
    'unit': ['A', 'A', 'B', 'B', 'B'],
    'service': ['energy', 'raise_6s', 'energy',
                'raise_6s', 'raise_reg'],
    '1': [50.0, 35.0, 60.0, 20.0, 30.0],  # $/MW
})

print(price_bids)
#   unit    service     1
# 0    A     energy  50.0
# 1    A   raise_6s  35.0
# 2    B     energy  60.0
# 3    B   raise_6s  20.0
# 4    B  raise_reg  30.0

# Participant defined operational constraints on FCAS enablement.
fcas_trapeziums = pd.DataFrame({
    'unit': ['B', 'B', 'A'],
    'service': ['raise_reg', 'raise_6s', 'raise_6s'],
    'max_availability': [15.0, 15.0, 10.0],
    'enablement_min': [50.0, 50.0, 70.0],
    'low_break_point': [65.0, 65.0, 80.0],
    'high_break_point': [95.0, 95.0, 100.0],
    'enablement_max': [110.0, 110.0, 110.0]
})

print(fcas_trapeziums)
#   unit    service  max_availability  enablement_min  low_break_point  high_break_point  enablement_max
# 0    B  raise_reg              15.0            50.0             65.0              95.0           110.0
# 1    B   raise_6s              15.0            50.0             65.0              95.0           110.0
# 2    A   raise_6s              10.0            70.0             80.0             100.0           110.0

# Unit locations.
unit_info = pd.DataFrame({
    'unit': ['A', 'B'],
    'region': ['NSW', 'NSW']
})

print(unit_info)
#   unit region
# 0    A    NSW
# 1    B    NSW

# The demand in the region\s being dispatched.
demand = pd.DataFrame({
    'region': ['NSW'],
    'demand': [195.0]  # MW
})

print(demand)
#   region  demand
# 0    NSW   195.0

# FCAS requirement in the region\s being dispatched.
fcas_requirements = pd.DataFrame({
    'set': ['nsw_regulation_requirement', 'nsw_raise_6s_requirement'],
    'region': ['NSW', 'NSW'],
    'service': ['raise_reg', 'raise_6s'],
    'volume': [10.0, 10.0]  # MW
})

print(fcas_requirements)
#                           set region    service  volume
# 0  nsw_regulation_requirement    NSW  raise_reg    10.0
# 1    nsw_raise_6s_requirement    NSW   raise_6s    10.0

# Create the market model with unit service bids.
market = markets.SpotMarket(unit_info=unit_info,
                            market_regions=['NSW'])
market.set_unit_volume_bids(volume_bids)
market.set_unit_price_bids(price_bids)

# Create constraints that enforce the top of the FCAS trapezium.
fcas_availability = fcas_trapeziums.loc[:, ['unit', 'service', 'max_availability']]
market.set_fcas_max_availability(fcas_availability)

# Create constraints that enforce the lower and upper slope of the FCAS regulation
# service trapeziums.
regulation_trapeziums = fcas_trapeziums[fcas_trapeziums['service'] == 'raise_reg']
market.set_energy_and_regulation_capacity_constraints(regulation_trapeziums)

# Create constraints that enforce the lower and upper slope of the FCAS contingency
# trapezium. These constrains also scale slopes of the trapezium to ensure the
# co-dispatch of contingency and regulation services is technically feasible.
contingency_trapeziums = fcas_trapeziums[fcas_trapeziums['service'] == 'raise_6s']
market.set_joint_capacity_constraints(contingency_trapeziums)

# Set the demand for energy.
market.set_demand_constraints(demand)

# Set the required volume of FCAS services.
market.set_fcas_requirements_constraints(fcas_requirements)

# Calculate dispatch and pricing
market.dispatch()

# Return the total dispatch of each unit in MW.
print(market.get_unit_dispatch())
#   unit    service  dispatch
# 0    A     energy     100.0
# 1    A   raise_6s       5.0
# 2    B     energy      95.0
# 3    B   raise_6s       5.0
# 4    B  raise_reg      10.0

# Return the price of energy.
print(market.get_energy_prices())
#   region  price
# 0    NSW   75.0

# Note:
#  A marginal unit of energy would have to come from unit B, as unit A is fully
#  dispatch, this would cost 60 $/MW/h. However, to turn unit B up, you would
#  need it to dispatch less raise_6s, this would cost - 20 $/MW/h, and the
#  extra FCAS would have to come from unit A, this would cost 35 $/MW/h.
#  Therefore the marginal cost of energy is 60 - 20 + 35 = 75 $/MW/h

# Return the price of regulation FCAS.
print(market.get_fcas_prices())
#   region    service  price
# 0    NSW   raise_6s   35.0
# 1    NSW  raise_reg   45.0

# Note:
# A marginal unit of raise_reg would have to come from unit B as it is the only
# provider, this would cost 30 $/MW/h. It would also require unit B to provide
# less raise_6s, this would cost -20 $/MW/h, extra raise_6s would then be
# required from unit A costing 35 $/MW/h. This gives a total marginal cost of
# 30 - 20 + 35 = 45 $/MW/h.
#
# A marginal unit of raise_6s would be provided by unit A at a cost of 35$/MW/h/.

Simple recreation of historical dispatch

Demonstrates using nempy to recreate historical dispatch intervals by implementing a simple energy market with unit bids, unit maximum capacity constraints and interconnector models, all sourced from historical data published by AEMO.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
# Notice: this script downloads large volumes of historical market data from AEMO's nemweb portal.

import sqlite3
import pandas as pd
from nempy import markets
from nempy.historical_inputs import loaders, mms_db, \
    xml_cache, units, demand, interconnectors

con = sqlite3.connect('market_management_system.db')
mms_db_manager = mms_db.DBManager(connection=con)

xml_cache_manager = xml_cache.XMLCacheManager('cache_directory')

# The second time this example is run on a machine this flag can
# be set to false to save downloading the data again.
download_inputs = True

if download_inputs:
    # This requires approximately 5 GB of storage.
    mms_db_manager.populate(start_year=2019, start_month=1,
                            end_year=2019, end_month=1)

    # This requires approximately 60 GB of storage.
    xml_cache_manager.populate(start_year=2019, start_month=1,
                               end_year=2019, end_month=1)

raw_inputs_loader = loaders.RawInputsLoader(
    nemde_xml_cache_manager=xml_cache_manager,
    market_management_system_database=mms_db_manager)

# A list of intervals we want to recreate historical dispatch for.
dispatch_intervals = ['2019/01/01 12:00:00',
                      '2019/01/01 12:05:00',
                      '2019/01/01 12:10:00',
                      '2019/01/01 12:15:00',
                      '2019/01/01 12:20:00',
                      '2019/01/01 12:25:00',
                      '2019/01/01 12:30:00']

# List for saving outputs to.
outputs = []

# Create and dispatch the spot market for each dispatch interval.
for interval in dispatch_intervals:
    raw_inputs_loader.set_interval(interval)
    unit_inputs = units.UnitData(raw_inputs_loader)
    demand_inputs = demand.DemandData(raw_inputs_loader)
    interconnector_inputs = \
        interconnectors.InterconnectorData(raw_inputs_loader)

    unit_info = unit_inputs.get_unit_info()
    market = markets.SpotMarket(market_regions=['QLD1', 'NSW1', 'VIC1',
                                                'SA1', 'TAS1'],
                                unit_info=unit_info)

    volume_bids, price_bids = unit_inputs.get_processed_bids()
    market.set_unit_volume_bids(volume_bids)
    market.set_unit_price_bids(price_bids)

    unit_bid_limit = unit_inputs.get_unit_bid_availability()
    market.set_unit_bid_capacity_constraints(unit_bid_limit)

    unit_uigf_limit = unit_inputs.get_unit_uigf_limits()
    market.set_unconstrained_intermitent_generation_forecast_constraint(
        unit_uigf_limit)

    regional_demand = demand_inputs.get_operational_demand()
    market.set_demand_constraints(regional_demand)

    interconnectors_definitions = \
        interconnector_inputs.get_interconnector_definitions()
    loss_functions, interpolation_break_points = \
        interconnector_inputs.get_interconnector_loss_model()
    market.set_interconnectors(interconnectors_definitions)
    market.set_interconnector_losses(loss_functions,
                                     interpolation_break_points)
    market.dispatch()

    # Save prices from this interval
    prices = market.get_energy_prices()
    prices['time'] = interval
    outputs.append(prices.loc[:, ['time', 'region', 'price']])

con.close()
print(pd.concat(outputs))
#                   time region      price
# 0  2019/01/01 12:00:00   NSW1  91.857666
# 1  2019/01/01 12:00:00   QLD1  76.180429
# 2  2019/01/01 12:00:00    SA1  85.126914
# 3  2019/01/01 12:00:00   TAS1  85.948523
# 4  2019/01/01 12:00:00   VIC1  83.250703
# 0  2019/01/01 12:05:00   NSW1  88.357224
# 1  2019/01/01 12:05:00   QLD1  72.255334
# 2  2019/01/01 12:05:00    SA1  82.417720
# 3  2019/01/01 12:05:00   TAS1  83.451561
# 4  2019/01/01 12:05:00   VIC1  80.621103
# 0  2019/01/01 12:10:00   NSW1  91.857666
# 1  2019/01/01 12:10:00   QLD1  75.665675
# 2  2019/01/01 12:10:00    SA1  85.680310
# 3  2019/01/01 12:10:00   TAS1  86.715499
# 4  2019/01/01 12:10:00   VIC1  83.774337
# 0  2019/01/01 12:15:00   NSW1  88.343034
# 1  2019/01/01 12:15:00   QLD1  71.746786
# 2  2019/01/01 12:15:00    SA1  82.379539
# 3  2019/01/01 12:15:00   TAS1  83.451561
# 4  2019/01/01 12:15:00   VIC1  80.621103
# 0  2019/01/01 12:20:00   NSW1  91.864122
# 1  2019/01/01 12:20:00   QLD1  75.052319
# 2  2019/01/01 12:20:00    SA1  85.722028
# 3  2019/01/01 12:20:00   TAS1  86.576848
# 4  2019/01/01 12:20:00   VIC1  83.859306
# 0  2019/01/01 12:25:00   NSW1  91.864122
# 1  2019/01/01 12:25:00   QLD1  75.696247
# 2  2019/01/01 12:25:00    SA1  85.746024
# 3  2019/01/01 12:25:00   TAS1  86.613642
# 4  2019/01/01 12:25:00   VIC1  83.894945
# 0  2019/01/01 12:30:00   NSW1  91.870167
# 1  2019/01/01 12:30:00   QLD1  75.188735
# 2  2019/01/01 12:30:00    SA1  85.694071
# 3  2019/01/01 12:30:00   TAS1  86.560602
# 4  2019/01/01 12:30:00   VIC1  83.843570

Detailed recreation of historical dispatch

Demonstrates using nempy to recreate historical dispatch intervals by implementing a simple energy market using all the features of the nempy market model, all inputs sourced from historical data published by AEMO. Note each interval is dispatched as a standalone simulation and the results from one dispatch interval are not carried over to be the initial conditions of the next interval, rather the historical initial conditions are always used.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
# Notice: this script downloads large volumes of historical market data from AEMO's nemweb portal.

import sqlite3
import pandas as pd
from nempy import markets
from nempy.historical_inputs import loaders, mms_db, \
    xml_cache, units, demand, interconnectors, \
    constraints

con = sqlite3.connect('market_management_system.db')
mms_db_manager = mms_db.DBManager(connection=con)

xml_cache_manager = xml_cache.XMLCacheManager('cache_directory')

# The second time this example is run on a machine this flag can
# be set to false to save downloading the data again.
download_inputs = True

if download_inputs:
    # This requires approximately 5 GB of storage.
    mms_db_manager.populate(start_year=2019, start_month=1,
                            end_year=2019, end_month=1)

    # This requires approximately 60 GB of storage.
    xml_cache_manager.populate(start_year=2019, start_month=1,
                               end_year=2019, end_month=1)

raw_inputs_loader = loaders.RawInputsLoader(
    nemde_xml_cache_manager=xml_cache_manager,
    market_management_system_database=mms_db_manager)

# A list of intervals we want to recreate historical dispatch for.
dispatch_intervals = ['2019/01/01 12:00:00',
                      '2019/01/01 12:05:00',
                      '2019/01/01 12:10:00',
                      '2019/01/01 12:15:00',
                      '2019/01/01 12:20:00',
                      '2019/01/01 12:25:00',
                      '2019/01/01 12:30:00']

# List for saving outputs to.
outputs = []

# Create and dispatch the spot market for each dispatch interval.
for interval in dispatch_intervals:
    raw_inputs_loader.set_interval(interval)
    unit_inputs = units.UnitData(raw_inputs_loader)
    interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
    constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
    demand_inputs = demand.DemandData(raw_inputs_loader)

    unit_info = unit_inputs.get_unit_info()
    market = markets.SpotMarket(market_regions=['QLD1', 'NSW1', 'VIC1',
                                                'SA1', 'TAS1'],
                                unit_info=unit_info)

    # Set bids
    volume_bids, price_bids = unit_inputs.get_processed_bids()
    market.set_unit_volume_bids(volume_bids)
    market.set_unit_price_bids(price_bids)

    # Set bid in capacity limits
    unit_bid_limit = unit_inputs.get_unit_bid_availability()
    market.set_unit_bid_capacity_constraints(unit_bid_limit)
    cost = constraint_inputs.get_constraint_violation_prices()['unit_capacity']
    market.make_constraints_elastic('unit_bid_capacity', violation_cost=cost)

    # Set limits provided by the unconstrained intermittent generation
    # forecasts. Primarily for wind and solar.
    unit_uigf_limit = unit_inputs.get_unit_uigf_limits()
    market.set_unconstrained_intermitent_generation_forecast_constraint(
        unit_uigf_limit)
    cost = constraint_inputs.get_constraint_violation_prices()['uigf']
    market.make_constraints_elastic('uigf_capacity', violation_cost=cost)

    # Set unit ramp rates.
    ramp_rates = unit_inputs.get_ramp_rates_used_for_energy_dispatch()
    market.set_unit_ramp_up_constraints(
        ramp_rates.loc[:, ['unit', 'initial_output', 'ramp_up_rate']])
    market.set_unit_ramp_down_constraints(
        ramp_rates.loc[:, ['unit', 'initial_output', 'ramp_down_rate']])
    cost = constraint_inputs.get_constraint_violation_prices()['ramp_rate']
    market.make_constraints_elastic('ramp_up', violation_cost=cost)
    market.make_constraints_elastic('ramp_down', violation_cost=cost)

    # Set unit FCAS trapezium constraints.
    unit_inputs.add_fcas_trapezium_constraints()
    cost = constraint_inputs.get_constraint_violation_prices()['fcas_max_avail']
    fcas_availability = unit_inputs.get_fcas_max_availability()
    market.set_fcas_max_availability(fcas_availability)
    market.make_constraints_elastic('fcas_max_availability', cost)
    cost = constraint_inputs.get_constraint_violation_prices()['fcas_profile']
    regulation_trapeziums = unit_inputs.get_fcas_regulation_trapeziums()
    market.set_energy_and_regulation_capacity_constraints(regulation_trapeziums)
    market.make_constraints_elastic('energy_and_regulation_capacity', cost)
    scada_ramp_down_rates = unit_inputs.get_scada_ramp_down_rates_of_lower_reg_units()
    market.set_joint_ramping_constraints_lower_reg(scada_ramp_down_rates)
    market.make_constraints_elastic('joint_ramping_lower_reg', cost)
    scada_ramp_up_rates = unit_inputs.get_scada_ramp_up_rates_of_raise_reg_units()
    market.set_joint_ramping_constraints_raise_reg(scada_ramp_up_rates)
    market.make_constraints_elastic('joint_ramping_raise_reg', cost)
    contingency_trapeziums = unit_inputs.get_contingency_services()
    market.set_joint_capacity_constraints(contingency_trapeziums)
    market.make_constraints_elastic('joint_capacity', cost)

    # Set interconnector definitions, limits and loss models.
    interconnectors_definitions = \
        interconnector_inputs.get_interconnector_definitions()
    loss_functions, interpolation_break_points = \
        interconnector_inputs.get_interconnector_loss_model()
    market.set_interconnectors(interconnectors_definitions)
    market.set_interconnector_losses(loss_functions,
                                     interpolation_break_points)

    # Add generic constraints and FCAS market constraints.
    fcas_requirements = constraint_inputs.get_fcas_requirements()
    market.set_fcas_requirements_constraints(fcas_requirements)
    violation_costs = constraint_inputs.get_violation_costs()
    market.make_constraints_elastic('fcas', violation_cost=violation_costs)
    generic_rhs = constraint_inputs.get_rhs_and_type_excluding_regional_fcas_constraints()
    market.set_generic_constraints(generic_rhs)
    market.make_constraints_elastic('generic', violation_cost=violation_costs)
    unit_generic_lhs = constraint_inputs.get_unit_lhs()
    market.link_units_to_generic_constraints(unit_generic_lhs)
    interconnector_generic_lhs = constraint_inputs.get_interconnector_lhs()
    market.link_interconnectors_to_generic_constraints(
        interconnector_generic_lhs)

    # Set the operational demand to be met by dispatch.
    regional_demand = demand_inputs.get_operational_demand()
    market.set_demand_constraints(regional_demand)
    
    # Get unit dispatch without fast start constraints and use it to
    # make fast start unit commitment decisions.
    market.dispatch()
    dispatch = market.get_unit_dispatch()
    fast_start_profiles = unit_inputs.get_fast_start_profiles_for_dispatch(dispatch)
    market.set_fast_start_constraints(fast_start_profiles)
    if 'fast_start' in market.get_constraint_set_names():
        cost = constraint_inputs.get_constraint_violation_prices()['fast_start']
        market.make_constraints_elastic('fast_start', violation_cost=cost)

    # If AEMO historical used the over constrained dispatch rerun
    # process then allow it to be used in dispatch. This is needed
    # because sometimes the conditions for over constrained dispatch
    # are present but the rerun process isn't used.
    if constraint_inputs.is_over_constrained_dispatch_rerun():
        market.dispatch(allow_over_constrained_dispatch_re_run=True,
                        energy_market_floor_price=-1000.0,
                        energy_market_ceiling_price=14500.0,
                        fcas_market_ceiling_price=1000.0)
    else:
        # The market price ceiling and floor are not needed here
        # because they are only used for the over constrained
        # dispatch rerun process.
        market.dispatch(allow_over_constrained_dispatch_re_run=False)

    # Save prices from this interval
    prices = market.get_energy_prices()
    prices['time'] = interval
    outputs.append(prices.loc[:, ['time', 'region', 'price']])

con.close()
print(pd.concat(outputs))
#                   time region      price
# 0  2019/01/01 12:00:00   NSW1  91.870167
# 1  2019/01/01 12:00:00   QLD1  76.190796
# 2  2019/01/01 12:00:00    SA1  86.899534
# 3  2019/01/01 12:00:00   TAS1  89.805037
# 4  2019/01/01 12:00:00   VIC1  84.984255
# 0  2019/01/01 12:05:00   NSW1  91.870496
# 1  2019/01/01 12:05:00   QLD1  64.991736
# 2  2019/01/01 12:05:00    SA1  87.462599
# 3  2019/01/01 12:05:00   TAS1  90.178036
# 4  2019/01/01 12:05:00   VIC1  85.556009
# 0  2019/01/01 12:10:00   NSW1  91.870496
# 1  2019/01/01 12:10:00   QLD1  64.991736
# 2  2019/01/01 12:10:00    SA1  86.868556
# 3  2019/01/01 12:10:00   TAS1  89.983716
# 4  2019/01/01 12:10:00   VIC1  84.936150
# 0  2019/01/01 12:15:00   NSW1  91.870496
# 1  2019/01/01 12:15:00   QLD1  64.776456
# 2  2019/01/01 12:15:00    SA1  86.844540
# 3  2019/01/01 12:15:00   TAS1  89.582288
# 4  2019/01/01 12:15:00   VIC1  84.990796
# 0  2019/01/01 12:20:00   NSW1  91.870496
# 1  2019/01/01 12:20:00   QLD1  64.776456
# 2  2019/01/01 12:20:00    SA1  87.496112
# 3  2019/01/01 12:20:00   TAS1  90.291144
# 4  2019/01/01 12:20:00   VIC1  85.594840
# 0  2019/01/01 12:25:00   NSW1  91.870167
# 1  2019/01/01 12:25:00   QLD1  64.991736
# 2  2019/01/01 12:25:00    SA1  87.519993
# 3  2019/01/01 12:25:00   TAS1  90.488064
# 4  2019/01/01 12:25:00   VIC1  85.630617
# 0  2019/01/01 12:30:00   NSW1  91.870496
# 1  2019/01/01 12:30:00   QLD1  64.991736
# 2  2019/01/01 12:30:00    SA1  87.462000
# 3  2019/01/01 12:30:00   TAS1  90.196284
# 4  2019/01/01 12:30:00   VIC1  85.573321

Time sequential recreation of historical dispatch

Demonstrates using nempy to recreate historical dispatch in a dynamic or time sequential manner, this means the outputs of one interval become the initial conditions for the next dispatch interval. Note, currently there is not the infrastructure in place to include features such as generic constraints in the time sequential model as the rhs values of many constraints would need to be re-calculated based on the dynamic system state. Similarly, using historical bids in this example is some what problematic as participants also dynamically change their bids based on market conditions. However, for sake of demonstrating how nempy can be used to create time sequential models, historical bids are used in this example.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
# Notice: this script downloads large volumes of historical market data from AEMO's nemweb portal.

import sqlite3
import pandas as pd
from nempy import markets, time_sequential
from nempy.historical_inputs import loaders, mms_db, \
    xml_cache, units, demand, interconnectors, constraints

con = sqlite3.connect('market_management_system.db')
mms_db_manager = mms_db.DBManager(connection=con)

xml_cache_manager = xml_cache.XMLCacheManager('cache_directory')

# The second time this example is run on a machine this flag can
# be set to false to save downloading the data again.
download_inputs = False

if download_inputs:
    # This requires approximately 5 GB of storage.
    mms_db_manager.populate(start_year=2019, start_month=1,
                            end_year=2019, end_month=1)

    # This requires approximately 60 GB of storage.
    xml_cache_manager.populate(start_year=2019, start_month=1,
                               end_year=2019, end_month=1)

raw_inputs_loader = loaders.RawInputsLoader(
    nemde_xml_cache_manager=xml_cache_manager,
    market_management_system_database=mms_db_manager)

# A list of intervals we want to recreate historical dispatch for.
dispatch_intervals = ['2019/01/01 12:00:00',
                      '2019/01/01 12:05:00',
                      '2019/01/01 12:10:00',
                      '2019/01/01 12:15:00',
                      '2019/01/01 12:20:00',
                      '2019/01/01 12:25:00',
                      '2019/01/01 12:30:00']

# List for saving outputs to.
outputs = []

unit_dispatch = None
from time import time

t0 = time()
# Create and dispatch the spot market for each dispatch interval.
for interval in dispatch_intervals:
    print(interval)
    raw_inputs_loader.set_interval(interval)
    unit_inputs = units.UnitData(raw_inputs_loader)
    demand_inputs = demand.DemandData(raw_inputs_loader)
    interconnector_inputs = \
        interconnectors.InterconnectorData(raw_inputs_loader)
    constraint_inputs = \
        constraints.ConstraintData(raw_inputs_loader)

    unit_info = unit_inputs.get_unit_info()
    market = markets.SpotMarket(market_regions=['QLD1', 'NSW1', 'VIC1',
                                                'SA1', 'TAS1'],
                                unit_info=unit_info)

    volume_bids, price_bids = unit_inputs.get_processed_bids()
    market.set_unit_volume_bids(volume_bids)
    market.set_unit_price_bids(price_bids)

    violation_cost = \
        constraint_inputs.get_constraint_violation_prices()['unit_capacity']
    unit_bid_limit = unit_inputs.get_unit_bid_availability()
    market.set_unit_bid_capacity_constraints(unit_bid_limit)
    market.make_constraints_elastic('unit_bid_capacity', violation_cost)

    unit_uigf_limit = unit_inputs.get_unit_uigf_limits()
    market.set_unconstrained_intermitent_generation_forecast_constraint(
        unit_uigf_limit)

    ramp_rates = unit_inputs.get_as_bid_ramp_rates()

    # This is the part that makes it time sequential.
    if unit_dispatch is None:
        # For the first dispatch interval we use historical values
        # as initial conditions.
        historical_dispatch = unit_inputs.get_initial_unit_output()
        ramp_rates = time_sequential.create_seed_ramp_rate_parameters(
            historical_dispatch, ramp_rates)
    else:
        # For subsequent dispatch intervals we use the output levels
        # determined by the last dispatch as the new initial conditions
        ramp_rates = time_sequential.construct_ramp_rate_parameters(
            unit_dispatch, ramp_rates)

    violation_cost = \
        constraint_inputs.get_constraint_violation_prices()['ramp_rate']
    market.set_unit_ramp_up_constraints(
        ramp_rates.loc[:, ['unit', 'initial_output', 'ramp_up_rate']])
    market.make_constraints_elastic('ramp_up', violation_cost)
    market.set_unit_ramp_down_constraints(
        ramp_rates.loc[:, ['unit', 'initial_output', 'ramp_down_rate']])
    market.make_constraints_elastic('ramp_down', violation_cost)

    regional_demand = demand_inputs.get_operational_demand()
    market.set_demand_constraints(regional_demand)

    interconnectors_definitions = \
        interconnector_inputs.get_interconnector_definitions()
    loss_functions, interpolation_break_points = \
        interconnector_inputs.get_interconnector_loss_model()
    market.set_interconnectors(interconnectors_definitions)
    market.set_interconnector_losses(loss_functions,
                                     interpolation_break_points)
    market.dispatch()

    # Save prices from this interval
    prices = market.get_energy_prices()
    prices['time'] = interval
    outputs.append(prices.loc[:, ['time', 'region', 'price']])

    unit_dispatch = market.get_unit_dispatch()
print("Run time per interval {}".format((time()-t0)/len(dispatch_intervals)))
con.close()
print(pd.concat(outputs))
#                   time region      price
# 0  2019/01/01 12:00:00   NSW1  91.857666
# 1  2019/01/01 12:00:00   QLD1  76.716642
# 2  2019/01/01 12:00:00    SA1  85.126914
# 3  2019/01/01 12:00:00   TAS1  86.173481
# 4  2019/01/01 12:00:00   VIC1  83.250703
# 0  2019/01/01 12:05:00   NSW1  88.357224
# 1  2019/01/01 12:05:00   QLD1  72.255334
# 2  2019/01/01 12:05:00    SA1  82.417720
# 3  2019/01/01 12:05:00   TAS1  83.451561
# 4  2019/01/01 12:05:00   VIC1  80.621103
# 0  2019/01/01 12:10:00   NSW1  91.857666
# 1  2019/01/01 12:10:00   QLD1  75.665675
# 2  2019/01/01 12:10:00    SA1  85.680310
# 3  2019/01/01 12:10:00   TAS1  86.715499
# 4  2019/01/01 12:10:00   VIC1  83.774337
# 0  2019/01/01 12:15:00   NSW1  88.113012
# 1  2019/01/01 12:15:00   QLD1  71.559977
# 2  2019/01/01 12:15:00    SA1  82.165045
# 3  2019/01/01 12:15:00   TAS1  83.451561
# 4  2019/01/01 12:15:00   VIC1  80.411187
# 0  2019/01/01 12:20:00   NSW1  91.864122
# 1  2019/01/01 12:20:00   QLD1  75.052319
# 2  2019/01/01 12:20:00    SA1  85.722028
# 3  2019/01/01 12:20:00   TAS1  86.576848
# 4  2019/01/01 12:20:00   VIC1  83.859306
# 0  2019/01/01 12:25:00   NSW1  91.864122
# 1  2019/01/01 12:25:00   QLD1  75.696247
# 2  2019/01/01 12:25:00    SA1  85.746024
# 3  2019/01/01 12:25:00   TAS1  86.613642
# 4  2019/01/01 12:25:00   VIC1  83.894945
# 0  2019/01/01 12:30:00   NSW1  91.870167
# 1  2019/01/01 12:30:00   QLD1  75.188735
# 2  2019/01/01 12:30:00    SA1  85.694071
# 3  2019/01/01 12:30:00   TAS1  86.560602
# 4  2019/01/01 12:30:00   VIC1  83.843570