diff --git a/.github/workflows/light-ci.yml b/.github/workflows/light-ci.yml index 0db4b9d..feaf37a 100644 --- a/.github/workflows/light-ci.yml +++ b/.github/workflows/light-ci.yml @@ -16,12 +16,18 @@ jobs: - name: Set up Python 3.8 uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.11' - name: Install dependencies run: | - python -m pip install --upgrade "pip<24.1" "setuptools<66" "wheel<0.38" - python -m pip install --no-build-isolation -r requirements-ci.txt + python -m pip install --upgrade pip + python -m pip install -c requirements-ci.txt -e .[dev] + + - name: Ruff lint + run: ruff check . + + - name: Ruff format check + run: ruff format --check . - name: Run lightweight test suite run: pytest -q -m "not powerflow" diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 7cb5bac..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include rl_adn/data_sources/network_data/node_123/*.csv -include rl_adn/data_sources/network_data/node_25/*.csv -include rl_adn/data_sources/network_data/node_34/*.csv -include rl_adn/data_sources/network_data/node_69/*.csv -include rl_adn/data_sources/time_series_data/*.csv diff --git a/README.md b/README.md index af44e74..4159197 100644 --- a/README.md +++ b/README.md @@ -8,25 +8,25 @@ RL-ADN now supports `topology-as-scenario` for the `34-bus` and `69-bus` feeders ## Quickstart -Install runtime dependencies: +Install the package: ```bash -py -3 -m pip install -r requirements.txt +py -3 -m pip install . ``` Install the development toolchain: ```bash -py -3 -m pip install -r requirements-dev.txt +py -3 -m pip install -e .[dev] ``` -Run the lightweight test suite: +Run the lightweight verification suite: ```bash -py -3 -m pytest tests -q -m "not powerflow" +py -3 -m pytest -q -m "not powerflow" ``` -Some power-flow validation tests require `pandapower`. If it is not installed, those tests are skipped automatically. +Power-flow validation tests require the optional `pandapower` extra and are skipped when it is not installed. ## First Import @@ -35,7 +35,7 @@ from rl_adn import PowerNetEnv, make_env_config config = make_env_config() env = PowerNetEnv(config) -state, info = env.reset(return_info=True) +state, info = env.reset(seed=2026) ``` Run the script-style quickstart: @@ -58,7 +58,7 @@ from rl_adn import PowerNetEnv, make_env_config config = make_env_config(node=34, topology_scenario="TP4", return_graph=True) env = PowerNetEnv(config) -state, info = env.reset(return_info=True) +state, info = env.reset(seed=2026) print(info["topology_scenario"]) ``` @@ -72,7 +72,7 @@ config = make_env_config( return_graph=True, ) env = PowerNetEnv(config) -state, info = env.reset(return_info=True) +state, info = env.reset(seed=2026) ``` Inspect the active topology for later GNN work: @@ -84,6 +84,27 @@ graph = env.get_graph_data() `metadata` includes feeder id, scenario id, node count, edge count, and active edges. `graph` returns plain NumPy/Python structures such as adjacency and edge index. +## Public API + +The stable package surface is: + +- `Battery` +- `BatteryConfig` +- `EnvConfig` +- `TopologyConfig` +- `GeneralPowerDataManager` +- `PowerNetEnv` +- `make_env_config(...)` + +`make_env_config(...)` now returns a typed `EnvConfig` dataclass rather than a loose dictionary. + +`PowerNetEnv` follows Gymnasium semantics: + +```python +obs, info = env.reset(seed=2026) +next_obs, reward, terminated, truncated, info = env.step(action) +``` + ## Repository Structure - `rl_adn/`: package source code @@ -93,7 +114,7 @@ graph = env.get_graph_data() ## Highlights -- Flexible active distribution network environment modeling +- Gymnasium-style active distribution network environment - Laurent power flow solver for faster training-time simulation - DRL algorithms and optimization baselines in the same repository - Bundled network and time-series datasets for reproducible experiments @@ -107,9 +128,9 @@ The library was originally released alongside the RL-ADN research paper on optim ## Recommended Learning Path 1. Run `examples/quickstart_env.py` for the minimal package-backed environment flow. -2. Open `examples/Customize_env.ipynb` to understand configuration customization. -3. Open the DDPG training notebook once the environment baseline is clear. -4. Try fixed and pooled topology scenarios before moving on to GNN-based experiments. +2. Read the typed config surface through `rl_adn.make_env_config(...)`. +3. Try fixed and pooled topology scenarios before moving on to GNN-based experiments. +4. Use notebooks only as supplementary material after the script workflow is clear. ## Current Limits diff --git a/docs/conf.py b/docs/conf.py index 127b692..319be51 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,18 +12,17 @@ # - import os import sys -sys.path.insert(0, os.path.abspath('..')) +sys.path.insert(0, os.path.abspath("..")) # -- Project information ----------------------------------------------------- -project = 'RL-ADN' -copyright = '2023, Shengren' -author = 'Shengren' +project = "RL-ADN" +copyright = "2023, Shengren" +author = "Shengren" # -- General configuration --------------------------------------------------- @@ -31,24 +30,20 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.coverage', - 'sphinx.ext.napoleon' -] +extensions = ["sphinx.ext.autodoc", "sphinx.ext.coverage", "sphinx.ext.napoleon"] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] autodoc_default_options = { - 'show-inheritance': True, + "show-inheritance": True, } -autodoc_mock_imports = ['gym', 'pandapower', 'pandapower.topology', 'psutil'] +autodoc_mock_imports = ["gym", "pandapower", "pandapower.topology", "psutil", "pyomo", "pyomo.environ"] # -- Options for HTML output ------------------------------------------------- @@ -57,7 +52,7 @@ # a list of builtin themes. # # html_theme = 'alabaster' -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Add any paths that contain custom static files (such as style sheets) here, diff --git a/docs/modules.rst b/docs/modules.rst index d1ee640..ffe7f78 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -5,11 +5,9 @@ rl_adn :maxdepth: 4 rl_adn - rl_adn.data_manager + rl_adn.data rl_adn.data_sources - rl_adn.data_sources.network_data - rl_adn.data_sources.time_series_data - rl_adn.DRL_algorithms + rl_adn.algorithms rl_adn.environments - rl_adn.utility - rl_adn.benckmark_algorithms + rl_adn.network + rl_adn.benchmarks diff --git a/docs/requirements.txt b/docs/requirements.txt index e0a8f5e..5bd8f93 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1 +1 @@ --r ../requirements-dev.txt +-e ..[dev] diff --git a/docs/rl_adn.DRL_algorithms.rst b/docs/rl_adn.DRL_algorithms.rst deleted file mode 100644 index 89504b4..0000000 --- a/docs/rl_adn.DRL_algorithms.rst +++ /dev/null @@ -1,15 +0,0 @@ -rl_adn.DRL_algorithms package -============================= - -Algorithm modules are documented on dedicated subpages to avoid duplicate -autodoc entries while preserving the full module tree. - -.. toctree:: - :maxdepth: 2 - - rl_adn.DRL_algorithms.Agent - rl_adn.DRL_algorithms.DDPG - rl_adn.DRL_algorithms.PPO - rl_adn.DRL_algorithms.SAC - rl_adn.DRL_algorithms.TD3 - rl_adn.DRL_algorithms.utility diff --git a/docs/rl_adn.DRL_algorithms.Agent.rst b/docs/rl_adn.algorithms.Agent.rst similarity index 51% rename from docs/rl_adn.DRL_algorithms.Agent.rst rename to docs/rl_adn.algorithms.Agent.rst index 9bc2fbf..8ab7cf8 100644 --- a/docs/rl_adn.DRL_algorithms.Agent.rst +++ b/docs/rl_adn.algorithms.Agent.rst @@ -1,7 +1,7 @@ -rl_adn.DRL_algorithms.Agent module +rl_adn.algorithms.Agent module ================================== -.. automodule:: rl_adn.DRL_algorithms.Agent +.. automodule:: rl_adn.algorithms.Agent :members: :show-inheritance: :no-index: diff --git a/docs/rl_adn.DRL_algorithms.DDPG.rst b/docs/rl_adn.algorithms.DDPG.rst similarity index 52% rename from docs/rl_adn.DRL_algorithms.DDPG.rst rename to docs/rl_adn.algorithms.DDPG.rst index 955148f..eed5dcb 100644 --- a/docs/rl_adn.DRL_algorithms.DDPG.rst +++ b/docs/rl_adn.algorithms.DDPG.rst @@ -1,7 +1,7 @@ -rl_adn.DRL_algorithms.DDPG module +rl_adn.algorithms.DDPG module ================================= -.. automodule:: rl_adn.DRL_algorithms.DDPG +.. automodule:: rl_adn.algorithms.DDPG :members: :show-inheritance: :no-index: diff --git a/docs/rl_adn.DRL_algorithms.SAC.rst b/docs/rl_adn.algorithms.PPO.rst similarity index 52% rename from docs/rl_adn.DRL_algorithms.SAC.rst rename to docs/rl_adn.algorithms.PPO.rst index 422d103..c014144 100644 --- a/docs/rl_adn.DRL_algorithms.SAC.rst +++ b/docs/rl_adn.algorithms.PPO.rst @@ -1,7 +1,7 @@ -rl_adn.DRL_algorithms.SAC module +rl_adn.algorithms.PPO module ================================ -.. automodule:: rl_adn.DRL_algorithms.SAC +.. automodule:: rl_adn.algorithms.PPO :members: :show-inheritance: :no-index: diff --git a/docs/rl_adn.DRL_algorithms.TD3.rst b/docs/rl_adn.algorithms.SAC.rst similarity index 52% rename from docs/rl_adn.DRL_algorithms.TD3.rst rename to docs/rl_adn.algorithms.SAC.rst index 5570c6b..af0a9ea 100644 --- a/docs/rl_adn.DRL_algorithms.TD3.rst +++ b/docs/rl_adn.algorithms.SAC.rst @@ -1,7 +1,7 @@ -rl_adn.DRL_algorithms.TD3 module +rl_adn.algorithms.SAC module ================================ -.. automodule:: rl_adn.DRL_algorithms.TD3 +.. automodule:: rl_adn.algorithms.SAC :members: :show-inheritance: :no-index: diff --git a/docs/rl_adn.DRL_algorithms.PPO.rst b/docs/rl_adn.algorithms.TD3.rst similarity index 52% rename from docs/rl_adn.DRL_algorithms.PPO.rst rename to docs/rl_adn.algorithms.TD3.rst index a191c35..2e3f863 100644 --- a/docs/rl_adn.DRL_algorithms.PPO.rst +++ b/docs/rl_adn.algorithms.TD3.rst @@ -1,7 +1,7 @@ -rl_adn.DRL_algorithms.PPO module +rl_adn.algorithms.TD3 module ================================ -.. automodule:: rl_adn.DRL_algorithms.PPO +.. automodule:: rl_adn.algorithms.TD3 :members: :show-inheritance: :no-index: diff --git a/docs/rl_adn.algorithms.rst b/docs/rl_adn.algorithms.rst new file mode 100644 index 0000000..6d38850 --- /dev/null +++ b/docs/rl_adn.algorithms.rst @@ -0,0 +1,15 @@ +rl_adn.algorithms package +============================= + +Algorithm modules are documented on dedicated subpages to avoid duplicate +autodoc entries while preserving the full module tree. + +.. toctree:: + :maxdepth: 2 + + rl_adn.algorithms.Agent + rl_adn.algorithms.DDPG + rl_adn.algorithms.PPO + rl_adn.algorithms.SAC + rl_adn.algorithms.TD3 + rl_adn.algorithms.utility diff --git a/docs/rl_adn.DRL_algorithms.utility.rst b/docs/rl_adn.algorithms.utility.rst similarity index 69% rename from docs/rl_adn.DRL_algorithms.utility.rst rename to docs/rl_adn.algorithms.utility.rst index 43dece2..52dbbb9 100644 --- a/docs/rl_adn.DRL_algorithms.utility.rst +++ b/docs/rl_adn.algorithms.utility.rst @@ -1,8 +1,8 @@ -rl_adn.DRL_algorithms.utility module +rl_adn.algorithms.utility module ==================================== The utility module remains part of the docs tree, but member-level autodoc is deferred until its legacy docstrings are normalized. -.. automodule:: rl_adn.DRL_algorithms.utility +.. automodule:: rl_adn.algorithms.utility :show-inheritance: diff --git a/docs/rl_adn.benchmarks.rst b/docs/rl_adn.benchmarks.rst new file mode 100644 index 0000000..0c85714 --- /dev/null +++ b/docs/rl_adn.benchmarks.rst @@ -0,0 +1,7 @@ +rl_adn.benchmarks package +==================================== + +.. automodule:: rl_adn.benchmarks + :members: + :show-inheritance: + diff --git a/docs/rl_adn.benckmark_algorithms.rst b/docs/rl_adn.benckmark_algorithms.rst deleted file mode 100644 index 58bf6d2..0000000 --- a/docs/rl_adn.benckmark_algorithms.rst +++ /dev/null @@ -1,7 +0,0 @@ -rl_adn.benckmark_algorithms package -==================================== - -.. automodule:: rl_adn.benckmark_algorithms - :members: - :show-inheritance: - diff --git a/docs/rl_adn.data_manager.rst b/docs/rl_adn.data.rst similarity index 63% rename from docs/rl_adn.data_manager.rst rename to docs/rl_adn.data.rst index 85e7588..a28dc9f 100644 --- a/docs/rl_adn.data_manager.rst +++ b/docs/rl_adn.data.rst @@ -1,8 +1,8 @@ -rl_adn.data_manager package +rl_adn.data package =========================== The stable public entrypoint for the data manager is documented below. -.. automodule:: rl_adn.data_manager.data_manager +.. automodule:: rl_adn.data.manager :members: :show-inheritance: diff --git a/docs/rl_adn.environments.rst b/docs/rl_adn.environments.rst index 63959f9..6184a88 100644 --- a/docs/rl_adn.environments.rst +++ b/docs/rl_adn.environments.rst @@ -7,7 +7,19 @@ The package-level exports mirror the core environment entrypoints documented bel :members: :show-inheritance: -.. automodule:: rl_adn.environments.config +.. automodule:: rl_adn.config + :members: + :show-inheritance: + +.. automodule:: rl_adn.environments.observation + :members: + :show-inheritance: + +.. automodule:: rl_adn.environments.reward + :members: + :show-inheritance: + +.. automodule:: rl_adn.environments.solvers :members: :show-inheritance: diff --git a/docs/rl_adn.utility.rst b/docs/rl_adn.network.rst similarity index 60% rename from docs/rl_adn.utility.rst rename to docs/rl_adn.network.rst index 827dadf..3a2a792 100644 --- a/docs/rl_adn.utility.rst +++ b/docs/rl_adn.network.rst @@ -1,15 +1,15 @@ -rl_adn.utility package +rl_adn.network package ====================== Utility submodules are documented individually to avoid duplicating package-level aliases. -.. automodule:: rl_adn.utility.grid +.. automodule:: rl_adn.network.grid :show-inheritance: -.. automodule:: rl_adn.utility.numbarize +.. automodule:: rl_adn.network.numbarize :members: :show-inheritance: -.. automodule:: rl_adn.utility.utils +.. automodule:: rl_adn.network.utils :members: :show-inheritance: diff --git a/examples/Tutorial_DDPG_training_using_RL_ADN.ipynb b/examples/Tutorial_DDPG_training_using_RL_ADN.ipynb index aedf3a1..2b288b6 100644 --- a/examples/Tutorial_DDPG_training_using_RL_ADN.ipynb +++ b/examples/Tutorial_DDPG_training_using_RL_ADN.ipynb @@ -69,8 +69,8 @@ "source": [ "import torch\n", "from torch.nn.utils import clip_grad_norm_\n", - "from rl_adn.DRL_algorithms.Agent import AgentDDPG\n", - "from rl_adn.DRL_algorithms.utility import Config, ReplayBuffer, SumTree, build_mlp, get_episode_return, get_optim_param\n", + "from rl_adn.algorithms.Agent import AgentDDPG\n", + "from rl_adn.algorithms.utility import Config, ReplayBuffer, SumTree, build_mlp, get_episode_return, get_optim_param\n", "import time" ], "metadata": { diff --git a/examples/Tutorial_DDPG_training_using_RL_ADN_for_Colab.ipynb b/examples/Tutorial_DDPG_training_using_RL_ADN_for_Colab.ipynb index 99b378e..c58b6de 100644 --- a/examples/Tutorial_DDPG_training_using_RL_ADN_for_Colab.ipynb +++ b/examples/Tutorial_DDPG_training_using_RL_ADN_for_Colab.ipynb @@ -132,8 +132,8 @@ "source": [ "import torch\n", "from torch.nn.utils import clip_grad_norm_\n", - "from rl_adn.DRL_algorithms.Agent import AgentDDPG\n", - "from rl_adn.DRL_algorithms.utility import Config, ReplayBuffer, SumTree, build_mlp, get_episode_return, get_optim_param\n", + "from rl_adn.algorithms.Agent import AgentDDPG\n", + "from rl_adn.algorithms.utility import Config, ReplayBuffer, SumTree, build_mlp, get_episode_return, get_optim_param\n", "from rl_adn.environments.env import PowerNetEnv\n", "import time" ], diff --git a/examples/custom_env_config.py b/examples/custom_env_config.py new file mode 100644 index 0000000..4800c40 --- /dev/null +++ b/examples/custom_env_config.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from dataclasses import replace + +from rl_adn import PowerNetEnv, make_env_config + + +def main() -> None: + base_config = make_env_config(node=34, topology_scenario="TP3") + custom_config = replace( + base_config, + battery_nodes=(11, 15, 26), + battery=replace(base_config.battery, max_charge_kw=25.0, max_discharge_kw=25.0), + ) + + env = PowerNetEnv(custom_config) + state, info = env.reset(seed=2026) + + print("Battery nodes:", custom_config.battery_nodes) + print("Topology scenario:", info["topology_scenario"]) + print("State shape:", state.shape) + + +if __name__ == "__main__": + main() diff --git a/examples/quickstart_env.py b/examples/quickstart_env.py index 1e48c44..c77a659 100644 --- a/examples/quickstart_env.py +++ b/examples/quickstart_env.py @@ -6,15 +6,18 @@ def main() -> None: config = make_env_config() env = PowerNetEnv(config) - state = env.reset() + state, info = env.reset(seed=2026) - action = np.zeros((len(config["battery_list"]), 1), dtype=np.float32) - next_state, reward, done, _ = env.step(action) + action = np.zeros(len(config.battery_nodes), dtype=np.float32) + next_state, reward, terminated, truncated, step_info = env.step(action) print("Initial state shape:", state.shape) + print("Topology scenario:", info["topology_scenario"]) print("Next state shape:", next_state.shape) print("Reward:", reward) - print("Done:", done) + print("Terminated:", terminated) + print("Truncated:", truncated) + print("Saved money:", step_info["reward_breakdown"]["saved_money"]) if __name__ == "__main__": diff --git a/examples/topology_scenarios.py b/examples/topology_scenarios.py new file mode 100644 index 0000000..714a348 --- /dev/null +++ b/examples/topology_scenarios.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from rl_adn import PowerNetEnv, make_env_config + + +def main() -> None: + config = make_env_config( + node=34, + topology_mode="scenario_pool", + topology_pool=["TP2", "TP3", "TP4"], + return_graph=True, + ) + env = PowerNetEnv(config) + _state, info = env.reset(seed=2026) + metadata = env.get_topology_metadata() + graph = env.get_graph_data() + + print("Sampled scenario:", info["topology_scenario"]) + print("Active edge count:", metadata["edge_count"]) + print("Adjacency shape:", graph["adjacency"].shape) + + +if __name__ == "__main__": + main() diff --git a/examples/training_smoke.py b/examples/training_smoke.py new file mode 100644 index 0000000..1a097f8 --- /dev/null +++ b/examples/training_smoke.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from rl_adn import PowerNetEnv, make_env_config +from rl_adn.algorithms.DDPG import AgentDDPG +from rl_adn.algorithms.utility import Config + + +def main() -> None: + config = make_env_config() + env = PowerNetEnv(config) + args = Config() + args.num_envs = 1 + args.if_off_policy = True + args.batch_size = 8 + args.repeat_times = 1.0 + + agent = AgentDDPG( + net_dims=[32, 32], + state_dim=env.observation_space.shape[0], + action_dim=env.action_space.shape[0], + gpu_id=-1, + args=args, + ) + states, actions, rewards, undones = agent.explore_one_env(env, horizon_len=2, if_random=True) + + print("Collected states:", tuple(states.shape)) + print("Collected actions:", tuple(actions.shape)) + print("Collected rewards:", tuple(rewards.shape)) + print("Collected undones:", tuple(undones.shape)) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..bd538f2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,100 @@ +[build-system] +requires = ["setuptools>=69", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "rl-adn" +version = "0.2.0" +description = "RL-ADN: a modern Python toolkit for ESS dispatch research in active distribution networks." +readme = "README.md" +requires-python = ">=3.11" +license = "MIT" +authors = [ + { name = "Hou Shengren" }, + { name = "Gao Shuyi" }, + { name = "Pedro P. Vergara" }, +] +keywords = ["reinforcement-learning", "power-systems", "distribution-network", "energy-storage"] +dependencies = [ + "gymnasium>=1.2,<2", + "matplotlib>=3.8,<4", + "networkx>=3.2,<4", + "numba>=0.60,<0.65", + "numpy>=1.26,<3", + "pandas>=2.2,<3", + "psutil>=6,<7", + "scipy>=1.11,<2", + "tqdm>=4.66,<5", +] + +[project.optional-dependencies] +algorithms = [ + "torch>=2.2,<3", +] +augmentation = [ + "copulas>=0.12,<1", + "scikit-learn>=1.4,<2", + "seaborn>=0.13,<1", + "SciencePlots>=2.1,<3", +] +benchmarks = [ + "pyomo>=6.8,<7", +] +pandapower = [ + "pandapower>=2.14,<4", +] +dev = [ + "build>=1.2,<2", + "pytest>=8,<9", + "ruff>=0.11,<1", + "sphinx>=8,<9", + "sphinx-rtd-theme>=3,<4", +] + +[project.urls] +Homepage = "https://github.com/EnergyQuantResearch/RL-ADN" +Documentation = "https://github.com/EnergyQuantResearch/RL-ADN/wiki" +Repository = "https://github.com/EnergyQuantResearch/RL-ADN" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["rl_adn*"] + +[tool.setuptools.package-data] +rl_adn = [ + "data_sources/network_data/node_123/*.csv", + "data_sources/network_data/node_25/*.csv", + "data_sources/network_data/node_34/*.csv", + "data_sources/network_data/node_69/*.csv", + "data_sources/time_series_data/*.csv", +] + +[tool.pytest.ini_options] +markers = [ + "powerflow: tests that require optional pandapower support", +] +testpaths = ["tests"] + +[tool.ruff] +line-length = 240 +target-version = "py311" +extend-exclude = [ + "build", + "dist", + "docs", + "examples/*.ipynb", + "rl_adn/benchmarks", + "rl_adn.egg-info", +] + +[tool.ruff.lint] +select = ["E", "F", "I", "W"] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" + +[tool.ruff.lint.per-file-ignores] +"tests/*_node_network_powerflow_test.py" = ["E402"] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 0f85ec4..0000000 --- a/pytest.ini +++ /dev/null @@ -1,4 +0,0 @@ -[pytest] -testpaths = tests -markers = - powerflow: tests that require pandapower-backed power-flow validation diff --git a/requirements-ci.txt b/requirements-ci.txt index b5b9914..08b6cfb 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -1,10 +1,13 @@ -numpy==1.21.6 -pandas==1.3.5 -gym==0.21.0 -matplotlib==3.5.3 -networkx==2.6.3 -numba==0.53.0 +build==1.3.0 +gymnasium==1.2.3 +matplotlib==3.10.7 +networkx==3.5 +numba==0.64.0 +numpy==2.3.3 +pandas==2.3.2 psutil==6.1.1 -scipy==1.7.3 -tqdm==4.64.1 -pytest>=8,<9 +pytest==8.4.2 +ruff==0.15.7 +scipy==1.16.2 +setuptools==80.9.0 +tqdm==4.67.1 diff --git a/requirements-dev.txt b/requirements-dev.txt index 607d98e..aefbcb6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1 @@ --r requirements.txt -pytest>=8,<9 -sphinx>=7,<8 -sphinx-rtd-theme>=2,<3 -nbformat>=5,<6 +-e .[dev] diff --git a/requirements.txt b/requirements.txt index 362faf1..d6e1198 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1 @@ -copulas==0.9.0 -future==0.18.3 -gym==0.21.0 -matplotlib==3.5.3 -networkx==2.6.3 -numba==0.53.0 -numpy==1.21.6 -pandapower==2.10.1 -pandas==1.3.5 -psutil==6.1.1 -Pyomo==6.3.0 -SciencePlots==2.1.0 -scikit_learn==1.0.2 -scipy==1.7.3 -seaborn==0.11.2 -setuptools==65.5.1 -torch==1.11.0 -tqdm==4.64.1 +-e . diff --git a/rl_adn/DRL_algorithms/__init__.py b/rl_adn/DRL_algorithms/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/rl_adn/__init__.py b/rl_adn/__init__.py index dbf57a2..2844282 100644 --- a/rl_adn/__init__.py +++ b/rl_adn/__init__.py @@ -1,30 +1,32 @@ -"""Top-level package for RL-ADN.""" +"""Public package surface for RL-ADN.""" from importlib import import_module -__version__ = "0.1.3" +__version__ = "0.2.0" __all__ = [ "__version__", "Battery", - "battery_parameters", + "BatteryConfig", + "EnvConfig", "GeneralPowerDataManager", "PowerNetEnv", - "env_config", + "TopologyConfig", "make_env_config", ] _LAZY_EXPORTS = { - "Battery": ("rl_adn.environments.battery", "Battery"), - "battery_parameters": ("rl_adn.environments.battery", "battery_parameters"), - "GeneralPowerDataManager": ("rl_adn.data_manager", "GeneralPowerDataManager"), + "Battery": ("rl_adn.environments", "Battery"), + "BatteryConfig": ("rl_adn.config", "BatteryConfig"), + "EnvConfig": ("rl_adn.config", "EnvConfig"), + "GeneralPowerDataManager": ("rl_adn.data", "GeneralPowerDataManager"), "PowerNetEnv": ("rl_adn.environments", "PowerNetEnv"), - "env_config": ("rl_adn.environments", "env_config"), - "make_env_config": ("rl_adn.environments", "make_env_config"), + "TopologyConfig": ("rl_adn.config", "TopologyConfig"), + "make_env_config": ("rl_adn.config", "make_env_config"), } -def __getattr__(name): +def __getattr__(name: str): if name not in _LAZY_EXPORTS: raise AttributeError(f"module 'rl_adn' has no attribute {name!r}") diff --git a/rl_adn/DRL_algorithms/Agent.py b/rl_adn/algorithms/Agent.py similarity index 89% rename from rl_adn/DRL_algorithms/Agent.py rename to rl_adn/algorithms/Agent.py index 8eba3fc..e529777 100644 --- a/rl_adn/DRL_algorithms/Agent.py +++ b/rl_adn/algorithms/Agent.py @@ -1,33 +1,38 @@ -from torch import nn, Tensor -from typing import Tuple, Union -import torch import os from copy import deepcopy +from typing import Tuple, Union + import numpy as np -from rl_adn.DRL_algorithms.utility import Config, ReplayBuffer, build_mlp, \ - get_optim_param +import torch +from torch import Tensor, nn from torch.nn.utils import clip_grad_norm_ + +from rl_adn.algorithms.env_api import reset_env, step_env +from rl_adn.algorithms.utility import Config, ReplayBuffer, build_mlp, get_optim_param + + class AgentBase: """ - Base Agent class for handling basic agent functions. + Base Agent class for handling basic agent functions. + + Args: + net_dims (list): Network dimensions. + state_dim (int): Dimensionality of the state space. + action_dim (int): Dimensionality of the action space. + gpu_id (int): GPU ID. Default is 0. + args (Config): Configuration arguments. Default is `Config()`. + + Attributes: + gamma (float): Discount factor of future rewards. + num_envs (int): Number of sub-environments in a vectorized environment. + batch_size (int): Number of transitions sampled from replay buffer. + ... + save_attr_names (set): Attributes to be saved or loaded. + + Example: + >>> agent = AgentBase(net_dims=[64, 64], state_dim=10, action_dim=2) + """ - Args: - net_dims (list): Network dimensions. - state_dim (int): Dimensionality of the state space. - action_dim (int): Dimensionality of the action space. - gpu_id (int): GPU ID. Default is 0. - args (Config): Configuration arguments. Default is `Config()`. - - Attributes: - gamma (float): Discount factor of future rewards. - num_envs (int): Number of sub-environments in a vectorized environment. - batch_size (int): Number of transitions sampled from replay buffer. - ... - save_attr_names (set): Attributes to be saved or loaded. - - Example: - >>> agent = AgentBase(net_dims=[64, 64], state_dim=10, action_dim=2) - """ def __init__(self, net_dims: [int], state_dim: int, action_dim: int, gpu_id: int = 0, args: Config = Config()): self.gamma = args.gamma # discount factor of future rewards self.num_envs = args.num_envs # the number of sub envs in vectorized env. `num_envs=1` in single env. @@ -45,18 +50,17 @@ def __init__(self, net_dims: [int], state_dim: int, action_dim: int, gpu_id: int self.last_state = None # last state of the trajectory for training. last_state.shape == (num_envs, state_dim) self.device = torch.device(f"cuda:{gpu_id}" if (torch.cuda.is_available() and (gpu_id >= 0)) else "cpu") - '''network''' + """network""" act_class = getattr(self, "act_class", None) cri_class = getattr(self, "cri_class", None) self.act = self.act_target = act_class(net_dims, state_dim, action_dim).to(self.device) - self.cri = self.cri_target = cri_class(net_dims, state_dim, action_dim).to(self.device) \ - if cri_class else self.act + self.cri = self.cri_target = cri_class(net_dims, state_dim, action_dim).to(self.device) if cri_class else self.act - '''optimizer''' + """optimizer""" self.act_optimizer = torch.optim.AdamW(self.act.parameters(), self.learning_rate) - self.cri_optimizer = torch.optim.AdamW(self.cri.parameters(), self.learning_rate) \ - if cri_class else self.act_optimizer + self.cri_optimizer = torch.optim.AdamW(self.cri.parameters(), self.learning_rate) if cri_class else self.act_optimizer from types import MethodType # built-in package of Python3 + self.act_optimizer.parameters = MethodType(get_optim_param, self.act_optimizer) self.cri_optimizer.parameters = MethodType(get_optim_param, self.cri_optimizer) @@ -66,7 +70,7 @@ def __init__(self, net_dims: [int], state_dim: int, action_dim: int, gpu_id: int else: self.explore_env = self.explore_vec_env - self.if_use_per = getattr(args, 'if_use_per', None) # use PER (Prioritized Experience Replay) + self.if_use_per = getattr(args, "if_use_per", None) # use PER (Prioritized Experience Replay) if self.if_use_per: self.criterion = torch.nn.SmoothL1Loss(reduction="none") self.get_obj_critic = self.get_obj_critic_per @@ -75,7 +79,7 @@ def __init__(self, net_dims: [int], state_dim: int, action_dim: int, gpu_id: int self.get_obj_critic = self.get_obj_critic_raw """save and load""" - self.save_attr_names = {'act', 'act_target', 'act_optimizer', 'cri', 'cri_target', 'cri_optimizer'} + self.save_attr_names = {"act", "act_target", "act_optimizer", "cri", "cri_target", "cri_optimizer"} def explore_one_env(self, env, horizon_len: int, if_random: bool = False) -> Tuple[Tensor, ...]: """ @@ -103,8 +107,8 @@ def explore_one_env(self, env, horizon_len: int, if_random: bool = False) -> Tup states[t] = state ary_action = action[0].detach().cpu().numpy() - ary_state, reward, done, _ = env.step(ary_action) # next_state - ary_state = env.reset() if done else ary_state # ary_state.shape == (state_dim, ) + ary_state, reward, done, _ = step_env(env, ary_action) # next_state + ary_state = reset_env(env) if done else ary_state # ary_state.shape == (state_dim, ) state = torch.as_tensor(ary_state, dtype=torch.float32, device=self.device).unsqueeze(0) actions[t] = action @@ -138,11 +142,10 @@ def explore_vec_env(self, env, horizon_len: int, if_random: bool = False) -> Tup state = self.last_state # last_state.shape == (num_envs, state_dim) get_action = self.act.get_action for t in range(horizon_len): - action = torch.rand(self.num_envs, self.action_dim) * 2 - 1.0 if if_random \ - else get_action(state).detach() + action = torch.rand(self.num_envs, self.action_dim) * 2 - 1.0 if if_random else get_action(state).detach() states[t] = state # state.shape == (num_envs, state_dim) - state, reward, done, _ = env.step(action) # next_state + state, reward, done, _ = step_env(env, action) # next_state actions[t] = action rewards[t] = reward dones[t] = done @@ -276,7 +279,7 @@ def save_or_load_agent(self, cwd: str, if_save: bool): cwd: Current Working Directory. ElegantRL save training files in CWD. if_save: True: save files. False: load files. """ - assert self.save_attr_names.issuperset({'act', 'act_target', 'act_optimizer'}) + assert self.save_attr_names.issuperset({"act", "act_target", "act_optimizer"}) for attr_name in self.save_attr_names: file_path = f"{cwd}/{attr_name}.pth" @@ -286,6 +289,7 @@ def save_or_load_agent(self, cwd: str, if_save: bool): elif os.path.isfile(file_path): setattr(self, attr_name, torch.load(file_path, map_location=self.device)) + class Actor(nn.Module): """ Actor network for policy learning in Actor-Critic models. @@ -295,20 +299,26 @@ class Actor(nn.Module): state_dim (int): Dimensionality of the state space. action_dim (int): Dimensionality of the action space. """ + def __init__(self, dims: [int], state_dim: int, action_dim: int): super().__init__() self.net = build_mlp(dims=[state_dim, *dims, action_dim]) self.explore_noise_std = None # standard deviation of exploration action noise + def forward(self, state: Tensor) -> Tensor: return self.net(state).tanh() # action.tanh() + def get_action(self, state: Tensor) -> Tensor: # for exploration action = self.net(state).tanh() noise = (torch.randn_like(action) * self.explore_noise_std).clamp(-0.5, 0.5) return (action + noise).clamp(-1.0, 1.0) + def get_action_noise(self, state: Tensor, action_std: float) -> Tensor: action = self.net(state).tanh() noise = (torch.randn_like(action) * action_std).clamp(-0.5, 0.5) return (action + noise).clamp(-1.0, 1.0) + + class Critic(nn.Module): """ Critic network for evaluating the value of taking a particular action in a given state. @@ -321,11 +331,15 @@ class Critic(nn.Module): Example: >>> critic = Critic(dims=[64, 64], state_dim=10, action_dim=2) """ + def __init__(self, dims: [int], state_dim: int, action_dim: int): super().__init__() self.net = build_mlp(dims=[state_dim + action_dim, *dims, 1]) + def forward(self, value: Tensor) -> Tensor: return self.net(value) # Q value + + class AgentDDPG(AgentBase): """ Twin Delayed DDPG (Deep Deterministic Policy Gradient) algorithm agent. @@ -348,13 +362,13 @@ class AgentDDPG(AgentBase): """ def __init__(self, net_dims: [int], state_dim: int, action_dim: int, gpu_id: int = 0, args: Config = Config()): - self.act_class = getattr(self, 'act_class', Actor) - self.cri_class = getattr(self, 'cri_class', Critic) + self.act_class = getattr(self, "act_class", Actor) + self.cri_class = getattr(self, "cri_class", Critic) super().__init__(net_dims=net_dims, state_dim=state_dim, action_dim=action_dim, gpu_id=gpu_id, args=args) self.act_target = deepcopy(self.act) self.cri_target = deepcopy(self.cri) - '''comapre to TD3, there is no policy noise''' - self.explore_noise_std = getattr(args, 'explore_noise_std', 0.05) # standard deviation of exploration noise + """comapre to TD3, there is no policy noise""" + self.explore_noise_std = getattr(args, "explore_noise_std", 0.05) # standard deviation of exploration noise self.act.explore_noise_std = self.explore_noise_std # assign explore_noise_std for agent.act.get_action(state) def update_net(self, buffer: ReplayBuffer) -> Tuple[float, ...]: @@ -373,14 +387,14 @@ def update_net(self, buffer: ReplayBuffer) -> Tuple[float, ...]: obj_critics = 0.0 obj_actors = 0.0 # update_times = int(buffer.add_size * self.repeat_times) - update_times = int(buffer.cur_size * self.repeat_times/self.batch_size) + update_times = int(buffer.cur_size * self.repeat_times / self.batch_size) assert update_times >= 1 for update_c in range(update_times): obj_critic, state = self.get_obj_critic(buffer, self.batch_size) obj_critics += obj_critic.item() self.optimizer_update(self.cri_optimizer, obj_critic) self.soft_update(self.cri_target, self.cri, self.soft_update_tau) - '''compare with TD3, DDPG no policy delay update''' + """compare with TD3, DDPG no policy delay update""" action_pg = self.act(state) # policy gradient obj_actor = self.cri_target(torch.cat((state, action_pg), dim=1)).mean() # use cri_target is more stable than cri obj_actors += obj_actor.item() @@ -409,13 +423,13 @@ def get_obj_critic_raw(self, buffer: ReplayBuffer, batch_size: int) -> Tuple[Ten rewards = rewards.unsqueeze(-1) if undones.dim() != states.dim(): undones = undones.unsqueeze(-1) - '''compare with TD3 no policy noise''' - next_as = self.act_target(next_ss) # next actions - next_qs = self.cri_target(torch.cat((next_ss, next_as),dim=1)) # next q values + """compare with TD3 no policy noise""" + next_as = self.act_target(next_ss) # next actions + next_qs = self.cri_target(torch.cat((next_ss, next_as), dim=1)) # next q values q_labels = rewards + undones * self.gamma * next_qs - q_values = self.cri(torch.cat((states, actions),dim=1)) - obj_critic = self.criterion(q_values,q_labels) + q_values = self.cri(torch.cat((states, actions), dim=1)) + obj_critic = self.criterion(q_values, q_labels) return obj_critic, states def get_obj_critic_per(self, buffer: ReplayBuffer, batch_size: int) -> Tuple[Tensor, Tensor]: @@ -442,11 +456,11 @@ def get_obj_critic_per(self, buffer: ReplayBuffer, batch_size: int) -> Tuple[Ten undones = undones.unsqueeze(-1) next_as = self.act_target(next_ss) - next_qs = self.cri_target(torch.cat((next_ss, next_as),dim=1)) # next q values + next_qs = self.cri_target(torch.cat((next_ss, next_as), dim=1)) # next q values q_labels = rewards + undones * self.gamma * next_qs - q_values = self.cri(torch.cat((states, actions),dim=1)) + q_values = self.cri(torch.cat((states, actions), dim=1)) td_errors = self.criterion(q_values, q_labels) obj_critic = (td_errors * is_weights).mean() @@ -475,7 +489,7 @@ def explore_one_env(self, env, horizon_len: int, if_random: bool = False) -> [Te rewards = torch.zeros((horizon_len, self.num_envs), dtype=torch.float32).to(self.device) dones = torch.zeros((horizon_len, self.num_envs), dtype=torch.bool).to(self.device) - ary_state =env.reset() + ary_state = reset_env(env) get_action = self.act.get_action for i in range(horizon_len): state = torch.as_tensor(ary_state, dtype=torch.float32, device=self.device) @@ -485,8 +499,8 @@ def explore_one_env(self, env, horizon_len: int, if_random: bool = False) -> [Te actions[i] = action ary_action = action.detach().cpu().numpy() - next_state, reward, done,_ = env.step(ary_action) - ary_state = env.reset() if done else next_state + next_state, reward, done, _ = step_env(env, ary_action) + ary_state = reset_env(env) if done else next_state rewards[i] = reward dones[i] = done @@ -496,6 +510,7 @@ def explore_one_env(self, env, horizon_len: int, if_random: bool = False) -> [Te # undones = (1.0 - dones.type(torch.float32)).unsqueeze(1) return states, actions, rewards, undones + class ActorSAC(nn.Module): """ Actor network for Soft Actor-Critic (SAC) algorithm. @@ -540,18 +555,20 @@ def get_action_logprob(self, state: Tensor) -> [Tensor, Tensor]: action_std = action_log_std.exp() action_avg = self.dec_a_avg(state_tmp) - '''add noise to a_noise in stochastic policy''' + """add noise to a_noise in stochastic policy""" noise = torch.randn_like(action_avg, requires_grad=True) a_noise = action_avg + action_std * noise - '''compute log_prob according to mean and std of a_noise (stochastic policy)''' + """compute log_prob according to mean and std of a_noise (stochastic policy)""" # self.sqrt_2pi_log = np.log(np.sqrt(2 * np.pi)) log_prob = -action_log_std - noise.pow(2) * 0.5 - np.log(np.sqrt(2 * np.pi)) - '''fix logprob by adding the derivative of y=tanh(x)''' - log_prob -= (np.log(2.) - a_noise - self.soft_plus(-2. * a_noise)) * 2. + """fix logprob by adding the derivative of y=tanh(x)""" + log_prob -= (np.log(2.0) - a_noise - self.soft_plus(-2.0 * a_noise)) * 2.0 # logprob -= (1.000001 - action.tanh().pow(2)).log() return a_noise.tanh(), log_prob.sum(1, keepdim=True) + + class CriticTwin(nn.Module): """ Twin Critic network for algorithms like SAC and TD3. @@ -579,6 +596,8 @@ def forward(self, value: Tensor) -> Tensor: def get_q1_q2(self, value): sa_tmp = self.enc_sa(value) return self.dec_q1(sa_tmp), self.dec_q2(sa_tmp) # two Q valuesplease show me all + + class AgentSAC(AgentBase): """ Soft Actor-Critic (SAC) agent implementation. @@ -599,8 +618,8 @@ class AgentSAC(AgentBase): """ def __init__(self, net_dims: [int], state_dim: int, action_dim: int, gpu_id: int = 0, args: Config = Config()): - self.act_class = getattr(self, 'act_class', ActorSAC) # get the attribute of object `self` - self.cri_class = getattr(self, 'cri_class', CriticTwin) # get the attribute of object `self` + self.act_class = getattr(self, "act_class", ActorSAC) # get the attribute of object `self` + self.cri_class = getattr(self, "cri_class", CriticTwin) # get the attribute of object `self` super().__init__(net_dims, state_dim, action_dim, gpu_id, args) self.cri_target = deepcopy(self.cri) @@ -629,7 +648,7 @@ def explore_one_env(self, env, horizon_len: int, if_random: bool = False) -> [Te rewards = torch.zeros((horizon_len, self.num_envs), dtype=torch.float32).to(self.device) dones = torch.zeros((horizon_len, self.num_envs), dtype=torch.bool).to(self.device) - ary_state =env.reset() + ary_state = reset_env(env) get_action = self.act.get_action for i in range(horizon_len): state = torch.as_tensor(ary_state, dtype=torch.float32, device=self.device) @@ -639,8 +658,8 @@ def explore_one_env(self, env, horizon_len: int, if_random: bool = False) -> [Te actions[i] = action ary_action = action.detach().cpu().numpy() - next_state, reward, done,_ = env.step(ary_action) - ary_state = env.reset() if done else next_state + next_state, reward, done, _ = step_env(env, ary_action) + ary_state = reset_env(env) if done else next_state rewards[i] = reward dones[i] = done @@ -667,33 +686,32 @@ def update_net(self, buffer: ReplayBuffer) -> Tuple[float, ...]: obj_actors = 0.0 alphas = 0.0 - update_times = int(buffer.cur_size * self.repeat_times/self.batch_size) + update_times = int(buffer.cur_size * self.repeat_times / self.batch_size) assert update_times >= 1 for _ in range(update_times): - '''objective of critic (loss function of critic)''' + """objective of critic (loss function of critic)""" obj_critic, state = self.get_obj_critic(buffer, self.batch_size) obj_critics += obj_critic.item() self.optimizer_update(self.cri_optimizer, obj_critic) self.soft_update(self.cri_target, self.cri, self.soft_update_tau) - '''objective of alpha (temperature parameter automatic adjustment)''' + """objective of alpha (temperature parameter automatic adjustment)""" action_pg, log_prob = self.act.get_action_logprob(state) # policy gradient obj_alpha = (self.alpha_log * (-log_prob + self.target_entropy).detach()).mean() self.optimizer_update(self.alpha_optimizer, obj_alpha) - '''objective of actor''' + """objective of actor""" alpha = self.alpha_log.exp().detach() alphas += alpha.item() # # keep the log into clip domains. # with torch.no_grad(): # self.alpha_log[:] = self.alpha_log.clamp(-16, 2) - '''here is the objective of actor changes''' + """here is the objective of actor changes""" obj_actor = (self.cri(torch.cat((state, action_pg), dim=1)) - log_prob * alpha).mean() obj_actors += obj_actor.item() self.optimizer_update(self.act_optimizer, -obj_actor) return obj_critics / update_times, obj_actors / update_times, alphas / update_times - def get_obj_critic_raw(self, buffer, batch_size: int) -> Tuple[Tensor, Tensor]: with torch.no_grad(): states, actions, rewards, undones, next_ss = buffer.sample(batch_size) # next_ss: next states @@ -703,14 +721,14 @@ def get_obj_critic_raw(self, buffer, batch_size: int) -> Tuple[Tensor, Tensor]: if undones.dim() != states.dim(): undones = undones.unsqueeze(-1) next_as, next_logprobs = self.act.get_action_logprob(next_ss) # next actions - '''here is how to calculate the next qs and q labels''' - next_qs = torch.min(*self.cri_target.get_q1_q2(torch.cat((next_ss, next_as),dim=1))) + """here is how to calculate the next qs and q labels""" + next_qs = torch.min(*self.cri_target.get_q1_q2(torch.cat((next_ss, next_as), dim=1))) alpha = self.alpha_log.exp() q_labels = rewards + undones * self.gamma * (next_qs - next_logprobs * alpha) - q1, q2 = self.cri.get_q1_q2(torch.cat((states, actions),dim=1)) + q1, q2 = self.cri.get_q1_q2(torch.cat((states, actions), dim=1)) - obj_critic = self.criterion(q1, q_labels) + self.criterion(q2, q_labels)/2. # twin critics + obj_critic = self.criterion(q1, q_labels) + self.criterion(q2, q_labels) / 2.0 # twin critics return obj_critic, states def get_obj_critic_per(self, buffer: ReplayBuffer, batch_size: int) -> Tuple[Tensor, Tensor]: @@ -722,33 +740,39 @@ def get_obj_critic_per(self, buffer: ReplayBuffer, batch_size: int) -> Tuple[Ten if undones.dim() != states.dim(): undones = undones.unsqueeze(-1) next_as, next_logprobs = self.act.get_action_logprob(next_ss) - next_qs = torch.min(*self.cri_target.get_q1_q2(torch.cat((next_ss, next_as),dim=1))) + next_qs = torch.min(*self.cri_target.get_q1_q2(torch.cat((next_ss, next_as), dim=1))) alpha = self.alpha_log.exp() q_labels = rewards + undones * self.gamma * (next_qs - next_logprobs * alpha) - q1, q2 = self.cri.get_q1_q2(torch.cat((states, actions),dim=1)) + q1, q2 = self.cri.get_q1_q2(torch.cat((states, actions), dim=1)) - td_errors = (self.criterion(q1, q_labels) + self.criterion(q2, q_labels))/2. + td_errors = (self.criterion(q1, q_labels) + self.criterion(q2, q_labels)) / 2.0 obj_critic = (td_errors * is_weights).mean() buffer.td_error_update_for_per(is_indices.detach(), td_errors.detach()) return obj_critic, states -class Actor_TD3(nn.Module): + +class Actor_TD3(nn.Module): def __init__(self, dims: [int], state_dim: int, action_dim: int): super().__init__() self.net = build_mlp(dims=[state_dim, *dims, action_dim]) self.explore_noise_std = None # standard deviation of exploration action noise + def forward(self, state: Tensor) -> Tensor: return self.net(state).tanh() # action.tanh() + def get_action(self, state: Tensor) -> Tensor: # for exploration action = self.net(state).tanh() noise = (torch.randn_like(action) * self.explore_noise_std).clamp(-0.5, 0.5) return (action + noise).clamp(-1.0, 1.0) + def get_action_noise(self, state: Tensor, action_std: float) -> Tensor: action = self.net(state).tanh() noise = (torch.randn_like(action) * action_std).clamp(-0.5, 0.5) return (action + noise).clamp(-1.0, 1.0) + + class AgentTD3(AgentBase): """ Twin Delayed Deep Deterministic Policy Gradient (TD3) agent implementation. @@ -770,15 +794,15 @@ class AgentTD3(AgentBase): """ def __init__(self, net_dims: [int], state_dim: int, action_dim: int, gpu_id: int = 0, args: Config = Config()): - self.act_class = getattr(self, 'act_class', Actor_TD3) - self.cri_class = getattr(self, 'cri_class', CriticTwin) + self.act_class = getattr(self, "act_class", Actor_TD3) + self.cri_class = getattr(self, "cri_class", CriticTwin) super().__init__(net_dims=net_dims, state_dim=state_dim, action_dim=action_dim, gpu_id=gpu_id, args=args) self.act_target = deepcopy(self.act) self.cri_target = deepcopy(self.cri) - self.explore_noise_std = getattr(args, 'explore_noise_std', 0.05) # standard deviation of exploration noise - self.policy_noise_std = getattr(args, 'policy_noise_std', 0.10) # standard deviation of exploration noise - self.update_freq = getattr(args, 'update_freq', 2) # delay update frequency + self.explore_noise_std = getattr(args, "explore_noise_std", 0.05) # standard deviation of exploration noise + self.policy_noise_std = getattr(args, "policy_noise_std", 0.10) # standard deviation of exploration noise + self.update_freq = getattr(args, "update_freq", 2) # delay update frequency self.act.explore_noise_std = self.explore_noise_std # assign explore_noise_std for agent.act.get_action(state) @@ -798,7 +822,7 @@ def update_net(self, buffer: ReplayBuffer) -> Tuple[float, ...]: obj_critics = 0.0 obj_actors = 0.0 # update_times = int(buffer.add_size * self.repeat_times) - update_times = int(buffer.cur_size * self.repeat_times/self.batch_size) + update_times = int(buffer.cur_size * self.repeat_times / self.batch_size) assert update_times >= 1 for update_c in range(update_times): obj_critic, state = self.get_obj_critic(buffer, self.batch_size) @@ -823,11 +847,11 @@ def get_obj_critic_raw(self, buffer: ReplayBuffer, batch_size: int) -> Tuple[Ten if undones.dim() != states.dim(): undones = undones.unsqueeze(-1) next_as = self.act_target.get_action_noise(next_ss, self.policy_noise_std) # next actions - next_qs = torch.min(*self.cri_target.get_q1_q2(torch.cat((next_ss, next_as),dim=1))) + next_qs = torch.min(*self.cri_target.get_q1_q2(torch.cat((next_ss, next_as), dim=1))) q_labels = rewards + undones * self.gamma * next_qs - q1, q2 = self.cri.get_q1_q2(torch.cat((states, actions),dim=1)) + q1, q2 = self.cri.get_q1_q2(torch.cat((states, actions), dim=1)) obj_critic = self.criterion(q1, q_labels) + self.criterion(q2, q_labels) # twin critics return obj_critic, states @@ -841,10 +865,10 @@ def get_obj_critic_per(self, buffer: ReplayBuffer, batch_size: int) -> Tuple[Ten if undones.dim() != states.dim(): undones = undones.unsqueeze(-1) next_as = self.act_target.get_action_noise(next_ss, self.policy_noise_std) - next_qs = torch.min(*self.cri_target.get_q1_q2(torch.cat((next_ss, next_as),dim=1))) + next_qs = torch.min(*self.cri_target.get_q1_q2(torch.cat((next_ss, next_as), dim=1))) q_labels = rewards + undones * self.gamma * next_qs - q1, q2 = self.cri.get_q1_q2(torch.cat((states, actions),dim=1)) + q1, q2 = self.cri.get_q1_q2(torch.cat((states, actions), dim=1)) td_errors = self.criterion(q1, q_labels) + self.criterion(q2, q_labels) obj_critic = (td_errors * is_weights).mean() @@ -871,7 +895,7 @@ def explore_one_env(self, env, horizon_len: int, if_random: bool = False) -> [Te rewards = torch.zeros((horizon_len, self.num_envs), dtype=torch.float32).to(self.device) dones = torch.zeros((horizon_len, self.num_envs), dtype=torch.bool).to(self.device) - ary_state =env.reset() + ary_state = reset_env(env) get_action = self.act.get_action for i in range(horizon_len): state = torch.as_tensor(ary_state, dtype=torch.float32, device=self.device) @@ -881,8 +905,8 @@ def explore_one_env(self, env, horizon_len: int, if_random: bool = False) -> [Te actions[i] = action ary_action = action.detach().cpu().numpy() - next_state, reward, done,_ = env.step(ary_action) - ary_state = env.reset() if done else next_state + next_state, reward, done, _ = step_env(env, ary_action) + ary_state = reset_env(env) if done else next_state rewards[i] = reward dones[i] = done @@ -890,4 +914,4 @@ def explore_one_env(self, env, horizon_len: int, if_random: bool = False) -> [Te # rewards = rewards.unsqueeze(1) undones = 1.0 - dones.type(torch.float32) # undones = (1.0 - dones.type(torch.float32)).unsqueeze(1) - return states, actions, rewards, undones \ No newline at end of file + return states, actions, rewards, undones diff --git a/rl_adn/DRL_algorithms/DDPG.py b/rl_adn/algorithms/DDPG.py similarity index 88% rename from rl_adn/DRL_algorithms/DDPG.py rename to rl_adn/algorithms/DDPG.py index 910554a..29757f9 100644 --- a/rl_adn/DRL_algorithms/DDPG.py +++ b/rl_adn/algorithms/DDPG.py @@ -1,14 +1,15 @@ +from copy import deepcopy +from typing import Tuple + import torch import torch.onnx -import copy as cp -from copy import deepcopy -import os -from torch import nn, Tensor -from typing import Tuple, Union -from rl_adn.DRL_algorithms.utility import Config, ReplayBuffer, SumTree, build_mlp, get_episode_return, get_optim_param -from rl_adn.DRL_algorithms.Agent import AgentBase -from rl_adn.environments.env import PowerNetEnv, env_config -import time +from torch import Tensor, nn + +from rl_adn.algorithms.Agent import AgentBase +from rl_adn.algorithms.env_api import reset_env, step_env +from rl_adn.algorithms.utility import Config, ReplayBuffer, build_mlp + + class Actor(nn.Module): def __init__(self, dims: [int], state_dim: int, action_dim: int): """ @@ -26,6 +27,7 @@ def __init__(self, dims: [int], state_dim: int, action_dim: int): super().__init__() self.net = build_mlp(dims=[state_dim, *dims, action_dim]) self.explore_noise_std = None # standard deviation of exploration action noise + def forward(self, state: Tensor) -> Tensor: """ Defines the forward pass of the Actor network. @@ -37,6 +39,7 @@ def forward(self, state: Tensor) -> Tensor: Tensor: The output action tensor after applying the tanh activation function. """ return self.net(state).tanh() # action.tanh() + def get_action(self, state: Tensor) -> Tensor: """ Computes the action for a given state with added exploration noise. @@ -50,6 +53,7 @@ def get_action(self, state: Tensor) -> Tensor: action = self.net(state).tanh() noise = (torch.randn_like(action) * self.explore_noise_std).clamp(-0.5, 0.5) return (action + noise).clamp(-1.0, 1.0) + def get_action_noise(self, state: Tensor, action_std: float) -> Tensor: """ Computes the action for a given state with specified exploration noise. @@ -64,6 +68,8 @@ def get_action_noise(self, state: Tensor, action_std: float) -> Tensor: action = self.net(state).tanh() noise = (torch.randn_like(action) * action_std).clamp(-0.5, 0.5) return (action + noise).clamp(-1.0, 1.0) + + class Critic(nn.Module): def __init__(self, dims: [int], state_dim: int, action_dim: int): """ @@ -79,6 +85,7 @@ def __init__(self, dims: [int], state_dim: int, action_dim: int): """ super().__init__() self.net = build_mlp(dims=[state_dim + action_dim, *dims, 1]) + def forward(self, value: Tensor) -> Tensor: """ Defines the forward pass of the Critic network. @@ -90,6 +97,8 @@ def forward(self, value: Tensor) -> Tensor: Tensor: The output Q-value tensor. """ return self.net(value) # Q value + + class AgentDDPG(AgentBase): """ Implements the Twin Delayed Deep Deterministic Policy Gradient (TD3) algorithm. @@ -115,13 +124,13 @@ def __init__(self, net_dims: [int], state_dim: int, action_dim: int, gpu_id: int gpu_id (int): GPU ID for running the networks. Defaults to 0. args (Config): Configuration object with additional settings. """ - self.act_class = getattr(self, 'act_class', Actor) - self.cri_class = getattr(self, 'cri_class', Critic) + self.act_class = getattr(self, "act_class", Actor) + self.cri_class = getattr(self, "cri_class", Critic) super().__init__(net_dims=net_dims, state_dim=state_dim, action_dim=action_dim, gpu_id=gpu_id, args=args) self.act_target = deepcopy(self.act) self.cri_target = deepcopy(self.cri) - '''comapre to TD3, there is no policy noise''' - self.explore_noise_std = getattr(args, 'explore_noise_std', 0.05) # standard deviation of exploration noise + """comapre to TD3, there is no policy noise""" + self.explore_noise_std = getattr(args, "explore_noise_std", 0.05) # standard deviation of exploration noise self.act.explore_noise_std = self.explore_noise_std # assign explore_noise_std for agent.act.get_action(state) def update_net(self, buffer: ReplayBuffer) -> Tuple[float, ...]: @@ -137,14 +146,14 @@ def update_net(self, buffer: ReplayBuffer) -> Tuple[float, ...]: obj_critics = 0.0 obj_actors = 0.0 # update_times = int(buffer.add_size * self.repeat_times) - update_times = int(buffer.cur_size * self.repeat_times/self.batch_size) + update_times = int(buffer.cur_size * self.repeat_times / self.batch_size) assert update_times >= 1 for update_c in range(update_times): obj_critic, state = self.get_obj_critic(buffer, self.batch_size) obj_critics += obj_critic.item() self.optimizer_update(self.cri_optimizer, obj_critic) self.soft_update(self.cri_target, self.cri, self.soft_update_tau) - '''compare with TD3, DDPG no policy delay update''' + """compare with TD3, DDPG no policy delay update""" action_pg = self.act(state) # policy gradient obj_actor = self.cri_target(torch.cat((state, action_pg), dim=1)).mean() # use cri_target is more stable than cri obj_actors += obj_actor.item() @@ -171,13 +180,13 @@ def get_obj_critic_raw(self, buffer: ReplayBuffer, batch_size: int) -> Tuple[Ten rewards = rewards.unsqueeze(-1) if undones.dim() != states.dim(): undones = undones.unsqueeze(-1) - '''compare with TD3 no policy noise''' - next_as = self.act_target(next_ss) # next actions - next_qs = self.cri_target(torch.cat((next_ss, next_as),dim=1)) # next q values + """compare with TD3 no policy noise""" + next_as = self.act_target(next_ss) # next actions + next_qs = self.cri_target(torch.cat((next_ss, next_as), dim=1)) # next q values q_labels = rewards + undones * self.gamma * next_qs - q_values = self.cri(torch.cat((states, actions),dim=1)) - obj_critic = self.criterion(q_values,q_labels) + q_values = self.cri(torch.cat((states, actions), dim=1)) + obj_critic = self.criterion(q_values, q_labels) return obj_critic, states def get_obj_critic_per(self, buffer: ReplayBuffer, batch_size: int) -> Tuple[Tensor, Tensor]: @@ -201,16 +210,17 @@ def get_obj_critic_per(self, buffer: ReplayBuffer, batch_size: int) -> Tuple[Ten undones = undones.unsqueeze(-1) next_as = self.act_target(next_ss) - next_qs = self.cri_target(torch.cat((next_ss, next_as),dim=1)) # next q values + next_qs = self.cri_target(torch.cat((next_ss, next_as), dim=1)) # next q values q_labels = rewards + undones * self.gamma * next_qs - q_values = self.cri(torch.cat((states, actions),dim=1)) + q_values = self.cri(torch.cat((states, actions), dim=1)) td_errors = self.criterion(q_values, q_labels) obj_critic = (td_errors * is_weights).mean() buffer.td_error_update_for_per(is_indices.detach(), td_errors.detach()) return obj_critic, states + def explore_one_env(self, env, horizon_len: int, if_random: bool = False) -> [Tensor]: """ Explores the environment for a given number of steps. @@ -228,7 +238,7 @@ def explore_one_env(self, env, horizon_len: int, if_random: bool = False) -> [Te rewards = torch.zeros((horizon_len, self.num_envs), dtype=torch.float32).to(self.device) dones = torch.zeros((horizon_len, self.num_envs), dtype=torch.bool).to(self.device) - ary_state =env.reset() + ary_state = reset_env(env) get_action = self.act.get_action for i in range(horizon_len): state = torch.as_tensor(ary_state, dtype=torch.float32, device=self.device) @@ -238,8 +248,8 @@ def explore_one_env(self, env, horizon_len: int, if_random: bool = False) -> [Te actions[i] = action ary_action = action.detach().cpu().numpy() - next_state, reward, done,_ = env.step(ary_action) - ary_state = env.reset() if done else next_state + next_state, reward, done, _ = step_env(env, ary_action) + ary_state = reset_env(env) if done else next_state rewards[i] = reward dones[i] = done diff --git a/rl_adn/DRL_algorithms/PPO.py b/rl_adn/algorithms/PPO.py similarity index 91% rename from rl_adn/DRL_algorithms/PPO.py rename to rl_adn/algorithms/PPO.py index 9c395e8..8536b3b 100644 --- a/rl_adn/DRL_algorithms/PPO.py +++ b/rl_adn/algorithms/PPO.py @@ -1,17 +1,12 @@ +import os + import torch import torch.onnx - -import os -from torch import nn, Tensor +from torch import Tensor, nn from torch.distributions.normal import Normal -from typing import Tuple, Union -from rl_adn.DRL_algorithms.utility import Config, ReplayBuffer, SumTree, build_mlp, get_episode_return, get_optim_param -from rl_adn.DRL_algorithms.Agent import AgentBase -import time - -config_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../rl_adn', 'environments', 'env_config.py')) - +from rl_adn.algorithms.env_api import reset_env, step_env +from rl_adn.algorithms.utility import Config, build_mlp class ActorPPO(nn.Module): @@ -90,6 +85,7 @@ def convert_action_for_env(action: Tensor) -> Tensor: """ return action.tanh() + class CriticPPO(nn.Module): def __init__(self, dims: [int], state_dim: int, _action_dim: int): """ @@ -115,6 +111,7 @@ def forward(self, state: Tensor) -> Tensor: """ return self.net(state) # advantage value + class AgentBase: def __init__(self, net_dims: [int], state_dim: int, action_dim: int, gpu_id: int = 0, args: Config = Config()): self.state_dim = state_dim @@ -134,16 +131,14 @@ def __init__(self, net_dims: [int], state_dim: int, action_dim: int, gpu_id: int act_class = getattr(self, "act_class", None) cri_class = getattr(self, "cri_class", None) self.act = self.act_target = act_class(net_dims, state_dim, action_dim).to(self.device) - self.cri = self.cri_target = cri_class(net_dims, state_dim, action_dim).to(self.device) \ - if cri_class else self.act + self.cri = self.cri_target = cri_class(net_dims, state_dim, action_dim).to(self.device) if cri_class else self.act self.act_optimizer = torch.optim.Adam(self.act.parameters(), self.learning_rate) - self.cri_optimizer = torch.optim.Adam(self.cri.parameters(), self.learning_rate) \ - if cri_class else self.act_optimizer + self.cri_optimizer = torch.optim.Adam(self.cri.parameters(), self.learning_rate) if cri_class else self.act_optimizer self.criterion = torch.nn.SmoothL1Loss() """save and load""" - self.save_attr_names = {'act', 'act_target', 'act_optimizer', 'cri', 'cri_target', 'cri_optimizer'} + self.save_attr_names = {"act", "act_target", "act_optimizer", "cri", "cri_target", "cri_optimizer"} @staticmethod def optimizer_update(optimizer, objective: Tensor): @@ -156,13 +151,14 @@ def soft_update(target_net: torch.nn.Module, current_net: torch.nn.Module, tau: # assert target_net is not current_net for tar, cur in zip(target_net.parameters(), current_net.parameters()): tar.data.copy_(cur.data * tau + tar.data * (1.0 - tau)) + def save_or_load_agent(self, cwd: str, if_save: bool): """save or load training files for Agent cwd: Current Working Directory. ElegantRL save training files in CWD. if_save: True: save files. False: load files. """ - assert self.save_attr_names.issuperset({'act', 'act_target', 'act_optimizer'}) + assert self.save_attr_names.issuperset({"act", "act_target", "act_optimizer"}) for attr_name in self.save_attr_names: file_path = f"{cwd}/{attr_name}.pth" @@ -172,8 +168,8 @@ def save_or_load_agent(self, cwd: str, if_save: bool): elif os.path.isfile(file_path): setattr(self, attr_name, torch.load(file_path, map_location=self.device)) -class AgentPPO(AgentBase): +class AgentPPO(AgentBase): def __init__(self, net_dims: [int], state_dim: int, action_dim: int, gpu_id: int = 0, args: Config = Config()): """ Initializes the Proximal Policy Optimization (PPO) Agent. @@ -212,7 +208,7 @@ def explore_env(self, env, horizon_len: int) -> [Tensor]: rewards = torch.zeros(horizon_len, dtype=torch.float32).to(self.device) dones = torch.zeros(horizon_len, dtype=torch.bool).to(self.device) - ary_state = env.reset() + ary_state = reset_env(env) get_action = self.act.get_action convert = self.act.convert_action_for_env @@ -221,8 +217,8 @@ def explore_env(self, env, horizon_len: int) -> [Tensor]: action, logprob = [t.squeeze(0) for t in get_action(state.unsqueeze(0))[:2]] ary_action = convert(action).detach().cpu().numpy() - next_state, reward, done, _ = env.step(ary_action) - ary_state = env.reset() if done else next_state + next_state, reward, done, _ = step_env(env, ary_action) + ary_state = reset_env(env) if done else next_state states[i] = state actions[i] = action @@ -248,9 +244,9 @@ def update_net(self, buffer) -> [float]: states, actions, logprobs, rewards, undones = buffer buffer_size = states.shape[0] - '''get advantages reward_sums''' - bs =4096 # set a smaller 'batch_size' when out of GPU memory. - values = [self.cri(states[i:i + bs]) for i in range(0, buffer_size, bs)] + """get advantages reward_sums""" + bs = 4096 # set a smaller 'batch_size' when out of GPU memory. + values = [self.cri(states[i : i + bs]) for i in range(0, buffer_size, bs)] values = torch.cat(values, dim=0).squeeze(1) # values.shape == (buffer_size, ) advantages = self.get_advantages(rewards, undones, values) # advantages.shape == (buffer_size, ) @@ -260,7 +256,7 @@ def update_net(self, buffer) -> [float]: advantages = (advantages - advantages.mean()) / (advantages.std(dim=0) + 1e-5) assert logprobs.shape == advantages.shape == reward_sums.shape == (buffer_size,) - '''update network''' + """update network""" obj_critics = 0.0 obj_actors = 0.0 @@ -289,7 +285,7 @@ def update_net(self, buffer) -> [float]: obj_critics += obj_critic.item() obj_actors += obj_actor.item() - a_std_log = getattr(self.act, 'a_std_log', torch.zeros(1)).mean() + a_std_log = getattr(self.act, "a_std_log", torch.zeros(1)).mean() return obj_critics / update_times, obj_actors / update_times, a_std_log.item() def get_advantages(self, rewards: Tensor, undones: Tensor, values: Tensor) -> Tensor: diff --git a/rl_adn/DRL_algorithms/SAC.py b/rl_adn/algorithms/SAC.py similarity index 91% rename from rl_adn/DRL_algorithms/SAC.py rename to rl_adn/algorithms/SAC.py index c645906..e0151ae 100644 --- a/rl_adn/DRL_algorithms/SAC.py +++ b/rl_adn/algorithms/SAC.py @@ -1,14 +1,15 @@ -import torch +from copy import deepcopy +from typing import Tuple + import numpy as np -import torch.onnx +import torch import torch.nn as nn -import copy as cp -from copy import deepcopy -import os -from torch import nn, Tensor -from typing import Tuple, Union -from rl_adn.DRL_algorithms.utility import Config, ReplayBuffer, SumTree, build_mlp, get_episode_return, get_optim_param -from rl_adn.DRL_algorithms.Agent import AgentBase +import torch.onnx +from torch import Tensor + +from rl_adn.algorithms.Agent import AgentBase +from rl_adn.algorithms.env_api import reset_env, step_env +from rl_adn.algorithms.utility import Config, ReplayBuffer, build_mlp ## define net @@ -28,6 +29,7 @@ class ActorSAC(nn.Module): get_action(state): Computes the action for exploration. get_action_logprob(state): Computes the action and its log probability for a given state. """ + def __init__(self, dims: [int], state_dim: int, action_dim: int): """ Initializes the Actor network for SAC. @@ -93,18 +95,20 @@ def get_action_logprob(self, state: Tensor) -> Tuple[Tensor, Tensor]: action_std = action_log_std.exp() action_avg = self.dec_a_avg(state_tmp) - '''add noise to a_noise in stochastic policy''' + """add noise to a_noise in stochastic policy""" noise = torch.randn_like(action_avg, requires_grad=True) a_noise = action_avg + action_std * noise - '''compute log_prob according to mean and std of a_noise (stochastic policy)''' + """compute log_prob according to mean and std of a_noise (stochastic policy)""" # self.sqrt_2pi_log = np.log(np.sqrt(2 * np.pi)) log_prob = -action_log_std - noise.pow(2) * 0.5 - np.log(np.sqrt(2 * np.pi)) - '''fix logprob by adding the derivative of y=tanh(x)''' - log_prob -= (np.log(2.) - a_noise - self.soft_plus(-2. * a_noise)) * 2. + """fix logprob by adding the derivative of y=tanh(x)""" + log_prob -= (np.log(2.0) - a_noise - self.soft_plus(-2.0 * a_noise)) * 2.0 # logprob -= (1.000001 - action.tanh().pow(2)).log() return a_noise.tanh(), log_prob.sum(1, keepdim=True) + + class CriticTwin(nn.Module): """ Twin Critic network for algorithms like SAC and TD3. @@ -118,6 +122,7 @@ class CriticTwin(nn.Module): forward(value): Computes the first Q value for a given state-action pair. get_q1_q2(value): Computes both Q values for a given state-action pair. """ + def __init__(self, dims: [int], state_dim: int, action_dim: int): """ Initializes the Twin Critic network. @@ -158,6 +163,7 @@ def get_q1_q2(self, value: Tensor) -> Tuple[Tensor, Tensor]: sa_tmp = self.enc_sa(value) return self.dec_q1(sa_tmp), self.dec_q2(sa_tmp) # two Q values + class AgentSAC(AgentBase): """ Soft Actor-Critic (SAC) agent implementation. @@ -176,10 +182,11 @@ class AgentSAC(AgentBase): get_obj_critic_raw(buffer, batch_size): Computes the raw objective for the critic. get_obj_critic_per(buffer, batch_size): Computes the PER-adjusted objective for the critic. """ + def __init__(self, net_dims: [int], state_dim: int, action_dim: int, gpu_id: int = 0, args: Config = Config()): - self.act_class = getattr(self, 'act_class', ActorSAC) # get the attribute of object `self` - self.cri_class = getattr(self, 'cri_class', CriticTwin) # get the attribute of object `self` + self.act_class = getattr(self, "act_class", ActorSAC) # get the attribute of object `self` + self.cri_class = getattr(self, "cri_class", CriticTwin) # get the attribute of object `self` super().__init__(net_dims, state_dim, action_dim, gpu_id, args) self.cri_target = deepcopy(self.cri) @@ -207,7 +214,7 @@ def explore_one_env(self, env, horizon_len: int, if_random: bool = False) -> [Te rewards = torch.zeros((horizon_len, self.num_envs), dtype=torch.float32).to(self.device) dones = torch.zeros((horizon_len, self.num_envs), dtype=torch.bool).to(self.device) - ary_state =env.reset() + ary_state = reset_env(env) get_action = self.act.get_action for i in range(horizon_len): state = torch.as_tensor(ary_state, dtype=torch.float32, device=self.device) @@ -217,8 +224,8 @@ def explore_one_env(self, env, horizon_len: int, if_random: bool = False) -> [Te actions[i] = action ary_action = action.detach().cpu().numpy() - next_state, reward, done,_ = env.step(ary_action) - ary_state = env.reset() if done else next_state + next_state, reward, done, _ = step_env(env, ary_action) + ary_state = reset_env(env) if done else next_state rewards[i] = reward dones[i] = done @@ -240,32 +247,32 @@ def update_net(self, buffer: ReplayBuffer) -> Tuple[float, ...]: Returns: Tuple[float, float, float]: A tuple containing the average objective values for the critic, actor, and alpha (temperature parameter) updates. """ - '''update network''' + """update network""" obj_critics = 0.0 obj_actors = 0.0 alphas = 0.0 - update_times = int(buffer.cur_size * self.repeat_times/self.batch_size) + update_times = int(buffer.cur_size * self.repeat_times / self.batch_size) assert update_times >= 1 for _ in range(update_times): - '''objective of critic (loss function of critic)''' + """objective of critic (loss function of critic)""" obj_critic, state = self.get_obj_critic(buffer, self.batch_size) obj_critics += obj_critic.item() self.optimizer_update(self.cri_optimizer, obj_critic) self.soft_update(self.cri_target, self.cri, self.soft_update_tau) - '''objective of alpha (temperature parameter automatic adjustment)''' + """objective of alpha (temperature parameter automatic adjustment)""" action_pg, log_prob = self.act.get_action_logprob(state) # policy gradient obj_alpha = (self.alpha_log * (-log_prob + self.target_entropy).detach()).mean() self.optimizer_update(self.alpha_optimizer, obj_alpha) - '''objective of actor''' + """objective of actor""" alpha = self.alpha_log.exp().detach() alphas += alpha.item() # # keep the log into clip domains. # with torch.no_grad(): # self.alpha_log[:] = self.alpha_log.clamp(-16, 2) - '''here is the objective of actor changes''' + """here is the objective of actor changes""" obj_actor = (self.cri(torch.cat((state, action_pg), dim=1)) - log_prob * alpha).mean() obj_actors += obj_actor.item() self.optimizer_update(self.act_optimizer, -obj_actor) @@ -292,14 +299,14 @@ def get_obj_critic_raw(self, buffer, batch_size: int) -> Tuple[Tensor, Tensor]: if undones.dim() != states.dim(): undones = undones.unsqueeze(-1) next_as, next_logprobs = self.act.get_action_logprob(next_ss) # next actions - '''here is how to calculate the next qs and q labels''' - next_qs = torch.min(*self.cri_target.get_q1_q2(torch.cat((next_ss, next_as),dim=1))) + """here is how to calculate the next qs and q labels""" + next_qs = torch.min(*self.cri_target.get_q1_q2(torch.cat((next_ss, next_as), dim=1))) alpha = self.alpha_log.exp() q_labels = rewards + undones * self.gamma * (next_qs - next_logprobs * alpha) - q1, q2 = self.cri.get_q1_q2(torch.cat((states, actions),dim=1)) + q1, q2 = self.cri.get_q1_q2(torch.cat((states, actions), dim=1)) - obj_critic = self.criterion(q1, q_labels) + self.criterion(q2, q_labels)/2. # twin critics + obj_critic = self.criterion(q1, q_labels) + self.criterion(q2, q_labels) / 2.0 # twin critics return obj_critic, states def get_obj_critic_per(self, buffer: ReplayBuffer, batch_size: int) -> Tuple[Tensor, Tensor]: @@ -323,13 +330,13 @@ def get_obj_critic_per(self, buffer: ReplayBuffer, batch_size: int) -> Tuple[Ten if undones.dim() != states.dim(): undones = undones.unsqueeze(-1) next_as, next_logprobs = self.act.get_action_logprob(next_ss) - next_qs = torch.min(*self.cri_target.get_q1_q2(torch.cat((next_ss, next_as),dim=1))) + next_qs = torch.min(*self.cri_target.get_q1_q2(torch.cat((next_ss, next_as), dim=1))) alpha = self.alpha_log.exp() q_labels = rewards + undones * self.gamma * (next_qs - next_logprobs * alpha) - q1, q2 = self.cri.get_q1_q2(torch.cat((states, actions),dim=1)) + q1, q2 = self.cri.get_q1_q2(torch.cat((states, actions), dim=1)) - td_errors = (self.criterion(q1, q_labels) + self.criterion(q2, q_labels))/2. + td_errors = (self.criterion(q1, q_labels) + self.criterion(q2, q_labels)) / 2.0 obj_critic = (td_errors * is_weights).mean() buffer.td_error_update_for_per(is_indices.detach(), td_errors.detach()) diff --git a/rl_adn/DRL_algorithms/TD3.py b/rl_adn/algorithms/TD3.py similarity index 90% rename from rl_adn/DRL_algorithms/TD3.py rename to rl_adn/algorithms/TD3.py index 1a0a2f2..4a20cd4 100644 --- a/rl_adn/DRL_algorithms/TD3.py +++ b/rl_adn/algorithms/TD3.py @@ -1,14 +1,14 @@ +from copy import deepcopy +from typing import Tuple + import torch -import numpy as np -import torch.onnx import torch.nn as nn -import copy as cp -from copy import deepcopy -import os -from torch import nn, Tensor -from typing import Tuple, Union -from rl_adn.DRL_algorithms.utility import Config, ReplayBuffer, SumTree, build_mlp, get_episode_return, get_optim_param -from rl_adn.DRL_algorithms.Agent import AgentBase +import torch.onnx +from torch import Tensor + +from rl_adn.algorithms.Agent import AgentBase +from rl_adn.algorithms.env_api import reset_env, step_env +from rl_adn.algorithms.utility import Config, ReplayBuffer, build_mlp class CriticTwin(nn.Module): @@ -38,17 +38,22 @@ def forward(self, value: Tensor) -> Tensor: def get_q1_q2(self, value): sa_tmp = self.enc_sa(value) return self.dec_q1(sa_tmp), self.dec_q2(sa_tmp) # two Q values + + class Actor_TD3(nn.Module): def __init__(self, dims: [int], state_dim: int, action_dim: int): super().__init__() self.net = build_mlp(dims=[state_dim, *dims, action_dim]) self.explore_noise_std = None # standard deviation of exploration action noise + def forward(self, state: Tensor) -> Tensor: return self.net(state).tanh() # action.tanh() + def get_action(self, state: Tensor) -> Tensor: # for exploration action = self.net(state).tanh() noise = (torch.randn_like(action) * self.explore_noise_std).clamp(-0.5, 0.5) return (action + noise).clamp(-1.0, 1.0) + def get_action_noise(self, state: Tensor, action_std: float) -> Tensor: action = self.net(state).tanh() noise = (torch.randn_like(action) * action_std).clamp(-0.5, 0.5) @@ -76,15 +81,15 @@ class AgentTD3(AgentBase): """ def __init__(self, net_dims: [int], state_dim: int, action_dim: int, gpu_id: int = 0, args: Config = Config()): - self.act_class = getattr(self, 'act_class', Actor_TD3) - self.cri_class = getattr(self, 'cri_class', CriticTwin) + self.act_class = getattr(self, "act_class", Actor_TD3) + self.cri_class = getattr(self, "cri_class", CriticTwin) super().__init__(net_dims=net_dims, state_dim=state_dim, action_dim=action_dim, gpu_id=gpu_id, args=args) self.act_target = deepcopy(self.act) self.cri_target = deepcopy(self.cri) - self.explore_noise_std = getattr(args, 'explore_noise_std', 0.05) # standard deviation of exploration noise - self.policy_noise_std = getattr(args, 'policy_noise_std', 0.10) # standard deviation of exploration noise - self.update_freq = getattr(args, 'update_freq', 2) # delay update frequency + self.explore_noise_std = getattr(args, "explore_noise_std", 0.05) # standard deviation of exploration noise + self.policy_noise_std = getattr(args, "policy_noise_std", 0.10) # standard deviation of exploration noise + self.update_freq = getattr(args, "update_freq", 2) # delay update frequency self.act.explore_noise_std = self.explore_noise_std # assign explore_noise_std for agent.act.get_action(state) @@ -103,7 +108,7 @@ def update_net(self, buffer: ReplayBuffer) -> Tuple[float, ...]: obj_critics = 0.0 obj_actors = 0.0 - update_times = int(buffer.cur_size * self.repeat_times/self.batch_size) + update_times = int(buffer.cur_size * self.repeat_times / self.batch_size) assert update_times >= 1 for update_c in range(update_times): obj_critic, state = self.get_obj_critic(buffer, self.batch_size) @@ -127,11 +132,11 @@ def get_obj_critic_raw(self, buffer: ReplayBuffer, batch_size: int) -> Tuple[Ten if undones.dim() != states.dim(): undones = undones.unsqueeze(-1) next_as = self.act_target.get_action_noise(next_ss, self.policy_noise_std) # next actions - next_qs = torch.min(*self.cri_target.get_q1_q2(torch.cat((next_ss, next_as),dim=1))) + next_qs = torch.min(*self.cri_target.get_q1_q2(torch.cat((next_ss, next_as), dim=1))) q_labels = rewards + undones * self.gamma * next_qs - q1, q2 = self.cri.get_q1_q2(torch.cat((states, actions),dim=1)) + q1, q2 = self.cri.get_q1_q2(torch.cat((states, actions), dim=1)) obj_critic = self.criterion(q1, q_labels) + self.criterion(q2, q_labels) # twin critics return obj_critic, states @@ -143,15 +148,16 @@ def get_obj_critic_per(self, buffer: ReplayBuffer, batch_size: int) -> Tuple[Ten if undones.dim() != states.dim(): undones = undones.unsqueeze(-1) next_as = self.act_target.get_action_noise(next_ss, self.policy_noise_std) - next_qs = torch.min(*self.cri_target.get_q1_q2(torch.cat((next_ss, next_as),dim=1))) + next_qs = torch.min(*self.cri_target.get_q1_q2(torch.cat((next_ss, next_as), dim=1))) q_labels = rewards + undones * self.gamma * next_qs - q1, q2 = self.cri.get_q1_q2(torch.cat((states, actions),dim=1)) + q1, q2 = self.cri.get_q1_q2(torch.cat((states, actions), dim=1)) td_errors = self.criterion(q1, q_labels) + self.criterion(q2, q_labels) obj_critic = (td_errors * is_weights).mean() buffer.td_error_update_for_per(is_indices.detach(), td_errors.detach()) return obj_critic, states + def explore_one_env(self, env, horizon_len: int, if_random: bool = False) -> [Tensor]: """ Explores a given environment for a specified horizon length. @@ -172,7 +178,7 @@ def explore_one_env(self, env, horizon_len: int, if_random: bool = False) -> [Te rewards = torch.zeros((horizon_len, self.num_envs), dtype=torch.float32).to(self.device) dones = torch.zeros((horizon_len, self.num_envs), dtype=torch.bool).to(self.device) - ary_state =env.reset() + ary_state = reset_env(env) get_action = self.act.get_action for i in range(horizon_len): state = torch.as_tensor(ary_state, dtype=torch.float32, device=self.device) @@ -182,8 +188,8 @@ def explore_one_env(self, env, horizon_len: int, if_random: bool = False) -> [Te actions[i] = action ary_action = action.detach().cpu().numpy() - next_state, reward, done,_ = env.step(ary_action) - ary_state = env.reset() if done else next_state + next_state, reward, done, _ = step_env(env, ary_action) + ary_state = reset_env(env) if done else next_state rewards[i] = reward dones[i] = done diff --git a/rl_adn/algorithms/__init__.py b/rl_adn/algorithms/__init__.py new file mode 100644 index 0000000..39f3ec5 --- /dev/null +++ b/rl_adn/algorithms/__init__.py @@ -0,0 +1,13 @@ +"""RL algorithm exports for RL-ADN.""" + +from rl_adn.algorithms.DDPG import AgentDDPG +from rl_adn.algorithms.PPO import AgentPPO +from rl_adn.algorithms.SAC import AgentSAC +from rl_adn.algorithms.TD3 import AgentTD3 + +__all__ = [ + "AgentDDPG", + "AgentPPO", + "AgentSAC", + "AgentTD3", +] diff --git a/rl_adn/algorithms/env_api.py b/rl_adn/algorithms/env_api.py new file mode 100644 index 0000000..44d1375 --- /dev/null +++ b/rl_adn/algorithms/env_api.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from typing import Any + +import numpy as np + + +def reset_env(env) -> np.ndarray: + reset_result = env.reset() + if isinstance(reset_result, tuple) and len(reset_result) == 2: + observation, _info = reset_result + return observation + return reset_result + + +def step_env(env, action: np.ndarray) -> tuple[np.ndarray, float, bool, dict[str, Any]]: + step_result = env.step(action) + if len(step_result) == 5: + observation, reward, terminated, truncated, info = step_result + return observation, float(reward), bool(terminated or truncated), info + observation, reward, done, info = step_result + return observation, float(reward), bool(done), info diff --git a/rl_adn/DRL_algorithms/utility.py b/rl_adn/algorithms/utility.py similarity index 83% rename from rl_adn/DRL_algorithms/utility.py rename to rl_adn/algorithms/utility.py index 7e6147d..36709f8 100644 --- a/rl_adn/DRL_algorithms/utility.py +++ b/rl_adn/algorithms/utility.py @@ -1,10 +1,13 @@ -import numpy as np -import torch +import math import os -from torch import nn, Tensor from typing import Tuple -import math -from torch.nn.utils import clip_grad_norm_ + +import numpy as np +import torch +from torch import Tensor, nn + +from rl_adn.algorithms.env_api import reset_env, step_env + class Config: """ @@ -51,34 +54,37 @@ class Config: print(): Prints the configuration in a readable format. to_dict(): Converts the configuration to a dictionary. """ + def __init__(self, agent_class=None, env_class=None, env_args=None): self.num_envs = None self.agent_class = agent_class # agent = agent_class(...) self.if_off_policy = self.get_if_off_policy() # whether off-policy or on-policy of DRL algorithm - '''Argument of environment''' + """Argument of environment""" self.env_class = env_class # env = env_class(**env_args) self.env_args = env_args # env = env_class(**env_args) if env_args is None: # dummy env_args - env_args = {'env_name': None, - 'num_envs': 1, - 'max_step': 96, - 'state_dim': None, - 'action_dim': None, - 'if_discrete': None, } - env_args.setdefault('num_envs', 1) # `num_envs=1` in default in single env. - env_args.setdefault('max_step', 96) # `max_step=12345` in default, which is a large enough value. - self.env_name = env_args['env_name'] # the name of environment. Be used to set 'cwd'. - self.num_envs = env_args['num_envs'] # the number of sub envs in vectorized env. `num_envs=1` in single env. - self.max_step = env_args['max_step'] # the max step number of an episode. 'set as 12345 in default. - self.state_dim = env_args['state_dim'] # vector dimension (feature number) of state - self.action_dim = env_args['action_dim'] # vector dimension (feature number) of action - self.if_discrete = env_args['if_discrete'] # discrete or continuous action space - '''Arguments for reward shaping''' + env_args = { + "env_name": None, + "num_envs": 1, + "max_step": 96, + "state_dim": None, + "action_dim": None, + "if_discrete": None, + } + env_args.setdefault("num_envs", 1) # `num_envs=1` in default in single env. + env_args.setdefault("max_step", 96) # `max_step=12345` in default, which is a large enough value. + self.env_name = env_args["env_name"] # the name of environment. Be used to set 'cwd'. + self.num_envs = env_args["num_envs"] # the number of sub envs in vectorized env. `num_envs=1` in single env. + self.max_step = env_args["max_step"] # the max step number of an episode. 'set as 12345 in default. + self.state_dim = env_args["state_dim"] # vector dimension (feature number) of state + self.action_dim = env_args["action_dim"] # vector dimension (feature number) of action + self.if_discrete = env_args["if_discrete"] # discrete or continuous action space + """Arguments for reward shaping""" self.gamma = 0.99 # discount factor of future rewards - self.reward_scale = 2 ** 0 # an approximate target reward usually be closed to 256 + self.reward_scale = 2**0 # an approximate target reward usually be closed to 256 - '''Arguments for training''' + """Arguments for training""" self.net_dims = (64, 32) # the middle layer dimension of MLP (MultiLayer Perceptron) self.learning_rate = 6e-5 # the learning rate for network updating self.clip_grad_norm = 3.0 # 0.1 ~ 4.0, clip the gradient after normalization @@ -96,23 +102,24 @@ def __init__(self, agent_class=None, env_class=None, env_args=None): self.buffer_size = None # ReplayBuffer size. Empty the ReplayBuffer for on-policy. self.repeat_times = 8.0 # repeatedly update network using ReplayBuffer to keep critic's loss small self.if_use_vtrace = False # use V-trace + GAE (Generalized Advantage Estimation) for sparse reward - self.random_seed=521 - self.num_episode=2000 + self.random_seed = 521 + self.num_episode = 2000 self.buffer_size = 500000 # capacity of replay buffer - '''Arguments for device''' + """Arguments for device""" self.gpu_id = int(0) # `int` means the ID of single GPU, -1 means CPU self.num_workers = 2 # rollout workers number pre GPU (adjust it to get high GPU usage) self.num_threads = 8 # cpu_num for pytorch, `torch.set_num_threads(self.num_threads)` self.random_seed = 0 # initialize random seed in self.init_before_training() self.learner_gpus = 0 # `int` means the ID of single GPU, -1 means CPU - '''arguments for creating data storage directory''' + """arguments for creating data storage directory""" - self.run_name=None - '''Arguments for save and plot issues''' + self.run_name = None + """Arguments for save and plot issues""" self.cwd = None # current work directory. None means set automatically self.if_remove = True # remove the cwd folder? (True, False, None:ask me) - self.train=True + self.train = True + def init_before_training(self): np.random.seed(self.random_seed) torch.manual_seed(self.random_seed) @@ -120,29 +127,34 @@ def init_before_training(self): torch.set_default_dtype(torch.float32) if self.cwd is None: agent_name = self.agent_class.__name__[5:] - self.cwd = f'./{agent_name}/{self.run_name}' + self.cwd = f"./{agent_name}/{self.run_name}" - '''remove history''' + """remove history""" if self.if_remove is None: - self.if_remove = bool(input(f"| Arguments PRESS 'y' to REMOVE: {self.cwd}? ") == 'y') + self.if_remove = bool(input(f"| Arguments PRESS 'y' to REMOVE: {self.cwd}? ") == "y") if self.if_remove: import shutil + shutil.rmtree(self.cwd, ignore_errors=True) print(f"| Arguments Remove cwd: {self.cwd}") else: print(f"| Arguments Keep cwd: {self.cwd}") os.makedirs(self.cwd, exist_ok=True) + def get_if_off_policy(self) -> bool: - agent_name = self.agent_class.__name__ if self.agent_class else '' - on_policy_names = ('SARSA', 'VPG', 'A2C', 'A3C', 'TRPO', 'PPO', 'MPO') + agent_name = self.agent_class.__name__ if self.agent_class else "" + on_policy_names = ("SARSA", "VPG", "A2C", "A3C", "TRPO", "PPO", "MPO") return all([agent_name.find(s) == -1 for s in on_policy_names]) def print(self): from pprint import pprint + pprint(vars(self)) # prints out args in a neat, readable format def to_dict(self): return vars(self) + + def get_optim_param(optimizer: torch.optim) -> list: # backup """ Extracts parameters from the optimizer state. @@ -157,6 +169,8 @@ def get_optim_param(optimizer: torch.optim) -> list: # backup for params_dict in optimizer.state_dict()["state"].values(): params_list.extend([t for t in params_dict.values() if isinstance(t, torch.Tensor)]) return params_list + + def build_mlp(dims: [int]) -> nn.Sequential: # MLP (MultiLayer Perceptron) """ Builds a Multi-Layer Perceptron (MLP) network. @@ -172,6 +186,8 @@ def build_mlp(dims: [int]) -> nn.Sequential: # MLP (MultiLayer Perceptron) net_list.extend([nn.Linear(dims[i], dims[i + 1]), nn.ReLU()]) del net_list[-1] # remove the activation of output layer return nn.Sequential(*net_list) + + class ReplayBuffer: # for off-policy """ Replay Buffer for storing and sampling experiences for off-policy reinforcement learning algorithms. @@ -192,14 +208,17 @@ class ReplayBuffer: # for off-policy td_error_update_for_per(is_indices, td_error): Updates the priorities based on TD error. save_or_load_history(cwd, if_save): Saves or loads the buffer history. """ - def __init__(self, - max_size: int, - state_dim: int, - action_dim: int, - gpu_id: int = 0, - num_seqs: int = 1, - if_use_per: bool = False, - args: Config = Config()): + + def __init__( + self, + max_size: int, + state_dim: int, + action_dim: int, + gpu_id: int = 0, + num_seqs: int = 1, + if_use_per: bool = False, + args: Config = Config(), + ): self.p = 0 # pointer self.if_full = False self.cur_size = 0 @@ -208,12 +227,12 @@ def __init__(self, self.max_size = max_size self.num_seqs = num_seqs self.device = torch.device(f"cuda:{gpu_id}" if (torch.cuda.is_available() and (gpu_id >= 0)) else "cpu") - self.args=args + self.args = args """The struction of ReplayBuffer (for examples, num_seqs = num_workers * num_envs == 2*4 = 8 ReplayBuffer: - worker0 for env0: sequence of sub_env0.0 self.states = Tensor[s, s, ..., s, ..., s] - self.actions = Tensor[a, a, ..., a, ..., a] - self.rewards = Tensor[r, r, ..., r, ..., r] + worker0 for env0: sequence of sub_env0.0 self.states = Tensor[s, s, ..., s, ..., s] + self.actions = Tensor[a, a, ..., a, ..., a] + self.rewards = Tensor[r, r, ..., r, ..., r] self.undones = Tensor[d, d, ..., d, ..., d] <-----max_size-----> <-cur_size-> @@ -239,8 +258,8 @@ def __init__(self, self.if_use_per = if_use_per if if_use_per: self.sum_trees = [SumTree(buf_len=max_size) for _ in range(num_seqs)] - self.per_alpha = getattr(args, 'per_alpha', 0.6) # alpha = (Uniform:0, Greedy:1) - self.per_beta = getattr(args, 'per_beta', 0.4) # alpha = (Uniform:0, Greedy:1) + self.per_alpha = getattr(args, "per_alpha", 0.6) # alpha = (Uniform:0, Greedy:1) + self.per_beta = getattr(args, "per_beta", 0.4) # alpha = (Uniform:0, Greedy:1) """PER. Prioritized Experience Replay. Section 4 alpha, beta = 0.7, 0.5 for rank-based variant alpha, beta = 0.6, 0.4 for proportional variant @@ -286,20 +305,20 @@ def update(self, items: Tuple[Tensor, ...]): self.rewards[p0:p1], self.rewards[0:p] = rewards[:p2], rewards[-p:] self.undones[p0:p1], self.undones[0:p] = undones[:p2], undones[-p:] else: - self.states[self.p:p] = states - self.actions[self.p:p] = actions - self.rewards[self.p:p] = rewards - self.undones[self.p:p] = undones + self.states[self.p : p] = states + self.actions[self.p : p] = actions + self.rewards[self.p : p] = rewards + self.undones[self.p : p] = undones if self.if_use_per: - '''data_ids for single env''' + """data_ids for single env""" data_ids = torch.arange(self.p, p, dtype=torch.long, device=self.device) if p > self.max_size: data_ids = torch.fmod(data_ids, self.max_size) - '''apply data_ids for vectorized env''' + """apply data_ids for vectorized env""" for sum_tree in self.sum_trees: - sum_tree.update_ids(data_ids=data_ids.cpu(), prob=10.) + sum_tree.update_ids(data_ids=data_ids.cpu(), prob=10.0) self.p = p self.cur_size = self.max_size if self.if_full else self.p @@ -325,13 +344,15 @@ def sample(self, batch_size: int) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tenso ids = torch.randint(sample_len * self.num_seqs, size=(batch_size,), requires_grad=False) ids0 = torch.fmod(ids, sample_len) # ids % sample_len - ids1 = torch.div(ids, sample_len, rounding_mode='floor') # ids // sample_len + ids1 = torch.div(ids, sample_len, rounding_mode="floor") # ids // sample_len - return (self.states[ids0, ids1], - self.actions[ids0, ids1], - self.rewards[ids0, ids1], - self.undones[ids0, ids1], - self.states[ids0 + 1, ids1],) # next_state + return ( + self.states[ids0, ids1], + self.actions[ids0, ids1], + self.rewards[ids0, ids1], + self.undones[ids0, ids1], + self.states[ids0 + 1, ids1], + ) # next_state def sample_for_per(self, batch_size: int) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]: """ @@ -355,7 +376,7 @@ def sample_for_per(self, batch_size: int) -> Tuple[Tensor, Tensor, Tensor, Tenso beg = -self.max_size end = (self.cur_size - self.max_size) if (self.cur_size < self.max_size) else -1 - '''get is_indices, is_weights''' + """get is_indices, is_weights""" is_indices: list = [] is_weights: list = [] @@ -371,7 +392,7 @@ def sample_for_per(self, batch_size: int) -> Tuple[Tensor, Tensor, Tensor, Tenso is_weights: Tensor = torch.hstack(is_weights).to(self.device) ids0 = torch.fmod(is_indices, self.cur_size) # is_indices % sample_len - ids1 = torch.div(is_indices, self.cur_size, rounding_mode='floor') # is_indices // sample_len + ids1 = torch.div(is_indices, self.cur_size, rounding_mode="floor") # is_indices // sample_len return ( self.states[ids0, ids1], self.actions[ids0, ids1], @@ -430,9 +451,9 @@ def save_or_load_history(self, cwd: str, if_save: bool): if if_save: for item, name in item_names: if self.cur_size == self.p: - buf_item = item[:self.cur_size] + buf_item = item[: self.cur_size] else: - buf_item = torch.vstack((item[self.p:self.cur_size], item[0:self.p])) + buf_item = torch.vstack((item[self.p : self.cur_size], item[0 : self.p])) file_path = f"{cwd}/replay_buffer_{name}.pth" print(f"| buffer.save_or_load_history(): Save {file_path}") torch.save(buf_item, file_path) @@ -450,6 +471,8 @@ def save_or_load_history(self, cwd: str, if_save: bool): assert all([max_size == max_sizes[0] for max_size in max_sizes]) self.cur_size = self.p = max_sizes[0] self.if_full = self.cur_size == self.max_size + + class SumTree: """ Binary Search Tree for efficient sampling in Prioritized Experience Replay. @@ -466,6 +489,7 @@ class SumTree: get_leaf_id_and_value(v): Retrieves the leaf ID and value for a given value. important_sampling(batch_size, beg, end, per_beta): Performs important sampling for a batch. """ + def __init__(self, buf_len: int): """ Initializes the SumTree object. @@ -486,16 +510,16 @@ def __init__(self, buf_len: int): def update_id(self, data_id: int, prob=10): # 10 is max_prob """ - Updates the priority of a single data point in the SumTree. + Updates the priority of a single data point in the SumTree. - Args: - data_id (int): The index of the data point in the buffer. - prob (float, optional): The new priority value for the data point. Defaults to 10, which is considered the maximum priority. + Args: + data_id (int): The index of the data point in the buffer. + prob (float, optional): The new priority value for the data point. Defaults to 10, which is considered the maximum priority. - Description: - This method updates the priority of a single data point in the SumTree. It adjusts the values in the tree - to maintain the sum property after the update. This is used in PER to adjust the sampling probability of experiences. - """ + Description: + This method updates the priority of a single data point in the SumTree. It adjusts the values in the tree + to maintain the sum property after the update. This is used in PER to adjust the sampling probability of experiences. + """ tree_id = data_id + self.buf_len - 1 delta = prob - self.tree[tree_id] @@ -505,7 +529,7 @@ def update_id(self, data_id: int, prob=10): # 10 is max_prob tree_id = (tree_id - 1) // 2 # faster than the recursive loop self.tree[tree_id] += delta - def update_ids(self, data_ids: Tensor, prob: Tensor = 10.): # 10 is max_prob + def update_ids(self, data_ids: Tensor, prob: Tensor = 10.0): # 10 is max_prob """ Updates the priorities of multiple data points in the SumTree. @@ -522,7 +546,7 @@ def update_ids(self, data_ids: Tensor, prob: Tensor = 10.): # 10 is max_prob self.tree[l_ids] = prob for depth in range(self.depth - 2): # propagate the change through tree - p_ids = torch.div(l_ids - 1, 2, rounding_mode='floor').unique() # parent indices + p_ids = torch.div(l_ids - 1, 2, rounding_mode="floor").unique() # parent indices l_ids = p_ids * 2 + 1 # left children indices r_ids = l_ids + 1 # right children indices self.tree[p_ids] = self.tree[l_ids] + self.tree[r_ids] @@ -570,15 +594,17 @@ def important_sampling(self, batch_size: int, beg: int, end: int, per_beta: floa leaf_values = torch.tensor(leaf_values, dtype=torch.float32) indices = leaf_ids - (self.buf_len - 1) - if indices.min()<0: - print(f'the wrong indice is{indices.min()}') - print(f'the whole indices is {indices}') + if indices.min() < 0: + print(f"the wrong indice is{indices.min()}") + print(f"the whole indices is {indices}") # assert 0 <= indices.min() assert indices.max() < self.buf_len prob_ary = leaf_values / self.tree[beg:end].min() weights = torch.pow(prob_ary, -per_beta) return indices, weights + + def get_episode_return(env, act, device): """ Calculates the return of an episode. @@ -591,44 +617,46 @@ def get_episode_return(env, act, device): Returns: Tuple containing episode return, violation time, violation value, rewards for power, good actions, and penalties, and the list of states. """ - env.train=False + env.train = False episode_return = 0.0 violation_time = 0 reward_for_power = 0 reward_for_good_action = 0 reward_for_penalty = 0 - state_list=[] - state = env.reset() - print(f'the year:{env.year},month:{env.month},day:{env.day} is used for testing this episode') - - # Initialize arrays to store voltage values - v_before_control_array = np.zeros((96, len(env.battery_list))) - v_after_control_array = np.zeros((96, len(env.battery_list))) + state_list = [] + state = reset_env(env) + print(f"the year:{env.year},month:{env.month},day:{env.day} is used for testing this episode") violation_value = 0.0 - for i in range(96): + for i in range(env.episode_length): s_tensor = torch.as_tensor((state,), device=device, dtype=torch.float) a_tensor = act(s_tensor) - action = a_tensor.detach().cpu().numpy()[0] - next_state, reward, done,_ = env.step(action) + next_state, reward, done, info = step_env(env, action) state_list.append(state) - - for i in range(len(env.battery_list)): - violation = min(0, 0.05 - abs(1.0 - env.after_control[env.battery_list[i]])) + post_control_voltage = info["post_control_voltage_pu"] + for node_index in env.battery_nodes: + violation = min(0, 0.05 - abs(1.0 - post_control_voltage[node_index])) if violation < 0: violation_time += 1 violation_value += violation - - reward_for_power += env.reward_for_power + reward_for_power += info["reward_breakdown"]["economic"] reward_for_good_action += 0 - reward_for_penalty += env.reward_for_penalty + reward_for_penalty += info["reward_breakdown"]["voltage_penalty"] episode_return += reward state = next_state if done: break - return episode_return, violation_time, violation_value, reward_for_power, reward_for_good_action, reward_for_penalty, state_list \ No newline at end of file + return ( + episode_return, + violation_time, + violation_value, + reward_for_power, + reward_for_good_action, + reward_for_penalty, + state_list, + ) diff --git a/rl_adn/benchmarks/__init__.py b/rl_adn/benchmarks/__init__.py new file mode 100644 index 0000000..d9350b3 --- /dev/null +++ b/rl_adn/benchmarks/__init__.py @@ -0,0 +1,5 @@ +"""Benchmark exports for RL-ADN.""" + +from rl_adn.benchmarks.pyomo_timeseries_pandapower import construct_opf_model + +__all__ = ["construct_opf_model"] diff --git a/rl_adn/benchmarks/pyomo_timeseries_pandapower.py b/rl_adn/benchmarks/pyomo_timeseries_pandapower.py new file mode 100644 index 0000000..56f3784 --- /dev/null +++ b/rl_adn/benchmarks/pyomo_timeseries_pandapower.py @@ -0,0 +1,262 @@ +"""in this script, we consider constraints corresponding to the current multi battery environment, +that is to say the voltage regulation only considers that constrainting the limitation connected to the nodes connected to batteries, +Using 2020 11-6 as the example""" + +import numpy as np +import pandas as pd +from pyomo.environ import * + +from rl_adn import PowerNetEnv, make_env_config + + +def construct_opf_model(Vnom, Vmin, Vmax, Data_Network): + # Data Processing + battery_parameters = { + "capacity": 1.0, # MW.h + "max_charge": 0.3, # MW + "max_discharge": 0.3, # MW + "efficiency": 1, + "degradation": 0, # euro/kw + "max_soc": 0.8, + "min_soc": 0.2, + "initial_soc": 0.4, + } + TIMES = Data_Network["TIMES"] + NODES = Data_Network["NODES"] + LINES = Data_Network["LINES"] + Tb = Data_Network["Tb"] + PD = Data_Network["PD"] + QD = Data_Network["QD"] + R = Data_Network["R"] + X = Data_Network["X"] + BATTERY_NODES = Data_Network["BATTERY_NODES"] + # Type of Model + model = ConcreteModel() + + # Define Sets + model.NODES = Set(initialize=NODES) + model.LINES = Set(initialize=LINES) + model.TIMES = Set(initialize=TIMES) + + # Define Parameters + model.Vnom = Param(initialize=Vnom, mutable=False) + model.Vmin = Param(initialize=Vmin, mutable=False) + model.Vmax = Param(initialize=Vmax, mutable=False) + model.Tb = Param(model.NODES, initialize=Tb, mutable=True) + # model.PD = Param(model.TIMES,model.NODES, initialize=0, mutable=True) # Node demand + model.QD = Param(model.TIMES, model.NODES, initialize=0, mutable=True) # Node demand + model.R = Param(model.LINES, initialize=R, mutable=False) # Line resistance + model.X = Param(model.LINES, initialize=X, mutable=False) # Line resistance + ## define parameters for battery + model.battery_initial_soc = Param(default=battery_parameters["initial_soc"]) + model.battery_capacity = Param(default=battery_parameters["capacity"]) + model.battery_soc_max = Param(default=battery_parameters["max_soc"]) + model.battery_soc_min = Param(default=battery_parameters["min_soc"]) + model.battery_max_change = Param(default=battery_parameters["max_charge"]) + + # define initialize PD + def PD_init_rule(model, time, node): + model.PD[time, node] = PD[time, node] + return model.PD[time, node] + + model.PD = Param(model.TIMES, model.NODES, initialize=PD_init_rule) + + def R_init_rule(model, i, j): + return model.R[i, j] + + model.RM = Param(model.LINES, initialize=R_init_rule) # Line resistance + + def X_init_rule(model, i, j): + return model.X[i, j] + + model.XM = Param(model.LINES, initialize=X_init_rule) # Line resistance + + # Define Variables + model.P = Var(model.TIMES, model.LINES, initialize=0) # Acive power flowing in lines + model.Q = Var(model.TIMES, model.LINES, initialize=0) # Reacive power flowing in lines + model.I = Var(model.TIMES, model.LINES, initialize=0) # Current of lines + + model.SOC = Var( + model.TIMES, + model.NODES, + initialize=model.battery_initial_soc, + bounds=(model.battery_soc_min, model.battery_soc_max), + ) + + # we set energy>0 is discharge, also only when no slack bus we put battery + def energy_change_rule(model, time, i): + if i not in BATTERY_NODES: + tem = 0.0 + model.energy_change[time, i].fixed = True + else: + tem = 0.0 + model.energy_change[time, i].fixed = False + return tem + + model.energy_change = Var( + model.TIMES, + model.NODES, + initialize=energy_change_rule, + bounds=(-model.battery_max_change, model.battery_max_change), + ) + + def PS_init_rule(model, time, i): + # for time in model.TIMES: + if model.Tb[i].value == 0: + temp = 0.0 + model.PS[time, i].fixed = True + else: + temp = 0.0 + return temp + + model.PS = Var(model.TIMES, model.NODES, initialize=PS_init_rule) # Active power of the SS + + def QS_init_rule(model, time, i): + # for time in model.TIMES: + if model.Tb[i].value == 0: + temp = 0.0 + model.QS[time, i].fixed = True + else: + temp = 0.0 + return temp + + model.QS = Var(model.TIMES, model.NODES, initialize=QS_init_rule) # Reactive power of the SS + + # price init rule + def PRICE_init_rule(model, time): + return PRICE[time] + + model.PRICE = Param(model.TIMES, initialize=PRICE_init_rule, mutable=False) + + # Voltage of nodes + def Voltage_init(model, time, i): + # for time in model.TIMES: + if model.Tb[i].value == 1: + temp = model.Vnom + model.V[time, i].fixed = True + else: + temp = model.Vnom + model.V[time, i].fixed = False + return temp + + model.V = Var(model.TIMES, model.NODES, initialize=Voltage_init) + + # Define Objective Function,minimize the optimal power loss. Actually, when we only have one source from the grid, + """Since each element of model.LINES is a tuple of two integers, + you will need to use two indices to access the variable indexed by model.LINES. + For example, if you define a variable P indexed by both model.LINES and model.TIMES, + you would access the value of P for the line (1,2) at time t=1 using model.P[1, (1,2)].""" + # def act_loss(model): + # return (sum(sum(model.RM[i, j] * (model.I[time,(i, j)] ** 2) for i, j in model.LINES)for time in model.TIMES)) + + # here we create another objective: minimizing the imported power from external grid + # def min_power_ext_grid(model): + # + # return (sum(sum(model.PS[time,node]for node in model.NODES)for time in model.TIMES)) + + # Update the objective function to minimize the cost of buying energy from the external grid + def min_cost_ext_grid(model): + return sum(sum(model.PS[time, node] * model.PRICE[time] for node in model.NODES) for time in model.TIMES) + + model.obj = Objective(rule=min_cost_ext_grid, sense=minimize) + + # Update the objective function to earn money from battery dispatch + # def max_benefits_dispatch_battery(model): + # return sum(sum(model.energy_change[time, node] * model.PRICE[time] for node in model.NODES) for time in model.TIMES) + # + # model.obj = Objective(rule=max_benefits_dispatch_battery,sense=maximize) + + # model.obj = Objective(rule=min_power_ext_grid) + # model.obj = Objective(rule=act_loss) + # we need to revise this part for adding time constraint into here. + # %% Define Constraints + # define soc update constraint + + def soc_update_rule(model, time, node): + if node not in BATTERY_NODES: + return Constraint.Skip + if time == model.TIMES.first(): + return ( + model.SOC[time, node] + == model.battery_initial_soc - (model.energy_change[time, node] * 15.0 / 60.0) / model.battery_capacity + ) + else: + return ( + model.SOC[time, node] + == model.SOC[model.TIMES.prev(time), node] + - (model.energy_change[time, node] * 15.0 / 60.0) / model.battery_capacity + ) + + model.constaint_soc_update = Constraint(model.TIMES, model.NODES, rule=soc_update_rule) + + # for line k consumption == injection + def active_power_flow_rule(model, time, k): + + return ( + sum(model.P[time, (j, i)] for j, i in model.LINES if i == k) + - sum( + model.P[time, (i, j)] + model.RM[i, j] * (model.I[time, (i, j)] ** 2) for i, j in model.LINES if k == i + ) + + model.PS[time, k] + + model.energy_change[time, k] + == model.PD[time, k] + ) + + model.active_power_flow = Constraint(model.TIMES, model.NODES, rule=active_power_flow_rule) + + def reactive_power_flow_rule(model, time, k): + return ( + sum(model.Q[time, (j, i)] for j, i in model.LINES if i == k) + - sum( + model.Q[time, (i, j)] + model.XM[i, j] * (model.I[time, (i, j)] ** 2) for i, j in model.LINES if k == i + ) + + model.QS[time, k] + == model.QD[time, k] + ) + + model.reactive_power_flow = Constraint(model.TIMES, model.NODES, rule=reactive_power_flow_rule) + + ## role of voltage drop + def voltage_drop_rule(model, time, i, j): + return ( + model.V[time, i] ** 2 + - 2 * (model.RM[i, j] * model.P[time, (i, j)] + model.XM[i, j] * model.Q[time, (i, j)]) + - (model.RM[i, j] ** 2 + model.XM[i, j] ** 2) * model.I[time, (i, j)] ** 2 + - model.V[time, j] ** 2 + ) == 0 + + model.voltage_drop = Constraint(model.TIMES, model.LINES, rule=voltage_drop_rule) + + def define_current_rule(model, time, i, j): + return (model.I[time, (i, j)] ** 2) * (model.V[time, j] ** 2) == model.P[time, (i, j)] ** 2 + model.Q[ + time, (i, j) + ] ** 2 + + model.define_current = Constraint(model.TIMES, model.LINES, rule=define_current_rule) + + # here the current limit is over 0, representing that current can only from i to j, instead of versa. + # we change this step and try to calculate it according to the result three phase one + + def current_limit_rule(model, time, i, j): + return (0, model.I[time, (i, j)], None) + + # if we cancel this, then things to error + model.current_limit = Constraint(model.TIMES, model.LINES, rule=current_limit_rule) + + def voltage_limit_rule(model, time, i): + if i in BATTERY_NODES: + return (model.Vmin, model.V[time, i], model.Vmax) + return Constraint.Skip + + model.voltage_limit = Constraint(model.TIMES, model.NODES, rule=voltage_limit_rule) + + return model + + +def convert_dict_to_pd(data: dict): + df = pd.DataFrame(columns=list(set([k[1] for k in data.keys()]))) + for key, value in data.items(): + df.loc[key[0], key[1]] = value + # df=df.iloc[:,1:] + # df=df.drop(df.columns[0],axis=1) + return df diff --git a/rl_adn/benckmark_algorithms/__init__.py b/rl_adn/benckmark_algorithms/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/rl_adn/benckmark_algorithms/pyomo_timeseries_pandapower.py b/rl_adn/benckmark_algorithms/pyomo_timeseries_pandapower.py deleted file mode 100644 index 9e3a9bc..0000000 --- a/rl_adn/benckmark_algorithms/pyomo_timeseries_pandapower.py +++ /dev/null @@ -1,305 +0,0 @@ -'''in this script, we consider constraints corresponding to the current multi battery environment, -that is to say the voltage regulation only considers that constrainting the limitation connected to the nodes connected to batteries, -Using 2020 11-6 as the example''' -import numpy as np -from environments.env import PowerNetEnv -from pyomo.environ import * -import pandas as pd - - -def construct_opf_model(Vnom, Vmin, Vmax, Data_Network): - # Data Processing - battery_parameters = { - 'capacity': 1.0, # MW.h - 'max_charge': 0.3, # MW - 'max_discharge': 0.3, # MW - 'efficiency': 1, - 'degradation': 0, # euro/kw - 'max_soc': 0.8, - 'min_soc': 0.2, - 'initial_soc': 0.4} - TIMES=Data_Network['TIMES'] - NODES = Data_Network['NODES'] - LINES = Data_Network['LINES'] - Tb = Data_Network['Tb'] - PD = Data_Network['PD'] - QD = Data_Network['QD'] - R = Data_Network['R'] - X = Data_Network['X'] - BATTERY_NODES = Data_Network['BATTERY_NODES'] - # Type of Model - model = ConcreteModel() - - # Define Sets - model.NODES = Set(initialize=NODES) - model.LINES = Set(initialize=LINES) - model.TIMES = Set(initialize=TIMES) - - # Define Parameters - model.Vnom = Param(initialize=Vnom, mutable=False) - model.Vmin = Param(initialize=Vmin, mutable=False) - model.Vmax = Param(initialize=Vmax, mutable=False) - model.Tb = Param(model.NODES, initialize=Tb, mutable=True) - # model.PD = Param(model.TIMES,model.NODES, initialize=0, mutable=True) # Node demand - model.QD = Param(model.TIMES,model.NODES, initialize=0, mutable=True) # Node demand - model.R = Param(model.LINES, initialize=R, mutable=False) # Line resistance - model.X = Param(model.LINES, initialize=X, mutable=False) # Line resistance - ## define parameters for battery - model.battery_initial_soc=Param(default=battery_parameters['initial_soc']) - model.battery_capacity=Param(default=battery_parameters['capacity']) - model.battery_soc_max=Param(default=battery_parameters['max_soc']) - model.battery_soc_min=Param(default=battery_parameters['min_soc']) - model.battery_max_change=Param(default=battery_parameters['max_charge']) - - - - - - - # define initialize PD - def PD_init_rule(model,time,node): - model.PD[time,node]=PD[time,node] - return (model.PD[time,node]) - model.PD=Param(model.TIMES,model.NODES,initialize=PD_init_rule) - - def R_init_rule(model, i, j): - return (model.R[i, j]) - - model.RM = Param(model.LINES, initialize=R_init_rule) # Line resistance - - def X_init_rule(model, i, j): - return (model.X[i, j]) - - model.XM = Param(model.LINES, initialize=X_init_rule) # Line resistance - - - - - # Define Variables - model.P = Var(model.TIMES,model.LINES, initialize=0) # Acive power flowing in lines - model.Q = Var(model.TIMES,model.LINES, initialize=0) # Reacive power flowing in lines - model.I = Var(model.TIMES,model.LINES, initialize=0) # Current of lines - - model.SOC=Var(model.TIMES,model.NODES,initialize=model.battery_initial_soc,bounds=(model.battery_soc_min,model.battery_soc_max)) - # we set energy>0 is discharge, also only when no slack bus we put battery - def energy_change_rule(model, time, i): - if i not in BATTERY_NODES: - tem = 0.0 - model.energy_change[time, i].fixed = True - else: - tem = 0.0 - model.energy_change[time, i].fixed = False - return tem - - model.energy_change=Var(model.TIMES,model.NODES,initialize=energy_change_rule,bounds=(-model.battery_max_change,model.battery_max_change)) - - def PS_init_rule(model, time,i): - # for time in model.TIMES: - if model.Tb[i].value == 0: - temp = 0.0 - model.PS[time,i].fixed = True - else: - temp = 0.0 - return temp - model.PS = Var(model.TIMES,model.NODES, initialize=PS_init_rule) # Active power of the SS - - def QS_init_rule(model,time,i): - # for time in model.TIMES: - if model.Tb[i].value == 0: - temp = 0.0 - model.QS[time,i].fixed = True - else: - temp = 0.0 - return temp - model.QS = Var(model.TIMES,model.NODES, initialize=QS_init_rule) # Reactive power of the SS - # price init rule - def PRICE_init_rule(model, time): - return PRICE[time] - - model.PRICE = Param(model.TIMES, initialize=PRICE_init_rule, mutable=False) - # Voltage of nodes - def Voltage_init(model,time, i): - # for time in model.TIMES: - if model.Tb[i].value == 1: - temp = model.Vnom - model.V[time,i].fixed = True - else: - temp = model.Vnom - model.V[time,i].fixed = False - return temp - - model.V = Var(model.TIMES,model.NODES, initialize=Voltage_init) - - # Define Objective Function,minimize the optimal power loss. Actually, when we only have one source from the grid, - '''Since each element of model.LINES is a tuple of two integers, - you will need to use two indices to access the variable indexed by model.LINES. - For example, if you define a variable P indexed by both model.LINES and model.TIMES, - you would access the value of P for the line (1,2) at time t=1 using model.P[1, (1,2)].''' - # def act_loss(model): - # return (sum(sum(model.RM[i, j] * (model.I[time,(i, j)] ** 2) for i, j in model.LINES)for time in model.TIMES)) - - # here we create another objective: minimizing the imported power from external grid - # def min_power_ext_grid(model): - # - # return (sum(sum(model.PS[time,node]for node in model.NODES)for time in model.TIMES)) - - # Update the objective function to minimize the cost of buying energy from the external grid - def min_cost_ext_grid(model): - return sum(sum(model.PS[time, node] * model.PRICE[time] for node in model.NODES) for time in model.TIMES) - - model.obj = Objective(rule=min_cost_ext_grid,sense=minimize) - - # Update the objective function to earn money from battery dispatch - # def max_benefits_dispatch_battery(model): - # return sum(sum(model.energy_change[time, node] * model.PRICE[time] for node in model.NODES) for time in model.TIMES) - # - # model.obj = Objective(rule=max_benefits_dispatch_battery,sense=maximize) - - - - # model.obj = Objective(rule=min_power_ext_grid) - # model.obj = Objective(rule=act_loss) - #we need to revise this part for adding time constraint into here. - # %% Define Constraints - # define soc update constraint - - - def soc_update_rule(model, time, node): - if node not in BATTERY_NODES: - return Constraint.Skip - if time == model.TIMES.first(): - return (model.SOC[time, node] == model.battery_initial_soc - ( - model.energy_change[time, node] * 15.0 / 60.0) / model.battery_capacity) - else: - return (model.SOC[time, node] == model.SOC[model.TIMES.prev(time), node] - ( - model.energy_change[time, node] * 15.0 / 60.0) / model.battery_capacity) - - model.constaint_soc_update=Constraint(model.TIMES,model.NODES,rule=soc_update_rule) - - # for line k consumption == injection - def active_power_flow_rule(model, time,k): - - return (sum(model.P[time,(j, i)] for j, i in model.LINES if i == k) - sum( - model.P[time,(i, j)] + model.RM[i, j] * (model.I[time,(i, j)] ** 2) for i, j in model.LINES if k == i) + model.PS[time,k]+model.energy_change[time,k] == - model.PD[time,k]) - - model.active_power_flow = Constraint(model.TIMES,model.NODES, rule=active_power_flow_rule) - - def reactive_power_flow_rule(model,time, k): - return (sum(model.Q[time,(j, i)] for j, i in model.LINES if i == k) - sum( - model.Q[time,(i, j)] + model.XM[i, j] * (model.I[time,(i, j)] ** 2) for i, j in model.LINES if k == i) + model.QS[time,k] == - model.QD[time,k]) - - model.reactive_power_flow = Constraint(model.TIMES,model.NODES, rule=reactive_power_flow_rule) - - ## role of voltage drop - def voltage_drop_rule(model, time,i,j): - return ((model.V[time, i] ** 2 - 2 * ( - model.RM[i, j] * model.P[time, (i, j)] + model.XM[i, j] * model.Q[time, (i, j)]) - ( - model.RM[i, j] ** 2 + model.XM[i, j] ** 2) * model.I[time, (i, j)] ** 2 - model.V[ - time, j] ** 2 ) == 0) - - model.voltage_drop = Constraint(model.TIMES,model.LINES, rule=voltage_drop_rule) - - def define_current_rule(model, time,i, j): - return ((model.I[time,(i, j)] ** 2) * (model.V[time,j] ** 2) == model.P[time,(i, j)] ** 2 + model.Q[time,(i, j)] ** 2) - - model.define_current = Constraint(model.TIMES,model.LINES, rule=define_current_rule) - - # here the current limit is over 0, representing that current can only from i to j, instead of versa. - # we change this step and try to calculate it according to the result three phase one - - def current_limit_rule(model,time, i, j): - return (0, model.I[time,(i, j)], None) - # if we cancel this, then things to error - model.current_limit = Constraint(model.TIMES,model.LINES, rule=current_limit_rule) - - def voltage_limit_rule(model, time, i): - if i in BATTERY_NODES: - return (model.Vmin, model.V[time, i], model.Vmax) - return Constraint.Skip - - model.voltage_limit = Constraint(model.TIMES,model.NODES, rule=voltage_limit_rule) - - return model -def convert_dict_to_pd(data:dict): - df = pd.DataFrame(columns=list(set([k[1] for k in data.keys()]))) - for key, value in data.items(): - df.loc[key[0], key[1]] = value - # df=df.iloc[:,1:] - # df=df.drop(df.columns[0],axis=1) - return df -if __name__=='__main__': - # initialize environment and network data prepared to pyomo - env=PowerNetEnv() - env.reset() - net=env.net - ppc = net._ppc - branch = ppc['branch'] - bus=ppc['bus'] - # here we add 1, because when doing modification from PandaPower, the index of bus changed in here. - f = np.real(branch[:, 0]).astype(int) ## list of "from" buses - t = np.real(branch[:, 1]).astype(int) ## list of "to" buses - r = branch[:, 2] - x = branch[:, 3] - LINES = {(f[i], t[i]) for i in range(len(f))} - R = {(f[i], t[i]): np.real(r[i]) for i in range(len(f))} - X = {(f[i], t[i]): np.real(x[i]) for i in range(len(f))} - # change here and test - # NODES = [i for i in range(bus.shape[0])] - NODES = net.res_load.index.to_list() - TIMES = [i for i in range(96)] - Tb = dict() - for i, node in enumerate(NODES): - if i == 0: - Tb[i] = 1 - else: - Tb[i] = 0 - # only the last node is connected to battery - BATTERY_NODES = {11, 15, 26,29, 33} - # SORTED_BATTERY_NODES = [26, 15, 29, 11, 33] - SORTED_BATTERY_NODES = [11, 15, 26,29, 33] - - - # prepare netload data for all nodes - year=2020 - month=11 - day=6 - data_manager=env.data_manager - day_data=data_manager.select_day(year,month,day) - # day_data shape is (96,36), we now create our active power (PD) - active_power=day_data[:,0:34] - qg=day_data[:,-2] - price=day_data[:,-1] - pv_generation=qg.reshape(96,-1)*env.pv_parameters.reshape(1,-1) - netload=active_power-pv_generation - PD_array=netload/1000 - PRICE = price/1000 - Data_Network = {'TIMES': TIMES, 'NODES': NODES, 'LINES': LINES, 'Tb': Tb, 'PD': PD_array, 'QD': None, 'R': R, - 'X': X, 'BATTERY_NODES': BATTERY_NODES, 'PRICE': PRICE} - ## prepare other parameters - Vnom = 1.01 - Vmax = 1.05 - Vmin = 0.95 - # construct and solve model - model=construct_opf_model(Vnom, Vmin, Vmax, Data_Network) - # solver_path = '/home/hshengren/miniconda3/envs/caql/bin/ipopt' - - # solver = SolverFactory('ipopt',executable=solver_path) - solver = SolverFactory('ipopt') - - solver.options['constr_viol_tol'] = 1e-6 - solver.options['acceptable_tol'] = 1e-6 - solver.options['dual_inf_tol'] = 1e-6 - solver.solve(model, tee=True,) - - objective_value = model.obj() - print(objective_value) - ## prepare the results - voltage_after_control=convert_dict_to_pd(model.V.extract_values()) - active_power=convert_dict_to_pd(model.PD.extract_values()) - ext_grid_active_power=convert_dict_to_pd(model.PS.extract_values()) - ext_grid_reactive_power=convert_dict_to_pd(model.QS.extract_values()) - soc = convert_dict_to_pd(model.SOC.extract_values()) - energy_change=convert_dict_to_pd(model.energy_change.extract_values()) - print('result done') \ No newline at end of file diff --git a/rl_adn/config.py b/rl_adn/config.py new file mode 100644 index 0000000..4c2a49a --- /dev/null +++ b/rl_adn/config.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal, Optional + +from rl_adn.environments.topology_scenarios import get_topology_scenario + +PACKAGE_ROOT = Path(__file__).resolve().parent +DATA_ROOT = PACKAGE_ROOT / "data_sources" +NETWORK_ROOT = DATA_ROOT / "network_data" +TIME_SERIES_ROOT = DATA_ROOT / "time_series_data" + +DEFAULT_NODE_COUNT = 34 +CURATED_BATTERY_NODES = { + 34: (11, 15, 26, 29, 33), # paper nodes {12, 16, 27, 30, 34} + 69: (13, 15, 17, 19, 21, 23, 25, 26, 64), # paper nodes {14, 16, 18, 20, 22, 24, 26, 27, 65} +} + + +@dataclass(frozen=True) +class BatteryConfig: + capacity_kwh: float = 300.0 + max_charge_kw: float = 50.0 + max_discharge_kw: float = 50.0 + efficiency: float = 1.0 + degradation_eur_per_kw: float = 0.0 + max_soc: float = 0.8 + min_soc: float = 0.2 + initial_soc: float = 0.4 + time_interval_minutes: float = 15.0 + + def __post_init__(self) -> None: + if self.capacity_kwh <= 0: + raise ValueError("capacity_kwh must be positive") + if self.max_charge_kw <= 0 or self.max_discharge_kw <= 0: + raise ValueError("Battery charge and discharge limits must be positive") + if self.time_interval_minutes <= 0: + raise ValueError("time_interval_minutes must be positive") + if not 0 <= self.min_soc < self.max_soc <= 1: + raise ValueError("Battery SOC bounds must satisfy 0 <= min_soc < max_soc <= 1") + if not self.min_soc <= self.initial_soc <= self.max_soc: + raise ValueError("initial_soc must lie within [min_soc, max_soc]") + + @classmethod + def default(cls, *, time_interval_minutes: float = 15.0) -> "BatteryConfig": + return cls(time_interval_minutes=time_interval_minutes) + + +@dataclass(frozen=True) +class TopologyConfig: + mode: Literal["fixed", "scenario_pool"] = "fixed" + scenario_id: str = "TP1" + scenario_pool: tuple[str, ...] = () + return_graph: bool = False + + def __post_init__(self) -> None: + if self.mode not in {"fixed", "scenario_pool"}: + raise ValueError("TopologyConfig.mode must be 'fixed' or 'scenario_pool'") + if self.mode == "scenario_pool" and not self.scenario_pool: + raise ValueError("TopologyConfig.scenario_pool must not be empty when mode='scenario_pool'") + + +@dataclass(frozen=True) +class EnvConfig: + node_count: int + algorithm: Literal["Laurent", "PandaPower"] + battery_nodes: tuple[int, ...] + battery: BatteryConfig + year: int + month: int + day: int + train: bool + state_pattern: str + voltage_limits: tuple[float, float] + vm_pu: float + s_base: float + bus_info_file: str + branch_info_file: str + time_series_data_path: str + feeder_id: str + topology: TopologyConfig = field(default_factory=TopologyConfig) + + def __post_init__(self) -> None: + if self.algorithm not in {"Laurent", "PandaPower"}: + raise ValueError("algorithm must be 'Laurent' or 'PandaPower'") + if len(self.battery_nodes) == 0: + raise ValueError("battery_nodes must not be empty") + if self.state_pattern != "default": + raise ValueError("Only the 'default' state pattern is currently supported") + if self.voltage_limits[0] >= self.voltage_limits[1]: + raise ValueError("voltage_limits must be ordered as (low, high)") + if not Path(self.bus_info_file).exists(): + raise FileNotFoundError(f"Bus data file does not exist: {self.bus_info_file}") + if not Path(self.branch_info_file).exists(): + raise FileNotFoundError(f"Branch data file does not exist: {self.branch_info_file}") + + @property + def network_info(self) -> dict[str, object]: + return { + "vm_pu": self.vm_pu, + "s_base": self.s_base, + "bus_info_file": self.bus_info_file, + "branch_info_file": self.branch_info_file, + } + + +def _resolve_network_files(node_count: int) -> tuple[str, str]: + node_dir = NETWORK_ROOT / f"node_{node_count}" + bus_info_file = node_dir / f"Nodes_{node_count}.csv" + branch_info_file = node_dir / f"Lines_{node_count}.csv" + if not bus_info_file.exists() or not branch_info_file.exists(): + raise FileNotFoundError(f"Missing packaged network data for node_{node_count}") + return str(bus_info_file), str(branch_info_file) + + +def _resolve_time_series_data_path(node_count: int, override: Optional[str]) -> str: + if override is not None: + return str(Path(override)) + + candidate = TIME_SERIES_ROOT / f"{node_count}_node_time_series.csv" + if candidate.exists(): + return str(candidate) + + if node_count == DEFAULT_NODE_COUNT: + raise FileNotFoundError(f"Missing packaged time-series data for node_{node_count}") + + raise ValueError(f"No packaged time-series data is available for node_{node_count}; please provide time_series_data_path explicitly.") + + +def _resolve_battery_nodes(node_count: int, override: Optional[tuple[int, ...]]) -> tuple[int, ...]: + if override is not None: + return tuple(override) + try: + return CURATED_BATTERY_NODES[node_count] + except KeyError as exc: + raise ValueError(f"battery_nodes must be provided for node counts without a curated default (received node_count={node_count})") from exc + + +def make_env_config( + *, + node: int = DEFAULT_NODE_COUNT, + algorithm: Literal["Laurent", "PandaPower"] = "Laurent", + train: bool = True, + battery_nodes: Optional[tuple[int, ...]] = None, + battery: Optional[BatteryConfig] = None, + year: int = 2020, + month: int = 1, + day: int = 1, + state_pattern: str = "default", + vm_pu: float = 1.0, + s_base: float = 1000.0, + time_series_data_path: Optional[str] = None, + topology_mode: Literal["fixed", "scenario_pool"] = "fixed", + topology_scenario: Optional[str] = None, + topology_pool: Optional[list[str]] = None, + return_graph: bool = False, +) -> EnvConfig: + scenario_id = topology_scenario or "TP1" + get_topology_scenario(node, scenario_id) + if topology_pool is not None: + for topology_case in topology_pool: + get_topology_scenario(node, topology_case) + + bus_info_file, branch_info_file = _resolve_network_files(node) + resolved_time_series_path = _resolve_time_series_data_path(node, time_series_data_path) + resolved_battery_nodes = _resolve_battery_nodes(node, battery_nodes) + + resolved_battery = battery or BatteryConfig.default() + resolved_battery = BatteryConfig( + capacity_kwh=resolved_battery.capacity_kwh, + max_charge_kw=resolved_battery.max_charge_kw, + max_discharge_kw=resolved_battery.max_discharge_kw, + efficiency=resolved_battery.efficiency, + degradation_eur_per_kw=resolved_battery.degradation_eur_per_kw, + max_soc=resolved_battery.max_soc, + min_soc=resolved_battery.min_soc, + initial_soc=resolved_battery.initial_soc, + time_interval_minutes=resolved_battery.time_interval_minutes, + ) + + topology = TopologyConfig( + mode=topology_mode, + scenario_id=scenario_id, + scenario_pool=tuple(topology_pool or ()), + return_graph=return_graph, + ) + + return EnvConfig( + node_count=node, + algorithm=algorithm, + battery_nodes=resolved_battery_nodes, + battery=resolved_battery, + year=year, + month=month, + day=day, + train=train, + state_pattern=state_pattern, + voltage_limits=(0.95, 1.05), + vm_pu=vm_pu, + s_base=s_base, + bus_info_file=bus_info_file, + branch_info_file=branch_info_file, + time_series_data_path=resolved_time_series_path, + feeder_id=f"{node}-bus", + topology=topology, + ) + + +DEFAULT_ENV_CONFIG = make_env_config() diff --git a/rl_adn/data/__init__.py b/rl_adn/data/__init__.py new file mode 100644 index 0000000..ff09db9 --- /dev/null +++ b/rl_adn/data/__init__.py @@ -0,0 +1,5 @@ +"""Data access exports for RL-ADN.""" + +from rl_adn.data.manager import GeneralPowerDataManager + +__all__ = ["GeneralPowerDataManager"] diff --git a/rl_adn/data_manager/data_manager.py b/rl_adn/data/manager.py similarity index 69% rename from rl_adn/data_manager/data_manager.py rename to rl_adn/data/manager.py index 426038c..5fedeeb 100644 --- a/rl_adn/data_manager/data_manager.py +++ b/rl_adn/data/manager.py @@ -1,17 +1,11 @@ -""" -This module contains the DataManager class for managing and preprocessing -time-series data related to power systems. It includes functionalities for -data loading, cleaning, and basic manipulations. -""" +"""Time-series data loading and preprocessing for RL-ADN.""" - - - -from typing import List, Tuple -import pandas as pd -import numpy as np import random import re +from typing import List, Tuple + +import numpy as np +import pandas as pd class GeneralPowerDataManager: @@ -48,37 +42,21 @@ def __init__(self, datapath: str) -> None: # Prepare a datetime index and keep the DataFrame sorted for predictable slicing. data = self._prepare_datetime_index(data) - # Print data scale and initialize time interval min_date = data.index.min() max_date = data.index.max() - print(f"Data scale: from {min_date.strftime('%Y-%m-%d')} to {max_date.strftime('%Y-%m-%d')}") - self.time_interval = self._infer_time_interval(data.index) - print(f"Data time interval: {self.time_interval} minutes") + self.date_range = (min_date, max_date) # Initialize other attributes self.df = data self.data_array = data.values # Identify columns matching expected patterns for later use. - self.active_power_cols = self._get_columns_matching(r'active_power(_\w+)?') - self.reactive_power_cols = self._get_columns_matching(r'reactive_power(_\w+)?') - self.renewable_active_power_cols = self._get_columns_matching(r'renewable_active_power(_\w+)?') - self.renewable_reactive_power_cols = self._get_columns_matching(r'renewable_reactive_power(_\w+)?') - self.price_col = self._get_columns_matching(r'price(_\w+)?') - # Display dataset information - print(f"Dataset loaded from {datapath}") - print(f"Dataset dimensions: {self.df.shape}") - print(f"Dataset contains the following types of data:") - print( - f"Active power columns: {self.active_power_cols} (Indices: {[self.df.columns.get_loc(col) for col in self.active_power_cols]})") - print( - f"Reactive power columns: {self.reactive_power_cols} (Indices: {[self.df.columns.get_loc(col) for col in self.reactive_power_cols]})") - print( - f"Renewable active power columns: {self.renewable_active_power_cols} (Indices: {[self.df.columns.get_loc(col) for col in self.renewable_active_power_cols]})") - print( - f"Renewable reactive power columns: {self.renewable_reactive_power_cols} (Indices: {[self.df.columns.get_loc(col) for col in self.renewable_reactive_power_cols]})") - print(f"Price columns: {self.price_col} (Indices: {[self.df.columns.get_loc(col) for col in self.price_col]})") + self.active_power_cols = self._get_columns_matching(r"active_power(_\w+)?") + self.reactive_power_cols = self._get_columns_matching(r"reactive_power(_\w+)?") + self.renewable_active_power_cols = self._get_columns_matching(r"renewable_active_power(_\w+)?") + self.renewable_reactive_power_cols = self._get_columns_matching(r"renewable_reactive_power(_\w+)?") + self.price_col = self._get_columns_matching(r"price(_\w+)?") # Calculate max and min for each type of power self.active_power_max = self.df[self.active_power_cols].max().max() self.active_power_min = self.df[self.active_power_cols].min().min() @@ -86,15 +64,11 @@ def __init__(self, datapath: str) -> None: self.reactive_power_max = self.df[self.reactive_power_cols].max().max() if self.reactive_power_cols else None self.reactive_power_min = self.df[self.reactive_power_cols].min().min() if self.reactive_power_cols else None - self.renewable_active_power_max = self.df[ - self.renewable_active_power_cols].max().max() if self.renewable_active_power_cols else None - self.renewable_active_power_min = self.df[ - self.renewable_active_power_cols].min().min() if self.renewable_active_power_cols else None + self.renewable_active_power_max = self.df[self.renewable_active_power_cols].max().max() if self.renewable_active_power_cols else None + self.renewable_active_power_min = self.df[self.renewable_active_power_cols].min().min() if self.renewable_active_power_cols else None - self.renewable_reactive_power_max = self.df[ - self.renewable_reactive_power_cols].max().max() if self.renewable_reactive_power_cols else None - self.renewable_reactive_power_min = self.df[ - self.renewable_reactive_power_cols].min().min() if self.renewable_reactive_power_cols else None + self.renewable_reactive_power_max = self.df[self.renewable_reactive_power_cols].max().max() if self.renewable_reactive_power_cols else None + self.renewable_reactive_power_min = self.df[self.renewable_reactive_power_cols].min().min() if self.renewable_reactive_power_cols else None self.price_min = self.df[self.price_col].min().values[0] if self.price_col else None self.price_max = self.df[self.price_col].max().values[0] if self.price_col else None # split the train and test dates @@ -111,7 +85,7 @@ def _prepare_datetime_index(self, data: pd.DataFrame) -> pd.DataFrame: This keeps the index conversion and sorting in a single place so other methods can assume a consistent, timezone-aware index. """ - index_column = 'date_time' if 'date_time' in data.columns else data.columns[0] + index_column = "date_time" if "date_time" in data.columns else data.columns[0] # Copy to avoid mutating the caller's DataFrame unexpectedly. data = data.copy() @@ -145,7 +119,6 @@ def _get_columns_matching(self, pattern: str) -> List[str]: """Return column names that match the provided regex pattern.""" return [col for col in self.df.columns if re.fullmatch(pattern, col)] - def _replace_nan(self) -> None: """ Replace NaN values in the data with interpolated values or the average of the surrounding values. @@ -163,19 +136,18 @@ def _check_for_nan(self) -> None: def select_timeslot_data(self, year: int, month: int, day: int, timeslot: int) -> np.ndarray: """ - Select data for a specific timeslot on a specific day. - - Parameters: - year (int): The year of the date. - month (int): The month of the date. - day (int): The day of the date. - timeslot (int): The timeslot index. - - Returns: - np.ndarray: The data for the specified timeslot. - """ - dt = pd.Timestamp(year=year, month=month, day=day, hour=0, minute=0, second=0, tz='UTC') + pd.Timedelta( - minutes=self.time_interval * timeslot) + Select data for a specific timeslot on a specific day. + + Parameters: + year (int): The year of the date. + month (int): The month of the date. + day (int): The day of the date. + timeslot (int): The timeslot index. + + Returns: + np.ndarray: The data for the specified timeslot. + """ + dt = pd.Timestamp(year=year, month=month, day=day, hour=0, minute=0, second=0, tz="UTC") + pd.Timedelta(minutes=self.time_interval * timeslot) row = self.df.loc[dt] return row.values @@ -191,31 +163,32 @@ def select_day_data(self, year: int, month: int, day: int) -> np.ndarray: Returns: np.ndarray: The data for the specified day. """ - start_dt = pd.Timestamp(year=year, month=month, day=day, hour=0, minute=0, second=0, tz='UTC') + start_dt = pd.Timestamp(year=year, month=month, day=day, hour=0, minute=0, second=0, tz="UTC") end_dt = start_dt + pd.Timedelta(days=1) - day_data = self.df.loc[start_dt:end_dt - pd.Timedelta(minutes=1), :] + day_data = self.df.loc[start_dt : end_dt - pd.Timedelta(minutes=1), :] return day_data.values def list_dates(self) -> List[Tuple[int, int, int]]: """ - List all available dates in the data. + List all available dates in the data. - Returns: - List[Tuple[int, int, int]]: A list of available dates as (year, month, day). - """ + Returns: + List[Tuple[int, int, int]]: A list of available dates as (year, month, day). + """ normalized_dates = self.df.index.normalize().unique() return [(ts.year, ts.month, ts.day) for ts in normalized_dates] def random_date(self) -> Tuple[int, int, int]: """ - Randomly select a date from the available dates in the data. + Randomly select a date from the available dates in the data. - Returns: - Tuple[int, int, int]: The year, month, and day of the selected date. - """ + Returns: + Tuple[int, int, int]: The year, month, and day of the selected date. + """ dates = self.list_dates() year, month, day = random.choice(dates) return year, month, day + def split_data_set(self): """ Split the data into training and testing sets based on the date. @@ -262,4 +235,3 @@ def split_data_set(self): self.train_dates = train_dates self.test_dates = test_dates - diff --git a/rl_adn/data_augment/data_augment.py b/rl_adn/data_augment/data_augment.py index 2f73ae2..2329609 100644 --- a/rl_adn/data_augment/data_augment.py +++ b/rl_adn/data_augment/data_augment.py @@ -1,13 +1,16 @@ -import pandas as pd +import re +from datetime import timedelta + import numpy as np -from sklearn.mixture import GaussianMixture -from scipy.stats import norm -from scipy.optimize import brentq +import pandas as pd from copulas.multivariate import GaussianMultivariate -import re -from datetime import datetime, timedelta -from rl_adn.data_manager.data_manager import GeneralPowerDataManager from multicopula import EllipticalCopula +from scipy.optimize import brentq +from scipy.stats import norm +from sklearn.mixture import GaussianMixture + +from rl_adn.data import GeneralPowerDataManager + class ActivePowerDataManager(GeneralPowerDataManager): """ @@ -22,7 +25,7 @@ def __init__(self, datapath: str) -> None: if not datapath: raise ValueError("Please input the correct datapath") - self.df = pd.read_csv(datapath, index_col='date_time') + self.df = pd.read_csv(datapath, index_col="date_time") self.df.index = pd.to_datetime(self.df.index) # Assuming the data interval is consistent throughout the dataset @@ -33,28 +36,27 @@ def get_active_power_data(self) -> np.ndarray: """ Retrieve and preprocess active power data from the dataset. """ - self.df.interpolate(method='linear', inplace=True) - self.df['day'] = self.df.index.date - self.df['time'] = self.df.index.time + self.df.interpolate(method="linear", inplace=True) + self.df["day"] = self.df.index.date + self.df["time"] = self.df.index.time - count_per_day = self.df.groupby('day').size() + count_per_day = self.df.groupby("day").size() expected_time_steps = 24 * 60 / self.time_interval days_with_extra_steps = count_per_day[count_per_day > expected_time_steps] if not days_with_extra_steps.empty: - self.df = self.df[~self.df['day'].isin(days_with_extra_steps.index)] + self.df = self.df[~self.df["day"].isin(days_with_extra_steps.index)] - active_power_columns = [col for col in self.df.columns if re.fullmatch(r'active_power(_\w+)?', col)] + active_power_columns = [col for col in self.df.columns if re.fullmatch(r"active_power(_\w+)?", col)] active_power_df = self.df[active_power_columns].copy() - active_power_df['day'] = self.df['day'] - active_power_df['time'] = self.df['time'] + active_power_df["day"] = self.df["day"] + active_power_df["time"] = self.df["time"] - reshaped_active_power_df = active_power_df.set_index(['day', 'time']).stack().reset_index().rename( - columns={'level_2': 'node', 0: 'value'}) + reshaped_active_power_df = active_power_df.set_index(["day", "time"]).stack().reset_index().rename(columns={"level_2": "node", 0: "value"}) - grouped_active_power_df = reshaped_active_power_df.groupby('time')['value'].apply(list).reset_index() + grouped_active_power_df = reshaped_active_power_df.groupby("time")["value"].apply(list).reset_index() - reshaped_df = pd.DataFrame(grouped_active_power_df['value'].tolist(), index=grouped_active_power_df['time']) + reshaped_df = pd.DataFrame(grouped_active_power_df["value"].tolist(), index=grouped_active_power_df["time"]) active_power_array = reshaped_df.to_numpy().T active_power_array = active_power_array[~np.isnan(active_power_array).any(axis=1)] @@ -63,12 +65,12 @@ def get_active_power_data(self) -> np.ndarray: class TimeSeriesDataAugmentor: - def __init__(self, data_manager, augmentation_model_name='GMC'): + def __init__(self, data, augmentation_model_name="GMC"): """ Initialize the data augmentor with a data manager instance and the selected augmentation model. Additional parameters can be set here if required. """ - self.data_manager = data_manager + self.data = data self.augmentation_model_name = augmentation_model_name self.augmentation_model = None @@ -81,51 +83,46 @@ def _create_augmentation_model(self): GMM: Data augmentaiton using Gaussian Mixture models TC: Data augmentaiton using T Copulas """ - if self.augmentation_model_name == 'GMC': + if self.augmentation_model_name == "GMC": # Extract data from the data manager - active_power_array = self.data_manager.get_active_power_data() + active_power_array = self.data.get_active_power_data() # Determine the best number of components for each GMM model, for one day now it is 96 time steps - self.n_models = int(24.0 * 60.0 / self.data_manager.time_interval) + self.n_models = int(24.0 * 60.0 / self.data.time_interval) - # print(f'data manager time interval {self.data_manager.time_interval}') - best_components = [self._bic_value(active_power_array[:, i].reshape(-1, 1), 20) for i in - range(self.n_models)] + # print(f'data manager time interval {self.data.time_interval}') + best_components = [self._bic_value(active_power_array[:, i].reshape(-1, 1), 20) for i in range(self.n_models)] # Fit the GMM models - self.gmm_models = [GaussianMixture(n_components=bc).fit(active_power_array[:, i].reshape(-1, 1)) - for i, bc in enumerate(best_components)] + self.gmm_models = [GaussianMixture(n_components=bc).fit(active_power_array[:, i].reshape(-1, 1)) for i, bc in enumerate(best_components)] # Transform the data to standard format for copula fitting std_input_data = np.empty((active_power_array.shape[0], self.n_models)) for i in range(self.n_models): - std_input_data[:, i] = np.array( - [self._gmm_cdf(self.gmm_models[i], x) for x in active_power_array[:, i]]).reshape(1, -1) + std_input_data[:, i] = np.array([self._gmm_cdf(self.gmm_models[i], x) for x in active_power_array[:, i]]).reshape(1, -1) self.copula = GaussianMultivariate() self.copula.fit(std_input_data) # Assign the copula as the augmentation model self.augmentation_model = self.copula - if self.augmentation_model_name == 'GMM': - active_power_array = self.data_manager.get_active_power_data() + if self.augmentation_model_name == "GMM": + active_power_array = self.data.get_active_power_data() # Determine the best number of components for each GMM model, for one day now it is 96 time steps - self.n_models = int(24.0 * 60.0 / self.data_manager.time_interval) + self.n_models = int(24.0 * 60.0 / self.data.time_interval) - # print(f'data manager time interval {self.data_manager.time_interval}') - best_components = [self._bic_value(active_power_array[:, i].reshape(-1, 1), 20) for i in - range(self.n_models)] + # print(f'data manager time interval {self.data.time_interval}') + best_components = [self._bic_value(active_power_array[:, i].reshape(-1, 1), 20) for i in range(self.n_models)] # Fit the GMM models - self.gmm_models = [GaussianMixture(n_components=bc).fit(active_power_array[:, i].reshape(-1, 1)) - for i, bc in enumerate(best_components)] + self.gmm_models = [GaussianMixture(n_components=bc).fit(active_power_array[:, i].reshape(-1, 1)) for i, bc in enumerate(best_components)] self.augmentation_model = self.gmm_models - if self.augmentation_model_name == 'TC': - active_power_array = self.data_manager.get_active_power_data() - self.n_models = int(24.0 * 60.0 / self.data_manager.time_interval) + if self.augmentation_model_name == "TC": + active_power_array = self.data.get_active_power_data() + self.n_models = int(24.0 * 60.0 / self.data.time_interval) self.tc_model = EllipticalCopula(active_power_array.T) self.tc_model.fit() @@ -171,9 +168,9 @@ def augment_data(self, num_nodes, num_days, start_date): """ Perform data augmentation using the specified model and parameters. """ - if self.augmentation_model_name == 'GMC': + if self.augmentation_model_name == "GMC": num_samples = num_days * num_nodes - print('The number of samples is', num_samples) + print("The number of samples is", num_samples) generated_pesudo_obs = np.empty((0, self.n_models)) count = 0 @@ -188,14 +185,13 @@ def augment_data(self, num_nodes, num_days, start_date): # print(' the pesudo data is now sampled and next process is to transfer it to the realistic data') tran_samples = np.empty((generated_pesudo_obs.shape[0], generated_pesudo_obs.shape[1])) for i in range(self.n_models): - tran_samples[:, i] = np.array( - [self._inverse_gmm_cdf(self.gmm_models[i], u) for u in generated_pesudo_obs[:, i]]) - print(f'the {i} model columns now is calculated') + tran_samples[:, i] = np.array([self._inverse_gmm_cdf(self.gmm_models[i], u) for u in generated_pesudo_obs[:, i]]) + print(f"the {i} model columns now is calculated") tran_samples = tran_samples.flatten() - if self.augmentation_model_name == 'GMM': + if self.augmentation_model_name == "GMM": num_samples = num_days * num_nodes - print('The number of samples is', num_samples) + print("The number of samples is", num_samples) # generating the data gmm_samples = np.empty((num_samples, self.n_models)) @@ -204,9 +200,9 @@ def augment_data(self, num_nodes, num_days, start_date): tran_samples = gmm_samples.flatten() - if self.augmentation_model_name == 'TC': + if self.augmentation_model_name == "TC": num_samples = num_days * num_nodes - print('The number of samples is', num_samples) + print("The number of samples is", num_samples) # generating the data TC_samples = np.empty((0, self.n_models)) @@ -215,7 +211,7 @@ def augment_data(self, num_nodes, num_days, start_date): gen_one_sample = np.array(self.tc_model.sample(1)).reshape(1, -1) # cancel inf - if np.isinf(gen_one_sample).any() == False: + if not np.isinf(gen_one_sample).any(): count += 1 # print(count,num_samples) TC_samples = np.vstack((gen_one_sample, TC_samples)) @@ -231,71 +227,37 @@ def augment_data(self, num_nodes, num_days, start_date): for day in range(num_days): for node in range(1, num_nodes + 1): - time_step = timedelta(minutes=self.data_manager.time_interval) + time_step = timedelta(minutes=self.data.time_interval) timestamps.extend([start_date + timedelta(days=day) + i * time_step for i in range(self.n_models)]) - node_index.extend([f'active_power_node_{node}' for _ in range(self.n_models)]) + node_index.extend([f"active_power_node_{node}" for _ in range(self.n_models)]) # Create DataFrame - synthetic_data_df = pd.DataFrame({ - 'date_time': timestamps, - 'node': node_index, - 'value': tran_samples - }) + synthetic_data_df = pd.DataFrame({"date_time": timestamps, "node": node_index, "value": tran_samples}) # Pivot the DataFrame to get it into the desired format - augmented_df = synthetic_data_df.pivot(index='date_time', columns='node', values='value').reset_index() + augmented_df = synthetic_data_df.pivot(index="date_time", columns="node", values="value").reset_index() # Reorder the columns based on we need - active_power_cols = self.sort_columns(augmented_df.columns, r'active_power(_\w+)?') - reactive_power_cols = self.sort_columns(augmented_df.columns, r'reactive_power(_\w+)?') - renewable_active_power_cols = self.sort_columns(augmented_df.columns, r'renewable_active_power(_\w+)?') - renewable_reactive_power_cols = self.sort_columns(augmented_df.columns, r'renewable_reactive_power(_\w+)?') - price_cols = self.sort_columns(augmented_df.columns, r'price(_\w+)?') + active_power_cols = self.sort_columns(augmented_df.columns, r"active_power(_\w+)?") + reactive_power_cols = self.sort_columns(augmented_df.columns, r"reactive_power(_\w+)?") + renewable_active_power_cols = self.sort_columns(augmented_df.columns, r"renewable_active_power(_\w+)?") + renewable_reactive_power_cols = self.sort_columns(augmented_df.columns, r"renewable_reactive_power(_\w+)?") + price_cols = self.sort_columns(augmented_df.columns, r"price(_\w+)?") # Combine columns in the specified order - ordered_columns = ( - ['date_time'] + - active_power_cols + - reactive_power_cols + - renewable_active_power_cols + - renewable_reactive_power_cols + - price_cols - ) + ordered_columns = ["date_time"] + active_power_cols + reactive_power_cols + renewable_active_power_cols + renewable_reactive_power_cols + price_cols ordered_augmented_df = augmented_df[ordered_columns] return ordered_augmented_df def save_augmented_data(self, augmented_df, file_name): augmented_df.to_csv(file_name, index=False) - print('The data file is stored:', file_name) + print("The data file is stored:", file_name) def sort_columns(self, columns, pattern): def sort_key(col_name): - parts = col_name.split('_') + parts = col_name.split("_") if parts[-1].isdigit(): return int(parts[-1]) return 0 # Default sort value for non-numeric endings filtered_cols = [col for col in columns if re.fullmatch(pattern, col)] return sorted(filtered_cols, key=sort_key) - - -if __name__ == "__main__": - input_data_file = 'test_original_data.csv' # Replace with your actual file path - augmentation_model_name = 'GMM' - num_nodes = 3 # For examples, if you have 34 nodes - num_days = 3 # Assuming you want to generate data for a full year - - # Initialize the data manager with the input CSV file - data_manager = ActivePowerDataManager(input_data_file) - - # Initialize the TimeSeriesDataAugmentor with the data manager and model name - augmentor = TimeSeriesDataAugmentor(data_manager, augmentation_model_name) - - # Generate augmented data - augmented_df = augmentor.augment_data(num_nodes, num_days,start_date=datetime(2021, 1, 1, 0, 0)) - - # Define the file name where to save the augmented data - - # Save the augmented data to a CSV file - augmentor.save_augmented_data(augmented_df, 'test_generated_data.csv') - - print("Data augmentation completed and saved to file.") \ No newline at end of file diff --git a/rl_adn/data_manager/__init__.py b/rl_adn/data_manager/__init__.py deleted file mode 100644 index ba32e47..0000000 --- a/rl_adn/data_manager/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Stable data-manager exports for RL-ADN.""" - -from rl_adn.data_manager.data_manager import GeneralPowerDataManager - -__all__ = ["GeneralPowerDataManager"] diff --git a/rl_adn/environments/__init__.py b/rl_adn/environments/__init__.py index 9e6e3f9..58f79df 100644 --- a/rl_adn/environments/__init__.py +++ b/rl_adn/environments/__init__.py @@ -1,25 +1,25 @@ -"""Stable environment exports for RL-ADN.""" +"""Environment-facing exports for RL-ADN.""" from importlib import import_module __all__ = [ "Battery", - "battery_parameters", + "ObservationSnapshot", "PowerNetEnv", - "env_config", - "make_env_config", + "RewardBreakdown", + "StateScaler", ] _LAZY_EXPORTS = { "Battery": ("rl_adn.environments.battery", "Battery"), - "battery_parameters": ("rl_adn.environments.battery", "battery_parameters"), + "ObservationSnapshot": ("rl_adn.environments.observation", "ObservationSnapshot"), "PowerNetEnv": ("rl_adn.environments.env", "PowerNetEnv"), - "env_config": ("rl_adn.environments.config", "env_config"), - "make_env_config": ("rl_adn.environments.config", "make_env_config"), + "RewardBreakdown": ("rl_adn.environments.reward", "RewardBreakdown"), + "StateScaler": ("rl_adn.environments.observation", "StateScaler"), } -def __getattr__(name): +def __getattr__(name: str): if name not in _LAZY_EXPORTS: raise AttributeError(f"module 'rl_adn.environments' has no attribute {name!r}") diff --git a/rl_adn/environments/battery.py b/rl_adn/environments/battery.py index aca8fa4..a4e680b 100644 --- a/rl_adn/environments/battery.py +++ b/rl_adn/environments/battery.py @@ -1,125 +1,58 @@ -import numpy as np - -battery_parameters={ -'capacity':300,# kW.h -'max_charge':50, # kW -'max_discharge':50, #kW -'efficiency':1, -'degradation':0, #euro/kw -'max_soc':0.8, -'min_soc':0.2, -'initial_soc':0.4} +from __future__ import annotations -DEFAULT_TIME_INTERVAL_MINUTES = 15.0 +from dataclasses import replace -class Battery(): - """ - A simple battery model for energy storage and management. +import numpy as np - Attributes: - capacity (float): The total energy capacity of the battery in kW.h. - max_soc (float): The maximum state of charge (SOC) as a fraction of capacity. - initial_soc (float): The initial state of charge as a fraction of capacity. - min_soc (float): The minimum state of charge as a fraction of capacity. - degradation (float): The cost of battery degradation per kW. - max_charge (float): The maximum charging power of the battery in kW. - max_discharge (float): The maximum discharging power of the battery in kW. - efficiency (float): The efficiency of charging and discharging processes. - current_soc (float): The current state of charge of the battery. +from rl_adn.config import BatteryConfig - Args: - parameters (dict): A dictionary containing battery parameters. - Description: - This class simulates a simple battery for energy storage. It allows for charging and discharging - operations while considering constraints like maximum/minimum state of charge, charging/discharging - rates, and efficiency. It also calculates the cost associated with battery degradation. - """ +class Battery: + """Simple single-battery state tracker for ESS dispatch.""" - def __init__(self, parameters): - """ - Initializes the Battery object with given parameters. + def __init__(self, config: BatteryConfig) -> None: + self.config = config + self.last_power_kw = 0.0 + self.reset() - Args: - parameters (dict): A dictionary containing the battery's parameters such as capacity, state of charge limits, - degradation cost, maximum charge/discharge rates, and efficiency. - """ - self.capacity = parameters['capacity'] # 容量 - self.max_soc = parameters['max_soc'] # max soc 0.8 - self.initial_soc = parameters['initial_soc'] # initial soc 0.4 - self.min_soc = parameters['min_soc'] # 0.2 - self.degradation = parameters['degradation'] # degradation cost 0， - self.max_charge = parameters['max_charge'] # max charge ability - self.max_discharge = parameters['max_discharge'] # max discharge ability - self.efficiency = parameters['efficiency'] # charge and discharge efficiency - self.time_interval_minutes = float(parameters.get('time_interval_minutes', DEFAULT_TIME_INTERVAL_MINUTES)) - if self.time_interval_minutes <= 0: - raise ValueError("time_interval_minutes must be positive") + @property + def capacity_kwh(self) -> float: + return self.config.capacity_kwh - def step(self, action_battery): - """ - Executes a step of battery operation based on the given action. + @property + def time_interval_minutes(self) -> float: + return self.config.time_interval_minutes - Args: - action_battery (float): The action to be taken, typically representing the amount of energy to charge or discharge. + def with_time_interval(self, time_interval_minutes: float) -> "Battery": + return Battery(replace(self.config, time_interval_minutes=time_interval_minutes)) - Description: - This method updates the state of charge (SOC) of the battery based on the given action. It calculates the - energy change and updates the SOC while ensuring it stays within the defined minimum and maximum limits. - The energy change is also used to calculate the cost associated with battery operation. - """ - action_array = np.asarray(action_battery).reshape(-1) + def step(self, action: float | np.ndarray) -> float: + """Apply a normalized action in [-1, 1] and return realized battery power in kW.""" + action_array = np.asarray(action, dtype=np.float32).reshape(-1) if action_array.size != 1: raise ValueError("Battery.step expects a scalar action or a size-1 array") - action_battery = float(action_array[0]) - rated_power = self.max_discharge if action_battery >= 0 else self.max_charge - power_command = action_battery * rated_power - interval_hours = self.time_interval_minutes / 60.0 - updated_soc = max( - self.min_soc, - min( - self.max_soc, - (self.current_soc * self.capacity + power_command * interval_hours) / self.capacity, - ), - ) - - self.energy_change = (updated_soc - self.current_soc) * self.capacity / interval_hours # if charge, positive, if discharge, negative - self.current_soc = updated_soc # update capacity to current codition - - def _get_cost(self, energy): # calculate the cost depends on the energy change - """ - Calculates the cost associated with a given energy change. - Args: - energy (float): The amount of energy change in the battery. - - Returns: - float: The calculated cost based on the energy change. - - Description: - This method calculates the cost of operating the battery, which is a function of the absolute value of the energy change. - It is used internally to assess the cost implications of charging or discharging the battery. - """ - cost = np.abs(energy) - return cost - - def SOC(self): - """ - Returns the current state of charge (SOC) of the battery. + normalized_action = float(np.clip(action_array[0], -1.0, 1.0)) + rated_power_kw = self.config.max_discharge_kw if normalized_action >= 0 else self.config.max_charge_kw + requested_power_kw = normalized_action * rated_power_kw + interval_hours = self.config.time_interval_minutes / 60.0 + + current_energy_kwh = self.current_soc * self.capacity_kwh + requested_energy_kwh = requested_power_kw * interval_hours + updated_soc = np.clip( + (current_energy_kwh + requested_energy_kwh) / self.capacity_kwh, + self.config.min_soc, + self.config.max_soc, + ) - Returns: - float: The current SOC of the battery. + realized_energy_kwh = (updated_soc - self.current_soc) * self.capacity_kwh + self.last_power_kw = realized_energy_kwh / interval_hours + self.current_soc = float(updated_soc) + return self.last_power_kw - Description: - This method provides the current state of charge of the battery as a fraction of its total capacity. - """ + def SOC(self) -> float: return self.current_soc - def reset(self): - """ - Resets the state of charge (SOC) of the battery to its initial value. - - Description: - This method is used to reset the battery's state of charge to its initial value, typically used at the start of a new simulation or operational cycle. - """ - self.current_soc = self.initial_soc + def reset(self) -> None: + self.current_soc = float(self.config.initial_soc) + self.last_power_kw = 0.0 diff --git a/rl_adn/environments/config.py b/rl_adn/environments/config.py deleted file mode 100644 index 8e867a0..0000000 --- a/rl_adn/environments/config.py +++ /dev/null @@ -1,109 +0,0 @@ -from pathlib import Path -from typing import Dict, List, Optional - -from rl_adn.environments.topology_scenarios import get_topology_scenario - -PACKAGE_ROOT = Path(__file__).resolve().parents[1] -DATA_ROOT = PACKAGE_ROOT / "data_sources" -NETWORK_ROOT = DATA_ROOT / "network_data" -TIME_SERIES_ROOT = DATA_ROOT / "time_series_data" - -DEFAULT_NODE = 34 -DEFAULT_BATTERY_LISTS = { - # Paper battery placements are reported with 1-based node labels. - # RL-ADN stores controllable battery nodes as zero-based bus indices. - 34: [11, 15, 26, 29, 33], # paper nodes {12, 16, 27, 30, 34} - 69: [13, 15, 17, 19, 21, 23, 25, 26, 64], # paper nodes {14, 16, 18, 20, 22, 24, 26, 27, 65} -} - - -def _resolve_network_info(node: int, vm_pu: float, s_base: float) -> Dict[str, object]: - node_dir = NETWORK_ROOT / f"node_{node}" - bus_info_file = node_dir / f"Nodes_{node}.csv" - branch_info_file = node_dir / f"Lines_{node}.csv" - - if not bus_info_file.exists() or not branch_info_file.exists(): - raise FileNotFoundError(f"Missing packaged network data for node_{node}") - - return { - "vm_pu": vm_pu, - "s_base": s_base, - "bus_info_file": str(bus_info_file), - "branch_info_file": str(branch_info_file), - } - - -def _resolve_time_series_data_path(node: int, override: Optional[str]) -> str: - if override is not None: - return str(Path(override)) - - candidate = TIME_SERIES_ROOT / f"{node}_node_time_series.csv" - if candidate.exists(): - return str(candidate) - - if node == DEFAULT_NODE: - raise FileNotFoundError(f"Missing packaged time-series data for node_{node}") - - raise ValueError( - f"No packaged time-series data is available for node_{node}; please provide time_series_data_path explicitly." - ) - - -def make_env_config( - node: int = DEFAULT_NODE, - algorithm: str = "Laurent", - train: bool = True, - battery_list: Optional[List[int]] = None, - year: int = 2020, - month: int = 1, - day: int = 1, - state_pattern: str = "default", - vm_pu: float = 1.0, - s_base: float = 1000, - time_series_data_path: Optional[str] = None, - topology_mode: str = "fixed", - topology_scenario: Optional[str] = None, - topology_pool: Optional[List[str]] = None, - return_graph: bool = False, -) -> Dict[str, object]: - if topology_mode not in {"fixed", "scenario_pool"}: - raise ValueError("topology_mode must be either 'fixed' or 'scenario_pool'") - - if topology_pool is not None and len(topology_pool) == 0: - raise ValueError("topology_pool must not be empty when provided") - - if topology_scenario is not None: - get_topology_scenario(node, topology_scenario) - - if topology_pool is not None: - for scenario_id in topology_pool: - get_topology_scenario(node, scenario_id) - - if battery_list is None: - default_battery_list = DEFAULT_BATTERY_LISTS.get(node) - if default_battery_list is None: - raise ValueError( - f"battery_list must be provided for node counts without a curated default (received node={node})" - ) - battery_list = list(default_battery_list) - - return { - "voltage_limits": [0.95, 1.05], - "algorithm": algorithm, - "battery_list": list(battery_list), - "year": year, - "month": month, - "day": day, - "train": train, - "state_pattern": state_pattern, - "network_info": _resolve_network_info(node=node, vm_pu=vm_pu, s_base=s_base), - "time_series_data_path": _resolve_time_series_data_path(node=node, override=time_series_data_path), - "feeder_id": f"{node}-bus", - "topology_mode": topology_mode, - "topology_scenario": topology_scenario, - "topology_pool": list(topology_pool) if topology_pool is not None else None, - "return_graph": return_graph, - } - - -env_config = make_env_config() diff --git a/rl_adn/environments/env.py b/rl_adn/environments/env.py index e573ed8..41593b0 100644 --- a/rl_adn/environments/env.py +++ b/rl_adn/environments/env.py @@ -1,18 +1,21 @@ -import copy as cp -import random +from __future__ import annotations -import gym +from dataclasses import replace +from typing import Any + +import gymnasium as gym import numpy as np import pandas as pd -from gym import spaces - -from rl_adn.data_manager.data_manager import GeneralPowerDataManager -from rl_adn.environments.battery import Battery, battery_parameters -from rl_adn.environments.config import make_env_config, env_config +from gymnasium import spaces + +from rl_adn.config import DEFAULT_ENV_CONFIG, EnvConfig +from rl_adn.data import GeneralPowerDataManager +from rl_adn.environments.battery import Battery +from rl_adn.environments.observation import ObservationSnapshot, SlotFeatures, StateScaler, build_default_state +from rl_adn.environments.reward import RewardBreakdown, compute_default_reward +from rl_adn.environments.solvers import LaurentSolverAdapter, PandaPowerSolverAdapter, PowerFlowSnapshot from rl_adn.environments.topology_scenarios import get_topology_scenario -from rl_adn.utility.grid import GridTensor -from rl_adn.utility.utils import create_pandapower_net -from rl_adn.utility.topology import ( +from rl_adn.network.topology import ( apply_topology_scenario, build_adjacency_matrix, build_edge_index, @@ -21,288 +24,168 @@ ) -def _require_pandapower(): - try: - import pandapower as pp - except ImportError as exc: - raise ImportError("PandaPower support requires the optional dependency 'pandapower'.") from exc - return pp - - -class PowerNetEnv(gym.Env): - """ - Custom Environment for Power Network Management. - - The environment simulates a power network, and the agent's task is to - manage this network by controlling the batteries attached to various nodes. - - Attributes: - voltage_limits (tuple): Limits for the voltage. - algorithm (str): Algorithm choice. Can be 'Laurent' or 'PandaPower'. - battery_list (list): List of nodes where batteries are attached. - year (int): Current year in simulation. - month (int): Current month in simulation. - day (int): Current day in simulation. - train (bool): Whether the environment is in training mode. - state_pattern (str): Pattern for the state representation. - network_info (dict): Information about the network. - node_num (int): Number of nodes in the network. - action_space (gym.spaces.Box): Action space of the environment. - data_manager (GeneralPowerDataManager): Manager for the time-series data. - episode_length (int): Length of an episode. - state_length (int): Length of the state representation. - state_min (np.ndarray): Minimum values for each state element. - state_max (np.ndarray): Maximum values for each state element. - state_space (gym.spaces.Box): State space of the environment. - current_time (int): Current timestep in the episode. - after_control (np.ndarray): Voltages after control is applied. - - Args: - env_config_path (str): Path to the environment configuration file. - - """ - - def __init__(self, env_config: dict = env_config) -> None: - """ - Initialize the PowerNetEnv environment. - :param env_config_path: Path to the environment configuration file. Defaults to 'env_config.py'. - :type env_config_path: str - """ - config = env_config - - self.voltage_low_boundary = config['voltage_limits'][0] - self.voltage_high_boundary = config['voltage_limits'][1] - self.algorithm = config['algorithm'] - self.battery_list = config['battery_list'] - self.year = config['year'] - self.month = config['month'] - self.day = config['day'] - self.train = config['train'] - self.state_pattern = config['state_pattern'] - self.feeder_id = config.get('feeder_id', f"{config['network_info']['bus_info_file']}-feeder") - self.topology_mode = config.get('topology_mode', 'fixed') - self.topology_scenario = config.get('topology_scenario') - self.topology_pool = config.get('topology_pool') - self.return_graph = bool(config.get('return_graph', False)) - - self.network_info = cp.deepcopy(config['network_info']) - # network_info for building the network - if self.network_info == 'None': - print('create basic 34 node IEEE network, when initial data is not identified') - self.network_info = make_env_config()['network_info'] - self.s_base = 1000 - else: - self.s_base = self.network_info['s_base'] - - self._baseline_bus_info = pd.read_csv(self.network_info['bus_info_file']) - self._baseline_line_info = pd.read_csv(self.network_info['branch_info_file']) - self.node_num = len((self._baseline_bus_info.NODES)) +class PowerNetEnv(gym.Env[np.ndarray, np.ndarray]): + """Gymnasium environment for ESS dispatch in active distribution networks.""" + + metadata = {"render_modes": []} + + def __init__(self, config: EnvConfig = DEFAULT_ENV_CONFIG) -> None: + super().__init__() + self.config = config + self.algorithm = config.algorithm + self.train = config.train + self.state_pattern = config.state_pattern + self.feeder_id = config.feeder_id + self.node_count = config.node_count + self.battery_nodes = tuple(config.battery_nodes) + self.topology_config = config.topology + self.voltage_low_boundary, self.voltage_high_boundary = config.voltage_limits + self.year = config.year + self.month = config.month + self.day = config.day + + self._rng = np.random.default_rng() + self._episode_done = False + + self._baseline_bus_info = pd.read_csv(config.bus_info_file) + self._baseline_line_info = pd.read_csv(config.branch_info_file) + if len(self._baseline_bus_info) != self.node_count: + raise ValueError("EnvConfig.node_count does not match the packaged bus data") + + self.data_manager = GeneralPowerDataManager(config.time_series_data_path) + self.battery_config = replace(config.battery, time_interval_minutes=self.data_manager.time_interval) + self.batteries = {node_index: Battery(self.battery_config) for node_index in self.battery_nodes} + self.episode_length = int(24 * 60 / self.data_manager.time_interval) + self.state_scaler = StateScaler( + node_count=self.node_count, + battery_count=len(self.battery_nodes), + active_power_min=self.data_manager.active_power_min, + active_power_max=self.data_manager.active_power_max, + price_min=self.data_manager.price_min, + price_max=self.data_manager.price_max, + episode_length=self.episode_length, + min_soc=self.battery_config.min_soc, + max_soc=self.battery_config.max_soc, + ) + + self.action_space = spaces.Box( + low=-1.0, + high=1.0, + shape=(len(self.battery_nodes),), + dtype=np.float32, + ) + state_dim = self.node_count + 2 * len(self.battery_nodes) + 2 + self.observation_space = spaces.Box( + low=-2.0, + high=2.0, + shape=(state_dim,), + dtype=np.float32, + ) - self.current_scenario = None - self.active_line_info = None - self.active_bus_info = None - self.active_edges = None - self.adjacency_matrix = None - self.edge_index = None - self._apply_topology(self._determine_scenario_id(initial=True)) - - self.data_manager = GeneralPowerDataManager(config['time_series_data_path']) - if not self.battery_list: - raise ValueError("No batteries specified!") - battery_config = dict(battery_parameters) - battery_config["time_interval_minutes"] = self.data_manager.time_interval - for node_index in self.battery_list: - battery = Battery(battery_config) - setattr(self, f"battery_{node_index}", battery) - self.action_space = spaces.Box(low=-1, high=1, shape=(len(self.battery_list), 1), dtype=np.float32) self.active_power_indices = [self.data_manager.df.columns.get_loc(col) for col in self.data_manager.active_power_cols] - self.renewable_active_power_indices = [ - self.data_manager.df.columns.get_loc(col) for col in self.data_manager.renewable_active_power_cols - ] + self.renewable_active_power_indices = [self.data_manager.df.columns.get_loc(col) for col in self.data_manager.renewable_active_power_cols] self.price_index = self.data_manager.df.columns.get_loc(self.data_manager.price_col[0]) - self.episode_length: int = 24 * 60 / self.data_manager.time_interval - - if self.state_pattern == 'default': - self.state_length = len(self.battery_list) * 2 + self.node_num + 2 - print(self.data_manager.active_power_min) - print(self.data_manager.price_min) - self.state_min = np.array([self.data_manager.active_power_min, 0.2, self.data_manager.price_min, 0.0, 0.5]) - self.state_max = np.array( - [self.data_manager.active_power_max, 0.8, self.data_manager.price_max, self.episode_length - 1, 1.5]) - else: - raise ValueError("Invalid value for 'state_pattern'. Expected 'default' or define by yourself.") - self.state_space = spaces.Box(low=-2, high=2, shape=(self.state_length,), dtype=np.float32) - - def reset(self, return_info: bool = False): - self._apply_topology(self._determine_scenario_id()) - self._reset_date() - self._reset_time() + self.current_time = 0 + self.current_scenario = None + self.active_line_info: pd.DataFrame | None = None + self.active_bus_info: pd.DataFrame | None = None + self.active_edges: list[tuple[int, int]] = [] + self.adjacency_matrix: np.ndarray | None = None + self.edge_index: np.ndarray | None = None + self.solver = None + + self.current_slot_features: SlotFeatures | None = None + self.current_observation: ObservationSnapshot | None = None + self.current_precontrol_snapshot: PowerFlowSnapshot | None = None + self.last_reward_breakdown: RewardBreakdown | None = None + + self._apply_topology(self.topology_config.scenario_id) + + def reset(self, *, seed: int | None = None, options: dict[str, Any] | None = None) -> tuple[np.ndarray, dict[str, Any]]: + """Reset the environment and return the initial observation and metadata.""" + super().reset(seed=seed) + if seed is not None: + self._rng = np.random.default_rng(seed) + + scenario_id = self._sample_topology_scenario() + self._apply_topology(scenario_id) + self._select_episode_date() + self.current_time = 0 + self._episode_done = False self._reset_batteries() - state = self._build_state() - if return_info: - return state, self._build_info(state) - return state - - def _reset_date(self) -> None: - """ - Resets the date for the next episode. - """ - if self.train: - self.year, self.month, self.day = random.choice(self.data_manager.train_dates) - else: - self.year, self.month, self.day = random.choice(self.data_manager.test_dates) + self.current_observation, self.current_slot_features, self.current_precontrol_snapshot = self._observe_current_slot() + return self.current_observation.normalized_state.copy(), self._build_info() + + def step(self, action: np.ndarray) -> tuple[np.ndarray, float, bool, bool, dict[str, Any]]: + """Advance one environment step using a flat action vector in ``[-1, 1]``.""" + if self.current_observation is None or self.current_slot_features is None or self.current_precontrol_snapshot is None: + raise RuntimeError("reset() must be called before step()") + if self._episode_done: + raise RuntimeError("Episode has finished; call reset() before step() again") + + action_vector = np.asarray(action, dtype=np.float32).reshape(-1) + if action_vector.shape != (len(self.battery_nodes),): + raise ValueError(f"Expected action shape {(len(self.battery_nodes),)}, received {tuple(action_vector.shape)}") + + battery_dispatch_kw = np.array( + [self.batteries[node_index].step(action_component) for node_index, action_component in zip(self.battery_nodes, action_vector)], + dtype=np.float32, + ) + + net_load_kw = self.current_slot_features.active_power_kw - self.current_slot_features.renewable_active_power_kw + post_control_snapshot = self.solver.dispatch(net_load_kw, self.battery_nodes, battery_dispatch_kw) + reward_breakdown = compute_default_reward( + price=self.current_slot_features.price, + saved_power_kw=self.current_precontrol_snapshot.import_power_kw - post_control_snapshot.import_power_kw, + battery_voltages_pu=post_control_snapshot.node_voltages_pu[list(self.battery_nodes)], + ) + self.last_reward_breakdown = reward_breakdown + + truncated = self.current_time >= self.episode_length - 1 + terminated = False + info = self._build_info(post_control_snapshot=post_control_snapshot, battery_dispatch_kw=battery_dispatch_kw) + + if truncated: + self._episode_done = True + terminal_observation = self._build_observation(self.current_slot_features, post_control_snapshot) + return terminal_observation.normalized_state.copy(), reward_breakdown.total, terminated, True, info - def _reset_time(self) -> None: - """ - Resets the time for the next episode. - """ - self.current_time = 0 + self.current_time += 1 + self.current_observation, self.current_slot_features, self.current_precontrol_snapshot = self._observe_current_slot() + return self.current_observation.normalized_state.copy(), reward_breakdown.total, terminated, False, info - def _reset_batteries(self) -> None: - """ - Resets the batteries for the next episode. - """ - for node_index in self.battery_list: - getattr(self, f"battery_{node_index}").reset() - - def _build_state(self) -> np.ndarray: - """ - Builds the current state of the environment based on the current time and data from PowerDataManager. - - Returns: - normalized_state (np.ndarray): The current state of the environment, normalized between 0 and 1. - The state includes the following variables: - - Netload power - - SOC (State of Charge) of the last battery in the battery list - - Price of the energy - - Time state of the day - - Voltage from estimation - """ - # TODO: modify get state observation to fit new resources and data - obs = self._get_obs() - if self.state_pattern == 'default': - active_power = np.array(list(obs['node_data']['active_power'].values())) - price = float(obs['price']) - soc_list = np.array( - [obs['battery_data']['soc'][f'battery_{node_index}'] for node_index in self.battery_list]) - vm_pu_battery = np.array( - [obs['node_data']['voltage'][f'node_{node_index}'] for node_index in self.battery_list]) - state = np.concatenate((active_power, soc_list, [price], [self.current_time], vm_pu_battery)) - self.state = state - normalized_state = self._normalize_state(state) - self.normalized_state = normalized_state - return normalized_state - - def _split_state(self, state): - net_load_length = self.node_num - num_batteries = len(self.battery_list) - - soc_all_length = num_batteries - vm_pu_battery_nodes_length = num_batteries - - soc_all_start = net_load_length - price_start = soc_all_start + soc_all_length - current_time_start = price_start + 1 - vm_pu_battery_nodes_start = current_time_start + 1 - - net_load = state[:net_load_length] - soc_all = state[soc_all_start:soc_all_start + soc_all_length] - price = np.array([state[price_start]]) - current_time = np.array([state[current_time_start]]) - vm_pu_battery_nodes = state[vm_pu_battery_nodes_start:] - - return net_load, soc_all, price, current_time, vm_pu_battery_nodes - - def _normalize_state(self, state: np.ndarray) -> np.ndarray: - """ - Normalizes the state variables. - - Parameters: - state (np.ndarray): The current state of the environment. - - Returns: - np.ndarray: The normalized state of the environment. - """ - state[:self.node_num] = (state[:self.node_num] - self.state_min[0]) / (self.state_max[0] - self.state_min[0]) - state[self.node_num:self.node_num + len(self.battery_list)] = (state[self.node_num:self.node_num + len( - self.battery_list)] - self.state_min[1]) / (self.state_max[1] - self.state_min[1]) - state[self.node_num + len(self.battery_list):self.node_num + len(self.battery_list) + 1] = (state[ - self.node_num + len( - self.battery_list):self.node_num + len( - self.battery_list) + 1] - - self.state_min[ - 2]) / ( - self.state_max[ - 2] - - self.state_min[ - 2]) - state[self.node_num + len(self.battery_list) + 1:self.node_num + len(self.battery_list) + 2] = (state[ - self.node_num + len( - self.battery_list) + 1:self.node_num + len( - self.battery_list) + 2] - - self.state_min[ - 3]) / ( - self.state_max[ - 3] - - self.state_min[ - 3]) - normalized_state = state - return normalized_state - - def _denormalize_state(self, normalized_state: np.ndarray) -> np.ndarray: - """ - Denormalizes the state variables. - - Parameters: - normalized_state (np.ndarray): The normalized state of the environment. - - Returns: - np.ndarray: The denormalized state of the environment. - """ - normalized_state[:self.node_num] = normalized_state[:self.node_num] * (self.state_max[0] - self.state_min[0]) + \ - self.state_min[0] - - normalized_state[self.node_num:self.node_num + len(self.battery_list)] = normalized_state[ - self.node_num:self.node_num + len( - self.battery_list)] * ( - self.state_max[1] - - self.state_min[1]) + \ - self.state_min[1] - normalized_state[ - self.node_num + len(self.battery_list):self.node_num + len(self.battery_list) + 1] = normalized_state[ - self.node_num + len( - self.battery_list):self.node_num + len( - self.battery_list) + 1] * ( - self.state_max[ - 2] - - self.state_min[ - 2]) + \ - self.state_min[2] - normalized_state[ - self.node_num + len(self.battery_list) + 1:self.node_num + len(self.battery_list) + 2] = normalized_state[ - self.node_num + len( - self.battery_list) + 1:self.node_num + len( - self.battery_list) + 2] * ( - self.state_max[ - 3] - - self.state_min[ - 3]) + \ - self.state_min[3] - denormalized_state = normalized_state - return denormalized_state - - def _determine_scenario_id(self, initial: bool = False) -> str: - if self.topology_mode == 'fixed': - return self.topology_scenario or 'TP1' - if self.topology_mode == 'scenario_pool': - if not self.topology_pool: - raise ValueError("topology_pool must be provided when topology_mode='scenario_pool'") - return random.choice(self.topology_pool) - raise ValueError("Invalid topology_mode. Expected 'fixed' or 'scenario_pool'.") + def get_topology_metadata(self) -> dict[str, Any]: + return { + "feeder_id": self.feeder_id, + "scenario_id": self.current_scenario.scenario_id, + "node_count": self.node_count, + "edge_count": len(self.active_edges), + "active_edges": list(self.active_edges), + } + + def get_graph_data(self) -> dict[str, Any]: + if self.adjacency_matrix is None or self.edge_index is None or self.active_line_info is None: + raise RuntimeError("Topology has not been initialized") + return { + "adjacency": self.adjacency_matrix.copy(), + "edge_index": self.edge_index.copy(), + "node_ids": np.arange(1, self.node_count + 1, dtype=np.int64), + "active_line_data": self.active_line_info[["FROM", "TO", "R", "X", "B", "STATUS", "TAP"]].to_dict("records"), + } + + def render(self) -> None: + return None + + def _sample_topology_scenario(self) -> str: + if self.topology_config.mode == "fixed": + return self.topology_config.scenario_id + if self.topology_config.mode == "scenario_pool": + return str(self._rng.choice(self.topology_config.scenario_pool)) + raise ValueError("Unsupported topology mode") def _apply_topology(self, scenario_id: str) -> None: - scenario = get_topology_scenario(self.node_num, scenario_id) + scenario = get_topology_scenario(self.node_count, scenario_id) active_line_info = apply_topology_scenario(self._baseline_line_info, scenario) validation = validate_radial_topology(self._baseline_bus_info, active_line_info) if not (validation["is_connected"] and validation["is_radial"] and validation["slack_reaches_all"]): @@ -312,269 +195,98 @@ def _apply_topology(self, scenario_id: str) -> None: self.active_line_info = active_line_info self.active_bus_info = self._baseline_bus_info.copy(deep=True) self.active_edges = get_active_edges(active_line_info) - self.adjacency_matrix = build_adjacency_matrix(self.node_num, active_line_info) + self.adjacency_matrix = build_adjacency_matrix(self.node_count, active_line_info) self.edge_index = build_edge_index(active_line_info) if self.algorithm == "Laurent": - self.net = GridTensor( - node_file_path="", - lines_file_path="", - from_file=False, - nodes_frame=self.active_bus_info.copy(deep=True), - lines_frame=self.active_line_info.copy(deep=True), - s_base=self.s_base, + self.solver = LaurentSolverAdapter( + bus_info=self.active_bus_info, + line_info=self.active_line_info, + s_base=self.config.s_base, ) - self.net.Q_file = np.zeros(self.node_num - 1) - self.dense_Ybus = self.net._make_y_bus().toarray() elif self.algorithm == "PandaPower": - self.net = create_pandapower_net( - self.network_info, - branch_info=self.active_line_info.copy(deep=True), - bus_info=self.active_bus_info.copy(deep=True), + self.solver = PandaPowerSolverAdapter( + network_info=self.config.network_info, + bus_info=self.active_bus_info, + line_info=self.active_line_info, + s_base=self.config.s_base, ) else: - raise ValueError("Invalid algorithm choice. Please choose 'Laurent' or 'PandaPower'.") + raise ValueError("Unsupported algorithm") + + def _select_episode_date(self) -> None: + candidate_dates = self.data_manager.train_dates if self.train else self.data_manager.test_dates + if not candidate_dates: + raise ValueError("No episode dates are available for the requested split") + date_index = int(self._rng.integers(0, len(candidate_dates))) + self.year, self.month, self.day = candidate_dates[date_index] - def _extract_slot_features(self, one_slot_data: np.ndarray): - active_power = cp.copy(one_slot_data[self.active_power_indices]).astype(float) + def _reset_batteries(self) -> None: + for battery in self.batteries.values(): + battery.reset() + + def _extract_slot_features(self, timeslot: int) -> SlotFeatures: + one_slot_data = self.data_manager.select_timeslot_data(self.year, self.month, self.day, timeslot) + active_power = np.asarray(one_slot_data[self.active_power_indices], dtype=np.float32) renewable_active_power = np.zeros_like(active_power) if self.renewable_active_power_indices: - renewable_active_power = one_slot_data[self.renewable_active_power_indices].astype(float) + renewable_active_power = np.asarray(one_slot_data[self.renewable_active_power_indices], dtype=np.float32) price = float(one_slot_data[self.price_index]) - if active_power.shape[0] != self.node_num: - raise ValueError( - f"Time-series active power dimension {active_power.shape[0]} does not match feeder node count {self.node_num}" - ) - if renewable_active_power.shape[0] != self.node_num: - raise ValueError( - f"Time-series renewable power dimension {renewable_active_power.shape[0]} does not match feeder node count {self.node_num}" - ) - return active_power, renewable_active_power, price - - def _get_obs(self): - """ - Executes the power flow based on the chosen algorithm and returns the observations. - - Returns: - dict: The observation dictionary containing various state elements. - """ - if self.state_pattern == 'default': - one_slot_data = self.data_manager.select_timeslot_data(self.year, self.month, self.day, self.current_time) - active_power, renewable_active_power, price = self._extract_slot_features(one_slot_data) - - if self.algorithm == "Laurent": - self.active_power = (active_power - renewable_active_power)[1:self.node_num] - reactive_power = np.zeros(self.node_num - 1) - self.solution = self.net.run_pf(active_power=self.active_power) - - obs = {'node_data': {'voltage': {}, 'active_power': {}, 'reactive_power': {}, - 'renewable_active_power': {}}, - 'battery_data': {'soc': {}}, 'price': {}, 'aux': {}} - - for node_index in range(len(self.net.bus_info.NODES)): - if node_index == 0: - obs['node_data']['voltage'][f'node_{node_index}'] = 1.0 - obs['node_data']['active_power'][f'node_{node_index}'] = 0.0 - obs['node_data']['renewable_active_power'][f'node_{node_index}'] = 0.0 - else: - obs['node_data']['voltage'][f'node_{node_index}'] = abs( - self.solution['v'].T[node_index - 1]).squeeze() - obs['node_data']['active_power'][f'node_{node_index}'] = active_power[node_index - 1] - obs['node_data']['renewable_active_power'][f'node_{node_index}'] = renewable_active_power[ - node_index - 1] - for node_index in self.battery_list: - obs['battery_data']['soc'][f'battery_{node_index}'] = getattr(self, f'battery_{node_index}').SOC() - obs['price'] = price - else: - active_power[0] = 0 - renewable_active_power[0] = 0 - for bus_index in self.net.load.bus.index: - self.net.load.p_mw[bus_index] = (active_power[bus_index] - renewable_active_power[ - bus_index]) / self.s_base - self.net.load.q_mvar[bus_index] = 0 - _require_pandapower().runpp(self.net, algorithm='nr') - v_real = self.net.res_bus["vm_pu"].values * np.cos(np.deg2rad(self.net.res_bus["va_degree"].values)) - v_img = self.net.res_bus["vm_pu"].values * np.sin(np.deg2rad(self.net.res_bus["va_degree"].values)) - v_result = v_real + 1j * v_img - - obs = {'node_data': {'voltage': {}, 'active_power': {}, 'reactive_power': {}, - 'renewable_active_power': {}}, - 'battery_data': {'soc': {}}, 'price': {}, 'aux': {}} - - for node_index in self.net.load.bus.index: - bus_idx = self.net.load.at[node_index, 'bus'] - obs['node_data']['voltage'][f'node_{node_index}'] = self.net.res_bus.vm_pu.at[bus_idx] - obs['node_data']['active_power'][f'node_{node_index}'] = active_power[node_index] - obs['node_data']['reactive_power'][f'node_{node_index}'] = self.net.res_load.q_mvar[node_index] - obs['node_data']['renewable_active_power'][f'node_{node_index}'] = renewable_active_power[ - node_index] - for node_index in self.battery_list: - obs['battery_data']['soc'][f'battery_{node_index}'] = getattr(self, f'battery_{node_index}').SOC() - obs['price'] = price - else: - raise ValueError('please redesign the get obs function to fit the pattern you want') - return obs - - def _apply_battery_actions(self, action): - '''apply action to battery charge/discharge, update the battery condition, excute power flow, update the network condition''' - if self.state_pattern == 'default': - if self.algorithm == "Laurent": - v = self.solution["v"] - v_totall = np.insert(v, 0, 1) - current_each_node = np.matmul(self.dense_Ybus, v_totall) - power_imported_from_ex_grid_before = current_each_node[0].real - - for i, node_index in enumerate(self.battery_list): - getattr(self, f"battery_{node_index}").step(action[i]) - self.active_power[node_index - 1] += getattr(self, f"battery_{node_index}").energy_change - self.solution = self.net.run_pf(active_power=self.active_power) - - v = self.solution["v"] - v_totall = np.insert(v, 0, 1) - vm_pu_after_control = cp.deepcopy(abs(v_totall)) - vm_pu_after_control_bat = np.squeeze(vm_pu_after_control)[self.battery_list] - self.after_control = vm_pu_after_control - current_each_node = np.matmul(self.dense_Ybus, v_totall) - power_imported_from_ex_grid_after = current_each_node[0].real - saved_energy = power_imported_from_ex_grid_before - power_imported_from_ex_grid_after - else: - power_imported_from_ex_grid_before = cp.deepcopy(self.net.res_ext_grid['p_mw']) - - for i, node_index in enumerate(self.battery_list): - getattr(self, f"battery_{node_index}").step(action[i]) - self.net.load.p_mw[node_index] += getattr(self, f"battery_{node_index}").energy_change / 1000 - _require_pandapower().runpp(self.net, algorithm='nr') - vm_pu_after_control = cp.deepcopy(self.net.res_bus.vm_pu).to_numpy(dtype=float) - vm_pu_after_control_bat = vm_pu_after_control[self.battery_list] - - self.after_control = vm_pu_after_control - power_imported_from_ex_grid_after = self.net.res_ext_grid['p_mw'] - saved_energy = power_imported_from_ex_grid_before - power_imported_from_ex_grid_after - else: - raise ValueError('Expected default or define yourself based on the goal') - return saved_energy, vm_pu_after_control_bat - - def step(self, action: np.ndarray) -> tuple: - """ - Advance the environment by one timestep based on the provided action. - - :param action: Action to execute. - :type action: np.ndarray - :return: Tuple containing the next normalized observation, the reward, a boolean indicating if the episode has ended, and additional info. - :rtype: tuple - """ - - current_normalized_obs = self.normalized_state - info = self._build_info(current_normalized_obs) - - # Apply battery actions and get updated observations - saved_energy, vm_pu_after_control_bat = self._apply_battery_actions(action) - - reward = self._calculate_reward(current_normalized_obs, vm_pu_after_control_bat, saved_energy) - - finish = (self.current_time == self.episode_length - 1) - self.current_time += 1 - if finish: - self.current_time = 0 - next_normalized_obs = self.reset() - else: - next_normalized_obs = self._build_state() - return next_normalized_obs, float(reward), finish, info - - def get_topology_metadata(self): - return { + if active_power.shape[0] != self.node_count: + raise ValueError(f"Time-series active power dimension {active_power.shape[0]} does not match feeder node count {self.node_count}") + if renewable_active_power.shape[0] != self.node_count: + raise ValueError(f"Time-series renewable power dimension {renewable_active_power.shape[0]} does not match feeder node count {self.node_count}") + + return SlotFeatures( + active_power_kw=active_power, + renewable_active_power_kw=renewable_active_power, + price=price, + ) + + def _observe_current_slot(self) -> tuple[ObservationSnapshot, SlotFeatures, PowerFlowSnapshot]: + slot_features = self._extract_slot_features(self.current_time) + net_load_kw = slot_features.active_power_kw - slot_features.renewable_active_power_kw + precontrol_snapshot = self.solver.observe(net_load_kw) + observation = self._build_observation(slot_features, precontrol_snapshot) + return observation, slot_features, precontrol_snapshot + + def _build_observation( + self, + slot_features: SlotFeatures, + power_flow_snapshot: PowerFlowSnapshot, + ) -> ObservationSnapshot: + battery_soc = np.array([self.batteries[node_index].SOC() for node_index in self.battery_nodes], dtype=np.float32) + return build_default_state( + slot_features=slot_features, + battery_soc=battery_soc, + current_time=self.current_time, + node_voltages_pu=power_flow_snapshot.node_voltages_pu, + battery_nodes=self.battery_nodes, + scaler=self.state_scaler, + ) + + def _build_info( + self, + *, + post_control_snapshot: PowerFlowSnapshot | None = None, + battery_dispatch_kw: np.ndarray | None = None, + ) -> dict[str, Any]: + info: dict[str, Any] = { "feeder_id": self.feeder_id, - "scenario_id": self.current_scenario.scenario_id, - "node_count": self.node_num, - "edge_count": len(self.active_edges), - "active_edges": list(self.active_edges), - } - - def get_graph_data(self): - return { - "adjacency": self.adjacency_matrix.copy(), - "edge_index": self.edge_index.copy(), - "node_ids": np.arange(1, self.node_num + 1, dtype=np.int64), - "active_line_data": self.active_line_info[["FROM", "TO", "R", "X", "B", "STATUS", "TAP"]].to_dict("records"), - } - - def _build_info(self, current_normalized_obs=None): - info = { "topology_scenario": self.current_scenario.scenario_id, - "feeder_id": self.feeder_id, "active_edges_count": len(self.active_edges), + "current_time": self.current_time, } - if current_normalized_obs is not None: - info["current_normalized_obs"] = current_normalized_obs + if self.current_observation is not None: + info["current_normalized_obs"] = self.current_observation.normalized_state.copy() + if battery_dispatch_kw is not None: + info["battery_dispatch_kw"] = np.asarray(battery_dispatch_kw, dtype=np.float32).copy() + if post_control_snapshot is not None and self.last_reward_breakdown is not None: + info["post_control_voltage_pu"] = post_control_snapshot.node_voltages_pu.copy() + info["reward_breakdown"] = { + "economic": self.last_reward_breakdown.economic, + "voltage_penalty": self.last_reward_breakdown.voltage_penalty, + "saved_money": self.last_reward_breakdown.saved_money, + } return info - - def _calculate_reward(self, current_normalized_obs: np.ndarray, vm_pu_after_control_bat: np.ndarray, - saved_power: float) -> float: - """ - Calculate the reward based on the current observation and saved power. the default version is to calculate the battey saved energy - based on the current price - - Parameters: - current_normalized_obs (np.ndarray): The current normalized observations. - vm_pu_after_control_bat (np.ndarray): The voltage after control at battery locations. - saved_power (float): The amount of power saved. - - Returns: - float: Calculated reward. - """ - if self.state_pattern == 'default': - reward_for_power = 1 * current_normalized_obs[self.node_num + len(self.battery_list)] * float(saved_power) - reward_for_penalty = 0.0 - - for vm_pu_bat in vm_pu_after_control_bat: - reward_for_penalty += min(0, 100 * (0.05 - abs(1.0 - vm_pu_bat))) - - self.reward_for_power = reward_for_power - self.reward_for_penalty = reward_for_penalty - self.saved_money = -1 * self._denormalize_state(current_normalized_obs)[ - self.node_num + len(self.battery_list)] * float(saved_power) - - reward = reward_for_power + reward_for_penalty - else: - raise ValueError( - "Invalid value for 'state_pattern'. Expected 'default, or define by yourself based on different goal") - - return reward - - def render(self, current_obs, next_obs, reward, finish): - """ - Render the environment's current state. - - :param current_obs: Current observation. - :type current_obs: np.array - :param next_obs: Next observation. - :type next_obs: np.array - :param reward: Reward obtained from the last action. - :type reward: float - :param finish: Whether the episode has ended. - :type finish: bool - """ - print('state={}, next_state={}, reward={:.4f}, terminal={}\n'.format(current_obs, next_obs, reward, finish)) - - -if __name__ == '__main__': - power_net_env = PowerNetEnv(env_config=env_config) - power_net_env.reset() - - for j in range(1): - episode_reward = 0 - for i in range(1000): - # 1 is charge -1 is discharge - tem_action = np.ones(len(power_net_env.battery_list)) - # tem_action = power_net_env.action_space.sample() - print('year, month, day, current time', - (power_net_env.year, power_net_env.month, power_net_env.day, power_net_env.current_time)) - # print(f'current month is {power_net_env.month}, current day is {power_net_env.day}, current time is {power_net_env.current_time}') - next_obs, reward, finish, info = power_net_env.step(tem_action) - # print(power_net_env.reward_for_power) - print(power_net_env.reward_for_penalty) - # print('reward',reward) - episode_reward += reward - # power_net_env.render(current_obs, next_obs, reward, finish) - print(episode_reward) diff --git a/rl_adn/environments/observation.py b/rl_adn/environments/observation.py new file mode 100644 index 0000000..d0507bb --- /dev/null +++ b/rl_adn/environments/observation.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np + + +@dataclass(frozen=True) +class SlotFeatures: + active_power_kw: np.ndarray + renewable_active_power_kw: np.ndarray + price: float + + +@dataclass(frozen=True) +class ObservationSnapshot: + active_power_kw: np.ndarray + renewable_active_power_kw: np.ndarray + price: float + node_voltages_pu: np.ndarray + battery_soc: np.ndarray + raw_state: np.ndarray + normalized_state: np.ndarray + + +@dataclass(frozen=True) +class StateScaler: + node_count: int + battery_count: int + active_power_min: float + active_power_max: float + price_min: float + price_max: float + episode_length: int + min_soc: float + max_soc: float + + def normalize(self, raw_state: np.ndarray) -> np.ndarray: + state = raw_state.astype(np.float32, copy=True) + state[: self.node_count] = (state[: self.node_count] - self.active_power_min) / (self.active_power_max - self.active_power_min) + state[self.node_count : self.node_count + self.battery_count] = (state[self.node_count : self.node_count + self.battery_count] - self.min_soc) / (self.max_soc - self.min_soc) + price_index = self.node_count + self.battery_count + state[price_index] = (state[price_index] - self.price_min) / (self.price_max - self.price_min) + time_index = price_index + 1 + state[time_index] = state[time_index] / max(self.episode_length - 1, 1) + return state + + def denormalize(self, normalized_state: np.ndarray) -> np.ndarray: + state = normalized_state.astype(np.float32, copy=True) + state[: self.node_count] = (state[: self.node_count] * (self.active_power_max - self.active_power_min)) + self.active_power_min + state[self.node_count : self.node_count + self.battery_count] = (state[self.node_count : self.node_count + self.battery_count] * (self.max_soc - self.min_soc)) + self.min_soc + price_index = self.node_count + self.battery_count + state[price_index] = state[price_index] * (self.price_max - self.price_min) + self.price_min + time_index = price_index + 1 + state[time_index] = state[time_index] * max(self.episode_length - 1, 1) + return state + + +def build_default_state( + *, + slot_features: SlotFeatures, + battery_soc: np.ndarray, + current_time: int, + node_voltages_pu: np.ndarray, + battery_nodes: tuple[int, ...], + scaler: StateScaler, +) -> ObservationSnapshot: + raw_state = np.concatenate( + ( + slot_features.active_power_kw.astype(np.float32, copy=False), + battery_soc.astype(np.float32, copy=False), + np.array([slot_features.price, float(current_time)], dtype=np.float32), + node_voltages_pu[list(battery_nodes)].astype(np.float32, copy=False), + ) + ) + normalized_state = scaler.normalize(raw_state) + return ObservationSnapshot( + active_power_kw=slot_features.active_power_kw.astype(np.float32, copy=True), + renewable_active_power_kw=slot_features.renewable_active_power_kw.astype(np.float32, copy=True), + price=float(slot_features.price), + node_voltages_pu=node_voltages_pu.astype(np.float32, copy=True), + battery_soc=battery_soc.astype(np.float32, copy=True), + raw_state=raw_state, + normalized_state=normalized_state, + ) diff --git a/rl_adn/environments/reward.py b/rl_adn/environments/reward.py new file mode 100644 index 0000000..79b17b9 --- /dev/null +++ b/rl_adn/environments/reward.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np + + +@dataclass(frozen=True) +class RewardBreakdown: + total: float + economic: float + voltage_penalty: float + saved_money: float + + +def compute_default_reward( + *, + price: float, + saved_power_kw: float, + battery_voltages_pu: np.ndarray, + voltage_target_pu: float = 1.0, + voltage_band_pu: float = 0.05, + penalty_scale: float = 100.0, +) -> RewardBreakdown: + economic = float(price * saved_power_kw) + voltage_penalty = 0.0 + for voltage in battery_voltages_pu: + voltage_penalty += min(0.0, penalty_scale * (voltage_band_pu - abs(voltage_target_pu - float(voltage)))) + total = economic + voltage_penalty + return RewardBreakdown( + total=float(total), + economic=float(economic), + voltage_penalty=float(voltage_penalty), + saved_money=float(-economic), + ) diff --git a/rl_adn/environments/solvers.py b/rl_adn/environments/solvers.py new file mode 100644 index 0000000..86fb642 --- /dev/null +++ b/rl_adn/environments/solvers.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from rl_adn.network.grid import GridTensor +from rl_adn.network.utils import create_pandapower_net + + +@dataclass(frozen=True) +class PowerFlowSnapshot: + node_voltages_pu: np.ndarray + import_power_kw: float + + +class LaurentSolverAdapter: + def __init__(self, *, bus_info: pd.DataFrame, line_info: pd.DataFrame, s_base: float) -> None: + self.node_count = len(bus_info) + self.grid = GridTensor( + node_file_path="", + lines_file_path="", + from_file=False, + nodes_frame=bus_info.copy(deep=True), + lines_frame=line_info.copy(deep=True), + s_base=s_base, + ) + self.grid.Q_file = np.zeros(self.node_count - 1) + self.dense_ybus = self.grid._make_y_bus().toarray() + + def observe(self, net_load_kw: np.ndarray) -> PowerFlowSnapshot: + active_power = np.asarray(net_load_kw[1:], dtype=float) + solution = self.grid.run_pf(active_power=active_power) + voltages = np.ones(self.node_count, dtype=np.float32) + voltages[1:] = np.abs(solution["v"].T).reshape(-1).astype(np.float32) + v_total = np.insert(solution["v"], 0, 1) + import_power = float(np.matmul(self.dense_ybus, v_total)[0].real) + return PowerFlowSnapshot(node_voltages_pu=voltages, import_power_kw=import_power) + + def dispatch(self, net_load_kw: np.ndarray, battery_nodes: tuple[int, ...], battery_power_kw: np.ndarray) -> PowerFlowSnapshot: + adjusted_load = np.asarray(net_load_kw, dtype=float).copy() + for node_index, dispatch_kw in zip(battery_nodes, battery_power_kw): + adjusted_load[node_index] += float(dispatch_kw) + return self.observe(adjusted_load) + + +class PandaPowerSolverAdapter: + def __init__(self, *, network_info: dict[str, object], bus_info: pd.DataFrame, line_info: pd.DataFrame, s_base: float) -> None: + self.network_info = dict(network_info) + self.bus_info = bus_info.copy(deep=True) + self.line_info = line_info.copy(deep=True) + self.node_count = len(bus_info) + self.s_base = s_base + + def _run_power_flow(self, net_load_kw: np.ndarray) -> PowerFlowSnapshot: + pp, _ = _require_pandapower() + net = create_pandapower_net(self.network_info, branch_info=self.line_info, bus_info=self.bus_info) + net_load_mw = np.asarray(net_load_kw, dtype=float) / 1000.0 + for load_index, load_bus in enumerate(net.load.bus.values): + net.load.p_mw.iloc[load_index] = net_load_mw[int(load_bus)] + net.load.q_mvar.iloc[load_index] = 0.0 + pp.runpp(net, algorithm="nr") + voltages = net.res_bus.vm_pu.to_numpy(dtype=np.float32) + import_power = float(net.res_ext_grid["p_mw"].iloc[0] * 1000.0) + return PowerFlowSnapshot(node_voltages_pu=voltages, import_power_kw=import_power) + + def observe(self, net_load_kw: np.ndarray) -> PowerFlowSnapshot: + return self._run_power_flow(net_load_kw) + + def dispatch(self, net_load_kw: np.ndarray, battery_nodes: tuple[int, ...], battery_power_kw: np.ndarray) -> PowerFlowSnapshot: + adjusted_load = np.asarray(net_load_kw, dtype=float).copy() + for node_index, dispatch_kw in zip(battery_nodes, battery_power_kw): + adjusted_load[node_index] += float(dispatch_kw) + return self._run_power_flow(adjusted_load) + + +def _require_pandapower(): + try: + import pandapower as pp + import pandapower.topology as pandapower_topology + except ImportError as exc: + raise ImportError("PandaPower support requires the optional dependency 'pandapower'.") from exc + return pp, pandapower_topology diff --git a/rl_adn/environments/topology_scenarios.py b/rl_adn/environments/topology_scenarios.py index b721a38..8d2afbc 100644 --- a/rl_adn/environments/topology_scenarios.py +++ b/rl_adn/environments/topology_scenarios.py @@ -1,7 +1,6 @@ from dataclasses import dataclass from typing import Dict, List, Sequence, Tuple - Edge = Tuple[int, int] Rewire = Tuple[Edge, Edge] diff --git a/rl_adn/network/__init__.py b/rl_adn/network/__init__.py new file mode 100644 index 0000000..b9c6418 --- /dev/null +++ b/rl_adn/network/__init__.py @@ -0,0 +1,21 @@ +"""Network and solver utilities for RL-ADN.""" + +from rl_adn.network.grid import GridTensor +from rl_adn.network.topology import ( + apply_topology_scenario, + build_adjacency_matrix, + build_edge_index, + get_active_edges, + validate_radial_topology, +) +from rl_adn.network.utils import create_pandapower_net + +__all__ = [ + "GridTensor", + "apply_topology_scenario", + "build_adjacency_matrix", + "build_edge_index", + "create_pandapower_net", + "get_active_edges", + "validate_radial_topology", +] diff --git a/rl_adn/utility/grid.py b/rl_adn/network/grid.py similarity index 68% rename from rl_adn/utility/grid.py rename to rl_adn/network/grid.py index 32fc59b..53ee256 100644 --- a/rl_adn/utility/grid.py +++ b/rl_adn/network/grid.py @@ -1,189 +1,188 @@ -import pandas as pd +import os +import warnings +from time import perf_counter + import numpy as np -from scipy.sparse import csr_matrix, csc_matrix, diags +import pandas as pd +from numba import njit, set_num_threads +from scipy.sparse import csc_matrix, csr_matrix, diags from scipy.sparse.linalg import inv -from time import perf_counter -from numba import njit, set_num_threads -import warnings -from rl_adn.utility.utils import GPUPowerFlow -from rl_adn.utility.utils import generate_network -from rl_adn.utility.numbarize import (pre_power_flow_tensor, - power_flow_tensor, - power_flow_tensor_constant_power, - pre_power_flow_sam_sequential, - power_flow_sam_sequential, - power_flow_sam_sequential_constant_power_only) + +from rl_adn.network.numbarize import ( + power_flow_sam_sequential, + power_flow_sam_sequential_constant_power_only, + power_flow_tensor, + power_flow_tensor_constant_power, + pre_power_flow_sam_sequential, + pre_power_flow_tensor, +) +from rl_adn.network.utils import GPUPowerFlow, generate_network + try: import psutil except ImportError: psutil = None from tqdm import trange - - - -class GridTensor: - """ - Initializes the GridTensor object with grid parameters and data sources. - - Parameters: - node_file_path (str): Path to the file containing node data. Default is None. - lines_file_path (str): Path to the file containing line data. Default is None. - s_base (int): Base apparent power in kVA for one phase. Default is 1000 kVA. - v_base (float): Base voltage in kV for one phase. Default is 11 kV. - iterations (int): Maximum number of iterations for power flow calculations. Default is 100. - tolerance (float): Convergence tolerance for power flow calculations. Default is 1e-5. - from_file (bool): Flag to indicate whether to load data from files. Default is True. - nodes_frame (pd.DataFrame): DataFrame containing node data. Default is None. - lines_frame (pd.DataFrame): DataFrame containing line data. Default is None. - numba (bool): Flag to enable or disable Numba JIT compilation. Default is True. - gpu_mode (bool): Flag to enable or disable GPU mode. Default is False. - """ - def __init__(self, - node_file_path: str = None, - lines_file_path: str = None, - *, - s_base: int = 1000, # kVA - 1 phase - v_base: float = 11, # kV - 1 phase - iterations: int = 100, - tolerance: float = 1e-5, - from_file=True, - nodes_frame: pd.DataFrame = None, - lines_frame: pd.DataFrame = None, - numba=True, - gpu_mode=False): - - self.s_base = s_base - self.v_base = v_base # This is better loaded from the file (Extra column) - self.z_base = (self.v_base ** 2 * 1000) / self.s_base - self.i_base = self.s_base / (np.sqrt(3) * self.v_base) - - self.iterations = iterations - self.tolerance = tolerance - - if node_file_path is None and lines_file_path is None: - - print("no case is loaded") - - elif node_file_path is not None and lines_file_path is not None and from_file: - self.branch_info = pd.read_csv(lines_file_path) - self.bus_info = pd.read_csv(node_file_path) - - elif nodes_frame is not None and lines_frame is not None: - self.branch_info = lines_frame - self.bus_info = nodes_frame - else: - raise ValueError("Wrong input configuration") - - self._make_y_bus() - self._compute_alphas() - self.v_0 = None - self._F_ = None - self._W_ = None - - # Placeholder for the methods that are pre-compiled with numba. - self._power_flow_tensor_constant_power = None - self._pre_power_flow_tensor = None - self._power_flow_tensor = None - - self._pre_power_flow_sam_sequential = None - self._power_flow_sam_sequential = None - self._power_flow_sam_sequential_constant_power_only = None - - self.is_numba_enabled = False - self.is_gpu_enabled = False - - if np.all(self.alpha_P) and not np.any(self.alpha_Z) and not np.any(self.alpha_I): - self.constant_power_only = True - self.start_time_pre_pf_tensor_constant_power_only = perf_counter() - - # TODO: Change to sparse inverse. - self._K_ = np.array(-inv(self.Ydd_sparse).todense()) # Reduced version of -B^-1 (Reduced version of _F_) TODO: check it exist .toarray() - self._L_ = self._K_ @ self.Yds # Reduced version of _W_ - self.end_time_pre_pf_tensor_constant_power_only = perf_counter() - else: - self.constant_power_only = False - - if numba: - self.enable_numba() - self.is_numba_enabled = True - else: - warnings.warn("Numba NOT enabled. Performance is greatly reduced.", RuntimeWarning) - self.disable_numba() - self.is_numba_enabled = False - - if gpu_mode: - self.gpu_solver = GPUPowerFlow() - self.is_gpu_enabled = True - - - - - def enable_numba(self): - """ - Disables Numba JIT compilation, reverting to standard Python execution. - """ - parallel = True - - self._power_flow_tensor_constant_power = power_flow_tensor_constant_power - self._pre_power_flow_tensor = njit(pre_power_flow_tensor, parallel=parallel) - self._power_flow_tensor = njit(power_flow_tensor, parallel=parallel) - - self._pre_power_flow_sam_sequential = njit(pre_power_flow_sam_sequential, parallel=parallel) - self._power_flow_sam_sequential = njit(power_flow_sam_sequential, parallel=parallel) - self._power_flow_sam_sequential_constant_power_only = njit(power_flow_sam_sequential_constant_power_only, - parallel=parallel) - - - def disable_numba(self): - - - self._power_flow_tensor_constant_power = power_flow_tensor_constant_power - self._pre_power_flow_tensor = pre_power_flow_tensor - self._power_flow_tensor = power_flow_tensor - - self._pre_power_flow_sam_sequential = pre_power_flow_sam_sequential - self._power_flow_sam_sequential = power_flow_sam_sequential - self._power_flow_sam_sequential_constant_power_only = power_flow_sam_sequential_constant_power_only - - @classmethod - def generate_from_graph(cls, *, nodes=100, child=2, plot_graph=True, load_factor=2, line_factor=3, **kwargs): - """ - Generates a synthetic grid using the networkX package and returns a GridTensor object. - - Parameters: - nodes (int): Number of nodes in the synthetic grid. Default is 100. - child (int): Number of child nodes for each node in the grid. Default is 2. - plot_graph (bool): Flag to plot the generated graph. Default is True. - load_factor (int): Load factor for the grid. Default is 2. - line_factor (int): Line factor for the grid. Default is 3. - - Returns: - GridTensor: An instance of the GridTensor class. - """ - - nodes_frame, lines_frame = generate_network(nodes=nodes, - child=child, - plot_graph=plot_graph, - load_factor=load_factor, - line_factor=line_factor) - - return cls(node_file_path="", - lines_file_path="", - from_file=False, - nodes_frame=nodes_frame, - lines_frame=lines_frame, - **kwargs) - - def reset_start(self): - """ - Resets the starting voltage values for power flow calculations to default flat start values. - """ - # TODO - self.v_0 = np.ones((self.nb-1, 1), dtype="complex128") # Flat start #2D array - def _set_number_of_threads(self, threads): - """ - Sets the number of threads for parallel execution in Numba. - + + +class GridTensor: + """ + Initializes the GridTensor object with grid parameters and data sources. + + Parameters: + node_file_path (str): Path to the file containing node data. Default is None. + lines_file_path (str): Path to the file containing line data. Default is None. + s_base (int): Base apparent power in kVA for one phase. Default is 1000 kVA. + v_base (float): Base voltage in kV for one phase. Default is 11 kV. + iterations (int): Maximum number of iterations for power flow calculations. Default is 100. + tolerance (float): Convergence tolerance for power flow calculations. Default is 1e-5. + from_file (bool): Flag to indicate whether to load data from files. Default is True. + nodes_frame (pd.DataFrame): DataFrame containing node data. Default is None. + lines_frame (pd.DataFrame): DataFrame containing line data. Default is None. + numba (bool): Flag to enable or disable Numba JIT compilation. Default is True. + gpu_mode (bool): Flag to enable or disable GPU mode. Default is False. + """ + + def __init__( + self, + node_file_path: str = None, + lines_file_path: str = None, + *, + s_base: int = 1000, # kVA - 1 phase + v_base: float = 11, # kV - 1 phase + iterations: int = 100, + tolerance: float = 1e-5, + from_file=True, + nodes_frame: pd.DataFrame = None, + lines_frame: pd.DataFrame = None, + numba=True, + gpu_mode=False, + ): + + self.s_base = s_base + self.v_base = v_base # This is better loaded from the file (Extra column) + self.z_base = (self.v_base**2 * 1000) / self.s_base + self.i_base = self.s_base / (np.sqrt(3) * self.v_base) + + self.iterations = iterations + self.tolerance = tolerance + + if node_file_path is None and lines_file_path is None: + raise ValueError("A grid case must be loaded from files or DataFrames") + + elif node_file_path is not None and lines_file_path is not None and from_file: + self.branch_info = pd.read_csv(lines_file_path) + self.bus_info = pd.read_csv(node_file_path) + + elif nodes_frame is not None and lines_frame is not None: + self.branch_info = lines_frame + self.bus_info = nodes_frame + else: + raise ValueError("Wrong input configuration") + + self._make_y_bus() + self._compute_alphas() + self.v_0 = None + self._F_ = None + self._W_ = None + + # Placeholder for the methods that are pre-compiled with numba. + self._power_flow_tensor_constant_power = None + self._pre_power_flow_tensor = None + self._power_flow_tensor = None + + self._pre_power_flow_sam_sequential = None + self._power_flow_sam_sequential = None + self._power_flow_sam_sequential_constant_power_only = None + + self.is_numba_enabled = False + self.is_gpu_enabled = False + + if np.all(self.alpha_P) and not np.any(self.alpha_Z) and not np.any(self.alpha_I): + self.constant_power_only = True + self.start_time_pre_pf_tensor_constant_power_only = perf_counter() + + # TODO: Change to sparse inverse. + self._K_ = np.array(-inv(self.Ydd_sparse).todense()) # Reduced version of -B^-1 (Reduced version of _F_) TODO: check it exist .toarray() + self._L_ = self._K_ @ self.Yds # Reduced version of _W_ + self.end_time_pre_pf_tensor_constant_power_only = perf_counter() + else: + self.constant_power_only = False + + if numba: + self.enable_numba() + self.is_numba_enabled = True + else: + warnings.warn("Numba NOT enabled. Performance is greatly reduced.", RuntimeWarning) + self.disable_numba() + self.is_numba_enabled = False + + if gpu_mode: + self.gpu_solver = GPUPowerFlow() + self.is_gpu_enabled = True + + def enable_numba(self): + """ + Disables Numba JIT compilation, reverting to standard Python execution. + """ + parallel = True + + self._power_flow_tensor_constant_power = power_flow_tensor_constant_power + self._pre_power_flow_tensor = njit(pre_power_flow_tensor, parallel=parallel) + self._power_flow_tensor = njit(power_flow_tensor, parallel=parallel) + + self._pre_power_flow_sam_sequential = njit(pre_power_flow_sam_sequential, parallel=parallel) + self._power_flow_sam_sequential = njit(power_flow_sam_sequential, parallel=parallel) + self._power_flow_sam_sequential_constant_power_only = njit(power_flow_sam_sequential_constant_power_only, parallel=parallel) + + def disable_numba(self): + + self._power_flow_tensor_constant_power = power_flow_tensor_constant_power + self._pre_power_flow_tensor = pre_power_flow_tensor + self._power_flow_tensor = power_flow_tensor + + self._pre_power_flow_sam_sequential = pre_power_flow_sam_sequential + self._power_flow_sam_sequential = power_flow_sam_sequential + self._power_flow_sam_sequential_constant_power_only = power_flow_sam_sequential_constant_power_only + + @classmethod + def generate_from_graph(cls, *, nodes=100, child=2, plot_graph=True, load_factor=2, line_factor=3, **kwargs): + """ + Generates a synthetic grid using the networkX package and returns a GridTensor object. + + Parameters: + nodes (int): Number of nodes in the synthetic grid. Default is 100. + child (int): Number of child nodes for each node in the grid. Default is 2. + plot_graph (bool): Flag to plot the generated graph. Default is True. + load_factor (int): Load factor for the grid. Default is 2. + line_factor (int): Line factor for the grid. Default is 3. + + Returns: + GridTensor: An instance of the GridTensor class. + """ + + nodes_frame, lines_frame = generate_network(nodes=nodes, child=child, plot_graph=plot_graph, load_factor=load_factor, line_factor=line_factor) + + return cls( + node_file_path="", + lines_file_path="", + from_file=False, + nodes_frame=nodes_frame, + lines_frame=lines_frame, + **kwargs, + ) + + def reset_start(self): + """ + Resets the starting voltage values for power flow calculations to default flat start values. + """ + # TODO + self.v_0 = np.ones((self.nb - 1, 1), dtype="complex128") # Flat start #2D array + + def _set_number_of_threads(self, threads): + """ + Sets the number of threads for parallel execution in Numba. + Parameters: threads (int): Number of threads to be used. """ @@ -191,568 +190,572 @@ def _set_number_of_threads(self, threads): max_threads = psutil.cpu_count() if psutil is not None else (os.cpu_count() or 1) assert threads <= max_threads, "Number of threads must be lower of cpu count." set_num_threads(threads) - print(f"Number of threads set to: {threads}") - - - def _make_y_bus(self) -> None: - - """ - Compute Y_bus submatrices - - For each branch, compute the elements of the branch admittance matrix where - | Is | | Yss Ysd | | Vs | - | | = | | * | | - |-Id | | Yds Ydd | | Vd | - """ - + return None + + def _make_y_bus(self) -> None: + """ + Compute Y_bus submatrices + + For each branch, compute the elements of the branch admittance matrix where + | Is | | Yss Ysd | | Vs | + | | = | | * | | + |-Id | | Yds Ydd | | Vd | + """ + self.nb = self.bus_info.shape[0] # number of buses active_branch_info = self.branch_info[self.branch_info.iloc[:, 5].astype(float) != 0].reset_index(drop=True) self.nl = active_branch_info.shape[0] # number of active lines - sl = self.bus_info[self.bus_info['Tb'] == 1]['NODES'].tolist() # Slack node(s) + sl = self.bus_info[self.bus_info["Tb"] == 1]["NODES"].tolist() # Slack node(s) stat = active_branch_info.iloc[:, 5] # ones at in-service branches - Ys = stat / ((active_branch_info.iloc[:, 2] + 1j * active_branch_info.iloc[:, 3]) / ( - self.v_base ** 2 * 1000 / self.s_base)) # series admittance - Bc = stat * active_branch_info.iloc[:, 4] * (self.v_base ** 2 * 1000 / self.s_base) # line charging susceptance + Ys = stat / ((active_branch_info.iloc[:, 2] + 1j * active_branch_info.iloc[:, 3]) / (self.v_base**2 * 1000 / self.s_base)) # series admittance + Bc = stat * active_branch_info.iloc[:, 4] * (self.v_base**2 * 1000 / self.s_base) # line charging susceptance tap = active_branch_info.iloc[:, 6] # default tap ratio = 1 Ytt = Ys + 1j * Bc / 2 Yff = Ytt / tap - Yft = - Ys / tap + Yft = -Ys / tap Ytf = Yft # build connection matrices f = active_branch_info.iloc[:, 0].astype(int) - 1 # list of "from" buses t = active_branch_info.iloc[:, 1].astype(int) - 1 # list of "to" buses - - # connection matrix for line & from buses - Cf = csr_matrix((np.ones(self.nl), (range(self.nl), f)), (self.nl, self.nb)) - - # connection matrix for line & to buses - Ct = csr_matrix((np.ones(self.nl), (range(self.nl), t)), (self.nl, self.nb)) - - # build Yf and Yt such that Yf * V is the vector of complex branch currents injected - # at each branch's "from" bus, and Yt is the same for the "to" bus end - i = np.r_[range(self.nl), range(self.nl)] # double set of row indices - - Yf = csr_matrix((np.r_[Yff, Yft], (i, np.r_[f, t]))) - Yt = csr_matrix((np.r_[Ytf, Ytt], (i, np.r_[f, t]))) - - # build Ybus - Ybus = Cf.T * Yf + Ct.T * Yt # Full Ybus - - # Dense matrices - # TODO - # TODO: This takes a lot of memory. Check if I can save it always as sparse for all methods. - self._Ybus = Ybus.toarray() - self.Yss = csr_matrix(Ybus[sl[0] - 1, sl[0] - 1], shape=(len(sl), len(sl))).toarray() - self.Ysd = np.array(Ybus[0, 1:].toarray()) # TODO: Here assume the slack is the first one? - self.Yds = self.Ysd.T - self.Ydd = np.array(Ybus[1:, 1:].toarray()) # TODO: This consumes a huge amount of memory - - self._Ybus_sparse = Ybus - self.Yss_sparse = csr_matrix(Ybus[sl[0] - 1, sl[0] - 1], shape=(len(sl), len(sl))) - self.Ysd_sparse = Ybus[0, 1:] # TODO: Here assume the slack is the first one? - self.Yds_sparse = csc_matrix(self.Ysd.T) - self.Ydd_sparse = Ybus[1:, 1:] - - - - return Ybus - def _compute_alphas(self): - """ - Computes alpha values for different load types in the grid. Assume P-1 and Z,I=0 - """ - self.alpha_P = 1 - self.alpha_I = 0 - self.alpha_Z = 0 - - self.flag_all_constant_impedance_is_zero = not np.any(self.alpha_Z) - self.flag_all_constant_current_is_zero = not np.any(self.alpha_I) - self.flag_all_constant_powers_are_ones = np.all(self.alpha_P) - - - - def _check_2d_to_1d(self, active_power, reactive_power): - """ - Checks and converts 2D power matrices to 1D vectors if applicable. - - Parameters: - active_power (np.ndarray): Active power matrix. - reactive_power (np.ndarray): Reactive power matrix. - - Returns: - tuple: Tuple containing active and reactive power as 1D vectors. - """ - - assert active_power.ndim == reactive_power.ndim, "Active and reactive power must have same dimension." - - if (active_power.ndim == 2 and active_power.shape[0] == 1) and \ - (reactive_power.ndim == 2 and reactive_power.shape[0] == 1): - active_power = active_power.flatten() - reactive_power = reactive_power.flatten() - elif (active_power.ndim == 2 and active_power.shape[0] != 1) and \ - (reactive_power.ndim == 2 and reactive_power.shape[0] != 1): - raise ValueError("Active and reactive power tensors must have only one time step.") - - assert active_power.ndim == 1, "Array should be one dimensional." - assert reactive_power.ndim == 1, "Array should be one dimensional." - assert len(active_power) == len(reactive_power) == self.nb - 1, "All load nodes must have power values." - - return active_power, reactive_power - - def _compute_chunks(self, DIMENSION_BOUND, n_nodes, n_steps): - """ - Computes chunks for processing based on dimension bounds and grid parameters. - - Parameters: - DIMENSION_BOUND (int): The upper bound for the dimension of the matrices. - n_nodes (int): Number of nodes in the grid. - n_steps (int): Number of time steps for the simulation. - - Returns: - list: Indices for slicing the power consumption array. - Breaks the n_steps in chunks so it can fit in memory. - The ideas is that n_nodes * n_steps cannot be bigger than DIMENSION_BOUND - DIMENSION_BOUND is a empirically found value (should vary due to the computer's RAM). - - Return: - idx: list: All the ts indices to slice the power consumption array. - e.g., idx = [0, 1000, 2000, 2500]. 2500 time step requested, chunked in 1000 time steps (last item is - the reminder: 2500-2000=500). - """ - - - # DIMENSION_BOUND = 500 * 5_000 - # n_nodes = 4999 - # n_steps = 3000 - - TS_MAX = DIMENSION_BOUND // n_nodes - if n_steps > TS_MAX: # Chunk it - (quotient, reminder) = divmod(n_steps, TS_MAX) - idx = [i * TS_MAX for i in range(quotient + 1)] - - if reminder != 0: - idx = idx + [idx[-1] + reminder] - # if reminder == 0: - # idx = idx + [idx[-1] + TS_MAX] - - else: # The requested amount of TS is lower than the bound. So, everything is ok - idx = [0, n_steps] - - # print(idx) - - return idx - - def _make_big_sparse_matrices(self, S_nom, Ydd_sparse, Yds_sparse): - """ - Creates large sparse matrices for solving the sparse tensor power flow problem. - - Parameters: - S_nom (np.ndarray): Nominal power values. - Ydd_sparse (sparse matrix): Sparse Ydd matrix. - Yds_sparse (sparse matrix): Sparse Yds matrix. - - Returns: - tuple: Tuple containing the big M matrix and H vector as sparse matrices. - """ - - n_steps = S_nom.shape[0] - n_nodes = S_nom.shape[1] - - M = -diags(1 / np.conj(S_nom[0, :])).dot(Ydd_sparse).asformat("coo") - H = diags(1 / np.conj(S_nom[0, :])).dot(Yds_sparse).asformat("coo") - - # First iteration of M-matrix and H-vector - idx_1_M_col = M.col - idx_1_M_row = M.row - M_1_data = M.data - - idx_1_H_row = H.row - H_1_data = H.data - - # Placeholder for M-matrix - idx_col_M_temp = [] - idx_row_M_temp = [] - M_data_temp = [] - - # Placeholder for H-vector - idx_row_H_temp = [] - H_data_temp = [] - if n_steps > 1: - times_multiplying = [] - for ii in range(1, n_steps): - start_multiply = perf_counter() - M_temp = -diags(1 / np.conj(S_nom[ii, :])).dot(Ydd_sparse).asformat("coo") - H_temp = diags(1 / np.conj(S_nom[ii, :])).dot(Yds_sparse).asformat("coo") - - idx_col_M_temp.append(M_temp.col + ii * n_nodes) - idx_row_M_temp.append(M_temp.row + ii * n_nodes) - M_data_temp.append(M_temp.data) - - idx_row_H_temp.append(H_temp.row + ii * n_nodes) - H_data_temp.append(H_temp.data) - - times_multiplying.append(perf_counter() - start_multiply) - - M_big_idx_col = np.hstack([idx_1_M_col, np.hstack(idx_col_M_temp)]) - M_big_idx_row = np.hstack([idx_1_M_row, np.hstack(idx_row_M_temp)]) - M_big_idx_data = np.hstack([M_1_data, np.hstack(M_data_temp)]) - - H_big_idx_row = np.hstack([idx_1_H_row, np.hstack(idx_row_H_temp)]) - H_big_idx_col = np.zeros(H_big_idx_row.shape[0], dtype=np.int32) - H_big_data = np.hstack([H_1_data, np.hstack(H_data_temp)]) - - M_big = csr_matrix((M_big_idx_data, (M_big_idx_row, M_big_idx_col))) - H_big = csr_matrix((H_big_data, (H_big_idx_row, H_big_idx_col)), shape=(n_steps * n_nodes, 1)) - - else: - M_big = M - H_big = H - - return M_big, H_big - - def reshape_tensor(self, tensor_array): - """ - Reshapes a tensor array for power flow calculations. - - Parameters: - tensor_array (np.ndarray): The tensor array to be reshaped. - - Returns: - tuple: Reshaped tensor array and its original shape. - """ - original_shape = tensor_array.shape - tau = np.prod(original_shape[:-1]) - tensor_array.shape = (tau, original_shape[-1]) # This reshapes in place (No new memory use) - - return tensor_array, original_shape - - def run_pf(self, - active_power: np.ndarray = None, - reactive_power: np.ndarray = None, - flat_start: bool = True, - start_value: np.ndarray = None, - tolerance: float = 1e-6, - algorithm: str = "tensor", - sparse_solver: str = "scipy"): - - """ - Run a power-flow solve for the provided active/reactive power inputs. - - The method accepts either batched tensors or a single-step vector and dispatches - to the selected solver implementation. - "time_algorithm": Total time algorithm. time_algorithm = time_pre_pf + time_pf - "iterations": Total number of iterations to converge. - - "convergence": Boolean indicating: True: Algorithm converged, False: it didn't. - "iterations_log": NOT USED. - "time_pre_pf_log": NOT USED. - "time_pf_log": NOT USED. - "convergence_log": NOT USED. - } - """ - - is_tensor = False - if active_power is not None and reactive_power is not None: - assert active_power.shape == reactive_power.shape, "Active and reactive power arrays must have the " \ - "same shape." - original_shape = active_power.shape - - if active_power.ndim > 2: # Reshape form N-D to 2-D: - active_power, original_shape = self.reshape_tensor(active_power) - reactive_power, _ = self.reshape_tensor(reactive_power) - is_tensor = True - - self.P_file = active_power - kwargs = dict() - if algorithm == "hp": # Same as hp-tensor but receive 1-D vectors - pf_algorithm = self.run_pf_tensor_hp_laurent - kwargs.update(solver=sparse_solver) - elif algorithm == "sam": - pf_algorithm = self.run_pf_sam_sequential - elif algorithm == "sequential": # Same as tensor but receive 1-D vectors - pf_algorithm = self.run_pf_tensor - elif algorithm == "tensor": - pf_algorithm = self.run_pf_tensor - elif algorithm == "hp-tensor": - pf_algorithm = self.run_pf_tensor_hp_laurent - elif algorithm == "gpu-tensor": - pf_algorithm = self.run_pf_tensor - kwargs.update(compute="gpu") - else: - raise ValueError("Incorrect power flow algorithm selected") - - solutions = pf_algorithm(active_power=self.P_file, # 2-D Array - reactive_power=reactive_power, # 2-D Array - flat_start=flat_start, - start_value=start_value, - tolerance=tolerance, - **kwargs) - - if is_tensor: # Solutions from a 2-D array to an N-D array. - solutions["v"].shape = original_shape - active_power.shape = original_shape - reactive_power.shape = original_shape - - return solutions - - def run_pf_tensor(self, - active_power: np.ndarray , - reactive_power: np.ndarray = None, - *, - start_value=None, - iterations: int = 100, - tolerance: float = 1e-6, - flat_start: bool = True, - compute: str = "cpu") -> dict: - if (active_power is not None) and (reactive_power is not None): - print('ok') - assert len(active_power.shape) == 2, "Array must be two dimensional." - assert len(reactive_power.shape) == 2, "Array must be two dimensional." - assert active_power.shape[1] == reactive_power.shape[1] == self.nb - 1, "All nodes must have power values." - else: - #active_power = self.P_file[np.newaxis, : - reactive_power = self.Q_file[np.newaxis, :] - # print('zhong') - - self.ts_n = active_power.shape[0] # Time steps to be simulated - if flat_start: - self.v_0 = np.ones((self.ts_n, self.nb - 1)) + 1j * np.zeros((self.ts_n, self.nb - 1)) # Flat star - v0_solutions = [] - total_time_pre_pf_all = [] - total_time_pf_all = [] - total_time_algorithm_all = [] - iterations_all = [] - flag_convergence_all = [] - flag_convergence_bool_all = True - - active_power_pu = active_power / self.s_base # Vector with all active power except slack - reactive_power_pu = reactive_power / self.s_base # Vector with all reactive power except slack - - S_nom = active_power_pu + 1j * reactive_power_pu # (ts x nodes) - - n_steps = S_nom.shape[0] - n_nodes = S_nom.shape[1] - - - if compute == "cpu": - # print("CPU Solver selected") - self._power_flow_tensor_solver = self._power_flow_tensor_constant_power - elif compute == "gpu" and self.is_gpu_enabled is False: - warnings.warn("GPU library not found, falling back to CPU.") - self._power_flow_tensor_solver = self._power_flow_tensor_constant_power - elif compute == "gpu" and self.is_gpu_enabled is True: - # print("GPU Solver selected") - self._power_flow_tensor_solver = self.gpu_solver.power_flow_gpu - - if compute == "cpu": - DIMENSION_BOUND = 500 * 100_000 # 5_000 x 10_000 did work. Empirical value for my machine - else: - DIMENSION_BOUND = 500 * 125_000 # 5_000 x 15_000 did work. Empirical value for my machine - - - - idx = self._compute_chunks(DIMENSION_BOUND, n_nodes=n_nodes, n_steps=n_steps) - n_chunks = len(idx) - 1 - - t = trange(n_chunks, desc='Chunk', leave=False) - for ii in t: - t.set_description(f"Chunk: {ii + 1} of {n_chunks}", refresh=True) - - ts_chunk = idx[ii + 1] - idx[ii] # Size of the chunk - # TODO - - self.v_0 = np.ones((ts_chunk, self.nb - 1)) + 1j * np.zeros((ts_chunk, self.nb - 1)) # Flat start - - - S_chunk = S_nom[idx[ii]:idx[ii + 1]] - - - if self.constant_power_only: - start_time_pre_pf = self.start_time_pre_pf_tensor_constant_power_only - # No pre-computing (Already done when creating the object) - end_time_pre_pf = self.end_time_pre_pf_tensor_constant_power_only - - start_time_pf = perf_counter() - self.v_0, t_iterations = self._power_flow_tensor_solver(K=self._K_, - L=self._L_, - S=S_chunk, - v0=self.v_0, - ts=ts_chunk, - nb=self.nb, - iterations=iterations, - tolerance=tolerance) - end_time_pf = perf_counter() - - else: - # raise ValueError("This should not be running") - start_time_pre_pf = perf_counter() - self._F_, self._W_ = self._pre_power_flow_tensor(flag_all_constant_impedance_is_zero=self.flag_all_constant_impedance_is_zero, - flag_all_constant_current_is_zero=self.flag_all_constant_current_is_zero, - flag_all_constant_powers_are_ones=self.flag_all_constant_powers_are_ones, - ts_n=ts_chunk, - nb=self.nb, - S_nom=S_chunk, - alpha_Z=self.alpha_Z, - alpha_I=self.alpha_I, - alpha_P=self.alpha_P, - Yds=self.Yds, - Ydd=self.Ydd) - end_time_pre_pf = perf_counter() - - start_time_pf = perf_counter() - self.v_0, t_iterations = self._power_flow_tensor(_F_=self._F_, - _W_=self._W_, - v_0=self.v_0, - ts_n=ts_chunk, - nb=self.nb, - iterations=iterations, - tolerance=tolerance) - end_time_pf = perf_counter() - - if t_iterations == iterations: - flag_convergence = False - warnings.warn("Power flow did not converge.") - else: - flag_convergence = True - - total_time_pre_pf = end_time_pre_pf - start_time_pre_pf - total_time_pf = end_time_pf - start_time_pf - total_time_algorithm = total_time_pre_pf + total_time_pf - - total_time_pre_pf_all.append(total_time_pre_pf) - total_time_pf_all.append(total_time_pf) - total_time_algorithm_all.append(total_time_algorithm) - iterations_all.append(t_iterations) - flag_convergence_all.append(flag_convergence) - flag_convergence_bool_all = flag_convergence_bool_all & flag_convergence - - v0_solutions.append(self.v_0.copy()) - - self.v_0 = np.vstack(v0_solutions) - - solution = {"v": self.v_0, # 2D-Vector. Solution of voltage in complex numbers - "time_pre_pf": sum(total_time_pre_pf_all), - "time_pf": sum(total_time_pf_all), - "time_algorithm": sum(total_time_algorithm_all), - "iterations": np.floor(np.mean(iterations_all)), - - "convergence": flag_convergence_bool_all, - "iterations_log": iterations_all, - "time_pre_pf_log": total_time_pre_pf_all, - "time_pf_log": total_time_pf_all, - "convergence_log": flag_convergence_all - } - - return solution - - - def run_pf_sam_sequential(self, - active_power: np.ndarray = None, - reactive_power: np.ndarray = None, - flat_start: bool = True, - start_value: np.array = None): - - r""" - Single time step power flow with numba performance increase. - This is the implementation of [1], algorithm called SAM (Successive Approximation Method) - - V[k+1] = B^{-1} ( A[k] @ V[k]^{*} - C - D[k]) - - Where: - A[k] = np.diag(\alpha_p \odot V[k]^{* -2} * S_n^{*}), \odot == Hadamard product, * == complex conjugate - B = np.diag(\alpha_z \odot S_n^{*}) + Y_dd - C = Y_ds @ V_s + \alpha_i \odot S_n^{*} - D[k] = 2 \alpha_p \odot V[k]^{* -1} \odot S_n^{*} - - Please note that for constant power only. i.e., \alpha_p = 1, \alpha_i = 0, \alpha_z = 0. - The matrices reduces to: - - A[k] = np.diag(V[k]^{* -2} * S_n^{*}), \odot == Hadamard product, * == complex conjugate - B = Y_dd - C = Y_ds @ V_s - D[k] = 2 V[k]^{* -1} \odot S_n^{*} - - [1] Juan S. Giraldo, Oscar Danilo Montoya, Pedro P. Vergara, Federico Milano, "A fixed-point current injection - power flow for electric distribution systems using Laurent series", Electric Power Systems Research, - Volume 211, 2022. https://doi.org/10.1016/j.epsr.2022.108326. - - """ - - if (active_power is not None) and (reactive_power is not None): - active_power, reactive_power = self._check_2d_to_1d(active_power, reactive_power) - else: # Default case - active_power = self.P_file - reactive_power = self.Q_file - - if flat_start: - # TODO - self.v_0 = np.ones((self.nb - 1, 1), dtype="complex128") # 2D-Vector - elif start_value is not None: - # TODO: Check the dimensions of the flat start - self.v_0 = start_value # User's start value - - active_power_pu = active_power / self.s_base # Vector with all active power except slack - reactive_power_pu = reactive_power / self.s_base # Vector with all reactive power except slack - S_nom = (active_power_pu + 1j * reactive_power_pu).reshape(-1, ) - - - if self.constant_power_only: - start_time_pre_pf = perf_counter() - # No precomputing, the minimum matrix multiplication is done in the initialization of the object. - end_time_pre_pf = perf_counter() - - start_time_pf = perf_counter() - V, iteration = self._power_flow_sam_sequential_constant_power_only(B_inv=-self._K_, - C=self.Yds.flatten(), - v_0=self.v_0, - s_n=S_nom, - iterations=self.iterations, - tolerance=self.tolerance) - end_time_pf = perf_counter() - - else: - start_time_pre_pf = perf_counter() - B_inv, C, S_nom = self._pre_power_flow_sam_sequential(active_power, # TODO: Change the input to S_nom - reactive_power, - s_base=self.s_base, - alpha_Z=self.alpha_Z, - alpha_I=self.alpha_I, - Yds=self.Yds, - Ydd=self.Ydd, - nb=self.nb) - end_time_pre_pf = perf_counter() - - start_time_pf = perf_counter() - V, iteration = self._power_flow_sam_sequential(B_inv, - C, - v_0=self.v_0, - s_n=S_nom, - alpha_P=self.alpha_P, - iterations=self.iterations, - tolerance=self.tolerance) - end_time_pf = perf_counter() - - if iteration == self.iterations: - flag_convergence = False - else: - flag_convergence = True - - total_time_pre_pf = end_time_pre_pf - start_time_pre_pf - total_time_pf = end_time_pf - start_time_pf - total_time_algorithm = total_time_pre_pf + total_time_pf - - solution = {"v": V.flatten(), # 1D-Vector. Solution of voltage in complex numbers - "time_pre_pf": total_time_pre_pf, - "time_pf": total_time_pf, - "time_algorithm": total_time_algorithm, - "iterations": iteration, - "convergence": flag_convergence,} - - return solution - - - def line_currents(self, volt_solutions=None): - raise NotImplementedError + + # connection matrix for line & from buses + Cf = csr_matrix((np.ones(self.nl), (range(self.nl), f)), (self.nl, self.nb)) + + # connection matrix for line & to buses + Ct = csr_matrix((np.ones(self.nl), (range(self.nl), t)), (self.nl, self.nb)) + + # build Yf and Yt such that Yf * V is the vector of complex branch currents injected + # at each branch's "from" bus, and Yt is the same for the "to" bus end + i = np.r_[range(self.nl), range(self.nl)] # double set of row indices + + Yf = csr_matrix((np.r_[Yff, Yft], (i, np.r_[f, t]))) + Yt = csr_matrix((np.r_[Ytf, Ytt], (i, np.r_[f, t]))) + + # build Ybus + Ybus = Cf.T * Yf + Ct.T * Yt # Full Ybus + + # Dense matrices + # TODO + # TODO: This takes a lot of memory. Check if I can save it always as sparse for all methods. + self._Ybus = Ybus.toarray() + self.Yss = csr_matrix(Ybus[sl[0] - 1, sl[0] - 1], shape=(len(sl), len(sl))).toarray() + self.Ysd = np.array(Ybus[0, 1:].toarray()) # TODO: Here assume the slack is the first one? + self.Yds = self.Ysd.T + self.Ydd = np.array(Ybus[1:, 1:].toarray()) # TODO: This consumes a huge amount of memory + + self._Ybus_sparse = Ybus + self.Yss_sparse = csr_matrix(Ybus[sl[0] - 1, sl[0] - 1], shape=(len(sl), len(sl))) + self.Ysd_sparse = Ybus[0, 1:] # TODO: Here assume the slack is the first one? + self.Yds_sparse = csc_matrix(self.Ysd.T) + self.Ydd_sparse = Ybus[1:, 1:] + + return Ybus + + def _compute_alphas(self): + """ + Computes alpha values for different load types in the grid. Assume P-1 and Z,I=0 + """ + self.alpha_P = 1 + self.alpha_I = 0 + self.alpha_Z = 0 + + self.flag_all_constant_impedance_is_zero = not np.any(self.alpha_Z) + self.flag_all_constant_current_is_zero = not np.any(self.alpha_I) + self.flag_all_constant_powers_are_ones = np.all(self.alpha_P) + + def _check_2d_to_1d(self, active_power, reactive_power): + """ + Checks and converts 2D power matrices to 1D vectors if applicable. + + Parameters: + active_power (np.ndarray): Active power matrix. + reactive_power (np.ndarray): Reactive power matrix. + + Returns: + tuple: Tuple containing active and reactive power as 1D vectors. + """ + + assert active_power.ndim == reactive_power.ndim, "Active and reactive power must have same dimension." + + if (active_power.ndim == 2 and active_power.shape[0] == 1) and (reactive_power.ndim == 2 and reactive_power.shape[0] == 1): + active_power = active_power.flatten() + reactive_power = reactive_power.flatten() + elif (active_power.ndim == 2 and active_power.shape[0] != 1) and (reactive_power.ndim == 2 and reactive_power.shape[0] != 1): + raise ValueError("Active and reactive power tensors must have only one time step.") + + assert active_power.ndim == 1, "Array should be one dimensional." + assert reactive_power.ndim == 1, "Array should be one dimensional." + assert len(active_power) == len(reactive_power) == self.nb - 1, "All load nodes must have power values." + + return active_power, reactive_power + + def _compute_chunks(self, DIMENSION_BOUND, n_nodes, n_steps): + """ + Computes chunks for processing based on dimension bounds and grid parameters. + + Parameters: + DIMENSION_BOUND (int): The upper bound for the dimension of the matrices. + n_nodes (int): Number of nodes in the grid. + n_steps (int): Number of time steps for the simulation. + + Returns: + list: Indices for slicing the power consumption array. + Breaks the n_steps in chunks so it can fit in memory. + The ideas is that n_nodes * n_steps cannot be bigger than DIMENSION_BOUND + DIMENSION_BOUND is a empirically found value (should vary due to the computer's RAM). + + Return: + idx: list: All the ts indices to slice the power consumption array. + e.g., idx = [0, 1000, 2000, 2500]. 2500 time step requested, chunked in 1000 time steps (last item is + the reminder: 2500-2000=500). + """ + + # DIMENSION_BOUND = 500 * 5_000 + # n_nodes = 4999 + # n_steps = 3000 + + TS_MAX = DIMENSION_BOUND // n_nodes + if n_steps > TS_MAX: # Chunk it + (quotient, reminder) = divmod(n_steps, TS_MAX) + idx = [i * TS_MAX for i in range(quotient + 1)] + + if reminder != 0: + idx = idx + [idx[-1] + reminder] + # if reminder == 0: + # idx = idx + [idx[-1] + TS_MAX] + + else: # The requested amount of TS is lower than the bound. So, everything is ok + idx = [0, n_steps] + + # print(idx) + + return idx + + def _make_big_sparse_matrices(self, S_nom, Ydd_sparse, Yds_sparse): + """ + Creates large sparse matrices for solving the sparse tensor power flow problem. + + Parameters: + S_nom (np.ndarray): Nominal power values. + Ydd_sparse (sparse matrix): Sparse Ydd matrix. + Yds_sparse (sparse matrix): Sparse Yds matrix. + + Returns: + tuple: Tuple containing the big M matrix and H vector as sparse matrices. + """ + + n_steps = S_nom.shape[0] + n_nodes = S_nom.shape[1] + + M = -diags(1 / np.conj(S_nom[0, :])).dot(Ydd_sparse).asformat("coo") + H = diags(1 / np.conj(S_nom[0, :])).dot(Yds_sparse).asformat("coo") + + # First iteration of M-matrix and H-vector + idx_1_M_col = M.col + idx_1_M_row = M.row + M_1_data = M.data + + idx_1_H_row = H.row + H_1_data = H.data + + # Placeholder for M-matrix + idx_col_M_temp = [] + idx_row_M_temp = [] + M_data_temp = [] + + # Placeholder for H-vector + idx_row_H_temp = [] + H_data_temp = [] + if n_steps > 1: + times_multiplying = [] + for ii in range(1, n_steps): + start_multiply = perf_counter() + M_temp = -diags(1 / np.conj(S_nom[ii, :])).dot(Ydd_sparse).asformat("coo") + H_temp = diags(1 / np.conj(S_nom[ii, :])).dot(Yds_sparse).asformat("coo") + + idx_col_M_temp.append(M_temp.col + ii * n_nodes) + idx_row_M_temp.append(M_temp.row + ii * n_nodes) + M_data_temp.append(M_temp.data) + + idx_row_H_temp.append(H_temp.row + ii * n_nodes) + H_data_temp.append(H_temp.data) + + times_multiplying.append(perf_counter() - start_multiply) + + M_big_idx_col = np.hstack([idx_1_M_col, np.hstack(idx_col_M_temp)]) + M_big_idx_row = np.hstack([idx_1_M_row, np.hstack(idx_row_M_temp)]) + M_big_idx_data = np.hstack([M_1_data, np.hstack(M_data_temp)]) + + H_big_idx_row = np.hstack([idx_1_H_row, np.hstack(idx_row_H_temp)]) + H_big_idx_col = np.zeros(H_big_idx_row.shape[0], dtype=np.int32) + H_big_data = np.hstack([H_1_data, np.hstack(H_data_temp)]) + + M_big = csr_matrix((M_big_idx_data, (M_big_idx_row, M_big_idx_col))) + H_big = csr_matrix((H_big_data, (H_big_idx_row, H_big_idx_col)), shape=(n_steps * n_nodes, 1)) + + else: + M_big = M + H_big = H + + return M_big, H_big + + def reshape_tensor(self, tensor_array): + """ + Reshapes a tensor array for power flow calculations. + + Parameters: + tensor_array (np.ndarray): The tensor array to be reshaped. + + Returns: + tuple: Reshaped tensor array and its original shape. + """ + original_shape = tensor_array.shape + tau = np.prod(original_shape[:-1]) + tensor_array.shape = (tau, original_shape[-1]) # This reshapes in place (No new memory use) + + return tensor_array, original_shape + + def run_pf( + self, + active_power: np.ndarray = None, + reactive_power: np.ndarray = None, + flat_start: bool = True, + start_value: np.ndarray = None, + tolerance: float = 1e-6, + algorithm: str = "tensor", + sparse_solver: str = "scipy", + ): + """ + Run a power-flow solve for the provided active/reactive power inputs. + + The method accepts either batched tensors or a single-step vector and dispatches + to the selected solver implementation. + "time_algorithm": Total time algorithm. time_algorithm = time_pre_pf + time_pf + "iterations": Total number of iterations to converge. + + "convergence": Boolean indicating: True: Algorithm converged, False: it didn't. + "iterations_log": NOT USED. + "time_pre_pf_log": NOT USED. + "time_pf_log": NOT USED. + "convergence_log": NOT USED. + } + """ + + is_tensor = False + if active_power is not None and reactive_power is not None: + assert active_power.shape == reactive_power.shape, "Active and reactive power arrays must have the same shape." + original_shape = active_power.shape + + if active_power.ndim > 2: # Reshape form N-D to 2-D: + active_power, original_shape = self.reshape_tensor(active_power) + reactive_power, _ = self.reshape_tensor(reactive_power) + is_tensor = True + + self.P_file = active_power + kwargs = dict() + if algorithm == "hp": # Same as hp-tensor but receive 1-D vectors + pf_algorithm = self.run_pf_tensor_hp_laurent + kwargs.update(solver=sparse_solver) + elif algorithm == "sam": + pf_algorithm = self.run_pf_sam_sequential + elif algorithm == "sequential": # Same as tensor but receive 1-D vectors + pf_algorithm = self.run_pf_tensor + elif algorithm == "tensor": + pf_algorithm = self.run_pf_tensor + elif algorithm == "hp-tensor": + pf_algorithm = self.run_pf_tensor_hp_laurent + elif algorithm == "gpu-tensor": + pf_algorithm = self.run_pf_tensor + kwargs.update(compute="gpu") + else: + raise ValueError("Incorrect power flow algorithm selected") + + solutions = pf_algorithm( + active_power=self.P_file, # 2-D Array + reactive_power=reactive_power, # 2-D Array + flat_start=flat_start, + start_value=start_value, + tolerance=tolerance, + **kwargs, + ) + + if is_tensor: # Solutions from a 2-D array to an N-D array. + solutions["v"].shape = original_shape + active_power.shape = original_shape + reactive_power.shape = original_shape + + return solutions + + def run_pf_tensor( + self, + active_power: np.ndarray, + reactive_power: np.ndarray = None, + *, + start_value=None, + iterations: int = 100, + tolerance: float = 1e-6, + flat_start: bool = True, + compute: str = "cpu", + ) -> dict: + if (active_power is not None) and (reactive_power is not None): + print("ok") + assert len(active_power.shape) == 2, "Array must be two dimensional." + assert len(reactive_power.shape) == 2, "Array must be two dimensional." + assert active_power.shape[1] == reactive_power.shape[1] == self.nb - 1, "All nodes must have power values." + else: + # active_power = self.P_file[np.newaxis, : + reactive_power = self.Q_file[np.newaxis, :] + # print('zhong') + + self.ts_n = active_power.shape[0] # Time steps to be simulated + if flat_start: + self.v_0 = np.ones((self.ts_n, self.nb - 1)) + 1j * np.zeros((self.ts_n, self.nb - 1)) # Flat star + v0_solutions = [] + total_time_pre_pf_all = [] + total_time_pf_all = [] + total_time_algorithm_all = [] + iterations_all = [] + flag_convergence_all = [] + flag_convergence_bool_all = True + + active_power_pu = active_power / self.s_base # Vector with all active power except slack + reactive_power_pu = reactive_power / self.s_base # Vector with all reactive power except slack + + S_nom = active_power_pu + 1j * reactive_power_pu # (ts x nodes) + + n_steps = S_nom.shape[0] + n_nodes = S_nom.shape[1] + + if compute == "cpu": + # print("CPU Solver selected") + self._power_flow_tensor_solver = self._power_flow_tensor_constant_power + elif compute == "gpu" and self.is_gpu_enabled is False: + warnings.warn("GPU library not found, falling back to CPU.") + self._power_flow_tensor_solver = self._power_flow_tensor_constant_power + elif compute == "gpu" and self.is_gpu_enabled is True: + # print("GPU Solver selected") + self._power_flow_tensor_solver = self.gpu_solver.power_flow_gpu + + if compute == "cpu": + DIMENSION_BOUND = 500 * 100_000 # 5_000 x 10_000 did work. Empirical value for my machine + else: + DIMENSION_BOUND = 500 * 125_000 # 5_000 x 15_000 did work. Empirical value for my machine + + idx = self._compute_chunks(DIMENSION_BOUND, n_nodes=n_nodes, n_steps=n_steps) + n_chunks = len(idx) - 1 + + t = trange(n_chunks, desc="Chunk", leave=False) + for ii in t: + t.set_description(f"Chunk: {ii + 1} of {n_chunks}", refresh=True) + + ts_chunk = idx[ii + 1] - idx[ii] # Size of the chunk + # TODO + + self.v_0 = np.ones((ts_chunk, self.nb - 1)) + 1j * np.zeros((ts_chunk, self.nb - 1)) # Flat start + + S_chunk = S_nom[idx[ii] : idx[ii + 1]] + + if self.constant_power_only: + start_time_pre_pf = self.start_time_pre_pf_tensor_constant_power_only + # No pre-computing (Already done when creating the object) + end_time_pre_pf = self.end_time_pre_pf_tensor_constant_power_only + + start_time_pf = perf_counter() + self.v_0, t_iterations = self._power_flow_tensor_solver( + K=self._K_, + L=self._L_, + S=S_chunk, + v0=self.v_0, + ts=ts_chunk, + nb=self.nb, + iterations=iterations, + tolerance=tolerance, + ) + end_time_pf = perf_counter() + + else: + # raise ValueError("This should not be running") + start_time_pre_pf = perf_counter() + self._F_, self._W_ = self._pre_power_flow_tensor( + flag_all_constant_impedance_is_zero=self.flag_all_constant_impedance_is_zero, + flag_all_constant_current_is_zero=self.flag_all_constant_current_is_zero, + flag_all_constant_powers_are_ones=self.flag_all_constant_powers_are_ones, + ts_n=ts_chunk, + nb=self.nb, + S_nom=S_chunk, + alpha_Z=self.alpha_Z, + alpha_I=self.alpha_I, + alpha_P=self.alpha_P, + Yds=self.Yds, + Ydd=self.Ydd, + ) + end_time_pre_pf = perf_counter() + + start_time_pf = perf_counter() + self.v_0, t_iterations = self._power_flow_tensor( + _F_=self._F_, + _W_=self._W_, + v_0=self.v_0, + ts_n=ts_chunk, + nb=self.nb, + iterations=iterations, + tolerance=tolerance, + ) + end_time_pf = perf_counter() + + if t_iterations == iterations: + flag_convergence = False + warnings.warn("Power flow did not converge.") + else: + flag_convergence = True + + total_time_pre_pf = end_time_pre_pf - start_time_pre_pf + total_time_pf = end_time_pf - start_time_pf + total_time_algorithm = total_time_pre_pf + total_time_pf + + total_time_pre_pf_all.append(total_time_pre_pf) + total_time_pf_all.append(total_time_pf) + total_time_algorithm_all.append(total_time_algorithm) + iterations_all.append(t_iterations) + flag_convergence_all.append(flag_convergence) + flag_convergence_bool_all = flag_convergence_bool_all & flag_convergence + + v0_solutions.append(self.v_0.copy()) + + self.v_0 = np.vstack(v0_solutions) + + solution = { + "v": self.v_0, # 2D-Vector. Solution of voltage in complex numbers + "time_pre_pf": sum(total_time_pre_pf_all), + "time_pf": sum(total_time_pf_all), + "time_algorithm": sum(total_time_algorithm_all), + "iterations": np.floor(np.mean(iterations_all)), + "convergence": flag_convergence_bool_all, + "iterations_log": iterations_all, + "time_pre_pf_log": total_time_pre_pf_all, + "time_pf_log": total_time_pf_all, + "convergence_log": flag_convergence_all, + } + + return solution + + def run_pf_sam_sequential( + self, + active_power: np.ndarray = None, + reactive_power: np.ndarray = None, + flat_start: bool = True, + start_value: np.array = None, + ): + r""" + Single time step power flow with numba performance increase. + This is the implementation of [1], algorithm called SAM (Successive Approximation Method) + + V[k+1] = B^{-1} ( A[k] @ V[k]^{*} - C - D[k]) + + Where: + A[k] = np.diag(\alpha_p \odot V[k]^{* -2} * S_n^{*}), \odot == Hadamard product, * == complex conjugate + B = np.diag(\alpha_z \odot S_n^{*}) + Y_dd + C = Y_ds @ V_s + \alpha_i \odot S_n^{*} + D[k] = 2 \alpha_p \odot V[k]^{* -1} \odot S_n^{*} + + Please note that for constant power only. i.e., \alpha_p = 1, \alpha_i = 0, \alpha_z = 0. + The matrices reduces to: + + A[k] = np.diag(V[k]^{* -2} * S_n^{*}), \odot == Hadamard product, * == complex conjugate + B = Y_dd + C = Y_ds @ V_s + D[k] = 2 V[k]^{* -1} \odot S_n^{*} + + [1] Juan S. Giraldo, Oscar Danilo Montoya, Pedro P. Vergara, Federico Milano, "A fixed-point current injection + power flow for electric distribution systems using Laurent series", Electric Power Systems Research, + Volume 211, 2022. https://doi.org/10.1016/j.epsr.2022.108326. + + """ + + if (active_power is not None) and (reactive_power is not None): + active_power, reactive_power = self._check_2d_to_1d(active_power, reactive_power) + else: # Default case + active_power = self.P_file + reactive_power = self.Q_file + + if flat_start: + # TODO + self.v_0 = np.ones((self.nb - 1, 1), dtype="complex128") # 2D-Vector + elif start_value is not None: + # TODO: Check the dimensions of the flat start + self.v_0 = start_value # User's start value + + active_power_pu = active_power / self.s_base # Vector with all active power except slack + reactive_power_pu = reactive_power / self.s_base # Vector with all reactive power except slack + S_nom = (active_power_pu + 1j * reactive_power_pu).reshape( + -1, + ) + + if self.constant_power_only: + start_time_pre_pf = perf_counter() + # No precomputing, the minimum matrix multiplication is done in the initialization of the object. + end_time_pre_pf = perf_counter() + + start_time_pf = perf_counter() + V, iteration = self._power_flow_sam_sequential_constant_power_only( + B_inv=-self._K_, + C=self.Yds.flatten(), + v_0=self.v_0, + s_n=S_nom, + iterations=self.iterations, + tolerance=self.tolerance, + ) + end_time_pf = perf_counter() + + else: + start_time_pre_pf = perf_counter() + B_inv, C, S_nom = self._pre_power_flow_sam_sequential( + active_power, # TODO: Change the input to S_nom + reactive_power, + s_base=self.s_base, + alpha_Z=self.alpha_Z, + alpha_I=self.alpha_I, + Yds=self.Yds, + Ydd=self.Ydd, + nb=self.nb, + ) + end_time_pre_pf = perf_counter() + + start_time_pf = perf_counter() + V, iteration = self._power_flow_sam_sequential( + B_inv, + C, + v_0=self.v_0, + s_n=S_nom, + alpha_P=self.alpha_P, + iterations=self.iterations, + tolerance=self.tolerance, + ) + end_time_pf = perf_counter() + + if iteration == self.iterations: + flag_convergence = False + else: + flag_convergence = True + + total_time_pre_pf = end_time_pre_pf - start_time_pre_pf + total_time_pf = end_time_pf - start_time_pf + total_time_algorithm = total_time_pre_pf + total_time_pf + + solution = { + "v": V.flatten(), # 1D-Vector. Solution of voltage in complex numbers + "time_pre_pf": total_time_pre_pf, + "time_pf": total_time_pf, + "time_algorithm": total_time_algorithm, + "iterations": iteration, + "convergence": flag_convergence, + } + + return solution + + def line_currents(self, volt_solutions=None): + raise NotImplementedError diff --git a/rl_adn/utility/numbarize.py b/rl_adn/network/numbarize.py similarity index 70% rename from rl_adn/utility/numbarize.py rename to rl_adn/network/numbarize.py index eed2dcc..2446d70 100644 --- a/rl_adn/utility/numbarize.py +++ b/rl_adn/network/numbarize.py @@ -1,36 +1,30 @@ -from numba import prange import numpy as np +from numba import prange -def pre_power_flow_sam_sequential(active_power, - reactive_power, - s_base, - alpha_Z, - alpha_I, - Yds, - Ydd, - nb - ): +def pre_power_flow_sam_sequential(active_power, reactive_power, s_base, alpha_Z, alpha_I, Yds, Ydd, nb): + """ + Prepares the matrices for the SAM sequential power flow method. + + Parameters: + active_power (np.ndarray): Array of active power values. + reactive_power (np.ndarray): Array of reactive power values. + s_base (float): Base power value for per-unit conversion. + alpha_Z (np.ndarray): Array of constant impedance values. + alpha_I (np.ndarray): Array of constant current values. + Yds (np.ndarray): Admittance matrix between slack and load buses. + Ydd (np.ndarray): Admittance matrix between load buses. + nb (int): Number of buses in the network. + + Returns: + tuple: Tuple containing B_inv, C, and S_nom matrices used in the SAM sequential power flow method. """ - Prepares the matrices for the SAM sequential power flow method. - - Parameters: - active_power (np.ndarray): Array of active power values. - reactive_power (np.ndarray): Array of reactive power values. - s_base (float): Base power value for per-unit conversion. - alpha_Z (np.ndarray): Array of constant impedance values. - alpha_I (np.ndarray): Array of constant current values. - Yds (np.ndarray): Admittance matrix between slack and load buses. - Ydd (np.ndarray): Admittance matrix between load buses. - nb (int): Number of buses in the network. - - Returns: - tuple: Tuple containing B_inv, C, and S_nom matrices used in the SAM sequential power flow method. - """ active_power_pu = active_power / s_base # Vector with all active power except slack reactive_power_pu = reactive_power / s_base # Vector with all reactive power except slack - S_nom = (active_power_pu + 1j * reactive_power_pu).reshape(-1, ) + S_nom = (active_power_pu + 1j * reactive_power_pu).reshape( + -1, + ) if not np.any(alpha_Z): # \alpha_z is 0 B_inv = np.linalg.inv(Ydd) else: @@ -45,14 +39,16 @@ def pre_power_flow_sam_sequential(active_power, return B_inv, C, S_nom -def power_flow_sam_sequential(B_inv, - C, - v_0, - s_n, - alpha_P, - iterations, - tolerance, - ): + +def power_flow_sam_sequential( + B_inv, + C, + v_0, + s_n, + alpha_P, + iterations, + tolerance, +): """ Performs the SAM sequential power flow calculation. @@ -83,13 +79,14 @@ def power_flow_sam_sequential(B_inv, return v_0, iteration # Solution of voltage in complex numbers -def power_flow_sam_sequential_constant_power_only(B_inv, - C, - v_0, - s_n, - iterations, - tolerance, - ): +def power_flow_sam_sequential_constant_power_only( + B_inv, + C, + v_0, + s_n, + iterations, + tolerance, +): """ Performs the SAM sequential power flow calculation for constant power loads only. @@ -120,17 +117,19 @@ def power_flow_sam_sequential_constant_power_only(B_inv, return v_0, iteration # Solution of voltage in complex numbers -def pre_power_flow_tensor(flag_all_constant_impedance_is_zero, - flag_all_constant_current_is_zero, - flag_all_constant_powers_are_ones, - ts_n, - nb, - S_nom, - alpha_Z, - alpha_I, - alpha_P, - Yds, - Ydd): +def pre_power_flow_tensor( + flag_all_constant_impedance_is_zero, + flag_all_constant_current_is_zero, + flag_all_constant_powers_are_ones, + ts_n, + nb, + S_nom, + alpha_Z, + alpha_I, + alpha_P, + Yds, + Ydd, +): """ Prepares the matrices for the tensor-based power flow method. @@ -179,14 +178,16 @@ def pre_power_flow_tensor(flag_all_constant_impedance_is_zero, return _F_2, _W_2 -def power_flow_tensor(_F_, - _W_, - v_0, - ts_n, - nb, - iterations, - tolerance, - ): + +def power_flow_tensor( + _F_, + _W_, + v_0, + ts_n, + nb, + iterations, + tolerance, +): """ Performs the tensor-based power flow calculation. @@ -217,16 +218,7 @@ def power_flow_tensor(_F_, return v_0, iteration - -def power_flow_tensor_constant_power_numba_parallel_True(K, - L, - S, - v0, - ts, - nb, - iterations, - tolerance - ): +def power_flow_tensor_constant_power_numba_parallel_True(K, L, S, v0, ts, nb, iterations, tolerance): """ Performs the tensor-based power flow calculation with constant power loads, optimized for parallel execution. @@ -257,15 +249,7 @@ def power_flow_tensor_constant_power_numba_parallel_True(K, return v0, iteration -def power_flow_tensor_constant_power(K, - L, - S, - v0, - ts, - nb, - iterations, - tolerance - ): +def power_flow_tensor_constant_power(K, L, S, v0, ts, nb, iterations, tolerance): """ Performs the tensor-based power flow calculation for constant power loads. @@ -305,15 +289,7 @@ def power_flow_tensor_constant_power(K, return v0, iteration -def power_flow_tensor_constant_power_new(K, - L, - S, - v0, - ts, - nb, - iterations, - tolerance - ): +def power_flow_tensor_constant_power_new(K, L, S, v0, ts, nb, iterations, tolerance): """ A new version of the tensor-based power flow calculation for constant power loads, supporting parallel execution. @@ -346,11 +322,11 @@ def power_flow_tensor_constant_power_new(K, LAMBDA = np.conj(S.T * (1 / v0.T)) # Hadamard product ( (nb-1) x ts) Z = K @ LAMBDA # Matrix ( (nb-1) x ts ) Z = Z.T - for j in prange(ts): # This is a brodcasted sum ( (nb-1) x ts + (nb-1) x 1 => (nb-1) x ts ) + for j in prange(ts): # This is a brodcasted sum ( (nb-1) x ts + (nb-1) x 1 => (nb-1) x ts ) voltage_k[j] = Z[j] + W tol = np.max(np.abs(np.abs(voltage_k) - np.abs(v0))) v0 = voltage_k iteration += 1 - return v0, iteration \ No newline at end of file + return v0, iteration diff --git a/rl_adn/utility/topology.py b/rl_adn/network/topology.py similarity index 98% rename from rl_adn/utility/topology.py rename to rl_adn/network/topology.py index 6c36e14..e3ac65f 100644 --- a/rl_adn/utility/topology.py +++ b/rl_adn/network/topology.py @@ -1,4 +1,4 @@ -from typing import Dict, Iterable, List, Tuple +from typing import Dict, List, Tuple import networkx as nx import numpy as np @@ -6,7 +6,6 @@ from rl_adn.environments.topology_scenarios import TopologyScenario - Edge = Tuple[int, int] diff --git a/rl_adn/utility/utils.py b/rl_adn/network/utils.py similarity index 70% rename from rl_adn/utility/utils.py rename to rl_adn/network/utils.py index 1ccf67a..7d5fac7 100644 --- a/rl_adn/utility/utils.py +++ b/rl_adn/network/utils.py @@ -1,17 +1,13 @@ - -import numpy as np +import os +import sys +from ctypes import CDLL, POINTER, byref, c_bool, c_double, c_int from time import perf_counter -import pandas as pd -import os import matplotlib.pyplot as plt import networkx as nx -import os import numpy as np -from ctypes import CDLL, POINTER, c_int, byref, c_bool, c_double +import pandas as pd from numpy import ctypeslib -import sys -from time import perf_counter def _require_pandapower(): @@ -22,6 +18,7 @@ def _require_pandapower(): raise ImportError("pandapower helpers require the optional dependency 'pandapower'.") from exc return pp, pandapower_topology + def load_library(): """ Loads a shared library for GPU-based power flow calculations. @@ -63,16 +60,16 @@ def load_library(): tensor_power_flow.restype = None ctypes_dtype_complex = ctypeslib.ndpointer(np.complex64) tensor_power_flow.argtypes = [ - POINTER(ctypes_dtype_complex), # Matrix S, dimensions: S(m x m) - POINTER(ctypes_dtype_complex), # Matrix K, dimensions: S(m x m) - POINTER(ctypes_dtype_complex), # Matrix V0, dimensions: V0(m x p)\ - POINTER(ctypes_dtype_complex), # Matrix W, dimensions: W(m x 1) - POINTER(c_int), # m - POINTER(c_int), # p - POINTER(c_double), # tolerance - POINTER(c_int), # iterations - POINTER(c_bool) # convergence - ] + POINTER(ctypes_dtype_complex), # Matrix S, dimensions: S(m x m) + POINTER(ctypes_dtype_complex), # Matrix K, dimensions: S(m x m) + POINTER(ctypes_dtype_complex), # Matrix V0, dimensions: V0(m x p)\ + POINTER(ctypes_dtype_complex), # Matrix W, dimensions: W(m x 1) + POINTER(c_int), # m + POINTER(c_int), # p + POINTER(c_double), # tolerance + POINTER(c_int), # iterations + POINTER(c_bool), # convergence + ] return tensor_power_flow @@ -92,15 +89,17 @@ class GPUPowerFlow(object): def __init__(self): self.gpu_solver = load_library() - def power_flow_gpu(self, - K: np.ndarray, # (m x m) == (nodes-1 x nodes-1) - L: np.ndarray, # (p x 1) == (time_steps x 1) or just p - S: np.ndarray, # (p x m) == (time_steps x nodes) - v0: np.ndarray, # (p x m) == (time_steps x nodes) - ts: np.ndarray, - nb: int, - iterations: int = 100, - tolerance: float = None): + def power_flow_gpu( + self, + K: np.ndarray, # (m x m) == (nodes-1 x nodes-1) + L: np.ndarray, # (p x 1) == (time_steps x 1) or just p + S: np.ndarray, # (p x m) == (time_steps x nodes) + v0: np.ndarray, # (p x m) == (time_steps x nodes) + ts: np.ndarray, + nb: int, + iterations: int = 100, + tolerance: float = None, + ): """ Executes the GPU-based power flow calculation. @@ -125,7 +124,7 @@ def power_flow_gpu(self, if tolerance is None: tolerance_gpu = 1e-10 else: - tolerance_gpu = tolerance ** 2 # Heuristic, this match to the CPU tolerance. + tolerance_gpu = tolerance**2 # Heuristic, this match to the CPU tolerance. # ====================================================================== # Reshape/casting and making sure that the complex matrices are 32 bits. @@ -175,15 +174,7 @@ def power_flow_gpu(self, W_ca = W_c.ctypes.data_as(POINTER(ctypes_dtype_complex)) # start = perf_counter() - self.gpu_solver(S_ca, - K_ca, - V0_ca, - W_ca, - m_int, - p_int, - tolerance_int, - iterations_int, - convergence_int) + self.gpu_solver(S_ca, K_ca, V0_ca, W_ca, m_int, p_int, tolerance_int, iterations_int, convergence_int) # print(f"GPU Dynamic library execution: {perf_counter() - start} sec.") # print(f"Convergence: {convergence.value}") v_solution = V0_c.reshape(m, p, order="F") @@ -192,6 +183,7 @@ def power_flow_gpu(self, # Voltage solution is a matrix with dimensions (time_steps x (n_nodes-1))-> Including the transpose. return v_solution.T, iter_solution + def generate_network(nodes, child=3, plot_graph=False, load_factor=2, line_factor=3): """ Generates a network graph and corresponding data frames for buses and lines. @@ -215,7 +207,7 @@ def generate_network(nodes, child=3, plot_graph=False, load_factor=2, line_facto if plot_graph: fig, ax = plt.subplots(1, 1, figsize=(10, 10)) - nx.draw_kamada_kawai(G, node_size=100, with_labels=True, font_size='medium', ax=ax) + nx.draw_kamada_kawai(G, node_size=100, with_labels=True, font_size="medium", ax=ax) assert nodes == len(G.nodes) assert LINES == len(G.edges) @@ -225,17 +217,13 @@ def generate_network(nodes, child=3, plot_graph=False, load_factor=2, line_facto Tb, Pct, Ict, Zct = 0, PCT, ICT, ZCT nodes_ = pd.DataFrame(list(G.nodes), columns=["NODES"]) + 1 - active_ns = np.random.normal(50 * load_factor, scale=50, size=nodes).round(3) - reactive_ns = (active_ns * .1).round(3) + reactive_ns = (active_ns * 0.1).round(3) - power = pd.DataFrame({"PD": active_ns, - "QD": reactive_ns}) - nodes_properties_ = pd.DataFrame(np.tile([[Tb, Pct, Ict, Zct]], (nodes, 1)), - columns=["Tb", "Pct", "Ict", "Zct"]) + power = pd.DataFrame({"PD": active_ns, "QD": reactive_ns}) + nodes_properties_ = pd.DataFrame(np.tile([[Tb, Pct, Ict, Zct]], (nodes, 1)), columns=["Tb", "Pct", "Ict", "Zct"]) nodes_properties = pd.concat([power, nodes_properties_], axis=1) - nodes_properties = nodes_properties.astype( - {"Tb": int, "PD": float, "QD": float, "Pct": int, "Ict": int, "Zct": int}) + nodes_properties = nodes_properties.astype({"Tb": int, "PD": float, "QD": float, "Pct": int, "Ict": int, "Zct": int}) nodes_properties = nodes_properties[["Tb", "PD", "QD", "Pct", "Ict", "Zct"]] nodes_properties.loc[0] = 1, 0.0, 0.0, PCT, ICT, ZCT # Slack nodes_frame = pd.concat([nodes_, nodes_properties], axis=1) @@ -243,13 +231,13 @@ def generate_network(nodes, child=3, plot_graph=False, load_factor=2, line_facto # R, X = 0.3144, 0.054 R, X = 0.3144 / line_factor, 0.054 / line_factor lines = pd.DataFrame.from_records(list(G.edges), columns=["FROM", "TO"]) + 1 # Count starts from 1 - lines_properties = pd.DataFrame(np.tile([[R, X, 0, 1, 1]], (LINES, 1)), - columns=["R", "X", "B", "STATUS", "TAP"]) + lines_properties = pd.DataFrame(np.tile([[R, X, 0, 1, 1]], (LINES, 1)), columns=["R", "X", "B", "STATUS", "TAP"]) lines_properties = lines_properties.astype({"R": float, "X": float, "B": int, "STATUS": int, "TAP": int}) lines_frame = pd.concat([lines, lines_properties], axis=1) return nodes_frame, lines_frame + def create_pandapower_net(network_info: dict, branch_info: pd.DataFrame = None, bus_info: pd.DataFrame = None): """ Creates a pandapower network from given network information. @@ -264,27 +252,25 @@ def create_pandapower_net(network_info: dict, branch_info: pd.DataFrame = None, This function reads network information from provided CSV files and creates a pandapower network with buses, lines, loads, and an external grid connection. It sets up the network for power flow analysis. """ - vm_pu=network_info['vm_pu'] - s_base=network_info['s_base'] - branch_info_file=network_info['branch_info_file'] - bus_info_file=network_info['bus_info_file'] + vm_pu = network_info["vm_pu"] + branch_info_file = network_info["branch_info_file"] + bus_info_file = network_info["bus_info_file"] pp, _ = _require_pandapower() if branch_info is None: - branch_info = pd.read_csv(branch_info_file, encoding='utf-8') + branch_info = pd.read_csv(branch_info_file, encoding="utf-8") else: branch_info = branch_info.copy(deep=True) if bus_info is None: - bus_info = pd.read_csv(bus_info_file, encoding='utf-8') + bus_info = pd.read_csv(bus_info_file, encoding="utf-8") else: bus_info = bus_info.copy(deep=True) - start = perf_counter() net = pp.create_empty_network() # Add buses bus_dict = {} - for i, bus_name in enumerate(bus_info["NODES"]): - bus_dict[bus_name] = pp.create_bus(net, vn_kv=11., name=f"Bus {bus_name}") + for bus_name in bus_info["NODES"]: + bus_dict[bus_name] = pp.create_bus(net, vn_kv=11.0, name=f"Bus {bus_name}") # Slack bus_slack = bus_info[bus_info["Tb"] == 1]["NODES"].values @@ -292,20 +278,28 @@ def create_pandapower_net(network_info: dict, branch_info: pd.DataFrame = None, pp.create_ext_grid(net, bus=bus_dict[bus_slack.item()], vm_pu=vm_pu, name="Grid Connection") # Lines - for i, (idx, (from_bus, to_bus, res, x_react, b_susceptance)) in enumerate( - branch_info[["FROM", "TO", "R", "X", "B"]].iterrows()): - pp.create_line_from_parameters(net, - from_bus=bus_dict[from_bus], to_bus=bus_dict[to_bus], - length_km=1, r_ohm_per_km=res, x_ohm_per_km=x_react, c_nf_per_km=b_susceptance, - max_i_ka=10, name=f"Line {i + 1}") + active_branches = branch_info[branch_info["STATUS"].astype(float) != 0].reset_index(drop=True) + for i, (_idx, (from_bus, to_bus, res, x_react, b_susceptance)) in enumerate(active_branches[["FROM", "TO", "R", "X", "B"]].iterrows()): + pp.create_line_from_parameters( + net, + from_bus=bus_dict[from_bus], + to_bus=bus_dict[to_bus], + length_km=1, + r_ohm_per_km=res, + x_ohm_per_km=x_react, + c_nf_per_km=b_susceptance, + max_i_ka=10, + name=f"Line {i + 1}", + ) # Loads: - for i, node in enumerate(bus_info['NODES']): - pp.create_load(net, bus=bus_dict[node], p_mw=0.02, q_mvar=0.0, name=f"Load") + for node in bus_info["NODES"]: + pp.create_load(net, bus=bus_dict[node], p_mw=0.02, q_mvar=0.0, name="Load") # print(f"Create net time: {perf_counter() - start}") return net + def plot_pandapower_net(net): """ Plots a pandapower network. @@ -319,37 +313,38 @@ def plot_pandapower_net(net): """ _, pandapower_topology = _require_pandapower() # Create a graph from the pandapower network - G = pandapower_topology.create_nxgraph(net, respect_switches = False) + G = pandapower_topology.create_nxgraph(net, respect_switches=False) # Set node positions based on bus coordinates - pos = {bus: (net.bus_geodata.at[bus, 'x'], net.bus_geodata.at[bus, 'y']) for bus in G.nodes} + pos = {bus: (net.bus_geodata.at[bus, "x"], net.bus_geodata.at[bus, "y"]) for bus in G.nodes} # Draw buses - buses = [bus for bus in G.nodes if net.bus.at[bus, 'type'] == 'b'] - nx.draw_networkx_nodes(G, pos, nodelist=buses, node_color='red', node_size=200, label='Buses') + buses = [bus for bus in G.nodes if net.bus.at[bus, "type"] == "b"] + nx.draw_networkx_nodes(G, pos, nodelist=buses, node_color="red", node_size=200, label="Buses") # Draw loads - loads = [bus for bus in G.nodes if net.bus.at[bus, 'type'] == 'l'] - nx.draw_networkx_nodes(G, pos, nodelist=loads, node_color='blue', node_size=200, label='Loads') + loads = [bus for bus in G.nodes if net.bus.at[bus, "type"] == "l"] + nx.draw_networkx_nodes(G, pos, nodelist=loads, node_color="blue", node_size=200, label="Loads") # Draw PV generations - pv_generations = [bus for bus in G.nodes if net.bus.at[bus, 'type'] == 's'] - nx.draw_networkx_nodes(G, pos, nodelist=pv_generations, node_color='green', node_size=200, label='PV Generations') + pv_generations = [bus for bus in G.nodes if net.bus.at[bus, "type"] == "s"] + nx.draw_networkx_nodes(G, pos, nodelist=pv_generations, node_color="green", node_size=200, label="PV Generations") # Draw lines nx.draw_networkx_edges(G, pos, width=1.0, alpha=0.5) # Add labels to the nodes - node_labels = {bus: bus.split(' ')[-1] for bus in G.nodes} + node_labels = {bus: bus.split(" ")[-1] for bus in G.nodes} nx.draw_networkx_labels(G, pos, labels=node_labels, font_size=8) # Add a legend plt.legend() # Display the plot - plt.axis('off') + plt.axis("off") plt.show() + def net_test(net): """ Tests the pandapower network with different power flow algorithms. @@ -366,17 +361,41 @@ def net_test(net): """ pp, _ = _require_pandapower() - v_solution = [0.98965162 + 0.00180549j, 0.98060256 + 0.00337785j, 0.96828145 + 0.00704551j, - 0.95767051 + 0.01019764j, 0.94765203 + 0.01316654j, 0.94090964 + 0.01600068j, - 0.93719984 + 0.01754998j, 0.93283877 + 0.01937559j, 0.93073823 + 0.02026054j, - 0.9299309 + 0.02058985j, 0.92968994 + 0.02068728j, 0.98003142 + 0.00362498j, - 0.97950885 + 0.00385019j, 0.97936712 + 0.00391065j, 0.97935604 + 0.0039148j, - 0.93971131 + 0.01547898j, 0.93309482 + 0.01739656j, 0.92577912 + 0.01988823j, - 0.91988489 + 0.02188907j, 0.91475251 + 0.02362566j, 0.90888169 + 0.02596304j, - 0.90404908 + 0.02788248j, 0.89950353 + 0.02968449j, 0.89731375 + 0.03055177j, - 0.89647201 + 0.03088507j, 0.89622055 + 0.03098473j, 0.94032081 + 0.01625577j, - 0.93992817 + 0.01642583j, 0.93973182 + 0.01651086j, 0.9301316 + 0.02052908j, - 0.92952481 + 0.02079761j, 0.92922137 + 0.02093188j, 0.92912022 + 0.02097663j] + v_solution = [ + 0.98965162 + 0.00180549j, + 0.98060256 + 0.00337785j, + 0.96828145 + 0.00704551j, + 0.95767051 + 0.01019764j, + 0.94765203 + 0.01316654j, + 0.94090964 + 0.01600068j, + 0.93719984 + 0.01754998j, + 0.93283877 + 0.01937559j, + 0.93073823 + 0.02026054j, + 0.9299309 + 0.02058985j, + 0.92968994 + 0.02068728j, + 0.98003142 + 0.00362498j, + 0.97950885 + 0.00385019j, + 0.97936712 + 0.00391065j, + 0.97935604 + 0.0039148j, + 0.93971131 + 0.01547898j, + 0.93309482 + 0.01739656j, + 0.92577912 + 0.01988823j, + 0.91988489 + 0.02188907j, + 0.91475251 + 0.02362566j, + 0.90888169 + 0.02596304j, + 0.90404908 + 0.02788248j, + 0.89950353 + 0.02968449j, + 0.89731375 + 0.03055177j, + 0.89647201 + 0.03088507j, + 0.89622055 + 0.03098473j, + 0.94032081 + 0.01625577j, + 0.93992817 + 0.01642583j, + 0.93973182 + 0.01651086j, + 0.9301316 + 0.02052908j, + 0.92952481 + 0.02079761j, + 0.92922137 + 0.02093188j, + 0.92912022 + 0.02097663j, + ] v_solution = np.array(v_solution, dtype="complex128") for pf_algorithm in ["nr", "bfsw"]: @@ -390,7 +409,7 @@ def net_test(net): v_real = net.res_bus["vm_pu"].values * np.cos(np.deg2rad(net.res_bus["va_degree"].values)) v_img = net.res_bus["vm_pu"].values * np.sin(np.deg2rad(net.res_bus["va_degree"].values)) v_result = v_real + 1j * v_img - print('here starts the complex printing') + print("here starts the complex printing") print(v_result) # print(f"NR. Iterations: {net._ppc['iterations']}. PF time: {net._ppc['et']}") print(f"Total pf time: {perf_counter() - start}.") @@ -413,38 +432,31 @@ def test_create_pandapower_net(network_info=None): """ _require_pandapower() if network_info is None: - data_dir = os.path.abspath( - os.path.join(os.path.dirname(__file__), '..', 'data_sources', - 'network_data', 'node_34') - ) + data_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "data_sources", "network_data", "node_34")) network_info = { - 'branch_info_file': os.path.join(data_dir, 'Lines_34.csv'), - 'bus_info_file': os.path.join(data_dir, 'Nodes_34.csv'), - 'vm_pu': 1.0, - 's_base': 1000, + "branch_info_file": os.path.join(data_dir, "Lines_34.csv"), + "bus_info_file": os.path.join(data_dir, "Nodes_34.csv"), + "vm_pu": 1.0, + "s_base": 1000, } - print('Network configuration:') + print("Network configuration:") for key, value in network_info.items(): print(f" {key}: {value}") net = create_pandapower_net(network_info) - print('\nCreated pandapower network:') + print("\nCreated pandapower network:") print(net) - print('\nBus data:') + print("\nBus data:") print(net.bus) - print('\nLine data:') + print("\nLine data:") print(net.line) - print('\nLoad data:') + print("\nLoad data:") print(net.load) return net + def test_plot_pandapower_net(net): """Simple wrapper to plot a pandapower network.""" - print('Plotting pandapower network...') + print("Plotting pandapower network...") plot_pandapower_net(net) - - -if __name__ == "__main__": - test_net = test_create_pandapower_net() - test_plot_pandapower_net(test_net) diff --git a/rl_adn/utility/__init__.py b/rl_adn/utility/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/setup.py b/setup.py deleted file mode 100644 index bd64624..0000000 --- a/setup.py +++ /dev/null @@ -1,51 +0,0 @@ -from pathlib import Path - -from setuptools import find_packages -from setuptools import setup - - -ROOT = Path(__file__).resolve().parent -VERSION = "0.1.3" - - -def read_requirements() -> list[str]: - requirements_path = ROOT / "requirements.txt" - if not requirements_path.exists(): - return [] - - requirements = [] - for raw_line in requirements_path.read_text(encoding="utf-8").splitlines(): - line = raw_line.strip() - if "#" in line: - line = line[: line.find("#")].strip() - if line: - requirements.append(line) - return requirements - - -LONG_DESCRIPTION = (ROOT / "README.md").read_text(encoding="utf-8") - - -setup( - name="RL-ADN", - version=VERSION, - packages=find_packages(), - package_data={ - "rl_adn": [ - "data_sources/network_data/node_123/*.csv", - "data_sources/network_data/node_25/*.csv", - "data_sources/network_data/node_34/*.csv", - "data_sources/network_data/node_69/*.csv", - "data_sources/time_series_data/*.csv", - ], - }, - install_requires=read_requirements(), - author="Hou Shengren, Gao Shuyi, Pedro Vargara", - author_email="houshengren97@gmail.com", - description="RL-ADN: A Benchmark Framework for DRL-based Battery Energy Arbitrage in Distribution Networks", - long_description=LONG_DESCRIPTION, - long_description_content_type="text/markdown", - url="https://github.com/EnergyQuantResearch/RL-ADN", - license="MIT", - keywords="DRL energy arbitrage", -) diff --git a/tests/123_node_network_powerflow_test.py b/tests/123_node_network_powerflow_test.py index 8674161..3614af3 100644 --- a/tests/123_node_network_powerflow_test.py +++ b/tests/123_node_network_powerflow_test.py @@ -1,45 +1,153 @@ import os import time + import numpy as np import pytest pp = pytest.importorskip("pandapower") pytestmark = pytest.mark.powerflow -from rl_adn.utility.grid import GridTensor -from rl_adn.utility.utils import create_pandapower_net +from rl_adn.network.grid import GridTensor +from rl_adn.network.utils import create_pandapower_net -ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '../rl_adn', 'data_sources')) +ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../rl_adn", "data_sources")) CONFIG = { - 'branch_info_file': os.path.join(ROOT_DIR, 'network_data/node_123', 'Lines_123.csv'), - 'bus_info_file': os.path.join(ROOT_DIR, 'network_data/node_123', 'Nodes_123.csv'), - 'vm_pu': 1.0, - 's_base': 1000, + "branch_info_file": os.path.join(ROOT_DIR, "network_data/node_123", "Lines_123.csv"), + "bus_info_file": os.path.join(ROOT_DIR, "network_data/node_123", "Nodes_123.csv"), + "vm_pu": 1.0, + "s_base": 1000, } -P_FILE = np.array([ - 387.09, 0., 387.09, 387.09, 0., 0., 387.09, 387.09, - 0., 387.09, 230.571, 121.176, 121.176, 121.176, 22.7205, 387.09, - 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, - 387.09, 0., 387.09, 387.09, 0., 0., 387.09, 387.09, - 0., 387.09, 230.571, 121.176, 121.176, 121.176, 22.7205, 387.09, - 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, - 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, - 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, - 0., 387.09, 230.571, 121.176, 121.176, 121.176, 22.7205, 387.09, - 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, - 387.09, 0., 387.09, 387.09, 0., 0., 387.09, 387.09, - 0., 387.09, 230.571, 121.176, 121.176, 121.176, 22.7205, 387.09, - 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, - 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, - 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, - 387.09, 387.09, -]) +P_FILE = np.array( + [ + 387.09, + 0.0, + 387.09, + 387.09, + 0.0, + 0.0, + 387.09, + 387.09, + 0.0, + 387.09, + 230.571, + 121.176, + 121.176, + 121.176, + 22.7205, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 0.0, + 387.09, + 387.09, + 0.0, + 0.0, + 387.09, + 387.09, + 0.0, + 387.09, + 230.571, + 121.176, + 121.176, + 121.176, + 22.7205, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 0.0, + 387.09, + 230.571, + 121.176, + 121.176, + 121.176, + 22.7205, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 0.0, + 387.09, + 387.09, + 0.0, + 0.0, + 387.09, + 387.09, + 0.0, + 387.09, + 230.571, + 121.176, + 121.176, + 121.176, + 22.7205, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + ] +) def _run_powerflow(config, p_file): - network = GridTensor(node_file_path=config['bus_info_file'], - lines_file_path=config['branch_info_file']) + network = GridTensor(node_file_path=config["bus_info_file"], lines_file_path=config["branch_info_file"]) network.Q_file = np.zeros(len(p_file)) start_laurent = time.time() @@ -56,7 +164,7 @@ def _run_powerflow(config, p_file): net.load.q_mvar[bus_index - 1] = 0 start_panda = time.time() - pp.runpp(net, algorithm='nr', max_iteration=100) + pp.runpp(net, algorithm="nr", max_iteration=100) time_panda = time.time() - start_panda v_laurent = solution_laurent["v"] diff --git a/tests/25_node_network_powerflow_test.py b/tests/25_node_network_powerflow_test.py index dd949f5..09b7b5f 100644 --- a/tests/25_node_network_powerflow_test.py +++ b/tests/25_node_network_powerflow_test.py @@ -1,33 +1,55 @@ import os import time + import numpy as np import pytest pp = pytest.importorskip("pandapower") pytestmark = pytest.mark.powerflow -from rl_adn.utility.grid import GridTensor -from rl_adn.utility.utils import create_pandapower_net - +from rl_adn.network.grid import GridTensor +from rl_adn.network.utils import create_pandapower_net -ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '../rl_adn', 'data_sources')) +ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../rl_adn", "data_sources")) CONFIG = { - 'branch_info_file': os.path.join(ROOT_DIR, 'network_data/node_25', 'Lines_25.csv'), - 'bus_info_file': os.path.join(ROOT_DIR, 'network_data/node_25', 'Nodes_25.csv'), - 'vm_pu': 1.0, - 's_base': 1000, + "branch_info_file": os.path.join(ROOT_DIR, "network_data/node_25", "Lines_25.csv"), + "bus_info_file": os.path.join(ROOT_DIR, "network_data/node_25", "Nodes_25.csv"), + "vm_pu": 1.0, + "s_base": 1000, } -P_FILE = np.array([ - 387.09, 0., 387.09, 387.09, 0., 0., 387.09, 387.09, - 0., 387.09, 230.571, 121.176, 121.176, 121.176, 22.7205, 387.09, - 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, -]) +P_FILE = np.array( + [ + 387.09, + 0.0, + 387.09, + 387.09, + 0.0, + 0.0, + 387.09, + 387.09, + 0.0, + 387.09, + 230.571, + 121.176, + 121.176, + 121.176, + 22.7205, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + ] +) def _run_powerflow(config, p_file): - network = GridTensor(node_file_path=config['bus_info_file'], - lines_file_path=config['branch_info_file']) + network = GridTensor(node_file_path=config["bus_info_file"], lines_file_path=config["branch_info_file"]) network.Q_file = np.zeros(len(p_file)) start_laurent = time.time() diff --git a/tests/34_node_network_powerflow_test.py b/tests/34_node_network_powerflow_test.py index dfa9496..0c72bb9 100644 --- a/tests/34_node_network_powerflow_test.py +++ b/tests/34_node_network_powerflow_test.py @@ -1,34 +1,64 @@ import os import time + import numpy as np import pytest pp = pytest.importorskip("pandapower") pytestmark = pytest.mark.powerflow -from rl_adn.utility.grid import GridTensor -from rl_adn.utility.utils import create_pandapower_net +from rl_adn.network.grid import GridTensor +from rl_adn.network.utils import create_pandapower_net -ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '../rl_adn', 'data_sources')) +ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../rl_adn", "data_sources")) CONFIG = { - 'branch_info_file': os.path.join(ROOT_DIR, 'network_data/node_34', 'Lines_34.csv'), - 'bus_info_file': os.path.join(ROOT_DIR, 'network_data/node_34', 'Nodes_34.csv'), - 'vm_pu': 1.0, - 's_base': 1000, + "branch_info_file": os.path.join(ROOT_DIR, "network_data/node_34", "Lines_34.csv"), + "bus_info_file": os.path.join(ROOT_DIR, "network_data/node_34", "Nodes_34.csv"), + "vm_pu": 1.0, + "s_base": 1000, } -P_FILE = np.array([ - 387.09, 0., 387.09, 387.09, 0., 0., 387.09, 387.09, - 0., 387.09, 230.571, 121.176, 121.176, 121.176, 22.7205, 387.09, - 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, - 387.09, 230.571, 126.225, 126.225, 126.225, 95.931, 95.931, 95.931, - 95.931, -]) +P_FILE = np.array( + [ + 387.09, + 0.0, + 387.09, + 387.09, + 0.0, + 0.0, + 387.09, + 387.09, + 0.0, + 387.09, + 230.571, + 121.176, + 121.176, + 121.176, + 22.7205, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 230.571, + 126.225, + 126.225, + 126.225, + 95.931, + 95.931, + 95.931, + 95.931, + ] +) def _run_powerflow(config, p_file): - network = GridTensor(node_file_path=config['bus_info_file'], - lines_file_path=config['branch_info_file']) + network = GridTensor(node_file_path=config["bus_info_file"], lines_file_path=config["branch_info_file"]) network.Q_file = np.zeros(len(p_file)) start_laurent = time.time() diff --git a/tests/69_node_network_powerflow_test.py b/tests/69_node_network_powerflow_test.py index be43a57..5abd84d 100644 --- a/tests/69_node_network_powerflow_test.py +++ b/tests/69_node_network_powerflow_test.py @@ -1,38 +1,99 @@ import os import time + import numpy as np import pytest pp = pytest.importorskip("pandapower") pytestmark = pytest.mark.powerflow -from rl_adn.utility.grid import GridTensor -from rl_adn.utility.utils import create_pandapower_net +from rl_adn.network.grid import GridTensor +from rl_adn.network.utils import create_pandapower_net -ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '../rl_adn', 'data_sources')) +ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../rl_adn", "data_sources")) CONFIG = { - 'branch_info_file': os.path.join(ROOT_DIR, 'network_data/node_69', 'Lines_69.csv'), - 'bus_info_file': os.path.join(ROOT_DIR, 'network_data/node_69', 'Nodes_69.csv'), - 'vm_pu': 1.0, - 's_base': 1000, + "branch_info_file": os.path.join(ROOT_DIR, "network_data/node_69", "Lines_69.csv"), + "bus_info_file": os.path.join(ROOT_DIR, "network_data/node_69", "Nodes_69.csv"), + "vm_pu": 1.0, + "s_base": 1000, } -P_FILE = np.array([ - 387.09, 0., 387.09, 387.09, 0., 0., 387.09, 387.09, - 0., 387.09, 230.571, 121.176, 121.176, 121.176, 22.7205, 387.09, - 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, - 387.09, 0., 387.09, 387.09, 0., 0., 387.09, 387.09, - 0., 387.09, 230.571, 121.176, 121.176, 121.176, 22.7205, 387.09, - 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, - 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, - 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, 387.09, - 387.09, 387.09, 387.09, 387.09, -]) +P_FILE = np.array( + [ + 387.09, + 0.0, + 387.09, + 387.09, + 0.0, + 0.0, + 387.09, + 387.09, + 0.0, + 387.09, + 230.571, + 121.176, + 121.176, + 121.176, + 22.7205, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 0.0, + 387.09, + 387.09, + 0.0, + 0.0, + 387.09, + 387.09, + 0.0, + 387.09, + 230.571, + 121.176, + 121.176, + 121.176, + 22.7205, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + 387.09, + ] +) def _run_powerflow(config, p_file): - network = GridTensor(node_file_path=config['bus_info_file'], - lines_file_path=config['branch_info_file']) + network = GridTensor(node_file_path=config["bus_info_file"], lines_file_path=config["branch_info_file"]) network.Q_file = np.zeros(len(p_file)) start_laurent = time.time() diff --git a/tests/data_manager_test.py b/tests/data_manager_test.py index 61b71e7..def07d6 100644 --- a/tests/data_manager_test.py +++ b/tests/data_manager_test.py @@ -1,54 +1,56 @@ -from rl_adn.data_manager.data_manager import GeneralPowerDataManager -import pandas as pd -import numpy as np import os import warnings +import numpy as np +import pandas as pd + +from rl_adn.data import GeneralPowerDataManager + def test_GeneralPowerDataManager(tmp_path): # Generate sample data sample_data = { - 'date_time': pd.date_range(start='2021-01-01', periods=24 * 30, freq='h', tz='UTC'), - 'active_power_node_1': np.random.rand(24 * 30), - 'active_power_node_2': np.random.rand(24 * 30), - 'reactive_power_node_1': np.random.rand(24 * 30), - 'price_node_1': np.random.rand(24 * 30) + "date_time": pd.date_range(start="2021-01-01", periods=24 * 30, freq="h", tz="UTC"), + "active_power_node_1": np.random.rand(24 * 30), + "active_power_node_2": np.random.rand(24 * 30), + "reactive_power_node_1": np.random.rand(24 * 30), + "price_node_1": np.random.rand(24 * 30), } df = pd.DataFrame(sample_data) - datapath = tmp_path / 'sample_data.csv' + datapath = tmp_path / "sample_data.csv" df.to_csv(datapath, index=False) # Test initialization with warnings.catch_warnings(record=True) as caught_warnings: warnings.simplefilter("always") - data_manager = GeneralPowerDataManager(datapath) + data = GeneralPowerDataManager(datapath) future_warnings = [w for w in caught_warnings if issubclass(w.category, FutureWarning)] assert not future_warnings, "GeneralPowerDataManager emitted FutureWarning during initialization" - assert isinstance(data_manager.df, pd.DataFrame), "DataFrame not initialized" - assert len(data_manager.active_power_cols) == 2, "Active power columns not detected correctly" - assert len(data_manager.reactive_power_cols) == 1, "Reactive power columns not detected correctly" - assert len(data_manager.price_col) == 1, "Price columns not detected correctly" + assert isinstance(data.df, pd.DataFrame), "DataFrame not initialized" + assert len(data.active_power_cols) == 2, "Active power columns not detected correctly" + assert len(data.reactive_power_cols) == 1, "Reactive power columns not detected correctly" + assert len(data.price_col) == 1, "Price columns not detected correctly" # Test select_timeslot_data - data = data_manager.select_timeslot_data(2021, 1, 1, 0) - assert len(data) == 4, "Timeslot data not fetched correctly" + timeslot_data = data.select_timeslot_data(2021, 1, 1, 0) + assert len(timeslot_data) == 4, "Timeslot data not fetched correctly" # Test select_day_data - day_data = data_manager.select_day_data(2021, 1, 1) + day_data = data.select_day_data(2021, 1, 1) assert day_data.shape[0] == 24, "Day data not fetched correctly" # Test list_dates - dates = data_manager.list_dates() + dates = data.list_dates() assert len(dates) == 30, "List dates not working correctly" # Test random_date - date = data_manager.random_date() + date = data.random_date() assert isinstance(date, tuple) and len(date) == 3, "Random date function not working correctly" # Test split_data_set - train_dates = data_manager.train_dates - test_dates = data_manager.test_dates + train_dates = data.train_dates + test_dates = data.test_dates assert len(train_dates) == 22, "Training dates not split correctly" assert len(test_dates) == 8, "Testing dates not split correctly" diff --git a/tests/test_algorithm_imports.py b/tests/test_algorithm_imports.py index f4e977d..c2e4d2c 100644 --- a/tests/test_algorithm_imports.py +++ b/tests/test_algorithm_imports.py @@ -2,18 +2,17 @@ import pytest - -pytest.importorskip("gym") +pytest.importorskip("gymnasium") pytest.importorskip("torch") @pytest.mark.parametrize( "module_name", [ - "rl_adn.DRL_algorithms.DDPG", - "rl_adn.DRL_algorithms.PPO", - "rl_adn.DRL_algorithms.SAC", - "rl_adn.DRL_algorithms.TD3", + "rl_adn.algorithms.DDPG", + "rl_adn.algorithms.PPO", + "rl_adn.algorithms.SAC", + "rl_adn.algorithms.TD3", ], ) def test_drl_algorithm_modules_import(module_name): diff --git a/tests/test_battery_smoke.py b/tests/test_battery_smoke.py index a81baec..9bb9fcf 100644 --- a/tests/test_battery_smoke.py +++ b/tests/test_battery_smoke.py @@ -1,13 +1,12 @@ import numpy as np import pytest -from rl_adn.environments.battery import Battery, battery_parameters -from rl_adn.environments.config import make_env_config +from rl_adn import BatteryConfig, make_env_config +from rl_adn.environments.battery import Battery def test_battery_soc_remains_scalar_after_vector_action(): - battery = Battery(battery_parameters) - battery.reset() + battery = Battery(BatteryConfig.default()) battery.step(np.array([0.0], dtype=np.float32)) @@ -15,8 +14,7 @@ def test_battery_soc_remains_scalar_after_vector_action(): def test_battery_rejects_multi_value_action(): - battery = Battery(battery_parameters) - battery.reset() + battery = Battery(BatteryConfig.default()) try: battery.step(np.array([0.0, 1.0], dtype=np.float32)) @@ -27,33 +25,31 @@ def test_battery_rejects_multi_value_action(): def test_battery_uses_15_minute_interval_by_default(): - battery = Battery(battery_parameters) - battery.reset() + battery = Battery(BatteryConfig.default()) battery.step(np.array([1.0], dtype=np.float32)) assert np.isclose(battery.SOC(), 0.4 + (50.0 * 0.25) / 300.0) - assert np.isclose(battery.energy_change, 50.0) + assert np.isclose(battery.last_power_kw, 50.0) def test_battery_respects_custom_time_interval(): - battery = Battery({**battery_parameters, "time_interval_minutes": 5}) - battery.reset() + battery = Battery(BatteryConfig.default(time_interval_minutes=5.0)) battery.step(np.array([1.0], dtype=np.float32)) assert np.isclose(battery.SOC(), 0.4 + (50.0 * (5.0 / 60.0)) / 300.0) - assert np.isclose(battery.energy_change, 50.0) + assert np.isclose(battery.last_power_kw, 50.0) def test_environment_batteries_inherit_dataset_time_interval(): - pytest.importorskip("gym") - from rl_adn.environments.env import PowerNetEnv + pytest.importorskip("gymnasium") + from rl_adn import PowerNetEnv env = PowerNetEnv(make_env_config()) try: - battery = getattr(env, f"battery_{env.battery_list[0]}") + battery = env.batteries[env.battery_nodes[0]] assert np.isclose(battery.time_interval_minutes, env.data_manager.time_interval) finally: del env diff --git a/tests/test_examples_smoke.py b/tests/test_examples_smoke.py index 0ca139d..6158941 100644 --- a/tests/test_examples_smoke.py +++ b/tests/test_examples_smoke.py @@ -1,18 +1,26 @@ -from pathlib import Path import runpy +from pathlib import Path import pytest - ROOT = Path(__file__).resolve().parents[1] -QUICKSTART_SCRIPT = ROOT / "examples" / "quickstart_env.py" +EXAMPLE_SCRIPTS = [ + ROOT / "examples" / "quickstart_env.py", + ROOT / "examples" / "custom_env_config.py", + ROOT / "examples" / "topology_scenarios.py", + ROOT / "examples" / "training_smoke.py", +] -def test_quickstart_script_exists(): - assert QUICKSTART_SCRIPT.exists() +def test_example_scripts_exist(): + for script in EXAMPLE_SCRIPTS: + assert script.exists() -def test_quickstart_script_runs_when_gym_is_installed(): - pytest.importorskip("gym") +@pytest.mark.parametrize("script_path", EXAMPLE_SCRIPTS) +def test_example_script_runs_when_gymnasium_is_installed(script_path: Path): + pytest.importorskip("gymnasium") + if script_path.name == "training_smoke.py": + pytest.importorskip("torch") - runpy.run_path(str(QUICKSTART_SCRIPT), run_name="__main__") + runpy.run_path(str(script_path), run_name="__main__") diff --git a/tests/test_public_api_smoke.py b/tests/test_public_api_smoke.py index 8e50382..a551f43 100644 --- a/tests/test_public_api_smoke.py +++ b/tests/test_public_api_smoke.py @@ -8,17 +8,17 @@ def test_make_env_config_returns_existing_packaged_paths(): config = rl_adn.make_env_config() - network_info = config["network_info"] - assert network_info["bus_info_file"] - assert network_info["branch_info_file"] - assert Path(network_info["bus_info_file"]).exists() - assert Path(network_info["branch_info_file"]).exists() - assert Path(config["time_series_data_path"]).exists() + assert config.bus_info_file + assert config.branch_info_file + assert Path(config.bus_info_file).exists() + assert Path(config.branch_info_file).exists() + assert Path(config.time_series_data_path).exists() -def test_powernet_env_is_available_when_gym_is_installed(): - pytest.importorskip("gym") +def test_powernet_env_is_available_when_gymnasium_is_installed(): + pytest.importorskip("gymnasium") env = rl_adn.PowerNetEnv(rl_adn.make_env_config()) - state = env.reset() + state, info = env.reset(seed=2026) assert state is not None + assert info["feeder_id"] == "34-bus" diff --git a/tests/test_topology_scenarios.py b/tests/test_topology_scenarios.py index 02aa853..c4ce9d8 100644 --- a/tests/test_topology_scenarios.py +++ b/tests/test_topology_scenarios.py @@ -1,17 +1,16 @@ from pathlib import Path -import random import numpy as np import pandas as pd import pytest -from rl_adn.environments.config import make_env_config +from rl_adn.config import make_env_config from rl_adn.environments.topology_scenarios import ( get_topology_scenario, list_topology_scenario_ids, ) -from rl_adn.utility.grid import GridTensor -from rl_adn.utility.topology import ( +from rl_adn.network.grid import GridTensor +from rl_adn.network.topology import ( apply_topology_scenario, build_adjacency_matrix, build_edge_index, @@ -19,7 +18,6 @@ validate_radial_topology, ) - ROOT = Path(__file__).resolve().parents[1] NETWORK_ROOT = ROOT / "rl_adn" / "data_sources" / "network_data" @@ -80,20 +78,19 @@ def test_each_topology_scenario_supports_laurent_initialization(node, scenario_i def test_make_env_config_preserves_old_behavior_when_topology_fields_omitted(): config = make_env_config() - assert config["algorithm"] == "Laurent" - assert "topology_mode" in config - assert config["topology_mode"] == "fixed" - assert config["topology_scenario"] is None - assert config["topology_pool"] is None - assert config["return_graph"] is False + assert config.algorithm == "Laurent" + assert config.topology.mode == "fixed" + assert config.topology.scenario_id == "TP1" + assert config.topology.scenario_pool == () + assert config.topology.return_graph is False def test_make_env_config_accepts_fixed_topology_scenario(): config = make_env_config(node=34, topology_scenario="TP3") - assert config["feeder_id"] == "34-bus" - assert config["topology_mode"] == "fixed" - assert config["topology_scenario"] == "TP3" + assert config.feeder_id == "34-bus" + assert config.topology.mode == "fixed" + assert config.topology.scenario_id == "TP3" def test_make_env_config_provides_curated_defaults_for_supported_feeders(): @@ -104,8 +101,8 @@ def test_make_env_config_provides_curated_defaults_for_supported_feeders(): time_series_data_path="synthetic.csv", ) - assert config_34["battery_list"] == [11, 15, 26, 29, 33] - assert config_69["battery_list"] == [13, 15, 17, 19, 21, 23, 25, 26, 64] + assert config_34.battery_nodes == (11, 15, 26, 29, 33) + assert config_69.battery_nodes == (13, 15, 17, 19, 21, 23, 25, 26, 64) def test_make_env_config_rejects_empty_scenario_pool(): @@ -114,13 +111,13 @@ def test_make_env_config_rejects_empty_scenario_pool(): def test_fixed_topology_reset_is_reproducible_and_exposes_metadata(): - pytest.importorskip("gym") - from rl_adn.environments.env import PowerNetEnv + pytest.importorskip("gymnasium") + from rl_adn import PowerNetEnv env = PowerNetEnv(make_env_config(node=34, topology_scenario="TP4", return_graph=True)) - state_a, info_a = env.reset(return_info=True) - state_b, info_b = env.reset(return_info=True) + state_a, info_a = env.reset(seed=2026) + state_b, info_b = env.reset(seed=2026) assert state_a.shape == state_b.shape assert info_a["topology_scenario"] == "TP4" @@ -130,23 +127,21 @@ def test_fixed_topology_reset_is_reproducible_and_exposes_metadata(): def test_scenario_pool_sampling_is_deterministic_under_fixed_seed(): - pytest.importorskip("gym") - from rl_adn.environments.env import PowerNetEnv + pytest.importorskip("gymnasium") + from rl_adn import PowerNetEnv - random.seed(2026) env_a = PowerNetEnv(make_env_config(node=34, topology_mode="scenario_pool", topology_pool=["TP2", "TP3", "TP4"])) - seq_a = [env_a.reset(return_info=True)[1]["topology_scenario"] for _ in range(3)] + seq_a = [env_a.reset(seed=2026 + index)[1]["topology_scenario"] for index in range(3)] - random.seed(2026) env_b = PowerNetEnv(make_env_config(node=34, topology_mode="scenario_pool", topology_pool=["TP2", "TP3", "TP4"])) - seq_b = [env_b.reset(return_info=True)[1]["topology_scenario"] for _ in range(3)] + seq_b = [env_b.reset(seed=2026 + index)[1]["topology_scenario"] for index in range(3)] assert seq_a == seq_b def test_graph_exports_match_active_topology(): - pytest.importorskip("gym") - from rl_adn.environments.env import PowerNetEnv + pytest.importorskip("gymnasium") + from rl_adn import PowerNetEnv env = PowerNetEnv(make_env_config(node=34, topology_scenario="TP2", return_graph=True)) env.reset() @@ -167,8 +162,8 @@ def test_graph_exports_match_active_topology(): def test_69_bus_env_supports_custom_timeseries_with_fixed_topology(tmp_path: Path): - pytest.importorskip("gym") - from rl_adn.environments.env import PowerNetEnv + pytest.importorskip("gymnasium") + from rl_adn import PowerNetEnv synthetic_path = tmp_path / "69_node_time_series.csv" _write_synthetic_timeseries(synthetic_path, node_count=69) @@ -181,7 +176,7 @@ def test_69_bus_env_supports_custom_timeseries_with_fixed_topology(tmp_path: Pat ) ) - state, info = env.reset(return_info=True) + state, info = env.reset(seed=2026) assert state is not None assert info["topology_scenario"] == "TP2" assert info["feeder_id"] == "69-bus"