From fe5fa5572879c8ecce3edaed908283c6328eead1 Mon Sep 17 00:00:00 2001 From: ram-from-tvl Date: Sat, 10 Jan 2026 00:35:02 +0530 Subject: [PATCH 1/3] Refactor: Move nwp module to scripts/nwp_data directory --- src/open_data_pvnet/{nwp => scripts/nwp_data}/__init__.py | 0 src/open_data_pvnet/{nwp => scripts/nwp_data}/dwd.py | 0 src/open_data_pvnet/{nwp => scripts/nwp_data}/gfs.py | 0 src/open_data_pvnet/{nwp => scripts/nwp_data}/gfs_dataset.py | 0 src/open_data_pvnet/{nwp => scripts/nwp_data}/met_office.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename src/open_data_pvnet/{nwp => scripts/nwp_data}/__init__.py (100%) rename src/open_data_pvnet/{nwp => scripts/nwp_data}/dwd.py (100%) rename src/open_data_pvnet/{nwp => scripts/nwp_data}/gfs.py (100%) rename src/open_data_pvnet/{nwp => scripts/nwp_data}/gfs_dataset.py (100%) rename src/open_data_pvnet/{nwp => scripts/nwp_data}/met_office.py (100%) diff --git a/src/open_data_pvnet/nwp/__init__.py b/src/open_data_pvnet/scripts/nwp_data/__init__.py similarity index 100% rename from src/open_data_pvnet/nwp/__init__.py rename to src/open_data_pvnet/scripts/nwp_data/__init__.py diff --git a/src/open_data_pvnet/nwp/dwd.py b/src/open_data_pvnet/scripts/nwp_data/dwd.py similarity index 100% rename from src/open_data_pvnet/nwp/dwd.py rename to src/open_data_pvnet/scripts/nwp_data/dwd.py diff --git a/src/open_data_pvnet/nwp/gfs.py b/src/open_data_pvnet/scripts/nwp_data/gfs.py similarity index 100% rename from src/open_data_pvnet/nwp/gfs.py rename to src/open_data_pvnet/scripts/nwp_data/gfs.py diff --git a/src/open_data_pvnet/nwp/gfs_dataset.py b/src/open_data_pvnet/scripts/nwp_data/gfs_dataset.py similarity index 100% rename from src/open_data_pvnet/nwp/gfs_dataset.py rename to src/open_data_pvnet/scripts/nwp_data/gfs_dataset.py diff --git a/src/open_data_pvnet/nwp/met_office.py b/src/open_data_pvnet/scripts/nwp_data/met_office.py similarity index 100% rename from src/open_data_pvnet/nwp/met_office.py rename to src/open_data_pvnet/scripts/nwp_data/met_office.py From de056008e3619a8d1367b5bbb01705d5413f0907 Mon Sep 17 00:00:00 2001 From: ram-from-tvl Date: Sat, 10 Jan 2026 00:46:16 +0530 Subject: [PATCH 2/3] Update all import paths to reflect nwp module relocation --- src/open_data_pvnet/main.py | 4 +- src/open_data_pvnet/scripts/archive.py | 6 +-- tests/test_dwd.py | 26 ++++++------- tests/test_met_office.py | 54 +++++++++++++------------- 4 files changed, 45 insertions(+), 45 deletions(-) diff --git a/src/open_data_pvnet/main.py b/src/open_data_pvnet/main.py index f8d733e..d782217 100644 --- a/src/open_data_pvnet/main.py +++ b/src/open_data_pvnet/main.py @@ -13,8 +13,8 @@ from typing import List, Tuple from open_data_pvnet.utils.data_uploader import upload_monthly_zarr, upload_to_huggingface from open_data_pvnet.scripts.archive import handle_archive -from open_data_pvnet.nwp.met_office import CONFIG_PATHS -from open_data_pvnet.nwp.dwd import process_dwd_data +from open_data_pvnet.scripts.nwp_data.met_office import CONFIG_PATHS +from open_data_pvnet.scripts.nwp_data.dwd import process_dwd_data logger = logging.getLogger(__name__) diff --git a/src/open_data_pvnet/scripts/archive.py b/src/open_data_pvnet/scripts/archive.py index 4a2df6f..adcf665 100644 --- a/src/open_data_pvnet/scripts/archive.py +++ b/src/open_data_pvnet/scripts/archive.py @@ -1,8 +1,8 @@ import logging from typing import Optional -from open_data_pvnet.nwp.met_office import process_met_office_data -from open_data_pvnet.nwp.gfs import process_gfs_data -from open_data_pvnet.nwp.dwd import process_dwd_data +from open_data_pvnet.scripts.nwp_data.met_office import process_met_office_data +from open_data_pvnet.scripts.nwp_data.gfs import process_gfs_data +from open_data_pvnet.scripts.nwp_data.dwd import process_dwd_data logger = logging.getLogger(__name__) diff --git a/tests/test_dwd.py b/tests/test_dwd.py index baf9293..3022c81 100644 --- a/tests/test_dwd.py +++ b/tests/test_dwd.py @@ -2,7 +2,7 @@ from unittest.mock import Mock from pathlib import Path -from open_data_pvnet.nwp.dwd import ( +from open_data_pvnet.scripts.nwp_data.dwd import ( generate_variable_url, fetch_dwd_data, process_dwd_data, @@ -36,9 +36,9 @@ def test_generate_variable_url(): def test_fetch_dwd_data_success(mocker, mock_config, tmp_path): """Test successful fetching of DWD data.""" # Setup mocks - mocker.patch("open_data_pvnet.nwp.dwd.PROJECT_BASE", str(tmp_path)) - mocker.patch("open_data_pvnet.nwp.dwd.CONFIG_PATH", "test_config.yaml") - mocker.patch("open_data_pvnet.nwp.dwd.load_config", return_value=mock_config) + mocker.patch("open_data_pvnet.scripts.nwp_data.dwd.PROJECT_BASE", str(tmp_path)) + mocker.patch("open_data_pvnet.scripts.nwp_data.dwd.CONFIG_PATH", "test_config.yaml") + mocker.patch("open_data_pvnet.scripts.nwp_data.dwd.load_config", return_value=mock_config) # Mock HTML content html_content = b""" @@ -76,9 +76,9 @@ def test_fetch_dwd_data_success(mocker, mock_config, tmp_path): def test_fetch_dwd_data_no_files(mocker, mock_config, tmp_path): """Test fetching DWD data when no files are available.""" # Setup mocks - mocker.patch("open_data_pvnet.nwp.dwd.PROJECT_BASE", str(tmp_path)) - mocker.patch("open_data_pvnet.nwp.dwd.CONFIG_PATH", "test_config.yaml") - mocker.patch("open_data_pvnet.nwp.dwd.load_config", return_value=mock_config) + mocker.patch("open_data_pvnet.scripts.nwp_data.dwd.PROJECT_BASE", str(tmp_path)) + mocker.patch("open_data_pvnet.scripts.nwp_data.dwd.CONFIG_PATH", "test_config.yaml") + mocker.patch("open_data_pvnet.scripts.nwp_data.dwd.load_config", return_value=mock_config) # Mock empty HTML response mock_head = mocker.patch("requests.head") @@ -94,10 +94,10 @@ def test_fetch_dwd_data_no_files(mocker, mock_config, tmp_path): def test_process_dwd_data_success(mocker, mock_config, tmp_path): """Test successful processing of DWD data.""" # Setup mocks - mocker.patch("open_data_pvnet.nwp.dwd.PROJECT_BASE", str(tmp_path)) - mocker.patch("open_data_pvnet.nwp.dwd.CONFIG_PATH", "test_config.yaml") - mocker.patch("open_data_pvnet.nwp.dwd.load_config", return_value=mock_config) - mock_fetch = mocker.patch("open_data_pvnet.nwp.dwd.fetch_dwd_data", return_value=3) + mocker.patch("open_data_pvnet.scripts.nwp_data.dwd.PROJECT_BASE", str(tmp_path)) + mocker.patch("open_data_pvnet.scripts.nwp_data.dwd.CONFIG_PATH", "test_config.yaml") + mocker.patch("open_data_pvnet.scripts.nwp_data.dwd.load_config", return_value=mock_config) + mock_fetch = mocker.patch("open_data_pvnet.scripts.nwp_data.dwd.fetch_dwd_data", return_value=3) # Mock xarray operations mock_ds = mocker.MagicMock() @@ -129,8 +129,8 @@ def test_process_dwd_data_success(mocker, mock_config, tmp_path): def test_process_dwd_data_no_files(mocker, mock_config): """Test processing when no files are downloaded.""" - mocker.patch("open_data_pvnet.nwp.dwd.load_config", return_value=mock_config) - mock_fetch = mocker.patch("open_data_pvnet.nwp.dwd.fetch_dwd_data", return_value=0) + mocker.patch("open_data_pvnet.scripts.nwp_data.dwd.load_config", return_value=mock_config) + mock_fetch = mocker.patch("open_data_pvnet.scripts.nwp_data.dwd.fetch_dwd_data", return_value=0) process_dwd_data(2023, 1, 1, 0) diff --git a/tests/test_met_office.py b/tests/test_met_office.py index 1f9250e..22ac7ab 100644 --- a/tests/test_met_office.py +++ b/tests/test_met_office.py @@ -1,7 +1,7 @@ import pytest from unittest.mock import Mock -from open_data_pvnet.nwp.met_office import ( +from open_data_pvnet.scripts.nwp_data.met_office import ( generate_prefix, fetch_met_office_data, process_met_office_data, @@ -36,17 +36,17 @@ def test_generate_prefix_global(): def test_process_met_office_data_success(mocker, mock_config, tmp_path): # Setup mocks - mocker.patch("open_data_pvnet.nwp.met_office.PROJECT_BASE", str(tmp_path)) - mocker.patch("open_data_pvnet.nwp.met_office.CONFIG_PATHS", {"uk": "test_config.yaml"}) - mocker.patch("open_data_pvnet.nwp.met_office.load_config", return_value=mock_config) + mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.PROJECT_BASE", str(tmp_path)) + mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.CONFIG_PATHS", {"uk": "test_config.yaml"}) + mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.load_config", return_value=mock_config) mock_fetch = mocker.patch( - "open_data_pvnet.nwp.met_office.fetch_met_office_data", return_value=3 + "open_data_pvnet.scripts.nwp_data.met_office.fetch_met_office_data", return_value=3 ) mock_convert = mocker.patch( - "open_data_pvnet.nwp.met_office.convert_nc_to_zarr", return_value=(3, 1000) + "open_data_pvnet.scripts.nwp_data.met_office.convert_nc_to_zarr", return_value=(3, 1000) ) - mock_upload = mocker.patch("open_data_pvnet.nwp.met_office.upload_to_huggingface") - mock_rmtree = mocker.patch("open_data_pvnet.nwp.met_office.shutil.rmtree") + mock_upload = mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.upload_to_huggingface") + mock_rmtree = mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.shutil.rmtree") # Call function with default archive_type process_met_office_data(2023, 12, 25, 0, "uk", overwrite=False) @@ -68,17 +68,17 @@ def test_process_met_office_data_success(mocker, mock_config, tmp_path): def test_process_met_office_data_with_tar(mocker, mock_config, tmp_path): # Setup mocks - mocker.patch("open_data_pvnet.nwp.met_office.PROJECT_BASE", str(tmp_path)) - mocker.patch("open_data_pvnet.nwp.met_office.CONFIG_PATHS", {"uk": "test_config.yaml"}) - mocker.patch("open_data_pvnet.nwp.met_office.load_config", return_value=mock_config) + mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.PROJECT_BASE", str(tmp_path)) + mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.CONFIG_PATHS", {"uk": "test_config.yaml"}) + mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.load_config", return_value=mock_config) mock_fetch = mocker.patch( - "open_data_pvnet.nwp.met_office.fetch_met_office_data", return_value=3 + "open_data_pvnet.scripts.nwp_data.met_office.fetch_met_office_data", return_value=3 ) mock_convert = mocker.patch( - "open_data_pvnet.nwp.met_office.convert_nc_to_zarr", return_value=(3, 1000) + "open_data_pvnet.scripts.nwp_data.met_office.convert_nc_to_zarr", return_value=(3, 1000) ) - mock_upload = mocker.patch("open_data_pvnet.nwp.met_office.upload_to_huggingface") - mock_rmtree = mocker.patch("open_data_pvnet.nwp.met_office.shutil.rmtree") + mock_upload = mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.upload_to_huggingface") + mock_rmtree = mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.shutil.rmtree") # Call function with tar archive_type process_met_office_data(2023, 12, 25, 0, "uk", overwrite=True, archive_type="tar") @@ -100,14 +100,14 @@ def test_process_met_office_data_with_tar(mocker, mock_config, tmp_path): def test_process_met_office_data_no_files(mocker, mock_config, tmp_path): # Setup mocks - mocker.patch("open_data_pvnet.nwp.met_office.PROJECT_BASE", str(tmp_path)) - mocker.patch("open_data_pvnet.nwp.met_office.CONFIG_PATHS", {"uk": "test_config.yaml"}) - mocker.patch("open_data_pvnet.nwp.met_office.load_config", return_value=mock_config) + mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.PROJECT_BASE", str(tmp_path)) + mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.CONFIG_PATHS", {"uk": "test_config.yaml"}) + mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.load_config", return_value=mock_config) mock_fetch = mocker.patch( - "open_data_pvnet.nwp.met_office.fetch_met_office_data", return_value=0 + "open_data_pvnet.scripts.nwp_data.met_office.fetch_met_office_data", return_value=0 ) - mock_convert = mocker.patch("open_data_pvnet.nwp.met_office.convert_nc_to_zarr") - mock_upload = mocker.patch("open_data_pvnet.nwp.met_office.upload_to_huggingface") + mock_convert = mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.convert_nc_to_zarr") + mock_upload = mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.upload_to_huggingface") # Call function process_met_office_data(2023, 12, 25, 0, "uk") @@ -120,10 +120,10 @@ def test_process_met_office_data_no_files(mocker, mock_config, tmp_path): def test_fetch_met_office_data_success(mocker, mock_config): # Setup mocks - mocker.patch("open_data_pvnet.nwp.met_office.CONFIG_PATHS", {"uk": "test_config.yaml"}) - mocker.patch("open_data_pvnet.nwp.met_office.load_config", return_value=mock_config) + mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.CONFIG_PATHS", {"uk": "test_config.yaml"}) + mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.load_config", return_value=mock_config) mock_s3 = Mock() - mocker.patch("open_data_pvnet.nwp.met_office.boto3.client", return_value=mock_s3) + mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.boto3.client", return_value=mock_s3) # Mock S3 response mock_s3.list_objects_v2.return_value = { @@ -146,10 +146,10 @@ def test_fetch_met_office_data_success(mocker, mock_config): def test_fetch_met_office_data_no_files(mocker, mock_config): # Setup mocks - mocker.patch("open_data_pvnet.nwp.met_office.CONFIG_PATHS", {"uk": "test_config.yaml"}) - mocker.patch("open_data_pvnet.nwp.met_office.load_config", return_value=mock_config) + mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.CONFIG_PATHS", {"uk": "test_config.yaml"}) + mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.load_config", return_value=mock_config) mock_s3 = Mock() - mocker.patch("open_data_pvnet.nwp.met_office.boto3.client", return_value=mock_s3) + mocker.patch("open_data_pvnet.scripts.nwp_data.met_office.boto3.client", return_value=mock_s3) # Mock empty S3 response mock_s3.list_objects_v2.return_value = {} From f7d6435024e498c5916db62ed7c4e30b404238eb Mon Sep 17 00:00:00 2001 From: ram-from-tvl Date: Wed, 21 Jan 2026 10:07:26 +0530 Subject: [PATCH 3/3] Move main.py to scripts/nwp_data as requested by maintainer --- pyproject.toml | 2 +- .../{ => scripts/nwp_data}/main.py | 0 tests/test_main.py | 20 +++++++++---------- 3 files changed, 11 insertions(+), 11 deletions(-) rename src/open_data_pvnet/{ => scripts/nwp_data}/main.py (100%) diff --git a/pyproject.toml b/pyproject.toml index a75b285..80a601a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ dev = [ ] [project.scripts] -open-data-pvnet = "open_data_pvnet.main:main" +open-data-pvnet = "open_data_pvnet.scripts.nwp_data.main:main" [tool.black] line-length = 100 diff --git a/src/open_data_pvnet/main.py b/src/open_data_pvnet/scripts/nwp_data/main.py similarity index 100% rename from src/open_data_pvnet/main.py rename to src/open_data_pvnet/scripts/nwp_data/main.py diff --git a/tests/test_main.py b/tests/test_main.py index cf42ff4..ab0da4c 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,6 +1,6 @@ import pytest from unittest.mock import patch -from open_data_pvnet.main import ( +from open_data_pvnet.scripts.nwp_data.main import ( configure_parser, load_env_and_setup_logger, main, @@ -98,23 +98,23 @@ def test_configure_parser(): assert not args.overwrite -@patch("open_data_pvnet.main.load_environment_variables") -@patch("open_data_pvnet.main.logging.basicConfig") +@patch("open_data_pvnet.scripts.nwp_data.main.load_environment_variables") +@patch("open_data_pvnet.scripts.nwp_data.main.logging.basicConfig") def test_load_env_and_setup_logger_success(mock_logging, mock_load_env): load_env_and_setup_logger() mock_load_env.assert_called_once() mock_logging.assert_called_once() -@patch("open_data_pvnet.main.load_environment_variables") +@patch("open_data_pvnet.scripts.nwp_data.main.load_environment_variables") def test_load_env_and_setup_logger_failure(mock_load_env): mock_load_env.side_effect = FileNotFoundError("Config file not found") with pytest.raises(FileNotFoundError): load_env_and_setup_logger() -@patch("open_data_pvnet.main.handle_load") -@patch("open_data_pvnet.main.load_env_and_setup_logger") +@patch("open_data_pvnet.scripts.nwp_data.main.handle_load") +@patch("open_data_pvnet.scripts.nwp_data.main.load_env_and_setup_logger") def test_main_metoffice_load(mock_load_env, mock_handle_load): # Test metoffice load command test_args = [ @@ -148,8 +148,8 @@ def test_main_metoffice_load(mock_load_env, mock_handle_load): ) -@patch("open_data_pvnet.main.print") -@patch("open_data_pvnet.main.load_env_and_setup_logger") +@patch("open_data_pvnet.scripts.nwp_data.main.print") +@patch("open_data_pvnet.scripts.nwp_data.main.load_env_and_setup_logger") def test_main_list_providers(mock_load_env, mock_print): # Test --list providers test_args = ["--list", "providers"] @@ -158,8 +158,8 @@ def test_main_list_providers(mock_load_env, mock_print): assert mock_print.call_count == 4 # One for header + three providers -@patch("open_data_pvnet.main.handle_load") -@patch("open_data_pvnet.main.load_env_and_setup_logger") +@patch("open_data_pvnet.scripts.nwp_data.main.handle_load") +@patch("open_data_pvnet.scripts.nwp_data.main.load_env_and_setup_logger") def test_main_metoffice_load_remote(mock_load_env, mock_handle_load): # Test metoffice load command with remote option test_args = [