From 2f3489d72e846fbe78b3662d7ac23a83246c71a7 Mon Sep 17 00:00:00 2001 From: Eric Yvorchuk Date: Mon, 14 Feb 2022 10:56:55 -0800 Subject: [PATCH 01/15] Temporarily allow slow tests for python-ci and use fileServer for THREDDS root --- .github/workflows/python-ci.yml | 2 +- tests/conftest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 23dd9a6..f396974 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -30,7 +30,7 @@ jobs: - name: Test with pytest (fast) if: github.ref != 'refs/heads/master' run: | - py.test -m "not slow and not online" + py.test -m "not online" - name: Code format check run: | black . --check diff --git a/tests/conftest.py b/tests/conftest.py index 41af35e..7cd076b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,7 +29,7 @@ def mock_thredds_url_root(monkeypatch): monkeypatch.setenv( "THREDDS_URL_ROOT", - "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets", + "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets", ) From ce72869e1db692d660fedf4df07f02a3e66afad2 Mon Sep 17 00:00:00 2001 From: Eric Yvorchuk Date: Mon, 14 Feb 2022 11:13:32 -0800 Subject: [PATCH 02/15] Use dodsC instead of fileServer for THREDDS root and mock urls --- tests/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 7cd076b..a36f88c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,7 +29,7 @@ def mock_thredds_url_root(monkeypatch): monkeypatch.setenv( "THREDDS_URL_ROOT", - "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets", + "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets", ) @@ -426,14 +426,14 @@ def mock_urls(requests_mock): ) requests_mock.register_uri( "GET", - "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets" + "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets" "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/" "tasmin_sClimMean_anusplin_historical_19710101-20001231.nc", content=tasmin_data, ) requests_mock.register_uri( "GET", - "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets" + "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets" "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/" "tasmax_sClimMean_anusplin_historical_19710101-20001231.nc", content=tasmax_data, From 30d6b07d6567edd72f7cc1049810e6e5a63dbb39 Mon Sep 17 00:00:00 2001 From: Eric Yvorchuk Date: Wed, 16 Feb 2022 12:25:28 -0800 Subject: [PATCH 03/15] Use different databases for local and THREDDS files --- tests/conftest.py | 223 ++++++++++++++++++++++++++++++----- tests/test_file_collector.py | 6 +- tests/test_resolver.py | 38 +++++- 3 files changed, 227 insertions(+), 40 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a36f88c..e398c61 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,28 @@ from .mock_data import geoserver_data, tasmin_data, tasmax_data +# Functions to create data for database + + +def make_data_file( + filename=None, run=None, +): + if not filename.startswith("/"): + filename = resource_filename("tests", "data/{}".format(filename)) + return DataFile( + filename=filename, + unique_id=filename, + first_1mib_md5sum="xxxx", + x_dim_name="lon", + y_dim_name="lat", + index_time=datetime.utcnow(), + run=run, + ) + + +# Fixtures + + @pytest.fixture def mock_thredds_url_root(monkeypatch): monkeypatch.setenv( @@ -72,9 +94,7 @@ def cleandb(app,): @pytest.fixture -def populateddb(cleandb,): - - now = datetime.utcnow() +def populateddb_thredds(cleandb,): populateable_db = cleandb sesh = populateable_db.session @@ -87,42 +107,23 @@ def populateddb(cleandb,): ] # Emissions + historical = Emission(short_name="historical") historical_rcp85 = Emission(short_name="historical,rcp85") # Runs + run1 = Run(name="r1i1p1", emission=historical) run2 = Run(name="r1i1p1", emission=historical_rcp85) # Models - bnu = Model(short_name="BNU-ESM", type="GCM", runs=[run1], organization="BNU") anusplin = Model(short_name="anusplin", type="GCM", runs=[run1], organization="") canesm2 = Model(short_name="CanESM2", type="GCM", runs=[run2], organization="") - models = [bnu, anusplin, canesm2] + models = [anusplin, canesm2] # Data files - def make_data_file( - filename=None, run=None, - ): - if not filename.startswith("/"): - filename = resource_filename("ce", "tests/data/{}".format(filename)) - return DataFile( - filename=filename, - unique_id=filename, - first_1mib_md5sum="xxxx", - x_dim_name="lon", - y_dim_name="lat", - index_time=now, - run=run, - ) - - df_bnu_seasonal = make_data_file( - filename="tasmin_sClim_BNU-ESM_historical_r1i1p1_19650101-19701230.nc", - run=run1, - ) # Only local file (only used to test file collector). All other files from THREDDS. - storage_root_anusplin = ( "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/" ) @@ -240,6 +241,7 @@ def make_data_file_variable( "pr": pr, "flow_direction": flow_direction, }[var_name] + return DataFileVariableGridded( file=file, netcdf_variable_name=var_name, @@ -250,9 +252,6 @@ def make_data_file_variable( variable_cell_methods=cell_methods, ) - tmax_bnu = make_data_file_variable( - df_bnu_seasonal, cell_methods="time: maximum", var_name="tasmin", - ) tmin_anusplin_seasonal = make_data_file_variable( df_anusplin_tasmin_seasonal, cell_methods="time: minimum time: mean over days", @@ -397,7 +396,6 @@ def make_data_file_variable( ], ) ts_hist_seasonal.files = [ - df_bnu_seasonal, df_anusplin_tasmin_seasonal, df_anusplin_tasmax_seasonal, df_anusplin_pr_seasonal, @@ -417,6 +415,169 @@ def make_data_file_variable( return populateable_db +@pytest.fixture() +def populateddb_local(cleandb,): + + populateable_db = cleandb + sesh = populateable_db.session + + # Ensembles + + p2a_rules = Ensemble(name="p2a_rules", version=1.0, changes="", description="") + ensembles = [ + p2a_rules, + ] + + # Emissions + + historical = Emission(short_name="historical") + + # Runs + + run1 = Run(name="r1i1p1", emission=historical) + + # Models + + anusplin = Model(short_name="anusplin", type="GCM", runs=[run1], organization="") + models = [anusplin] + + # Data files + + df_anusplin_tasmin_seasonal = make_data_file( + filename="tasmin_sClimMean_anusplin_historical_19710101-20001231.nc", run=run1, + ) + df_anusplin_tasmax_seasonal = make_data_file( + filename="tasmax_sClimMean_anusplin_historical_19710101-20001231.nc", run=run1, + ) + data_files = [v for k, v in locals().items() if k.startswith("df")] + + # VariableAlias + + tasmin = VariableAlias( + long_name="Daily Minimum Temperature", + standard_name="air_temperature", + units="degC", + ) + tasmax = VariableAlias( + long_name="Daily Maximum Temperature", + standard_name="air_temperature", + units="degC", + ) + pr = VariableAlias( + long_name="Precipitation", + standard_name="precipitation_flux", + units="kg d-1 m-2", + ) + flow_direction = VariableAlias( + long_name="Flow Direction", standard_name="flow_direction", units="1", + ) + variable_aliases = [ + tasmin, + tasmax, + pr, + flow_direction, + ] + + # Grids + + grid_anuspline = Grid( + name="Canada ANUSPLINE", + xc_grid_step=0.0833333, + yc_grid_step=0.0833333, + xc_origin=-140.958, + yc_origin=41.0417, + xc_count=1068, + yc_count=510, + xc_units="degrees_east", + yc_units="degrees_north", + evenly_spaced_y=True, + ) + grids = [grid_anuspline] + + # Add all the above + + sesh.add_all(ensembles) + sesh.add_all(models) + sesh.add_all(data_files) + sesh.add_all(variable_aliases) + sesh.add_all(grids) + sesh.flush() + + # DataFileVariable + + def make_data_file_variable( + file, cell_methods, var_name=None, grid=grid_anuspline, + ): + var_name_to_alias = { + "tasmin": tasmin, + "tasmax": tasmax, + "pr": pr, + "flow_direction": flow_direction, + }[var_name] + + return DataFileVariableGridded( + file=file, + netcdf_variable_name=var_name, + range_min=0, + range_max=50, + variable_alias=var_name_to_alias, + grid=grid, + variable_cell_methods=cell_methods, + ) + + tmin_anusplin_seasonal = make_data_file_variable( + df_anusplin_tasmin_seasonal, + cell_methods="time: minimum time: mean over days", + var_name="tasmin", + ) + tmax_anusplin_seasonal = make_data_file_variable( + df_anusplin_tasmax_seasonal, + cell_methods="time: maximum time: mean over days", + var_name="tasmax", + ) + var_names = ("tmin", "tmax") + data_file_variables = [v for k, v in locals().items() if k.startswith(var_names)] + + sesh.add_all(data_file_variables) + sesh.flush() + + # Associate to Ensembles + + for dfv in data_file_variables: + p2a_rules.data_file_variables.append(dfv) + sesh.add_all(sesh.dirty) + + # TimeSets + + ts_hist_seasonal = TimeSet( + calendar="standard", + start_date=datetime(1971, 1, 1), + end_date=datetime(2000, 12, 31), + multi_year_mean=True, + num_times=4, + time_resolution="seasonal", + times=[ + Time(time_idx=i, timestep=datetime(1986, 3 * i + 1, 16)) for i in range(4) + ], + climatological_times=[ + ClimatologicalTime( + time_idx=i, + time_start=datetime(1971, 3 * i + 1, 1) - relativedelta(months=1), + time_end=datetime(2000, 3 * i + 1, 1) + relativedelta(months=2), + ) + for i in range(4) + ], + ) + ts_hist_seasonal.files = [ + df_anusplin_tasmin_seasonal, + df_anusplin_tasmax_seasonal, + ] + sesh.add_all(sesh.dirty) + + sesh.commit() + return populateable_db + + @pytest.fixture() def mock_urls(requests_mock): requests_mock.register_uri( @@ -426,14 +587,14 @@ def mock_urls(requests_mock): ) requests_mock.register_uri( "GET", - "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets" + "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets" "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/" "tasmin_sClimMean_anusplin_historical_19710101-20001231.nc", content=tasmin_data, ) requests_mock.register_uri( "GET", - "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets" + "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets" "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/" "tasmax_sClimMean_anusplin_historical_19710101-20001231.nc", content=tasmax_data, diff --git a/tests/test_file_collector.py b/tests/test_file_collector.py index a199eea..a0db3c4 100644 --- a/tests/test_file_collector.py +++ b/tests/test_file_collector.py @@ -33,12 +33,12 @@ ], ), ) -def test_get_paths_by_var(populateddb, ensemble, date, area, variables): - sesh = populateddb.session +def test_get_paths_by_var(populateddb_local, ensemble, date, area, variables): + sesh = populateddb_local.session logger = setup_logging("ERROR") for name, values in variables.items(): paths = get_paths_by_var(sesh, values, ensemble, date, area, False, logger) for path in paths: - assert "/ce/tests/data/" in path or "/storage/data/" in path + assert "p2a-rule-engine/tests/data/" in path diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 5e72ab4..abe930f 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -20,7 +20,7 @@ ], ) def test_resolve_rules_basic( - populateddb, + populateddb_thredds, mock_thredds_url_root, mock_urls, csv, @@ -30,7 +30,7 @@ def test_resolve_rules_basic( ensemble, thredds, ): - sesh = populateddb.session + sesh = populateddb_thredds.session rules = resolve_rules( csv, date_range, get_region(region, geoserver), ensemble, sesh, thredds ) @@ -54,7 +54,7 @@ def test_resolve_rules_basic( ) @pytest.mark.parametrize("date_range", ["2050", "2080"]) def test_resolve_rules_multi_percentile( - populateddb, + populateddb_thredds, mock_thredds_url_root, csv, date_range, @@ -63,7 +63,7 @@ def test_resolve_rules_multi_percentile( ensemble, thredds, ): - sesh = populateddb.session + sesh = populateddb_thredds.session rules = resolve_rules( csv, date_range, get_region(region, geoserver), ensemble, sesh, thredds ) @@ -91,7 +91,7 @@ def test_resolve_rules_multi_percentile( ], ) def test_resolve_rules_multi_var( - populateddb, + populateddb_thredds, mock_thredds_url_root, csv, date_range, @@ -100,9 +100,35 @@ def test_resolve_rules_multi_var( ensemble, thredds, ): - sesh = populateddb.session + sesh = populateddb_thredds.session rules = resolve_rules( csv, date_range, get_region(region, geoserver), ensemble, sesh, thredds ) expected_rules = {"rule_shm": 65.807} assert round(rules["rule_shm"], 3) == expected_rules["rule_shm"] + + +@pytest.mark.slow +@pytest.mark.online +@pytest.mark.parametrize( + ("csv", "date_range", "region", "geoserver", "ensemble", "thredds"), + [ + ( + resource_filename("tests", "data/rules-basic.csv"), + "hist", + "vancouver_island", + "http://docker-dev01.pcic.uvic.ca:30123/geoserver/bc_regions/ows", + "p2a_rules", + False, + ), + ], +) +def test_resolve_rules_local( + populateddb_local, csv, date_range, region, geoserver, ensemble, thredds, +): + sesh = populateddb_local.session + rules = resolve_rules( + csv, date_range, get_region(region, geoserver), ensemble, sesh, thredds + ) + expected_rules = {"rule_snow": True, "rule_hybrid": True, "rule_rain": True} + assert rules == expected_rules From 9f67ee51a5b67a6257cb18877835c1197011d2ec Mon Sep 17 00:00:00 2001 From: Eric Yvorchuk Date: Wed, 16 Feb 2022 12:58:40 -0800 Subject: [PATCH 04/15] Add temporary test to ensure mock_url is registered --- tests/test_resolver.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_resolver.py b/tests/test_resolver.py index abe930f..7bb0b5d 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -3,6 +3,8 @@ from p2a_impacts.resolver import resolve_rules from p2a_impacts.utils import get_region +import requests +from .mock_data import tasmin_data @pytest.mark.slow @@ -132,3 +134,14 @@ def test_resolve_rules_local( ) expected_rules = {"rule_snow": True, "rule_hybrid": True, "rule_rain": True} assert rules == expected_rules + + +def test_mock_urls(mock_urls): + assert ( + requests.get( + "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets" + "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/" + "tasmin_sClimMean_anusplin_historical_19710101-20001231.nc" + ).content + == tasmin_data + ) From ad6fbfda20a7ef580a8ee0295700d61b986cd9ae Mon Sep 17 00:00:00 2001 From: Eric Yvorchuk Date: Wed, 16 Feb 2022 13:09:58 -0800 Subject: [PATCH 05/15] Mock out both http and OpenDAP requests --- tests/conftest.py | 14 ++++++++++++++ tests/test_resolver.py | 8 +++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e398c61..aee4177 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -585,6 +585,13 @@ def mock_urls(requests_mock): "http://docker-dev01.pcic.uvic.ca:30123/geoserver/bc_regions/ows", content=geoserver_data, ) + requests_mock.register_uri( + "GET", + "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets" + "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/" + "tasmin_sClimMean_anusplin_historical_19710101-20001231.nc", + content=tasmin_data, + ) requests_mock.register_uri( "GET", "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets" @@ -592,6 +599,13 @@ def mock_urls(requests_mock): "tasmin_sClimMean_anusplin_historical_19710101-20001231.nc", content=tasmin_data, ) + requests_mock.register_uri( + "GET", + "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets" + "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/" + "tasmax_sClimMean_anusplin_historical_19710101-20001231.nc", + content=tasmax_data, + ) requests_mock.register_uri( "GET", "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets" diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 7bb0b5d..565f0b6 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -3,6 +3,8 @@ from p2a_impacts.resolver import resolve_rules from p2a_impacts.utils import get_region + +import os import requests from .mock_data import tasmin_data @@ -136,11 +138,11 @@ def test_resolve_rules_local( assert rules == expected_rules -def test_mock_urls(mock_urls): +def test_mock_urls(mock_thredds_url_root, mock_urls): assert ( requests.get( - "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets" - "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/" + os.getenv("THREDDS_URL_ROOT") + + "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/" "tasmin_sClimMean_anusplin_historical_19710101-20001231.nc" ).content == tasmin_data From 29dd4b79d15dec589631547ed3b917f6b04455f3 Mon Sep 17 00:00:00 2001 From: Eric Yvorchuk Date: Wed, 16 Feb 2022 13:22:58 -0800 Subject: [PATCH 06/15] Mock out geoserver url for local test --- tests/test_resolver.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 565f0b6..ec71916 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -113,7 +113,6 @@ def test_resolve_rules_multi_var( @pytest.mark.slow -@pytest.mark.online @pytest.mark.parametrize( ("csv", "date_range", "region", "geoserver", "ensemble", "thredds"), [ @@ -128,7 +127,7 @@ def test_resolve_rules_multi_var( ], ) def test_resolve_rules_local( - populateddb_local, csv, date_range, region, geoserver, ensemble, thredds, + populateddb_local, mock_urls, csv, date_range, region, geoserver, ensemble, thredds, ): sesh = populateddb_local.session rules = resolve_rules( From 02248a82efb0e4f1a5b3899f024f47f31a832a39 Mon Sep 17 00:00:00 2001 From: Eric Yvorchuk Date: Fri, 18 Feb 2022 10:00:38 -0800 Subject: [PATCH 07/15] Temporarily omit python 3.6 and 3.7 for CI --- .github/workflows/python-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index f396974..c84528e 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.8] steps: - uses: actions/checkout@v2 From 2537e72254f07767de34a06857fc25f4ddb95acf Mon Sep 17 00:00:00 2001 From: Eric Yvorchuk Date: Thu, 3 Mar 2022 12:10:24 -0800 Subject: [PATCH 08/15] Use mock.patch to mock openDAP requests using netCDF4.Dataset --- test_requirements.txt | 1 + tests/conftest.py | 14 -------------- tests/mock_data.py | 7 +++++-- tests/test_resolver.py | 40 ++++++++++++++++++++++++++++++++-------- 4 files changed, 38 insertions(+), 24 deletions(-) diff --git a/test_requirements.txt b/test_requirements.txt index 6542db0..a7255ec 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -3,3 +3,4 @@ pre-commit==2.7.1 pytest==6.0.2 pytest-cov==2.10.1 requests_mock==1.9 +mock==4.0.3 diff --git a/tests/conftest.py b/tests/conftest.py index aee4177..e398c61 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -585,13 +585,6 @@ def mock_urls(requests_mock): "http://docker-dev01.pcic.uvic.ca:30123/geoserver/bc_regions/ows", content=geoserver_data, ) - requests_mock.register_uri( - "GET", - "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets" - "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/" - "tasmin_sClimMean_anusplin_historical_19710101-20001231.nc", - content=tasmin_data, - ) requests_mock.register_uri( "GET", "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets" @@ -599,13 +592,6 @@ def mock_urls(requests_mock): "tasmin_sClimMean_anusplin_historical_19710101-20001231.nc", content=tasmin_data, ) - requests_mock.register_uri( - "GET", - "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets" - "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/" - "tasmax_sClimMean_anusplin_historical_19710101-20001231.nc", - content=tasmax_data, - ) requests_mock.register_uri( "GET", "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets" diff --git a/tests/mock_data.py b/tests/mock_data.py index d156f12..8369c35 100644 --- a/tests/mock_data.py +++ b/tests/mock_data.py @@ -1,4 +1,5 @@ from pkg_resources import resource_filename +from netCDF4 import Dataset geoserver_data = open(resource_filename("tests", "data/geoserver_van.txt"), "rb").read() @@ -10,5 +11,7 @@ def get_nc_data(filename): return filedata -tasmin_data = get_nc_data("tasmin_sClimMean_anusplin_historical_19710101-20001231.nc") -tasmax_data = get_nc_data("tasmax_sClimMean_anusplin_historical_19710101-20001231.nc") +tasmin_filename = "tasmin_sClimMean_anusplin_historical_19710101-20001231.nc" +tasmax_filename = "tasmax_sClimMean_anusplin_historical_19710101-20001231.nc" +tasmin_data = get_nc_data(tasmin_filename) +tasmax_data = get_nc_data(tasmax_filename) diff --git a/tests/test_resolver.py b/tests/test_resolver.py index ec71916..e1d40b2 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -6,7 +6,25 @@ import os import requests -from .mock_data import tasmin_data +import mock +import netCDF4 +from netCDF4 import Dataset +from .mock_data import tasmin_data, tasmax_data + + +def mock_opendap_request(path, mode="r"): + return Dataset(resource_filename("tests", f"data/{os.path.basename(path)}"), "r") + + +@mock.patch("netCDF4.Dataset", side_effect=mock_opendap_request) +def test_mock_opendap_request(mock_opendap_request): + base_path_tasmin = "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/tasmin_sClimMean_anusplin_historical_19710101-20001231.nc" + dods_path_tasmin = os.getenv("THREDDS_URL_ROOT") + base_path_tasmin + tasmin = netCDF4.Dataset(dods_path_tasmin) + base_path_tasmax = "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/tasmax_sClimMean_anusplin_historical_19710101-20001231.nc" + dods_path_tasmax = os.getenv("THREDDS_URL_ROOT") + base_path_tasmax + tasmax = netCDF4.Dataset(dods_path_tasmax) + assert mock_opendap_request.call_count == 2 @pytest.mark.slow @@ -23,7 +41,9 @@ ), ], ) +@mock.patch("ce.api.util.Dataset", side_effect=mock_opendap_request) def test_resolve_rules_basic( + mock_opendap_request, populateddb_thredds, mock_thredds_url_root, mock_urls, @@ -138,11 +158,15 @@ def test_resolve_rules_local( def test_mock_urls(mock_thredds_url_root, mock_urls): - assert ( - requests.get( - os.getenv("THREDDS_URL_ROOT") - + "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/" - "tasmin_sClimMean_anusplin_historical_19710101-20001231.nc" - ).content - == tasmin_data + base_path_tasmin = "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/tasmin_sClimMean_anusplin_historical_19710101-20001231.nc" + fileserver_path_tasmin = ( + "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets" + + base_path_tasmin + ) + base_path_tasmax = "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/tasmax_sClimMean_anusplin_historical_19710101-20001231.nc" + fileserver_path_tasmax = ( + "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets" + + base_path_tasmax ) + assert requests.get(fileserver_path_tasmin).content == tasmin_data + assert requests.get(fileserver_path_tasmax).content == tasmax_data From b855d1c764a33f5041cce79165bbbd890cdd088d Mon Sep 17 00:00:00 2001 From: Eric Yvorchuk Date: Thu, 3 Mar 2022 12:23:33 -0800 Subject: [PATCH 09/15] Reuse python 3.6 and 3.7 for CI --- .github/workflows/python-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index c84528e..f396974 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [3.8] + python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 From 791f2b59df4fa07d78e6442f45fcb5ef1d936f93 Mon Sep 17 00:00:00 2001 From: Eric Yvorchuk Date: Thu, 3 Mar 2022 12:30:53 -0800 Subject: [PATCH 10/15] Use mock THREDDS root to test mock openDAP requests --- tests/test_resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_resolver.py b/tests/test_resolver.py index e1d40b2..5269e40 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -17,7 +17,7 @@ def mock_opendap_request(path, mode="r"): @mock.patch("netCDF4.Dataset", side_effect=mock_opendap_request) -def test_mock_opendap_request(mock_opendap_request): +def test_mock_opendap_request(mock_opendap_request, mock_thredds_url_root): base_path_tasmin = "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/tasmin_sClimMean_anusplin_historical_19710101-20001231.nc" dods_path_tasmin = os.getenv("THREDDS_URL_ROOT") + base_path_tasmin tasmin = netCDF4.Dataset(dods_path_tasmin) From 95be08367b74c38762e8a07e120545f1d870a46f Mon Sep 17 00:00:00 2001 From: Eric Yvorchuk Date: Thu, 3 Mar 2022 14:06:44 -0800 Subject: [PATCH 11/15] Clean up mock databases in conftest.py --- tests/conftest.py | 271 ++++++++++++++++------------------------------ 1 file changed, 96 insertions(+), 175 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e398c61..ae30785 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,15 +24,80 @@ from .mock_data import geoserver_data, tasmin_data, tasmax_data +# Objects used by both mock databases -# Functions to create data for database +# Ensembles + +p2a_rules = Ensemble(name="p2a_rules", version=1.0, changes="", description="") +ensembles = [ + p2a_rules, +] + +# Emissions + +historical = Emission(short_name="historical") +historical_rcp85 = Emission(short_name="historical,rcp85") + +# Runs + +run1 = Run(name="r1i1p1", emission=historical) +run2 = Run(name="r1i1p1", emission=historical_rcp85) + +# Models + +anusplin = Model(short_name="anusplin", type="GCM", runs=[run1], organization="") +canesm2 = Model(short_name="CanESM2", type="GCM", runs=[run2], organization="") + +# VariableAlias + +tasmin = VariableAlias( + long_name="Daily Minimum Temperature", + standard_name="air_temperature", + units="degC", +) +tasmax = VariableAlias( + long_name="Daily Maximum Temperature", + standard_name="air_temperature", + units="degC", +) +pr = VariableAlias( + long_name="Precipitation", standard_name="precipitation_flux", units="kg d-1 m-2", +) +flow_direction = VariableAlias( + long_name="Flow Direction", standard_name="flow_direction", units="1", +) +variable_aliases = [ + tasmin, + tasmax, + pr, + flow_direction, +] + +# Grids + +grid_anuspline = Grid( + name="Canada ANUSPLINE", + xc_grid_step=0.0833333, + yc_grid_step=0.0833333, + xc_origin=-140.958, + yc_origin=41.0417, + xc_count=1068, + yc_count=510, + xc_units="degrees_east", + yc_units="degrees_north", + evenly_spaced_y=True, +) +grids = [grid_anuspline] + + +# Functions to create data for mock databases def make_data_file( filename=None, run=None, ): if not filename.startswith("/"): - filename = resource_filename("tests", "data/{}".format(filename)) + filename = resource_filename("tests", f"data/{filename}") return DataFile( filename=filename, unique_id=filename, @@ -44,6 +109,27 @@ def make_data_file( ) +def make_data_file_variable( + file, cell_methods, var_name=None, grid=grid_anuspline, +): + var_name_to_alias = { + "tasmin": tasmin, + "tasmax": tasmax, + "pr": pr, + "flow_direction": flow_direction, + }[var_name] + + return DataFileVariableGridded( + file=file, + netcdf_variable_name=var_name, + range_min=0, + range_max=50, + variable_alias=var_name_to_alias, + grid=grid, + variable_cell_methods=cell_methods, + ) + + # Fixtures @@ -64,19 +150,19 @@ def ce_response(): } -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def sessiondir(request,): dir = py.path.local(tempfile.mkdtemp()) request.addfinalizer(lambda: dir.remove(rec=1)) return dir -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def dsn(sessiondir,): - return "sqlite:///{}".format(sessiondir.join("test.sqlite").realpath()) + return f"sqlite:///{sessiondir.join('test.sqlite').realpath()}" -@pytest.fixture +@pytest.fixture(scope="session") def app(dsn,): app = get_app() app.config["TESTING"] = True @@ -85,7 +171,7 @@ def app(dsn,): return app -@pytest.fixture +@pytest.fixture(scope="session") def cleandb(app,): db = SQLAlchemy(app) metadata.create_all(bind=db.engine) @@ -93,33 +179,12 @@ def cleandb(app,): return db -@pytest.fixture +@pytest.fixture(scope="session") def populateddb_thredds(cleandb,): populateable_db = cleandb sesh = populateable_db.session - # Ensembles - - p2a_rules = Ensemble(name="p2a_rules", version=1.0, changes="", description="") - ensembles = [ - p2a_rules, - ] - - # Emissions - - historical = Emission(short_name="historical") - historical_rcp85 = Emission(short_name="historical,rcp85") - - # Runs - - run1 = Run(name="r1i1p1", emission=historical) - run2 = Run(name="r1i1p1", emission=historical_rcp85) - - # Models - - anusplin = Model(short_name="anusplin", type="GCM", runs=[run1], organization="") - canesm2 = Model(short_name="CanESM2", type="GCM", runs=[run2], organization="") models = [anusplin, canesm2] # Data files @@ -178,49 +243,6 @@ def populateddb_thredds(cleandb,): ) data_files = [v for k, v in locals().items() if k.startswith("df")] - # VariableAlias - - tasmin = VariableAlias( - long_name="Daily Minimum Temperature", - standard_name="air_temperature", - units="degC", - ) - tasmax = VariableAlias( - long_name="Daily Maximum Temperature", - standard_name="air_temperature", - units="degC", - ) - pr = VariableAlias( - long_name="Precipitation", - standard_name="precipitation_flux", - units="kg d-1 m-2", - ) - flow_direction = VariableAlias( - long_name="Flow Direction", standard_name="flow_direction", units="1", - ) - variable_aliases = [ - tasmin, - tasmax, - pr, - flow_direction, - ] - - # Grids - - grid_anuspline = Grid( - name="Canada ANUSPLINE", - xc_grid_step=0.0833333, - yc_grid_step=0.0833333, - xc_origin=-140.958, - yc_origin=41.0417, - xc_count=1068, - yc_count=510, - xc_units="degrees_east", - yc_units="degrees_north", - evenly_spaced_y=True, - ) - grids = [grid_anuspline] - # Add all the above sesh.add_all(ensembles) @@ -230,27 +252,7 @@ def populateddb_thredds(cleandb,): sesh.add_all(grids) sesh.flush() - # DataFileVariable - - def make_data_file_variable( - file, cell_methods, var_name=None, grid=grid_anuspline, - ): - var_name_to_alias = { - "tasmin": tasmin, - "tasmax": tasmax, - "pr": pr, - "flow_direction": flow_direction, - }[var_name] - - return DataFileVariableGridded( - file=file, - netcdf_variable_name=var_name, - range_min=0, - range_max=50, - variable_alias=var_name_to_alias, - grid=grid, - variable_cell_methods=cell_methods, - ) + # DataFileVariables tmin_anusplin_seasonal = make_data_file_variable( df_anusplin_tasmin_seasonal, @@ -415,30 +417,12 @@ def make_data_file_variable( return populateable_db -@pytest.fixture() +@pytest.fixture(scope="session") def populateddb_local(cleandb,): populateable_db = cleandb sesh = populateable_db.session - # Ensembles - - p2a_rules = Ensemble(name="p2a_rules", version=1.0, changes="", description="") - ensembles = [ - p2a_rules, - ] - - # Emissions - - historical = Emission(short_name="historical") - - # Runs - - run1 = Run(name="r1i1p1", emission=historical) - - # Models - - anusplin = Model(short_name="anusplin", type="GCM", runs=[run1], organization="") models = [anusplin] # Data files @@ -451,49 +435,6 @@ def populateddb_local(cleandb,): ) data_files = [v for k, v in locals().items() if k.startswith("df")] - # VariableAlias - - tasmin = VariableAlias( - long_name="Daily Minimum Temperature", - standard_name="air_temperature", - units="degC", - ) - tasmax = VariableAlias( - long_name="Daily Maximum Temperature", - standard_name="air_temperature", - units="degC", - ) - pr = VariableAlias( - long_name="Precipitation", - standard_name="precipitation_flux", - units="kg d-1 m-2", - ) - flow_direction = VariableAlias( - long_name="Flow Direction", standard_name="flow_direction", units="1", - ) - variable_aliases = [ - tasmin, - tasmax, - pr, - flow_direction, - ] - - # Grids - - grid_anuspline = Grid( - name="Canada ANUSPLINE", - xc_grid_step=0.0833333, - yc_grid_step=0.0833333, - xc_origin=-140.958, - yc_origin=41.0417, - xc_count=1068, - yc_count=510, - xc_units="degrees_east", - yc_units="degrees_north", - evenly_spaced_y=True, - ) - grids = [grid_anuspline] - # Add all the above sesh.add_all(ensembles) @@ -505,26 +446,6 @@ def populateddb_local(cleandb,): # DataFileVariable - def make_data_file_variable( - file, cell_methods, var_name=None, grid=grid_anuspline, - ): - var_name_to_alias = { - "tasmin": tasmin, - "tasmax": tasmax, - "pr": pr, - "flow_direction": flow_direction, - }[var_name] - - return DataFileVariableGridded( - file=file, - netcdf_variable_name=var_name, - range_min=0, - range_max=50, - variable_alias=var_name_to_alias, - grid=grid, - variable_cell_methods=cell_methods, - ) - tmin_anusplin_seasonal = make_data_file_variable( df_anusplin_tasmin_seasonal, cell_methods="time: minimum time: mean over days", From 2b964ce5229ac5e5d6b928fe34596697bc766cbf Mon Sep 17 00:00:00 2001 From: Eric Yvorchuk Date: Fri, 4 Mar 2022 11:24:19 -0800 Subject: [PATCH 12/15] Create separate function to create modelmeta objects for both databases --- tests/conftest.py | 215 +++++++++++++++++++++++++++------------------- 1 file changed, 128 insertions(+), 87 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ae30785..d9838a7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,73 +24,88 @@ from .mock_data import geoserver_data, tasmin_data, tasmax_data -# Objects used by both mock databases -# Ensembles +# Functions to create data for mock databases -p2a_rules = Ensemble(name="p2a_rules", version=1.0, changes="", description="") -ensembles = [ - p2a_rules, -] -# Emissions +def create_modelmeta_objects(): + """Create modelmeta objects used for both mock databases""" -historical = Emission(short_name="historical") -historical_rcp85 = Emission(short_name="historical,rcp85") + objects = {} -# Runs + # Ensembles -run1 = Run(name="r1i1p1", emission=historical) -run2 = Run(name="r1i1p1", emission=historical_rcp85) + p2a_rules = Ensemble(name="p2a_rules", version=1.0, changes="", description="") + ensembles = [ + p2a_rules, + ] + objects["ensembles"] = ensembles -# Models + # Emissions -anusplin = Model(short_name="anusplin", type="GCM", runs=[run1], organization="") -canesm2 = Model(short_name="CanESM2", type="GCM", runs=[run2], organization="") + historical = Emission(short_name="historical") + historical_rcp85 = Emission(short_name="historical,rcp85") -# VariableAlias + # Runs -tasmin = VariableAlias( - long_name="Daily Minimum Temperature", - standard_name="air_temperature", - units="degC", -) -tasmax = VariableAlias( - long_name="Daily Maximum Temperature", - standard_name="air_temperature", - units="degC", -) -pr = VariableAlias( - long_name="Precipitation", standard_name="precipitation_flux", units="kg d-1 m-2", -) -flow_direction = VariableAlias( - long_name="Flow Direction", standard_name="flow_direction", units="1", -) -variable_aliases = [ - tasmin, - tasmax, - pr, - flow_direction, -] - -# Grids - -grid_anuspline = Grid( - name="Canada ANUSPLINE", - xc_grid_step=0.0833333, - yc_grid_step=0.0833333, - xc_origin=-140.958, - yc_origin=41.0417, - xc_count=1068, - yc_count=510, - xc_units="degrees_east", - yc_units="degrees_north", - evenly_spaced_y=True, -) -grids = [grid_anuspline] + run1 = Run(name="r1i1p1", emission=historical) + run2 = Run(name="r1i1p1", emission=historical_rcp85) + objects["run1"] = run1 + objects["run2"] = run2 + # Models -# Functions to create data for mock databases + anusplin = Model(short_name="anusplin", type="GCM", runs=[run1], organization="") + canesm2 = Model(short_name="CanESM2", type="GCM", runs=[run2], organization="") + objects["anusplin"] = anusplin + objects["canesm2"] = canesm2 + + # VariableAliases + + tasmin = VariableAlias( + long_name="Daily Minimum Temperature", + standard_name="air_temperature", + units="degC", + ) + tasmax = VariableAlias( + long_name="Daily Maximum Temperature", + standard_name="air_temperature", + units="degC", + ) + pr = VariableAlias( + long_name="Precipitation", + standard_name="precipitation_flux", + units="kg d-1 m-2", + ) + flow_direction = VariableAlias( + long_name="Flow Direction", standard_name="flow_direction", units="1", + ) + variable_aliases = [ + tasmin, + tasmax, + pr, + flow_direction, + ] + objects["variable_aliases"] = variable_aliases + + # Grids + + grid_anuspline = Grid( + name="Canada ANUSPLINE", + xc_grid_step=0.0833333, + yc_grid_step=0.0833333, + xc_origin=-140.958, + yc_origin=41.0417, + xc_count=1068, + yc_count=510, + xc_units="degrees_east", + yc_units="degrees_north", + evenly_spaced_y=True, + ) + grids = [grid_anuspline] + objects["grids"] = grids + + return objects def make_data_file( @@ -110,13 +125,13 @@ def make_data_file( def make_data_file_variable( - file, cell_methods, var_name=None, grid=grid_anuspline, + file, cell_methods, variable_aliases, var_name=None, grid=None, ): var_name_to_alias = { - "tasmin": tasmin, - "tasmax": tasmax, - "pr": pr, - "flow_direction": flow_direction, + "tasmin": variable_aliases[0], + "tasmax": variable_aliases[1], + "pr": variable_aliases[2], + "flow_direction": variable_aliases[3], }[var_name] return DataFileVariableGridded( @@ -150,19 +165,19 @@ def ce_response(): } -@pytest.fixture(scope="session") +@pytest.fixture() def sessiondir(request,): dir = py.path.local(tempfile.mkdtemp()) request.addfinalizer(lambda: dir.remove(rec=1)) return dir -@pytest.fixture(scope="session") +@pytest.fixture() def dsn(sessiondir,): return f"sqlite:///{sessiondir.join('test.sqlite').realpath()}" -@pytest.fixture(scope="session") +@pytest.fixture() def app(dsn,): app = get_app() app.config["TESTING"] = True @@ -171,7 +186,7 @@ def app(dsn,): return app -@pytest.fixture(scope="session") +@pytest.fixture() def cleandb(app,): db = SQLAlchemy(app) metadata.create_all(bind=db.engine) @@ -179,13 +194,14 @@ def cleandb(app,): return db -@pytest.fixture(scope="session") +@pytest.fixture() def populateddb_thredds(cleandb,): populateable_db = cleandb sesh = populateable_db.session + objects = create_modelmeta_objects() - models = [anusplin, canesm2] + models = [objects["anusplin"], objects["canesm2"]] # Data files @@ -199,57 +215,57 @@ def populateddb_thredds(cleandb,): df_anusplin_tasmin_seasonal = make_data_file( filename=storage_root_anusplin + "tasmin_sClimMean_anusplin_historical_19710101-20001231.nc", - run=run1, + run=objects["run1"], ) df_anusplin_tasmax_seasonal = make_data_file( filename=storage_root_anusplin + "tasmax_sClimMean_anusplin_historical_19710101-20001231.nc", - run=run1, + run=objects["run1"], ) df_anusplin_tasmin_mon = make_data_file( filename=storage_root_anusplin + "tasmin_mClimMean_anusplin_historical_19710101-20001231.nc", - run=run1, + run=objects["run1"], ) df_anusplin_tasmax_mon = make_data_file( filename=storage_root_anusplin + "tasmax_mClimMean_anusplin_historical_19710101-20001231.nc", - run=run1, + run=objects["run1"], ) df_anusplin_pr_seasonal = make_data_file( filename=storage_root_anusplin + "pr_sClimMean_anusplin_historical_19710101-20001231.nc", - run=run1, + run=objects["run1"], ) df_canesm2_tasmin_2050_seasonal = make_data_file( filename=canesm2_tasmin_root + "tasmin_sClim_BCCAQv2_CanESM2_historical+rcp85_r1i1p1_20400101-20691231_Canada.nc", - run=run2, + run=objects["run2"], ) df_canesm2_tasmax_2050_seasonal = make_data_file( filename=canesm2_tasmax_root + "tasmax_sClim_BCCAQv2_CanESM2_historical+rcp85_r1i1p1_20400101-20691231_Canada.nc", - run=run2, + run=objects["run2"], ) df_canesm2_tasmin_2080_seasonal = make_data_file( filename=canesm2_tasmin_root + "tasmin_sClim_BCCAQv2_CanESM2_historical+rcp85_r1i1p1_20700101-20991231_Canada.nc", - run=run2, + run=objects["run2"], ) df_canesm2_tasmax_2080_seasonal = make_data_file( filename=canesm2_tasmax_root + "tasmax_sClim_BCCAQv2_CanESM2_historical+rcp85_r1i1p1_20700101-20991231_Canada.nc", - run=run2, + run=objects["run2"], ) data_files = [v for k, v in locals().items() if k.startswith("df")] # Add all the above - sesh.add_all(ensembles) + sesh.add_all(objects["ensembles"]) sesh.add_all(models) sesh.add_all(data_files) - sesh.add_all(variable_aliases) - sesh.add_all(grids) + sesh.add_all(objects["variable_aliases"]) + sesh.add_all(objects["grids"]) sesh.flush() # DataFileVariables @@ -257,47 +273,65 @@ def populateddb_thredds(cleandb,): tmin_anusplin_seasonal = make_data_file_variable( df_anusplin_tasmin_seasonal, cell_methods="time: minimum time: mean over days", + variable_aliases=objects["variable_aliases"], var_name="tasmin", + grid=objects["grids"][0], ) tmax_anusplin_seasonal = make_data_file_variable( df_anusplin_tasmax_seasonal, cell_methods="time: maximum time: mean over days", + variable_aliases=objects["variable_aliases"], var_name="tasmax", + grid=objects["grids"][0], ) tmin_anusplin_mon = make_data_file_variable( df_anusplin_tasmin_mon, cell_methods="time: minimum time: mean over days", + variable_aliases=objects["variable_aliases"], var_name="tasmin", + grid=objects["grids"][0], ) tmax_anusplin_mon = make_data_file_variable( df_anusplin_tasmax_mon, cell_methods="time: minimum time: mean over days", + variable_aliases=objects["variable_aliases"], var_name="tasmax", + grid=objects["grids"][0], ) pr_anusplin = make_data_file_variable( df_anusplin_pr_seasonal, cell_methods="time: mean time: mean over days", + variable_aliases=objects["variable_aliases"], var_name="pr", + grid=objects["grids"][0], ) tmin_canesm2_2050 = make_data_file_variable( df_canesm2_tasmin_2050_seasonal, cell_methods="time: minimum", + variable_aliases=objects["variable_aliases"], var_name="tasmin", + grid=objects["grids"][0], ) tmax_canesm2_2050 = make_data_file_variable( df_canesm2_tasmax_2050_seasonal, cell_methods="time: maximum", + variable_aliases=objects["variable_aliases"], var_name="tasmax", + grid=objects["grids"][0], ) tmin_canesm2_2080 = make_data_file_variable( df_canesm2_tasmin_2080_seasonal, cell_methods="time: minimum", + variable_aliases=objects["variable_aliases"], var_name="tasmin", + grid=objects["grids"][0], ) tmax_canesm2_2080 = make_data_file_variable( df_canesm2_tasmax_2080_seasonal, cell_methods="time: maximum", + variable_aliases=objects["variable_aliases"], var_name="tasmax", + grid=objects["grids"][0], ) var_names = ("tmin", "tmax") data_file_variables = [ @@ -310,7 +344,7 @@ def populateddb_thredds(cleandb,): # Associate to Ensembles for dfv in data_file_variables: - p2a_rules.data_file_variables.append(dfv) + objects["ensembles"][0].data_file_variables.append(dfv) sesh.add_all(sesh.dirty) # TimeSets @@ -417,31 +451,34 @@ def populateddb_thredds(cleandb,): return populateable_db -@pytest.fixture(scope="session") +@pytest.fixture() def populateddb_local(cleandb,): populateable_db = cleandb sesh = populateable_db.session + objects = create_modelmeta_objects() - models = [anusplin] + models = [objects["anusplin"]] # Data files df_anusplin_tasmin_seasonal = make_data_file( - filename="tasmin_sClimMean_anusplin_historical_19710101-20001231.nc", run=run1, + filename="tasmin_sClimMean_anusplin_historical_19710101-20001231.nc", + run=objects["run1"], ) df_anusplin_tasmax_seasonal = make_data_file( - filename="tasmax_sClimMean_anusplin_historical_19710101-20001231.nc", run=run1, + filename="tasmax_sClimMean_anusplin_historical_19710101-20001231.nc", + run=objects["run1"], ) data_files = [v for k, v in locals().items() if k.startswith("df")] # Add all the above - sesh.add_all(ensembles) + sesh.add_all(objects["ensembles"]) sesh.add_all(models) sesh.add_all(data_files) - sesh.add_all(variable_aliases) - sesh.add_all(grids) + sesh.add_all(objects["variable_aliases"]) + sesh.add_all(objects["grids"]) sesh.flush() # DataFileVariable @@ -449,12 +486,16 @@ def populateddb_local(cleandb,): tmin_anusplin_seasonal = make_data_file_variable( df_anusplin_tasmin_seasonal, cell_methods="time: minimum time: mean over days", + variable_aliases=objects["variable_aliases"], var_name="tasmin", + grid=objects["grids"][0], ) tmax_anusplin_seasonal = make_data_file_variable( df_anusplin_tasmax_seasonal, cell_methods="time: maximum time: mean over days", + variable_aliases=objects["variable_aliases"], var_name="tasmax", + grid=objects["grids"][0], ) var_names = ("tmin", "tmax") data_file_variables = [v for k, v in locals().items() if k.startswith(var_names)] @@ -465,7 +506,7 @@ def populateddb_local(cleandb,): # Associate to Ensembles for dfv in data_file_variables: - p2a_rules.data_file_variables.append(dfv) + objects["ensembles"][0].data_file_variables.append(dfv) sesh.add_all(sesh.dirty) # TimeSets From 3a8d5cb7330c5f704e59c7df52d87df8dae3d0a9 Mon Sep 17 00:00:00 2001 From: Eric Yvorchuk Date: Fri, 4 Mar 2022 11:47:43 -0800 Subject: [PATCH 13/15] Omit slow tests for Python-CI in dev branches again --- .github/workflows/python-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index f396974..23dd9a6 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -30,7 +30,7 @@ jobs: - name: Test with pytest (fast) if: github.ref != 'refs/heads/master' run: | - py.test -m "not online" + py.test -m "not slow and not online" - name: Code format check run: | black . --check From ebc5b586234a371906536a83208cab369d321d32 Mon Sep 17 00:00:00 2001 From: Eric Yvorchuk Date: Fri, 4 Mar 2022 11:48:01 -0800 Subject: [PATCH 14/15] Remove slow markers for local and basic resolve rules tests --- tests/test_resolver.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 5269e40..f57fedb 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -27,7 +27,6 @@ def test_mock_opendap_request(mock_opendap_request, mock_thredds_url_root): assert mock_opendap_request.call_count == 2 -@pytest.mark.slow @pytest.mark.parametrize( ("csv", "date_range", "region", "geoserver", "ensemble", "thredds"), [ @@ -132,7 +131,6 @@ def test_resolve_rules_multi_var( assert round(rules["rule_shm"], 3) == expected_rules["rule_shm"] -@pytest.mark.slow @pytest.mark.parametrize( ("csv", "date_range", "region", "geoserver", "ensemble", "thredds"), [ From 70e59dc7418b4b02eff942a9edf36b888ecadf33 Mon Sep 17 00:00:00 2001 From: Eric Yvorchuk Date: Tue, 8 Mar 2022 09:18:09 -0800 Subject: [PATCH 15/15] Add docstrings for database fixtures and clean up conftest.py --- tests/conftest.py | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d9838a7..e1994f6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,7 +29,7 @@ def create_modelmeta_objects(): - """Create modelmeta objects used for both mock databases""" + """Create modelmeta objects used for both mock databases.""" objects = {} @@ -127,11 +127,12 @@ def make_data_file( def make_data_file_variable( file, cell_methods, variable_aliases, var_name=None, grid=None, ): + (tasmin, tasmax, pr, flow_direction) = variable_aliases[:] var_name_to_alias = { - "tasmin": variable_aliases[0], - "tasmax": variable_aliases[1], - "pr": variable_aliases[2], - "flow_direction": variable_aliases[3], + "tasmin": tasmin, + "tasmax": tasmax, + "pr": pr, + "flow_direction": flow_direction, }[var_name] return DataFileVariableGridded( @@ -196,10 +197,13 @@ def cleandb(app,): @pytest.fixture() def populateddb_thredds(cleandb,): + """Create mock database only containing data files from THREDDS.""" populateable_db = cleandb sesh = populateable_db.session objects = create_modelmeta_objects() + p2a_rules = objects["ensembles"][0] + grid_anuspline = objects["grids"][0] models = [objects["anusplin"], objects["canesm2"]] @@ -275,35 +279,35 @@ def populateddb_thredds(cleandb,): cell_methods="time: minimum time: mean over days", variable_aliases=objects["variable_aliases"], var_name="tasmin", - grid=objects["grids"][0], + grid=grid_anuspline, ) tmax_anusplin_seasonal = make_data_file_variable( df_anusplin_tasmax_seasonal, cell_methods="time: maximum time: mean over days", variable_aliases=objects["variable_aliases"], var_name="tasmax", - grid=objects["grids"][0], + grid=grid_anuspline, ) tmin_anusplin_mon = make_data_file_variable( df_anusplin_tasmin_mon, cell_methods="time: minimum time: mean over days", variable_aliases=objects["variable_aliases"], var_name="tasmin", - grid=objects["grids"][0], + grid=grid_anuspline, ) tmax_anusplin_mon = make_data_file_variable( df_anusplin_tasmax_mon, cell_methods="time: minimum time: mean over days", variable_aliases=objects["variable_aliases"], var_name="tasmax", - grid=objects["grids"][0], + grid=grid_anuspline, ) pr_anusplin = make_data_file_variable( df_anusplin_pr_seasonal, cell_methods="time: mean time: mean over days", variable_aliases=objects["variable_aliases"], var_name="pr", - grid=objects["grids"][0], + grid=grid_anuspline, ) tmin_canesm2_2050 = make_data_file_variable( df_canesm2_tasmin_2050_seasonal, @@ -317,21 +321,21 @@ def populateddb_thredds(cleandb,): cell_methods="time: maximum", variable_aliases=objects["variable_aliases"], var_name="tasmax", - grid=objects["grids"][0], + grid=grid_anuspline, ) tmin_canesm2_2080 = make_data_file_variable( df_canesm2_tasmin_2080_seasonal, cell_methods="time: minimum", variable_aliases=objects["variable_aliases"], var_name="tasmin", - grid=objects["grids"][0], + grid=grid_anuspline, ) tmax_canesm2_2080 = make_data_file_variable( df_canesm2_tasmax_2080_seasonal, cell_methods="time: maximum", variable_aliases=objects["variable_aliases"], var_name="tasmax", - grid=objects["grids"][0], + grid=grid_anuspline, ) var_names = ("tmin", "tmax") data_file_variables = [ @@ -344,7 +348,7 @@ def populateddb_thredds(cleandb,): # Associate to Ensembles for dfv in data_file_variables: - objects["ensembles"][0].data_file_variables.append(dfv) + p2a_rules.data_file_variables.append(dfv) sesh.add_all(sesh.dirty) # TimeSets @@ -453,10 +457,13 @@ def populateddb_thredds(cleandb,): @pytest.fixture() def populateddb_local(cleandb,): + """Create mock database only containing data files in this repository.""" populateable_db = cleandb sesh = populateable_db.session objects = create_modelmeta_objects() + p2a_rules = objects["ensembles"][0] + grid_anuspline = objects["grids"][0] models = [objects["anusplin"]] @@ -481,21 +488,21 @@ def populateddb_local(cleandb,): sesh.add_all(objects["grids"]) sesh.flush() - # DataFileVariable + # DataFileVariables tmin_anusplin_seasonal = make_data_file_variable( df_anusplin_tasmin_seasonal, cell_methods="time: minimum time: mean over days", variable_aliases=objects["variable_aliases"], var_name="tasmin", - grid=objects["grids"][0], + grid=grid_anuspline, ) tmax_anusplin_seasonal = make_data_file_variable( df_anusplin_tasmax_seasonal, cell_methods="time: maximum time: mean over days", variable_aliases=objects["variable_aliases"], var_name="tasmax", - grid=objects["grids"][0], + grid=grid_anuspline, ) var_names = ("tmin", "tmax") data_file_variables = [v for k, v in locals().items() if k.startswith(var_names)] @@ -506,7 +513,7 @@ def populateddb_local(cleandb,): # Associate to Ensembles for dfv in data_file_variables: - objects["ensembles"][0].data_file_variables.append(dfv) + p2a_rules.data_file_variables.append(dfv) sesh.add_all(sesh.dirty) # TimeSets