diff --git a/test_requirements.txt b/test_requirements.txt index 6542db0..a7255ec 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -3,3 +3,4 @@ pre-commit==2.7.1 pytest==6.0.2 pytest-cov==2.10.1 requests_mock==1.9 +mock==4.0.3 diff --git a/tests/conftest.py b/tests/conftest.py index 41af35e..e1994f6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,130 @@ from .mock_data import geoserver_data, tasmin_data, tasmax_data +# Functions to create data for mock databases + + +def create_modelmeta_objects(): + """Create modelmeta objects used for both mock databases.""" + + objects = {} + + # Ensembles + + p2a_rules = Ensemble(name="p2a_rules", version=1.0, changes="", description="") + ensembles = [ + p2a_rules, + ] + objects["ensembles"] = ensembles + + # Emissions + + historical = Emission(short_name="historical") + historical_rcp85 = Emission(short_name="historical,rcp85") + + # Runs + + run1 = Run(name="r1i1p1", emission=historical) + run2 = Run(name="r1i1p1", emission=historical_rcp85) + objects["run1"] = run1 + objects["run2"] = run2 + + # Models + + anusplin = Model(short_name="anusplin", type="GCM", runs=[run1], organization="") + canesm2 = Model(short_name="CanESM2", type="GCM", runs=[run2], organization="") + objects["anusplin"] = anusplin + objects["canesm2"] = canesm2 + + # VariableAliases + + tasmin = VariableAlias( + long_name="Daily Minimum Temperature", + standard_name="air_temperature", + units="degC", + ) + tasmax = VariableAlias( + long_name="Daily Maximum Temperature", + standard_name="air_temperature", + units="degC", + ) + pr = VariableAlias( + long_name="Precipitation", + standard_name="precipitation_flux", + units="kg d-1 m-2", + ) + flow_direction = VariableAlias( + long_name="Flow Direction", standard_name="flow_direction", units="1", + ) + variable_aliases = [ + tasmin, + tasmax, + pr, + flow_direction, + ] + objects["variable_aliases"] = variable_aliases + + # Grids + + grid_anuspline = Grid( + name="Canada ANUSPLINE", + xc_grid_step=0.0833333, + yc_grid_step=0.0833333, + xc_origin=-140.958, + yc_origin=41.0417, + xc_count=1068, + yc_count=510, + xc_units="degrees_east", + yc_units="degrees_north", + evenly_spaced_y=True, + ) + grids = [grid_anuspline] + objects["grids"] = grids + + return objects + + +def make_data_file( + filename=None, run=None, +): + if not filename.startswith("/"): + filename = resource_filename("tests", f"data/{filename}") + return DataFile( + filename=filename, + unique_id=filename, + first_1mib_md5sum="xxxx", + x_dim_name="lon", + y_dim_name="lat", + index_time=datetime.utcnow(), + run=run, + ) + + +def make_data_file_variable( + file, cell_methods, variable_aliases, var_name=None, grid=None, +): + (tasmin, tasmax, pr, flow_direction) = variable_aliases[:] + var_name_to_alias = { + "tasmin": tasmin, + "tasmax": tasmax, + "pr": pr, + "flow_direction": flow_direction, + }[var_name] + + return DataFileVariableGridded( + file=file, + netcdf_variable_name=var_name, + range_min=0, + range_max=50, + variable_alias=var_name_to_alias, + grid=grid, + variable_cell_methods=cell_methods, + ) + + +# Fixtures + + @pytest.fixture def mock_thredds_url_root(monkeypatch): monkeypatch.setenv( @@ -42,19 +166,19 @@ def ce_response(): } -@pytest.fixture(scope="function") +@pytest.fixture() def sessiondir(request,): dir = py.path.local(tempfile.mkdtemp()) request.addfinalizer(lambda: dir.remove(rec=1)) return dir -@pytest.fixture(scope="function") +@pytest.fixture() def dsn(sessiondir,): - return "sqlite:///{}".format(sessiondir.join("test.sqlite").realpath()) + return f"sqlite:///{sessiondir.join('test.sqlite').realpath()}" -@pytest.fixture +@pytest.fixture() def app(dsn,): app = get_app() app.config["TESTING"] = True @@ -63,7 +187,7 @@ def app(dsn,): return app -@pytest.fixture +@pytest.fixture() def cleandb(app,): db = SQLAlchemy(app) metadata.create_all(bind=db.engine) @@ -71,58 +195,20 @@ def cleandb(app,): return db -@pytest.fixture -def populateddb(cleandb,): - - now = datetime.utcnow() +@pytest.fixture() +def populateddb_thredds(cleandb,): + """Create mock database only containing data files from THREDDS.""" populateable_db = cleandb sesh = populateable_db.session + objects = create_modelmeta_objects() + p2a_rules = objects["ensembles"][0] + grid_anuspline = objects["grids"][0] - # Ensembles - - p2a_rules = Ensemble(name="p2a_rules", version=1.0, changes="", description="") - ensembles = [ - p2a_rules, - ] - - # Emissions - historical = Emission(short_name="historical") - historical_rcp85 = Emission(short_name="historical,rcp85") - - # Runs - run1 = Run(name="r1i1p1", emission=historical) - run2 = Run(name="r1i1p1", emission=historical_rcp85) - - # Models - - bnu = Model(short_name="BNU-ESM", type="GCM", runs=[run1], organization="BNU") - anusplin = Model(short_name="anusplin", type="GCM", runs=[run1], organization="") - canesm2 = Model(short_name="CanESM2", type="GCM", runs=[run2], organization="") - models = [bnu, anusplin, canesm2] + models = [objects["anusplin"], objects["canesm2"]] # Data files - def make_data_file( - filename=None, run=None, - ): - if not filename.startswith("/"): - filename = resource_filename("ce", "tests/data/{}".format(filename)) - return DataFile( - filename=filename, - unique_id=filename, - first_1mib_md5sum="xxxx", - x_dim_name="lon", - y_dim_name="lat", - index_time=now, - run=run, - ) - - df_bnu_seasonal = make_data_file( - filename="tasmin_sClim_BNU-ESM_historical_r1i1p1_19650101-19701230.nc", - run=run1, - ) # Only local file (only used to test file collector). All other files from THREDDS. - storage_root_anusplin = ( "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/" ) @@ -133,170 +219,123 @@ def make_data_file( df_anusplin_tasmin_seasonal = make_data_file( filename=storage_root_anusplin + "tasmin_sClimMean_anusplin_historical_19710101-20001231.nc", - run=run1, + run=objects["run1"], ) df_anusplin_tasmax_seasonal = make_data_file( filename=storage_root_anusplin + "tasmax_sClimMean_anusplin_historical_19710101-20001231.nc", - run=run1, + run=objects["run1"], ) df_anusplin_tasmin_mon = make_data_file( filename=storage_root_anusplin + "tasmin_mClimMean_anusplin_historical_19710101-20001231.nc", - run=run1, + run=objects["run1"], ) df_anusplin_tasmax_mon = make_data_file( filename=storage_root_anusplin + "tasmax_mClimMean_anusplin_historical_19710101-20001231.nc", - run=run1, + run=objects["run1"], ) df_anusplin_pr_seasonal = make_data_file( filename=storage_root_anusplin + "pr_sClimMean_anusplin_historical_19710101-20001231.nc", - run=run1, + run=objects["run1"], ) df_canesm2_tasmin_2050_seasonal = make_data_file( filename=canesm2_tasmin_root + "tasmin_sClim_BCCAQv2_CanESM2_historical+rcp85_r1i1p1_20400101-20691231_Canada.nc", - run=run2, + run=objects["run2"], ) df_canesm2_tasmax_2050_seasonal = make_data_file( filename=canesm2_tasmax_root + "tasmax_sClim_BCCAQv2_CanESM2_historical+rcp85_r1i1p1_20400101-20691231_Canada.nc", - run=run2, + run=objects["run2"], ) df_canesm2_tasmin_2080_seasonal = make_data_file( filename=canesm2_tasmin_root + "tasmin_sClim_BCCAQv2_CanESM2_historical+rcp85_r1i1p1_20700101-20991231_Canada.nc", - run=run2, + run=objects["run2"], ) df_canesm2_tasmax_2080_seasonal = make_data_file( filename=canesm2_tasmax_root + "tasmax_sClim_BCCAQv2_CanESM2_historical+rcp85_r1i1p1_20700101-20991231_Canada.nc", - run=run2, + run=objects["run2"], ) data_files = [v for k, v in locals().items() if k.startswith("df")] - # VariableAlias - - tasmin = VariableAlias( - long_name="Daily Minimum Temperature", - standard_name="air_temperature", - units="degC", - ) - tasmax = VariableAlias( - long_name="Daily Maximum Temperature", - standard_name="air_temperature", - units="degC", - ) - pr = VariableAlias( - long_name="Precipitation", - standard_name="precipitation_flux", - units="kg d-1 m-2", - ) - flow_direction = VariableAlias( - long_name="Flow Direction", standard_name="flow_direction", units="1", - ) - variable_aliases = [ - tasmin, - tasmax, - pr, - flow_direction, - ] - - # Grids - - grid_anuspline = Grid( - name="Canada ANUSPLINE", - xc_grid_step=0.0833333, - yc_grid_step=0.0833333, - xc_origin=-140.958, - yc_origin=41.0417, - xc_count=1068, - yc_count=510, - xc_units="degrees_east", - yc_units="degrees_north", - evenly_spaced_y=True, - ) - grids = [grid_anuspline] - # Add all the above - sesh.add_all(ensembles) + sesh.add_all(objects["ensembles"]) sesh.add_all(models) sesh.add_all(data_files) - sesh.add_all(variable_aliases) - sesh.add_all(grids) + sesh.add_all(objects["variable_aliases"]) + sesh.add_all(objects["grids"]) sesh.flush() - # DataFileVariable - - def make_data_file_variable( - file, cell_methods, var_name=None, grid=grid_anuspline, - ): - var_name_to_alias = { - "tasmin": tasmin, - "tasmax": tasmax, - "pr": pr, - "flow_direction": flow_direction, - }[var_name] - return DataFileVariableGridded( - file=file, - netcdf_variable_name=var_name, - range_min=0, - range_max=50, - variable_alias=var_name_to_alias, - grid=grid, - variable_cell_methods=cell_methods, - ) - - tmax_bnu = make_data_file_variable( - df_bnu_seasonal, cell_methods="time: maximum", var_name="tasmin", - ) + # DataFileVariables + tmin_anusplin_seasonal = make_data_file_variable( df_anusplin_tasmin_seasonal, cell_methods="time: minimum time: mean over days", + variable_aliases=objects["variable_aliases"], var_name="tasmin", + grid=grid_anuspline, ) tmax_anusplin_seasonal = make_data_file_variable( df_anusplin_tasmax_seasonal, cell_methods="time: maximum time: mean over days", + variable_aliases=objects["variable_aliases"], var_name="tasmax", + grid=grid_anuspline, ) tmin_anusplin_mon = make_data_file_variable( df_anusplin_tasmin_mon, cell_methods="time: minimum time: mean over days", + variable_aliases=objects["variable_aliases"], var_name="tasmin", + grid=grid_anuspline, ) tmax_anusplin_mon = make_data_file_variable( df_anusplin_tasmax_mon, cell_methods="time: minimum time: mean over days", + variable_aliases=objects["variable_aliases"], var_name="tasmax", + grid=grid_anuspline, ) pr_anusplin = make_data_file_variable( df_anusplin_pr_seasonal, cell_methods="time: mean time: mean over days", + variable_aliases=objects["variable_aliases"], var_name="pr", + grid=grid_anuspline, ) tmin_canesm2_2050 = make_data_file_variable( df_canesm2_tasmin_2050_seasonal, cell_methods="time: minimum", + variable_aliases=objects["variable_aliases"], var_name="tasmin", + grid=objects["grids"][0], ) tmax_canesm2_2050 = make_data_file_variable( df_canesm2_tasmax_2050_seasonal, cell_methods="time: maximum", + variable_aliases=objects["variable_aliases"], var_name="tasmax", + grid=grid_anuspline, ) tmin_canesm2_2080 = make_data_file_variable( df_canesm2_tasmin_2080_seasonal, cell_methods="time: minimum", + variable_aliases=objects["variable_aliases"], var_name="tasmin", + grid=grid_anuspline, ) tmax_canesm2_2080 = make_data_file_variable( df_canesm2_tasmax_2080_seasonal, cell_methods="time: maximum", + variable_aliases=objects["variable_aliases"], var_name="tasmax", + grid=grid_anuspline, ) var_names = ("tmin", "tmax") data_file_variables = [ @@ -397,7 +436,6 @@ def make_data_file_variable( ], ) ts_hist_seasonal.files = [ - df_bnu_seasonal, df_anusplin_tasmin_seasonal, df_anusplin_tasmax_seasonal, df_anusplin_pr_seasonal, @@ -417,6 +455,98 @@ def make_data_file_variable( return populateable_db +@pytest.fixture() +def populateddb_local(cleandb,): + """Create mock database only containing data files in this repository.""" + + populateable_db = cleandb + sesh = populateable_db.session + objects = create_modelmeta_objects() + p2a_rules = objects["ensembles"][0] + grid_anuspline = objects["grids"][0] + + models = [objects["anusplin"]] + + # Data files + + df_anusplin_tasmin_seasonal = make_data_file( + filename="tasmin_sClimMean_anusplin_historical_19710101-20001231.nc", + run=objects["run1"], + ) + df_anusplin_tasmax_seasonal = make_data_file( + filename="tasmax_sClimMean_anusplin_historical_19710101-20001231.nc", + run=objects["run1"], + ) + data_files = [v for k, v in locals().items() if k.startswith("df")] + + # Add all the above + + sesh.add_all(objects["ensembles"]) + sesh.add_all(models) + sesh.add_all(data_files) + sesh.add_all(objects["variable_aliases"]) + sesh.add_all(objects["grids"]) + sesh.flush() + + # DataFileVariables + + tmin_anusplin_seasonal = make_data_file_variable( + df_anusplin_tasmin_seasonal, + cell_methods="time: minimum time: mean over days", + variable_aliases=objects["variable_aliases"], + var_name="tasmin", + grid=grid_anuspline, + ) + tmax_anusplin_seasonal = make_data_file_variable( + df_anusplin_tasmax_seasonal, + cell_methods="time: maximum time: mean over days", + variable_aliases=objects["variable_aliases"], + var_name="tasmax", + grid=grid_anuspline, + ) + var_names = ("tmin", "tmax") + data_file_variables = [v for k, v in locals().items() if k.startswith(var_names)] + + sesh.add_all(data_file_variables) + sesh.flush() + + # Associate to Ensembles + + for dfv in data_file_variables: + p2a_rules.data_file_variables.append(dfv) + sesh.add_all(sesh.dirty) + + # TimeSets + + ts_hist_seasonal = TimeSet( + calendar="standard", + start_date=datetime(1971, 1, 1), + end_date=datetime(2000, 12, 31), + multi_year_mean=True, + num_times=4, + time_resolution="seasonal", + times=[ + Time(time_idx=i, timestep=datetime(1986, 3 * i + 1, 16)) for i in range(4) + ], + climatological_times=[ + ClimatologicalTime( + time_idx=i, + time_start=datetime(1971, 3 * i + 1, 1) - relativedelta(months=1), + time_end=datetime(2000, 3 * i + 1, 1) + relativedelta(months=2), + ) + for i in range(4) + ], + ) + ts_hist_seasonal.files = [ + df_anusplin_tasmin_seasonal, + df_anusplin_tasmax_seasonal, + ] + sesh.add_all(sesh.dirty) + + sesh.commit() + return populateable_db + + @pytest.fixture() def mock_urls(requests_mock): requests_mock.register_uri( diff --git a/tests/mock_data.py b/tests/mock_data.py index d156f12..8369c35 100644 --- a/tests/mock_data.py +++ b/tests/mock_data.py @@ -1,4 +1,5 @@ from pkg_resources import resource_filename +from netCDF4 import Dataset geoserver_data = open(resource_filename("tests", "data/geoserver_van.txt"), "rb").read() @@ -10,5 +11,7 @@ def get_nc_data(filename): return filedata -tasmin_data = get_nc_data("tasmin_sClimMean_anusplin_historical_19710101-20001231.nc") -tasmax_data = get_nc_data("tasmax_sClimMean_anusplin_historical_19710101-20001231.nc") +tasmin_filename = "tasmin_sClimMean_anusplin_historical_19710101-20001231.nc" +tasmax_filename = "tasmax_sClimMean_anusplin_historical_19710101-20001231.nc" +tasmin_data = get_nc_data(tasmin_filename) +tasmax_data = get_nc_data(tasmax_filename) diff --git a/tests/test_file_collector.py b/tests/test_file_collector.py index a199eea..a0db3c4 100644 --- a/tests/test_file_collector.py +++ b/tests/test_file_collector.py @@ -33,12 +33,12 @@ ], ), ) -def test_get_paths_by_var(populateddb, ensemble, date, area, variables): - sesh = populateddb.session +def test_get_paths_by_var(populateddb_local, ensemble, date, area, variables): + sesh = populateddb_local.session logger = setup_logging("ERROR") for name, values in variables.items(): paths = get_paths_by_var(sesh, values, ensemble, date, area, False, logger) for path in paths: - assert "/ce/tests/data/" in path or "/storage/data/" in path + assert "p2a-rule-engine/tests/data/" in path diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 5e72ab4..f57fedb 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -4,8 +4,29 @@ from p2a_impacts.resolver import resolve_rules from p2a_impacts.utils import get_region +import os +import requests +import mock +import netCDF4 +from netCDF4 import Dataset +from .mock_data import tasmin_data, tasmax_data + + +def mock_opendap_request(path, mode="r"): + return Dataset(resource_filename("tests", f"data/{os.path.basename(path)}"), "r") + + +@mock.patch("netCDF4.Dataset", side_effect=mock_opendap_request) +def test_mock_opendap_request(mock_opendap_request, mock_thredds_url_root): + base_path_tasmin = "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/tasmin_sClimMean_anusplin_historical_19710101-20001231.nc" + dods_path_tasmin = os.getenv("THREDDS_URL_ROOT") + base_path_tasmin + tasmin = netCDF4.Dataset(dods_path_tasmin) + base_path_tasmax = "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/tasmax_sClimMean_anusplin_historical_19710101-20001231.nc" + dods_path_tasmax = os.getenv("THREDDS_URL_ROOT") + base_path_tasmax + tasmax = netCDF4.Dataset(dods_path_tasmax) + assert mock_opendap_request.call_count == 2 + -@pytest.mark.slow @pytest.mark.parametrize( ("csv", "date_range", "region", "geoserver", "ensemble", "thredds"), [ @@ -19,8 +40,10 @@ ), ], ) +@mock.patch("ce.api.util.Dataset", side_effect=mock_opendap_request) def test_resolve_rules_basic( - populateddb, + mock_opendap_request, + populateddb_thredds, mock_thredds_url_root, mock_urls, csv, @@ -30,7 +53,7 @@ def test_resolve_rules_basic( ensemble, thredds, ): - sesh = populateddb.session + sesh = populateddb_thredds.session rules = resolve_rules( csv, date_range, get_region(region, geoserver), ensemble, sesh, thredds ) @@ -54,7 +77,7 @@ def test_resolve_rules_basic( ) @pytest.mark.parametrize("date_range", ["2050", "2080"]) def test_resolve_rules_multi_percentile( - populateddb, + populateddb_thredds, mock_thredds_url_root, csv, date_range, @@ -63,7 +86,7 @@ def test_resolve_rules_multi_percentile( ensemble, thredds, ): - sesh = populateddb.session + sesh = populateddb_thredds.session rules = resolve_rules( csv, date_range, get_region(region, geoserver), ensemble, sesh, thredds ) @@ -91,7 +114,7 @@ def test_resolve_rules_multi_percentile( ], ) def test_resolve_rules_multi_var( - populateddb, + populateddb_thredds, mock_thredds_url_root, csv, date_range, @@ -100,9 +123,48 @@ def test_resolve_rules_multi_var( ensemble, thredds, ): - sesh = populateddb.session + sesh = populateddb_thredds.session rules = resolve_rules( csv, date_range, get_region(region, geoserver), ensemble, sesh, thredds ) expected_rules = {"rule_shm": 65.807} assert round(rules["rule_shm"], 3) == expected_rules["rule_shm"] + + +@pytest.mark.parametrize( + ("csv", "date_range", "region", "geoserver", "ensemble", "thredds"), + [ + ( + resource_filename("tests", "data/rules-basic.csv"), + "hist", + "vancouver_island", + "http://docker-dev01.pcic.uvic.ca:30123/geoserver/bc_regions/ows", + "p2a_rules", + False, + ), + ], +) +def test_resolve_rules_local( + populateddb_local, mock_urls, csv, date_range, region, geoserver, ensemble, thredds, +): + sesh = populateddb_local.session + rules = resolve_rules( + csv, date_range, get_region(region, geoserver), ensemble, sesh, thredds + ) + expected_rules = {"rule_snow": True, "rule_hybrid": True, "rule_rain": True} + assert rules == expected_rules + + +def test_mock_urls(mock_thredds_url_root, mock_urls): + base_path_tasmin = "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/tasmin_sClimMean_anusplin_historical_19710101-20001231.nc" + fileserver_path_tasmin = ( + "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets" + + base_path_tasmin + ) + base_path_tasmax = "/storage/data/climate/downscale/BCCAQ2/ANUSPLIN/climatologies/tasmax_sClimMean_anusplin_historical_19710101-20001231.nc" + fileserver_path_tasmax = ( + "https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets" + + base_path_tasmax + ) + assert requests.get(fileserver_path_tasmin).content == tasmin_data + assert requests.get(fileserver_path_tasmax).content == tasmax_data