From be587cdc89e1676eb9b0cb5e0b8547e47ee8ab03 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Mon, 28 Jul 2025 13:48:42 -0600 Subject: [PATCH 1/8] Issue #72 - working on adding UNM GPR --- scripts/upload/add_unm_gpr.py | 76 ++++++------- snowex_db/point_data.py | 1 + .../point_primary_variable_overrides.yaml | 10 +- tests/data/csu_gpr.csv | 14 +++ tests/data/unm_gpr.csv | 2 - tests/points/test_gpr_unm.py | 103 ++++++++++++++++++ 6 files changed, 161 insertions(+), 45 deletions(-) create mode 100644 tests/data/csu_gpr.csv create mode 100644 tests/points/test_gpr_unm.py diff --git a/scripts/upload/add_unm_gpr.py b/scripts/upload/add_unm_gpr.py index 2d2b32e..f548b2e 100644 --- a/scripts/upload/add_unm_gpr.py +++ b/scripts/upload/add_unm_gpr.py @@ -10,65 +10,59 @@ import time from os.path import abspath, expanduser, join +from pathlib import Path import pandas as pd -from snowexsql.db import get_db +from snowexsql.db import get_db, db_session_with_credentials from snowex_db.upload import * +from snowex_db.upload.points import PointDataCSV +# Issue #72 + def main(): - filename = '../download/data/SNOWEX/SNEX20_UNM_GPR.001/2020.01.28/SNEX20_UNM_GPR.csv' + filename = ('../download/data/nsidc-cumulus-prod-protected/SNOWEX/' + 'SNEX20_UNM_GPR/1/2020/01/28/SNEX20_UNM_GPR.csv') kwargs = { - # Keyword argument to upload depth measurements - 'depth_is_metadata': False, - # Constant Metadata for the GPR data - 'site_name': 'Grand Mesa', - 'observers': 'Ryan Webb', - 'instrument': None, # See loop below - 'in_timezone': 'UTC', - 'out_timezone': 'UTC', + 'observer': 'Ryan Webb', 'doi': 'https://doi.org/10.5067/WE9GI1GVMQF6', - 'epsg': 26912 + 'campaign_name': 'Grand Mesa', + 'instrument': 'gpr', + # 'instrument_model': 'pulse EKKO Pro multi-polarization 1 GHz GPR', + 'timezone': 'UTC', + 'name': 'UNM GPR Data', } - # Break out the path and make it an absolute path - filename = abspath(expanduser(filename)) - # Grab a db connection to a local db named snowex - db_name = 'localhost/snowex' - engine, session = get_db(db_name, credentials='./credentials.json') + # Break out the path and make it an absolute path + file = Path(filename).expanduser().resolve() + # Make two files, filter by frequency # Read in for management of instruments df_raw = pd.read_csv(filename) - low_freq = df_raw['FREQ_MHz'] == 800 - hi_freq = df_raw['FREQ_MHz'] == 1600 - - # Instantiate the point uploader - csv = PointDataCSV(filename, **kwargs) - - # Convert depth to centimeters - csv.log.info('Converting depth to centimeters...') - csv.df['depth'] = csv.df['depth'].mul(100) - df_original = csv.df.copy() - - # Loop over the two insturments in the file and separate them for two submissions - for hz, ind in [(800, low_freq), (1600, hi_freq)]: - instrument = f'Mala {hz} MHz GPR' - csv.log.info(f'Isolating {instrument} data for upload.') - csv.df = df_original[ind].copy() - # Change the instrument. - csv.df['instrument'] = instrument - # Push it to the database - csv.submit(session) - - # Close out the session with the DB - session.close() - # return the number of errors for run.py can report it - return len(csv.errors) + # Grab a db connection + with db_session_with_credentials() as (_engine, session): + for freq in [800, 1600]: + new_file_name = file.parent.joinpath(f'{file.stem}_{freq}{file.suffix}') + # Filter the data by frequency + df_filtered = df_raw[df_raw['FREQ_MHz'] == freq] + # convert depth to cm + df_filtered['depth'] = df_filtered['depth'].mul(100) + # Save the filtered data to a new file + df_filtered.to_csv(new_file_name, index=False) + specific_kwargs = dict( + instrument_model=f'Mala {freq} MHz GPR' + ) + # Instantiate the point uploader + csv = PointDataCSV( + session, str(new_file_name), **{**kwargs, **specific_kwargs} + ) + # Push it to the database + csv.submit() if __name__ == '__main__': diff --git a/snowex_db/point_data.py b/snowex_db/point_data.py index bf29726..2e34b6c 100644 --- a/snowex_db/point_data.py +++ b/snowex_db/point_data.py @@ -245,6 +245,7 @@ def _read_csv( meta_parser.primary_variables.entries["UTCYEAR"], meta_parser.primary_variables.entries["UTM_ZONE"], meta_parser.primary_variables.entries["VERSION_NUMBER"], + meta_parser.primary_variables.entries["FREQUENCY"], ] shared_columns = [ diff --git a/snowex_db/point_primary_variable_overrides.yaml b/snowex_db/point_primary_variable_overrides.yaml index 8c21c19..7196d1b 100644 --- a/snowex_db/point_primary_variable_overrides.yaml +++ b/snowex_db/point_primary_variable_overrides.yaml @@ -13,7 +13,7 @@ BP: - bp_kpa_avg match_on_code: true DATE: - auto_remap: false + auto_remap: true code: date description: Measurement Date (only date column) map_from: @@ -167,7 +167,7 @@ SW_OUT: - sdn_avg match_on_code: true TIME: - auto_remap: false + auto_remap: true code: time description: Measurement time map_from: @@ -255,3 +255,9 @@ PIT_ID: - pit_id - pitid match_on_code: true +FREQUENCY: + auto_remap: true + code: frequency + description: Frequency of measurement + map_from: + - freq_mhz \ No newline at end of file diff --git a/tests/data/csu_gpr.csv b/tests/data/csu_gpr.csv new file mode 100644 index 0000000..7b8369c --- /dev/null +++ b/tests/data/csu_gpr.csv @@ -0,0 +1,14 @@ +Date [mmddyy],Time [HHMM],Longitude [DD],Latitude [DD],ElevationWGS84 [mae],Easting [m],Northing [m],UTM_Zone,TWT [ns],Depth [cm],SWE [mm] +020620,NaN,-108.176474337253,39.0192013494439,3047.4021717252,744448.66495925,4322701.1564092,12,7.348628,89.5675335280758,244.519366531647 +020620,NaN,-108.176477731632,39.0192008340717,3047.4004489707,744448.37281516,4322701.0900766,12,7.225466,88.0663939188338,240.421255398416 +020620,NaN,-108.176481162331,39.0192004664121,3047.3760773238,744448.07701698,4322701.0400424,12,7.102305,86.5652664979259,236.323177539338 +020620,NaN,-108.176484593025,39.0192000987253,3047.3517073054,744447.78121926,4322700.9900052,12,6.979144,85.0641390770181,232.225099680259 +020620,NaN,-108.17648792684,39.01919998347,3047.3289465061,744447.49294034,4322700.9682483,12,6.814929,83.0626317863629,226.760984776771 +020620,NaN,-108.176490159293,39.019201943951,3047.3229901536,744447.29287437,4322701.1798586,12,6.855982,83.5629994677761,228.126988547029 +020620,NaN,-108.176490747663,39.0192045818385,3047.3302044974,744447.23283544,4322701.4710812,12,6.855982,83.5629994677761,228.126988547029 +020620,NaN,-108.176490417839,39.0192072703924,3047.3457056512,744447.25212855,4322701.7703965,12,6.93809,84.5637592072707,230.859062635849 +020620,NaN,-108.176489927469,39.0192099392323,3047.3589162368,744447.28539142,4322702.0679552,12,7.102305,86.5652664979259,236.323177539338 +020620,NaN,-108.176489110114,39.0192125643129,3047.3633069293,744447.34711874,4322702.3615358,12,7.143359,87.0656463676733,237.689214583748 +020620,NaN,-108.176487628896,39.0192149999286,3047.3762636279,744447.46698324,4322702.6358708,12,7.225466,88.0663939188338,240.421255398416 +020620,NaN,-108.176485981931,39.0192173747429,3047.3837828919,744447.60140936,4322702.9039025,12,7.471789,91.0686609489837,248.617444390725 +020620,NaN,-108.176484243885,39.0192197112607,3047.3854843067,744447.7438542,4322703.1679282,12,7.759166,94.5713078488809,258.179670427445 \ No newline at end of file diff --git a/tests/data/unm_gpr.csv b/tests/data/unm_gpr.csv index 0be8121..2763001 100644 --- a/tests/data/unm_gpr.csv +++ b/tests/data/unm_gpr.csv @@ -3,5 +3,3 @@ DATE_dd_mmm_yy,TIME_GMT,FREQ_MHz,LONG,LAT,ELEV_m,NORTHING,EASTING,UTMzone,TWT_ns 29-Jan-20,20:35,800,-108.139506,39.014194,3106.94,4322247.041,747667.2868,12,7.34,0.9,244 30-Jan-20,22:20,800,-108.1622217,39.03095167,3110.8,4324045.489,745642.1231,12,9.55,1.17,318 31-Jan-20,20:04,800,-108.1762593,39.02676284,3090.19,4323542.671,744441.2436,12,8.73,1.07,291 -4-Feb-20,16:00,1600,-108.1391578,39.03117433,3141.17,4324132.814,747638.1351,12,7.27,0.89,242 -5-Feb-20,19:04,1600,-108.1715773,39.02401467,3103.3,4323250.231,744856.1064,12,6.15,0.75,205 diff --git a/tests/points/test_gpr_unm.py b/tests/points/test_gpr_unm.py new file mode 100644 index 0000000..c41ed43 --- /dev/null +++ b/tests/points/test_gpr_unm.py @@ -0,0 +1,103 @@ +from datetime import datetime, timezone, date + +import pytest +from geoalchemy2 import WKTElement +from snowexsql.tables import PointData, DOI, Campaign, Instrument, \ + MeasurementType, PointObservation +from snowexsql.tables.campaign_observation import CampaignObservation + +from snowex_db.upload.points import PointDataCSV + +from _base import PointBaseTesting + + +class TestGPR(PointBaseTesting): + """ + Test that a density file is uploaded correctly including sample + averaging for the main value. + """ + kwargs = { + 'timezone': "UTC", + 'doi': "some_gpr_point_doi", + "campaign_name": "Grand Mesa", + "name": "BSU GPR DATA", + "instrument": "gpr" + } + UploaderClass = PointDataCSV + TableClass = PointData + + @pytest.fixture(scope="class") + def uploaded_file(self, session, data_dir): + self.upload_file(session, str(data_dir.joinpath("bsu_gpr.csv"))) + + def filter_measurement_type(self, session, measurement_type, query=None): + if query is None: + query = session.query(self.TableClass) + + query = query.join( + self.TableClass.observation + ).join( + PointObservation.measurement_type + ).filter(MeasurementType.name == measurement_type) + return query + + @pytest.mark.parametrize( + "table, attribute, expected_value", [ + (Campaign, "name", "Grand Mesa"), + (Instrument, "name", "gpr"), + (Instrument, "model", None), + (MeasurementType, "name", ['two_way_travel', 'depth', "swe"]), + (MeasurementType, "units", ['ns', 'cm', 'mm']), + (MeasurementType, "derived", [False, False, False]), + (DOI, "doi", "some_gpr_point_doi"), + (CampaignObservation, "name", "BSU GPR DATA_gpr_two_way_travel"), + (PointData, "geom", + WKTElement('POINT (-108.190889311605 39.0343743775669)', srid=4326) + ), + (PointObservation, "date", date(2020, 1, 28)), + ] + ) + def test_metadata(self, table, attribute, expected_value, uploaded_file): + self._check_metadata(table, attribute, expected_value) + + @pytest.mark.parametrize( + "data_name, attribute_to_check, filter_attribute, filter_value, expected", [ + ('two_way_travel', 'value', 'date', date(2020, 1, 28), [8.3] * 8), + ('depth', 'value', 'date', date(2020, 1, 28), + [101.096735522092, 101.096735522092, 101.096735522092, 101.096735522092, 101.096735522092, 101.096735522092, 101.096735522092, 101.096735522092]), + ('swe', 'value', 'date', date(2020, 1, 28), + [275.994087975311, 275.994087975311, 275.994087975311, 275.994087975311, 275.994087975311, 275.994087975311, 275.994087975311, 275.994087975311]), + ] + ) + def test_value( + self, data_name, attribute_to_check, + filter_attribute, filter_value, expected, uploaded_file + ): + self.check_value( + data_name, attribute_to_check, + filter_attribute, filter_value, expected, + ) + + @pytest.mark.parametrize( + "data_name, expected", [ + ("depth", 12), + ("swe", 12), + ("two_way_travel", 12), + ("density", 0), # no measurements + ] + ) + def test_count(self, data_name, expected, uploaded_file): + n = self.check_count(data_name) + assert n == expected + + @pytest.mark.parametrize( + "data_name, attribute_to_count, expected", [ + ("depth", "value", 2), + ("swe", "value", 2), + ("swe", "units", 1) + ] + ) + def test_unique_count(self, data_name, attribute_to_count, expected, uploaded_file): + self.check_unique_count( + data_name, attribute_to_count, expected + ) From c676824e001354460a259105c9f5a59cabb3e3c8 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Mon, 28 Jul 2025 13:49:24 -0600 Subject: [PATCH 2/8] Testing #72 --- tests/points/test_gpr_unm.py | 41 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/tests/points/test_gpr_unm.py b/tests/points/test_gpr_unm.py index c41ed43..3c35ac1 100644 --- a/tests/points/test_gpr_unm.py +++ b/tests/points/test_gpr_unm.py @@ -17,18 +17,21 @@ class TestGPR(PointBaseTesting): averaging for the main value. """ kwargs = { - 'timezone': "UTC", - 'doi': "some_gpr_point_doi", - "campaign_name": "Grand Mesa", - "name": "BSU GPR DATA", - "instrument": "gpr" + # Constant Metadata for the GPR data + 'observer': 'Ryan Webb', + 'doi': 'https://doi.org/10.5067/WE9GI1GVMQF6', + 'campaign_name': 'Grand Mesa', + 'instrument': 'gpr', + 'instrument_model': f'Mala 800 MHz GPR', + 'timezone': 'UTC', + 'name': 'UNM GPR Data', } UploaderClass = PointDataCSV TableClass = PointData @pytest.fixture(scope="class") def uploaded_file(self, session, data_dir): - self.upload_file(session, str(data_dir.joinpath("bsu_gpr.csv"))) + self.upload_file(session, str(data_dir.joinpath("unm_gpr.csv"))) def filter_measurement_type(self, session, measurement_type, query=None): if query is None: @@ -45,14 +48,14 @@ def filter_measurement_type(self, session, measurement_type, query=None): "table, attribute, expected_value", [ (Campaign, "name", "Grand Mesa"), (Instrument, "name", "gpr"), - (Instrument, "model", None), + (Instrument, "model", "Mala 800 MHz GPR"), (MeasurementType, "name", ['two_way_travel', 'depth', "swe"]), (MeasurementType, "units", ['ns', 'cm', 'mm']), (MeasurementType, "derived", [False, False, False]), - (DOI, "doi", "some_gpr_point_doi"), - (CampaignObservation, "name", "BSU GPR DATA_gpr_two_way_travel"), + (DOI, "doi", "https://doi.org/10.5067/WE9GI1GVMQF6"), + (CampaignObservation, "name", "UNM GPR Data_gpr_Mala 800 MHz GPR_two_way_travel"), (PointData, "geom", - WKTElement('POINT (-108.190889311605 39.0343743775669)', srid=4326) + WKTElement('POINT (-108.1340183 39.0296597)', srid=4326) ), (PointObservation, "date", date(2020, 1, 28)), ] @@ -62,11 +65,9 @@ def test_metadata(self, table, attribute, expected_value, uploaded_file): @pytest.mark.parametrize( "data_name, attribute_to_check, filter_attribute, filter_value, expected", [ - ('two_way_travel', 'value', 'date', date(2020, 1, 28), [8.3] * 8), - ('depth', 'value', 'date', date(2020, 1, 28), - [101.096735522092, 101.096735522092, 101.096735522092, 101.096735522092, 101.096735522092, 101.096735522092, 101.096735522092, 101.096735522092]), - ('swe', 'value', 'date', date(2020, 1, 28), - [275.994087975311, 275.994087975311, 275.994087975311, 275.994087975311, 275.994087975311, 275.994087975311, 275.994087975311, 275.994087975311]), + ('two_way_travel', 'value', 'date', date(2020, 1, 28), [8.97]), + ('depth', 'value', 'date', date(2020, 1, 29), [0.9],), + ('swe', 'value', 'date', date(2020, 1, 31), [291]), ] ) def test_value( @@ -80,9 +81,9 @@ def test_value( @pytest.mark.parametrize( "data_name, expected", [ - ("depth", 12), - ("swe", 12), - ("two_way_travel", 12), + ("depth", 4), + ("swe", 4), + ("two_way_travel", 4), ("density", 0), # no measurements ] ) @@ -92,8 +93,8 @@ def test_count(self, data_name, expected, uploaded_file): @pytest.mark.parametrize( "data_name, attribute_to_count, expected", [ - ("depth", "value", 2), - ("swe", "value", 2), + ("depth", "value", 4), + ("swe", "value", 4), ("swe", "units", 1) ] ) From ff6013da6f93fbccac35f41a6483c8ee290456c7 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Mon, 28 Jul 2025 13:56:41 -0600 Subject: [PATCH 3/8] Issue #70 - testing CSU GPR upload --- scripts/download/nsidc_sources.txt | 4 +- scripts/upload/add_csu_gpr.py | 48 ++++++------- tests/data/csu_gpr.csv | 10 +-- tests/points/test_gpr_csu.py | 112 +++++++++++++++++++++++++++++ tests/points/test_gpr_unm.py | 2 +- 5 files changed, 137 insertions(+), 39 deletions(-) create mode 100644 tests/points/test_gpr_csu.py diff --git a/scripts/download/nsidc_sources.txt b/scripts/download/nsidc_sources.txt index 06888a9..195524f 100644 --- a/scripts/download/nsidc_sources.txt +++ b/scripts/download/nsidc_sources.txt @@ -3,8 +3,8 @@ https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_BSU_GPR.001/ https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_GM_SP.001/ https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_SMP.001/ https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_SD.001/ -https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_GM_CSU_GPR.001/2020.02.06/SNEX20_GM_CSU_GPR_1GHz_v01.csv -https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_UNM_GPR.001/2020.01.28/SNEX20_UNM_GPR.csv +https://data.nsidc.earthdatacloud.nasa.gov/nsidc-cumulus-prod-protected/SNOWEX/SNEX20_GM_CSU_GPR/1/2020/02/06/SNEX20_GM_CSU_GPR_1GHz_v01.csv +https://data.nsidc.earthdatacloud.nasa.gov/nsidc-cumulus-prod-protected/SNOWEX/SNEX20_UNM_GPR/1/2020/01/28/SNEX20_UNM_GPR.csv https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_SD_TLI.001/2019.09.29/SNEX20_SD_TLI_clean.csv https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX20_TS_SP.002/ https://n5eil01u.ecs.nsidc.org/SNOWEX/SNEX21_TS_SP.001/ diff --git a/scripts/upload/add_csu_gpr.py b/scripts/upload/add_csu_gpr.py index b2c30da..ccf9628 100644 --- a/scripts/upload/add_csu_gpr.py +++ b/scripts/upload/add_csu_gpr.py @@ -8,49 +8,43 @@ """ -import time -from os.path import abspath, expanduser, join +# Issue #70 -import pandas as pd +from os.path import abspath, expanduser -from snowexsql.db import get_db -from snowex_db.upload import * +from snowexsql.db import db_session_with_credentials +from snowex_db.upload.points import PointDataCSV def main(): - file = '../download/data/SNOWEX/SNEX20_GM_CSU_GPR.001/2020.02.06/SNEX20_GM_CSU_GPR_1GHz_v01.csv' + file = ( + '../download/data/nsidc-cumulus-prod-protected/SNOWEX/' + 'SNEX20_GM_CSU_GPR/1/2020/02/06/SNEX20_GM_CSU_GPR_1GHz_v01.csv' + ) kwargs = { - # Keyword argument to upload depth measurements - 'depth_is_metadata': False, - # Constant Metadata for the GPR data - 'site_name': 'Grand Mesa', - 'observers': 'Randall Bonnell', - 'instrument': 'pulse EKKO Pro multi-polarization 1 GHz GPR', - 'in_timezone': 'UTC', - 'out_timezone': 'UTC', + 'campaign_name': 'Grand Mesa', + 'observer': 'Randall Bonnell', + 'instrument': 'gpr', + 'instrument_model': 'pulse EKKO Pro multi-polarization 1 GHz GPR', + 'timezone': 'UTC', 'doi': 'https://doi.org/10.5067/S5EGFLCIAB18', - 'epsg': 26912 + 'name': 'CSU GPR Data', } # Break out the path and make it an absolute path file = abspath(expanduser(file)) - # Grab a db connection to a local db named snowex - db_name = 'localhost/snowex' - engine, session = get_db(db_name, credentials='./credentials.json') - - # Instantiate the point uploader - csv = PointDataCSV(file, **kwargs) - # Push it to the database - csv.submit(session) - - # Close out the session with the DB - session.close() + # Grab a db connection + with db_session_with_credentials() as (_engine, session): + # Instantiate the point uploader + csv = PointDataCSV(session, file, **kwargs) + # Push it to the database + csv.submit() # return the number of errors for run.py can report it - return len(csv.errors) + # return len(csv.errors) if __name__ == '__main__': diff --git a/tests/data/csu_gpr.csv b/tests/data/csu_gpr.csv index 7b8369c..3511d70 100644 --- a/tests/data/csu_gpr.csv +++ b/tests/data/csu_gpr.csv @@ -3,12 +3,4 @@ Date [mmddyy],Time [HHMM],Longitude [DD],Latitude [DD],ElevationWGS84 [mae],East 020620,NaN,-108.176477731632,39.0192008340717,3047.4004489707,744448.37281516,4322701.0900766,12,7.225466,88.0663939188338,240.421255398416 020620,NaN,-108.176481162331,39.0192004664121,3047.3760773238,744448.07701698,4322701.0400424,12,7.102305,86.5652664979259,236.323177539338 020620,NaN,-108.176484593025,39.0192000987253,3047.3517073054,744447.78121926,4322700.9900052,12,6.979144,85.0641390770181,232.225099680259 -020620,NaN,-108.17648792684,39.01919998347,3047.3289465061,744447.49294034,4322700.9682483,12,6.814929,83.0626317863629,226.760984776771 -020620,NaN,-108.176490159293,39.019201943951,3047.3229901536,744447.29287437,4322701.1798586,12,6.855982,83.5629994677761,228.126988547029 -020620,NaN,-108.176490747663,39.0192045818385,3047.3302044974,744447.23283544,4322701.4710812,12,6.855982,83.5629994677761,228.126988547029 -020620,NaN,-108.176490417839,39.0192072703924,3047.3457056512,744447.25212855,4322701.7703965,12,6.93809,84.5637592072707,230.859062635849 -020620,NaN,-108.176489927469,39.0192099392323,3047.3589162368,744447.28539142,4322702.0679552,12,7.102305,86.5652664979259,236.323177539338 -020620,NaN,-108.176489110114,39.0192125643129,3047.3633069293,744447.34711874,4322702.3615358,12,7.143359,87.0656463676733,237.689214583748 -020620,NaN,-108.176487628896,39.0192149999286,3047.3762636279,744447.46698324,4322702.6358708,12,7.225466,88.0663939188338,240.421255398416 -020620,NaN,-108.176485981931,39.0192173747429,3047.3837828919,744447.60140936,4322702.9039025,12,7.471789,91.0686609489837,248.617444390725 -020620,NaN,-108.176484243885,39.0192197112607,3047.3854843067,744447.7438542,4322703.1679282,12,7.759166,94.5713078488809,258.179670427445 \ No newline at end of file +020620,NaN,-108.17648792684,39.01919998347,3047.3289465061,744447.49294034,4322700.9682483,12,6.814929,83.0626317863629,226.760984776771 \ No newline at end of file diff --git a/tests/points/test_gpr_csu.py b/tests/points/test_gpr_csu.py new file mode 100644 index 0000000..1446304 --- /dev/null +++ b/tests/points/test_gpr_csu.py @@ -0,0 +1,112 @@ +from datetime import datetime, timezone, date + +import pytest +from geoalchemy2 import WKTElement +from snowexsql.tables import PointData, DOI, Campaign, Instrument, \ + MeasurementType, PointObservation +from snowexsql.tables.campaign_observation import CampaignObservation + +from snowex_db.upload.points import PointDataCSV + +from _base import PointBaseTesting + + +class TestCSUGPR(PointBaseTesting): + """ + Test that a density file is uploaded correctly including sample + averaging for the main value. + """ + kwargs = { + # Constant Metadata for the GPR data + 'campaign_name': 'Grand Mesa', + 'observer': 'Randall Bonnell', + 'instrument': 'gpr', + 'instrument_model': 'pulse EKKO Pro multi-polarization 1 GHz GPR', + 'timezone': 'UTC', + 'doi': 'https://doi.org/10.5067/S5EGFLCIAB18', + 'name': 'CSU GPR Data', + } + UploaderClass = PointDataCSV + TableClass = PointData + + @pytest.fixture(scope="class") + def uploaded_file(self, session, data_dir): + self.upload_file(session, str(data_dir.joinpath("csu_gpr.csv"))) + + def filter_measurement_type(self, session, measurement_type, query=None): + if query is None: + query = session.query(self.TableClass) + + query = query.join( + self.TableClass.observation + ).join( + PointObservation.measurement_type + ).filter(MeasurementType.name == measurement_type) + return query + + @pytest.mark.parametrize( + "table, attribute, expected_value", [ + (Campaign, "name", "Grand Mesa"), + (Instrument, "name", "gpr"), + (Instrument, "model", "pulse EKKO Pro multi-polarization 1 GHz GPR"), + (MeasurementType, "name", ['two_way_travel', 'depth', "swe"]), + (MeasurementType, "units", ['ns', 'cm', 'mm']), + (MeasurementType, "derived", [False, False, False]), + (DOI, "doi", "https://doi.org/10.5067/S5EGFLCIAB18"), + (CampaignObservation, "name", "CSU GPR Data_gpr_pulse EKKO Pro multi-polarization 1 GHz GPR_two_way_travel"), + (PointData, "geom", + WKTElement('POINT (-108.176474337253 39.0192013494439)', srid=4326) + ), + (PointObservation, "date", date(2020, 2, 6)), + ] + ) + def test_metadata(self, table, attribute, expected_value, uploaded_file): + self._check_metadata(table, attribute, expected_value) + + @pytest.mark.parametrize( + "data_name, attribute_to_check, filter_attribute, filter_value, expected", [ + ('two_way_travel', 'value', 'date', date(2020, 2, 6), [ + 7.348628, 7.225466, 7.102305, 6.979144, 6.814929 + ]), + ('depth', 'value', 'date', date(2020, 2, 6), [ + 89.5675335280758, 88.0663939188338, 86.5652664979259, + 85.0641390770181, 83.0626317863629 + ],), + ('swe', 'value', 'date', date(2020, 2, 6), [ + 244.519366531647, 240.421255398416, 236.323177539338, + 232.225099680259, 226.760984776771 + ]), + ] + ) + def test_value( + self, data_name, attribute_to_check, + filter_attribute, filter_value, expected, uploaded_file + ): + self.check_value( + data_name, attribute_to_check, + filter_attribute, filter_value, expected, + ) + + @pytest.mark.parametrize( + "data_name, expected", [ + ("depth", 5), + ("swe", 5), + ("two_way_travel", 5), + ("density", 0), # no measurements + ] + ) + def test_count(self, data_name, expected, uploaded_file): + n = self.check_count(data_name) + assert n == expected + + @pytest.mark.parametrize( + "data_name, attribute_to_count, expected", [ + ("depth", "value", 5), + ("swe", "value", 5), + ("swe", "units", 1) + ] + ) + def test_unique_count(self, data_name, attribute_to_count, expected, uploaded_file): + self.check_unique_count( + data_name, attribute_to_count, expected + ) diff --git a/tests/points/test_gpr_unm.py b/tests/points/test_gpr_unm.py index 3c35ac1..1a663fb 100644 --- a/tests/points/test_gpr_unm.py +++ b/tests/points/test_gpr_unm.py @@ -11,7 +11,7 @@ from _base import PointBaseTesting -class TestGPR(PointBaseTesting): +class TestUNMGPR(PointBaseTesting): """ Test that a density file is uploaded correctly including sample averaging for the main value. From 38e32860b13bbaa7b9195c27cbab9eb29ba0252c Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Mon, 28 Jul 2025 14:13:23 -0600 Subject: [PATCH 4/8] Issue #71 - I pulled the preliminary data here, still some questions --- scripts/upload/add_csu_gpr_AK.py | 50 ++++---- .../point_primary_variable_overrides.yaml | 2 + tests/data/csu_ak_gpr.csv | 6 + tests/points/test_gpr_csu_ak.py | 112 ++++++++++++++++++ 4 files changed, 143 insertions(+), 27 deletions(-) create mode 100644 tests/data/csu_ak_gpr.csv create mode 100644 tests/points/test_gpr_csu_ak.py diff --git a/scripts/upload/add_csu_gpr_AK.py b/scripts/upload/add_csu_gpr_AK.py index d56bb7b..2564530 100644 --- a/scripts/upload/add_csu_gpr_AK.py +++ b/scripts/upload/add_csu_gpr_AK.py @@ -7,12 +7,15 @@ """ +from os.path import abspath, expanduser from pathlib import Path -from snowexsql.db import get_db -from snowex_db.upload import PointDataCSV +from snowexsql.db import db_session_with_credentials +from snowex_db.upload.points import PointDataCSV import pandas as pd +# Issue #71 + def main(): file = Path('../download/data/SnowEx223_FLCF_1GHz_GPR_CSU.csv') @@ -23,43 +26,36 @@ def main(): modified = file.parent.joinpath(file.stem + f'_mod{file.suffix}') print(f"Removing Notes Column prior to upload. Writing to {modified}") - coi = [c for c in df.columns if c != 'Notes'] # No time is a problem. Use 12 AKST == 9pm (21:00) UTC - df['Time[HHMM]'] = '21:00' + # df['Time[HHMM]'] = '21:00' # Write out the modified version - df[coi].to_csv(modified, index=False) - + # df[coi].to_csv(modified, index=False) kwargs = { - # Keyword argument to upload depth measurements - 'depth_is_metadata': False, - # Constant Metadata for the GPR data - 'site_name': 'farmers-creamers', - 'observers': 'Randall Bonnell', - 'instrument': 'pulseEkko pro 1 GHz GPR', - 'in_timezone': 'UTC', - 'out_timezone': 'UTC', - 'doi': None, # Data is preliminary - 'epsg': 26906 + 'campaign_name': 'farmers-creamers', # TODO: should this be AK-something? + 'observer': 'Randall Bonnell', + 'instrument': 'gpr', + 'instrument_model': 'pulseEkko pro 1 GHz GPR', + 'timezone': 'UTC', + 'doi': None, # TODO: presumably this exists now? + 'name': 'CSU GPR Data', } - # Grab a db connection to a local db named snowex - db_name = 'localhost/snowex' - engine, session = get_db(db_name, credentials='./credentials.json') - - # Instantiate the point uploader - csv = PointDataCSV(modified, **kwargs) - # Push it to the database - csv.submit(session) + # Break out the path and make it an absolute path + file = abspath(expanduser(file)) - # Close out the session with the DB - session.close() + # Grab a db connection + with db_session_with_credentials() as (_engine, session): + # Instantiate the point uploader + csv = PointDataCSV(session, file, **kwargs) + # Push it to the database + csv.submit() # return the number of errors for run.py can report it - return len(csv.errors) + # return len(csv.errors) if __name__ == '__main__': diff --git a/snowex_db/point_primary_variable_overrides.yaml b/snowex_db/point_primary_variable_overrides.yaml index 7196d1b..6f6bbdb 100644 --- a/snowex_db/point_primary_variable_overrides.yaml +++ b/snowex_db/point_primary_variable_overrides.yaml @@ -235,6 +235,8 @@ CAMPAIGN: COMMENTS: code: comments description: Comments + map_from: + - notes match_on_code: true FLAGS: code: flags diff --git a/tests/data/csu_ak_gpr.csv b/tests/data/csu_ak_gpr.csv new file mode 100644 index 0000000..ddf9089 --- /dev/null +++ b/tests/data/csu_ak_gpr.csv @@ -0,0 +1,6 @@ +Date[mmddyy],Time[HHMM],Longitude[DD],Latitude[DD],ElevationWGS84[mae],Easting[m],Northing[m],UTM_Zone,TWT[ns],Depth[cm],SWE[mm],Density[kg m-3],Notes +030723,NaN,-147.737230798003,64.8638112503524,151.806903,465058.37445,7193480.256156,6,1.45,18.6201608827132,36.8679185477722,198,DN013 +030723,NaN,-147.737231429528,64.8638103942091,151.81797,465058.343408,7193480.161096,6,1.487265,19.098699017399,37.81542405445,198,DN013 +030723,NaN,-147.737232060693,64.8638095380227,151.829036,465058.312383,7193480.066031,6,1.524917,19.5822068088169,38.7727694814574,198,DN013 +030723,NaN,-147.73723268845,64.8638086814314,151.839986,465058.281519,7193479.970919,6,1.55,19.9043099091073,39.4105336200324,198,DN013 +030723,NaN,-147.737233145465,64.863807805451,151.844639,465058.258722,7193479.873552,6,1.562569,20.0657146002347,39.7301149084648,198,DN013 \ No newline at end of file diff --git a/tests/points/test_gpr_csu_ak.py b/tests/points/test_gpr_csu_ak.py new file mode 100644 index 0000000..6ee4148 --- /dev/null +++ b/tests/points/test_gpr_csu_ak.py @@ -0,0 +1,112 @@ +from datetime import datetime, timezone, date + +import pytest +from geoalchemy2 import WKTElement +from snowexsql.tables import PointData, DOI, Campaign, Instrument, \ + MeasurementType, PointObservation +from snowexsql.tables.campaign_observation import CampaignObservation + +from snowex_db.upload.points import PointDataCSV + +from _base import PointBaseTesting + + +class TestCSUAKGPR(PointBaseTesting): + """ + Test that a density file is uploaded correctly including sample + averaging for the main value. + """ + kwargs = { + # Constant Metadata for the GPR data + 'campaign_name': 'farmers-creamers', # TODO: should this be AK-something? + 'observer': 'Randall Bonnell', + 'instrument': 'gpr', + 'instrument_model': 'pulseEkko pro 1 GHz GPR', + 'timezone': 'UTC', + 'doi': "preliminary_gpr_ak_farmers-creamers", # TODO: presumably this exists now? + 'name': 'CSU GPR Data', + } + UploaderClass = PointDataCSV + TableClass = PointData + + @pytest.fixture(scope="class") + def uploaded_file(self, session, data_dir): + self.upload_file(session, str(data_dir.joinpath("csu_ak_gpr.csv"))) + + def filter_measurement_type(self, session, measurement_type, query=None): + if query is None: + query = session.query(self.TableClass) + + query = query.join( + self.TableClass.observation + ).join( + PointObservation.measurement_type + ).filter(MeasurementType.name == measurement_type) + return query + + @pytest.mark.parametrize( + "table, attribute, expected_value", [ + (Campaign, "name", "farmers-creamers"), + (Instrument, "name", "gpr"), + (Instrument, "model", "pulseEkko pro 1 GHz GPR"), + (MeasurementType, "name", ['two_way_travel', 'depth', "swe", "density"]), + (MeasurementType, "units", ['ns', 'cm', 'mm', "kg/m^3"]), + (MeasurementType, "derived", [False, False, False, False]), + (DOI, "doi", "preliminary_gpr_ak_farmers-creamers"), + (CampaignObservation, "name", "CSU GPR Data_gpr_pulseEkko pro 1 GHz GPR_two_way_travel"), + (PointData, "geom", + WKTElement('POINT (-147.737230798003 64.8638112503524)', srid=4326) + ), + (PointObservation, "date", date(2023, 3, 7)), + ] + ) + def test_metadata(self, table, attribute, expected_value, uploaded_file): + self._check_metadata(table, attribute, expected_value) + + @pytest.mark.parametrize( + "data_name, attribute_to_check, filter_attribute, filter_value, expected", [ + ('two_way_travel', 'value', 'date', date(2023, 3, 7), [ + 1.45, 1.487265, 1.524917, 1.55, 1.562569 + ]), + ('depth', 'value', 'date', date(2023, 3, 7), [ + 18.6201608827132, 19.098699017399, 19.5822068088169, + 19.9043099091073, 20.0657146002347 + ],), + ('swe', 'value', 'date', date(2023, 3, 7), [ + 36.8679185477722, 37.81542405445, 38.7727694814574, + 39.4105336200324, 39.7301149084648 + ]), + ] + ) + def test_value( + self, data_name, attribute_to_check, + filter_attribute, filter_value, expected, uploaded_file + ): + self.check_value( + data_name, attribute_to_check, + filter_attribute, filter_value, expected, + ) + + @pytest.mark.parametrize( + "data_name, expected", [ + ("depth", 5), + ("swe", 5), + ("two_way_travel", 5), + ("density", 5) + ] + ) + def test_count(self, data_name, expected, uploaded_file): + n = self.check_count(data_name) + assert n == expected + + @pytest.mark.parametrize( + "data_name, attribute_to_count, expected", [ + ("depth", "value", 5), + ("swe", "value", 5), + ("swe", "units", 1) + ] + ) + def test_unique_count(self, data_name, attribute_to_count, expected, uploaded_file): + self.check_unique_count( + data_name, attribute_to_count, expected + ) From c911eb492a68102accde2153e07c558d7672e626 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Mon, 28 Jul 2025 14:16:01 -0600 Subject: [PATCH 5/8] Clean up --- scripts/upload/add_csu_gpr_AK.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/scripts/upload/add_csu_gpr_AK.py b/scripts/upload/add_csu_gpr_AK.py index 2564530..3ba63fe 100644 --- a/scripts/upload/add_csu_gpr_AK.py +++ b/scripts/upload/add_csu_gpr_AK.py @@ -19,20 +19,6 @@ def main(): file = Path('../download/data/SnowEx223_FLCF_1GHz_GPR_CSU.csv') - # Fix quirks - df = pd.read_csv(file, dtype=str) - - # Upload is not able to handle the Notes col. So just remove it for now - modified = file.parent.joinpath(file.stem + f'_mod{file.suffix}') - print(f"Removing Notes Column prior to upload. Writing to {modified}") - - - # No time is a problem. Use 12 AKST == 9pm (21:00) UTC - # df['Time[HHMM]'] = '21:00' - - # Write out the modified version - # df[coi].to_csv(modified, index=False) - kwargs = { # Constant Metadata for the GPR data 'campaign_name': 'farmers-creamers', # TODO: should this be AK-something? From 08a67bfc275fdb7d89f79c97782219f74b8961a1 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Wed, 30 Jul 2025 10:58:17 -0600 Subject: [PATCH 6/8] Use absolute path --- scripts/upload/add_unm_gpr.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/upload/add_unm_gpr.py b/scripts/upload/add_unm_gpr.py index f548b2e..304159e 100644 --- a/scripts/upload/add_unm_gpr.py +++ b/scripts/upload/add_unm_gpr.py @@ -8,14 +8,11 @@ """ -import time -from os.path import abspath, expanduser, join from pathlib import Path import pandas as pd -from snowexsql.db import get_db, db_session_with_credentials -from snowex_db.upload import * +from snowexsql.db import db_session_with_credentials from snowex_db.upload.points import PointDataCSV @@ -36,9 +33,8 @@ def main(): 'name': 'UNM GPR Data', } - # Break out the path and make it an absolute path - file = Path(filename).expanduser().resolve() + file = Path(filename).absolute().resolve() # Make two files, filter by frequency # Read in for management of instruments From 43cc1d69f49ff64d2bb80310d955b429edf7a125 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Wed, 30 Jul 2025 11:04:52 -0600 Subject: [PATCH 7/8] Fix reference to depth in script --- scripts/upload/add_unm_gpr.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/upload/add_unm_gpr.py b/scripts/upload/add_unm_gpr.py index 304159e..d8e32a5 100644 --- a/scripts/upload/add_unm_gpr.py +++ b/scripts/upload/add_unm_gpr.py @@ -47,7 +47,9 @@ def main(): # Filter the data by frequency df_filtered = df_raw[df_raw['FREQ_MHz'] == freq] # convert depth to cm - df_filtered['depth'] = df_filtered['depth'].mul(100) + df_filtered.loc[:, ['DEPTH']] = df_filtered['DEPTH_m'].mul(100) + # Drop the original depth column + df_filtered = df_filtered.drop(columns=['DEPTH_m']) # Save the filtered data to a new file df_filtered.to_csv(new_file_name, index=False) specific_kwargs = dict( From 66d29605649b9b51e07cfb754ae4fc4a7c2c4137 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Thu, 31 Jul 2025 14:32:16 -0600 Subject: [PATCH 8/8] Change naming convention --- scripts/upload/add_csu_gpr.py | 4 ++-- scripts/upload/add_csu_gpr_AK.py | 4 ++-- scripts/upload/add_unm_gpr.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/upload/add_csu_gpr.py b/scripts/upload/add_csu_gpr.py index ccf9628..4bd267b 100644 --- a/scripts/upload/add_csu_gpr.py +++ b/scripts/upload/add_csu_gpr.py @@ -39,9 +39,9 @@ def main(): # Grab a db connection with db_session_with_credentials() as (_engine, session): # Instantiate the point uploader - csv = PointDataCSV(session, file, **kwargs) + uploader = PointDataCSV(session, file, **kwargs) # Push it to the database - csv.submit() + uploader.submit() # return the number of errors for run.py can report it # return len(csv.errors) diff --git a/scripts/upload/add_csu_gpr_AK.py b/scripts/upload/add_csu_gpr_AK.py index 3ba63fe..42e4d30 100644 --- a/scripts/upload/add_csu_gpr_AK.py +++ b/scripts/upload/add_csu_gpr_AK.py @@ -36,9 +36,9 @@ def main(): # Grab a db connection with db_session_with_credentials() as (_engine, session): # Instantiate the point uploader - csv = PointDataCSV(session, file, **kwargs) + uploader = PointDataCSV(session, file, **kwargs) # Push it to the database - csv.submit() + uploader.submit() # return the number of errors for run.py can report it # return len(csv.errors) diff --git a/scripts/upload/add_unm_gpr.py b/scripts/upload/add_unm_gpr.py index d8e32a5..8a80c79 100644 --- a/scripts/upload/add_unm_gpr.py +++ b/scripts/upload/add_unm_gpr.py @@ -56,11 +56,11 @@ def main(): instrument_model=f'Mala {freq} MHz GPR' ) # Instantiate the point uploader - csv = PointDataCSV( + uploader = PointDataCSV( session, str(new_file_name), **{**kwargs, **specific_kwargs} ) # Push it to the database - csv.submit() + uploader.submit() if __name__ == '__main__':