From e8577535cee6949a1feb84b281622ee6f31be411 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Tue, 1 Jul 2025 14:52:42 -0600 Subject: [PATCH 01/25] Working parsing of BSU GPR data --- snowex_db/point_data.py | 32 +++++------ snowex_db/upload/points.py | 23 ++++++-- tests/data/bsu_gpr.csv | 13 +++++ tests/points/test_gpr_bsu.py | 103 +++++++++++++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 24 deletions(-) create mode 100644 tests/data/bsu_gpr.csv create mode 100644 tests/points/test_gpr_bsu.py diff --git a/snowex_db/point_data.py b/snowex_db/point_data.py index 37cbd35..668f742 100644 --- a/snowex_db/point_data.py +++ b/snowex_db/point_data.py @@ -80,7 +80,8 @@ def read_csv_dataframe(profile_filename, columns, header_position): profile_filename, header=0, skiprows=header_position, names=columns, - encoding='latin' + encoding='latin', + dtype=str # treat all columns as strings to get weird date format ) if "flags" in df.columns: # Max length of the flags column @@ -106,9 +107,7 @@ def _get_location(self, row): else: raise RuntimeError("No valid location found in row or metadata.") - row["latitude"] = lat - row["longitude"] = lon - return row + return lat, lon def _get_datetime(self, row): """ @@ -142,18 +141,7 @@ def _get_datetime(self, row): result = self.metadata.date_time else: raise e - row["datetime"] = result - return row - - @classmethod - def _get_campaign(cls, row): - """ - fill in the campaign info for a row - Args: - row: pandas row - """ - row["campaign"] = row.get(YamlCodes.SITE_NAME) - return row + return result def _format_df(self, input_df): """ @@ -166,11 +154,17 @@ def _format_df(self, input_df): # Verify the sample column exists and rename to variable df = self._check_sample_columns(input_df) - df = df.apply(self._get_campaign, axis=1) + # Get the campaign name + df["campaign"] = df.get(YamlCodes.SITE_NAME) + # TODO: How do we speed this up? + # campaign should be very quick with a df level logic + # but the other ones will take morelogic # parse the location - df = df.apply(self._get_location, axis=1) + df[["latitude", "longitude"]] = df.apply( + self._get_location, axis=1, result_type="expand" + ) # Parse the datetime - df = df.apply(self._get_datetime, axis=1) + df["datetime"] = df.apply(self._get_datetime, axis=1, result_type="expand") location = gpd.points_from_xy( df["longitude"], df["latitude"] diff --git a/snowex_db/upload/points.py b/snowex_db/upload/points.py index 244957a..ce7983c 100644 --- a/snowex_db/upload/points.py +++ b/snowex_db/upload/points.py @@ -28,11 +28,6 @@ class DataValidationError(ValueError): pass -# TODO: do we need to make a SnowExPointDataCollection similar to -# SnowExProfileDataCollection, since some files will have more than one point -# measurement per file? This is true for GPR, summary swe, etc -# TODO: start with test datasets for simpler examples - class PointDataCSV(BaseUpload): """ @@ -51,6 +46,24 @@ class PointDataCSV(BaseUpload): } def __init__(self, profile_filename, timezone="US/Mountain", **kwargs): + """ + + Args: + profile_filename: + timezone: + **kwargs: + doi + instrument + header_sep + id + campaign_name + derived + instrument_model + comments + observer + name + row_based_timezone + """ self.filename = profile_filename self._timezone = timezone self._doi = kwargs.get("doi") diff --git a/tests/data/bsu_gpr.csv b/tests/data/bsu_gpr.csv new file mode 100644 index 0000000..a1adad0 --- /dev/null +++ b/tests/data/bsu_gpr.csv @@ -0,0 +1,13 @@ +Date,Time,Longitude,Latitude,ElevationWGS84,Easting,Northing,UTM_Zone,TWT,Depth,SWE +012820,161549.557,-108.190889311605,39.0343743775669,3040.469,743148.428,4324346.715,12,8.3,101.096735522092,275.994087975311 +012820,161549.59,-108.190889588925,39.0343743752416,3040.46,743148.404,4324346.714,12,8.3,101.096735522092,275.994087975311 +012820,161549.623,-108.190889843164,39.0343743723601,3040.451,743148.382,4324346.713,12,8.3,101.096735522092,275.994087975311 +012820,161549.656,-108.19089008622,39.0343743602,3040.442,743148.361,4324346.711,12,8.3,101.096735522092,275.994087975311 +012820,161549.689,-108.190890306195,39.0343743474837,3040.434,743148.342,4324346.709,12,8.3,101.096735522092,275.994087975311 +012820,161549.721,-108.190890514631,39.0343743344892,3040.427,743148.324,4324346.707,12,8.3,101.096735522092,275.994087975311 +012820,161549.754,-108.190890711883,39.0343743122161,3040.419,743148.307,4324346.704,12,8.3,101.096735522092,275.994087975311 +012820,161549.787,-108.190890897238,39.0343742986654,3040.413,743148.291,4324346.702,12,8.3,101.096735522092,275.994087975311 +020420,205415.639,-108.165976850641,39.0171321212163,3078.752,745364.801,4322499.769,12,8.4,102.31476848019,279.319317950918 +020420,205415.672,-108.16597604275,39.017132687161,3078.76,745364.869,4322499.834,12,8.4,102.31476848019,279.319317950918 +020420,205415.705,-108.165975246395,39.0171332533861,3078.768,745364.936,4322499.899,12,8.4,102.31476848019,279.319317950918 +020420,205415.738,-108.165974438503,39.0171338193308,3078.775,745365.004,4322499.964,12,8.4,102.31476848019,279.319317950918 \ No newline at end of file diff --git a/tests/points/test_gpr_bsu.py b/tests/points/test_gpr_bsu.py new file mode 100644 index 0000000..1582af1 --- /dev/null +++ b/tests/points/test_gpr_bsu.py @@ -0,0 +1,103 @@ +from datetime import datetime, timezone, date + +import pytest +from geoalchemy2 import WKTElement +from snowexsql.tables import PointData, DOI, Campaign, Instrument, \ + MeasurementType, PointObservation +from snowexsql.tables.campaign_observation import CampaignObservation + +from snowex_db.upload.points import PointDataCSV + +from _base import PointBaseTesting + + +class TestGPR(PointBaseTesting): + """ + Test that a density file is uploaded correctly including sample + averaging for the main value. + """ + kwargs = { + 'timezone': "UTC", + 'doi': "some_gpr_point_doi", + "campaign_name": "Grand Mesa", + "name": "BSU GPR DATA", + "instrument": "gpr" + } + UploaderClass = PointDataCSV + TableClass = PointData + + @pytest.fixture(scope="class") + def uploaded_file(self, db, data_dir): + self.upload_file(str(data_dir.joinpath("bsu_gpr.csv"))) + + def filter_measurement_type(self, session, measurement_type, query=None): + if query is None: + query = session.query(self.TableClass) + + query = query.join( + self.TableClass.observation + ).join( + PointObservation.measurement_type + ).filter(MeasurementType.name == measurement_type) + return query + + @pytest.mark.parametrize( + "table, attribute, expected_value", [ + (Campaign, "name", "Grand Mesa"), + (Instrument, "name", "gpr"), + (Instrument, "model", None), + (MeasurementType, "name", ['two_way_travel', 'depth', "swe"]), + (MeasurementType, "units", ['ns', 'cm', 'mm']), + (MeasurementType, "derived", [False, False, False]), + (DOI, "doi", "some_gpr_point_doi"), + (CampaignObservation, "name", "BSU GPR DATA_gpr_two_way_travel"), + (PointData, "geom", + WKTElement('POINT (-108.190889311605 39.0343743775669)', srid=4326) + ), + (PointObservation, "date", date(2020, 1, 28)), + ] + ) + def test_metadata(self, table, attribute, expected_value, uploaded_file): + self._check_metadata(table, attribute, expected_value) + + @pytest.mark.parametrize( + "data_name, attribute_to_check, filter_attribute, filter_value, expected", [ + ('two_way_travel', 'value', 'date', date(2020, 1, 28), [8.3] * 8), + ('depth', 'value', 'date', date(2020, 1, 28), + [101.096735522092, 101.096735522092, 101.096735522092, 101.096735522092, 101.096735522092, 101.096735522092, 101.096735522092, 101.096735522092]), + ('swe', 'value', 'date', date(2020, 1, 28), + [275.994087975311, 275.994087975311, 275.994087975311, 275.994087975311, 275.994087975311, 275.994087975311, 275.994087975311, 275.994087975311]), + ] + ) + def test_value( + self, data_name, attribute_to_check, + filter_attribute, filter_value, expected, uploaded_file + ): + self.check_value( + data_name, attribute_to_check, + filter_attribute, filter_value, expected, + ) + + @pytest.mark.parametrize( + "data_name, expected", [ + ("depth", 12), + ("swe", 12), + ("two_way_travel", 12), + ("density", 0), # no measurements + ] + ) + def test_count(self, data_name, expected, uploaded_file): + n = self.check_count(data_name) + assert n == expected + + @pytest.mark.parametrize( + "data_name, attribute_to_count, expected", [ + ("depth", "value", 2), + ("swe", "value", 2), + ("swe", "units", 1) + ] + ) + def test_unique_count(self, data_name, attribute_to_count, expected, uploaded_file): + self.check_unique_count( + data_name, attribute_to_count, expected + ) From 1da6bbfeff523894457b2ce80c9b48e381c547ee Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Tue, 1 Jul 2025 14:54:27 -0600 Subject: [PATCH 02/25] Issue #61 - script for bsu gpr --- scripts/upload/add_bsu_gpr.py | 48 ++++++++++++++--------------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/scripts/upload/add_bsu_gpr.py b/scripts/upload/add_bsu_gpr.py index 26d7e77..ed12a87 100644 --- a/scripts/upload/add_bsu_gpr.py +++ b/scripts/upload/add_bsu_gpr.py @@ -8,49 +8,39 @@ """ -import time -from os.path import abspath, expanduser, join +from os.path import abspath, expanduser -import pandas as pd - -from snowexsql.db import get_db -from snowex_db.upload import * +from snowexsql.db import db_session_with_credentials +from snowex_db.upload.points import PointDataCSV def main(): - file = '../download/data/SNOWEX/SNEX20_BSU_GPR.001/2020.01.28/SNEX20_BSU_GPR_pE_01282020_01292020_02042020.csv' + file = ('../download/data/SNOWEX/SNEX20_BSU_GPR.001/' + '2020.01.28/SNEX20_BSU_GPR_pE_01282020_01292020_02042020.csv') kwargs = { - # Keyword argument to upload depth measurements - 'depth_is_metadata': False, - # Constant Metadata for the GPR data - 'site_name': 'Grand Mesa', - 'observers': 'Tate Meehan', - 'instrument': 'pulse EKKO Pro multi-polarization 1 GHz GPR', - 'in_timezone': 'UTC', - 'out_timezone': 'UTC', - 'epsg': 26912, - 'doi': 'https://doi.org/10.5067/Q2LFK0QSVGS2' + 'campaign_name': 'Grand Mesa', + 'observer': 'Tate Meehan', + 'instrument': 'gpr', + 'instrument_model': 'pulse EKKO Pro multi-polarization 1 GHz GPR', + 'timezone': 'UTC', + 'doi': 'https://doi.org/10.5067/Q2LFK0QSVGS2', + 'name': 'BSU GPR Data', } # Break out the path and make it an absolute path file = abspath(expanduser(file)) - # Grab a db connection to a local db named snowex - db_name = 'localhost/snowex' - engine, session = get_db(db_name, credentials='./credentials.json') - - # Instantiate the point uploader - csv = PointDataCSV(file, **kwargs) - # Push it to the database - csv.submit(session) - - # Close out the session with the DB - session.close() + # Grab a db connection + with db_session_with_credentials() as (_engine, session): + # Instantiate the point uploader + csv = PointDataCSV(file, **kwargs) + # Push it to the database + csv.submit(session) # return the number of errors for run.py can report it - return len(csv.errors) + # return len(csv.errors) if __name__ == '__main__': From 0c3e2d32b2c604c05fd375bb6916ad8f90900f28 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Tue, 1 Jul 2025 15:42:15 -0600 Subject: [PATCH 03/25] Issue #60 - working on bulk properties scripts --- snowex_db/point_data.py | 11 ++++++--- .../point_primary_variable_overrides.yaml | 24 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/snowex_db/point_data.py b/snowex_db/point_data.py index 668f742..4a0eb42 100644 --- a/snowex_db/point_data.py +++ b/snowex_db/point_data.py @@ -1,7 +1,7 @@ import logging from pathlib import Path from typing import List - +from timezonefinder import TimezoneFinder import numpy as np import pandas as pd import geopandas as gpd @@ -118,7 +118,13 @@ def _get_datetime(self, row): tz = self._in_timezone if self._row_based_timezone: # TODO: do we have to look it up? - raise NotImplementedError("?") + # TODO: Look up the timezone for the location and apply that + tz = None + + timezone_str = TimezoneFinder().timezone_at( + lat=row["latitude"], lng=row["longitude"] + ) + tz = timezone_str # e.g., 'America/Denver' try: datetime = None # In case we found a date entry that has date and time @@ -219,7 +225,6 @@ def _read_csv( """ result = [] - # TODO: how does the metadata parser fit into this? if columns is None and header_pos is None: LOG.warning(f"File {fname} is empty of rows") df = pd.DataFrame() diff --git a/snowex_db/point_primary_variable_overrides.yaml b/snowex_db/point_primary_variable_overrides.yaml index 54ded94..9869784 100644 --- a/snowex_db/point_primary_variable_overrides.yaml +++ b/snowex_db/point_primary_variable_overrides.yaml @@ -67,6 +67,7 @@ IGNORE: - version_number - avgvelocity - count + - site match_on_code: true INSTRUMENT: auto_remap: true @@ -224,3 +225,26 @@ WIND_SPEED_10FT: map_from: - wsms_10ft_avg match_on_code: true +SITE_NAME: + auto_remap: true + code: site_name + description: Name of campaign site + map_from: + - location + match_on_code: true +NAME: + auto_remap: true + code: name + description: Name of the measurement + map_from: + - pit_id + - pitid + match_on_code: true +COMMENTS: + auto_remap: false + code: comments + description: Comments + map_from: + - comments + - flag + match_on_code: true \ No newline at end of file From bc07631ec6b31f1249caf6fe09ccc7f1d0a13468 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Wed, 2 Jul 2025 09:17:06 -0600 Subject: [PATCH 04/25] Issue #60 - Working on bulk property script --- snowex_db/point_data.py | 13 ++- .../point_primary_variable_overrides.yaml | 4 +- snowex_db/upload/points.py | 45 ++++--- tests/data/pit_summary_points.csv | 13 +++ tests/points/test_summary_pits.py | 110 ++++++++++++++++++ 5 files changed, 159 insertions(+), 26 deletions(-) create mode 100644 tests/data/pit_summary_points.csv create mode 100644 tests/points/test_summary_pits.py diff --git a/snowex_db/point_data.py b/snowex_db/point_data.py index 4a0eb42..13093da 100644 --- a/snowex_db/point_data.py +++ b/snowex_db/point_data.py @@ -117,10 +117,7 @@ def _get_datetime(self, row): """ tz = self._in_timezone if self._row_based_timezone: - # TODO: do we have to look it up? - # TODO: Look up the timezone for the location and apply that - tz = None - + # Look up the timezone for the location and apply that timezone_str = TimezoneFinder().timezone_at( lat=row["latitude"], lng=row["longitude"] ) @@ -161,7 +158,8 @@ def _format_df(self, input_df): df = self._check_sample_columns(input_df) # Get the campaign name - df["campaign"] = df.get(YamlCodes.SITE_NAME) + if "campaign" not in df.columns: + df["campaign"] = df.get(YamlCodes.SITE_NAME) # TODO: How do we speed this up? # campaign should be very quick with a df level logic # but the other ones will take morelogic @@ -234,6 +232,7 @@ def _read_csv( ) shared_column_options = [ + # TODO: could we make this a 'shared' option in the definition meta_parser.primary_variables.entries["INSTRUMENT"], meta_parser.primary_variables.entries["DATE"], meta_parser.primary_variables.entries["TIME"], @@ -247,7 +246,9 @@ def _read_csv( meta_parser.primary_variables.entries["NORTHING"], meta_parser.primary_variables.entries["ELEVATION"], meta_parser.primary_variables.entries["INSTRUMENT_MODEL"], - meta_parser.primary_variables.entries["UTM_ZONE"] + meta_parser.primary_variables.entries["UTM_ZONE"], + meta_parser.primary_variables.entries["NAME"], + meta_parser.primary_variables.entries["CAMPAIGN"], ] shared_columns = [ diff --git a/snowex_db/point_primary_variable_overrides.yaml b/snowex_db/point_primary_variable_overrides.yaml index 9869784..2fea758 100644 --- a/snowex_db/point_primary_variable_overrides.yaml +++ b/snowex_db/point_primary_variable_overrides.yaml @@ -225,9 +225,9 @@ WIND_SPEED_10FT: map_from: - wsms_10ft_avg match_on_code: true -SITE_NAME: +CAMPAIGN: auto_remap: true - code: site_name + code: campaign description: Name of campaign site map_from: - location diff --git a/snowex_db/upload/points.py b/snowex_db/upload/points.py index ce7983c..191e0ea 100644 --- a/snowex_db/upload/points.py +++ b/snowex_db/upload/points.py @@ -1,21 +1,14 @@ """ Module for classes that upload single files to the database. """ - -from pathlib import Path import pandas as pd import geopandas as gpd import logging -from typing import List from geoalchemy2 import WKTElement from snowexsql.tables import ( PointData, MeasurementType, Instrument, DOI, Campaign, Observer, PointObservation ) -from snowexsql.tables.campaign_observation import CampaignObservation - -from ..metadata import SnowExProfileMetadata -from ..point_metadata import PointSnowExMetadataParser from ..string_management import parse_none from ..point_data import PointDataCollection, SnowExPointData @@ -63,11 +56,19 @@ def __init__(self, profile_filename, timezone="US/Mountain", **kwargs): observer name row_based_timezone + instrument_map """ self.filename = profile_filename self._timezone = timezone self._doi = kwargs.get("doi") self._instrument = kwargs.get("instrument") + # a map of measurement type to instrument name + self._instrument_map = kwargs.get("instrument_map", {}) + if self._instrument_map and self._instrument: + raise ValueError( + "Cannot provide both 'instrument' and 'instrument_map'. " + "Please choose one." + ) self._header_sep = kwargs.get("header_sep", ",") self._id = kwargs.get("id") self._campaign_name = kwargs.get("campaign_name") @@ -163,6 +164,13 @@ def build_data(self, series: SnowExPointData) -> gpd.GeoDataFrame: if column_name not in columns: df[column_name] = [param] * len(df) + # Anywhere the instrument is None, use the instrument map + # based on the measurement name + if self._instrument_map and 'instrument' in df.columns: + df['instrument'] = df['instrument'].fillna( + df['type'].map(self._instrument_map) + ) + # Map the measurement names or default to original df["instrument"] = df['instrument'].map( lambda x: self.MEASUREMENT_NAMES.get(x, x) @@ -187,7 +195,6 @@ def submit(self, session): if not df.empty: # IMPORTANT: Add observations first, so we can use them in the # entries - # TODO: how do these link back? self._add_campaign_observation( session, df ) @@ -198,7 +205,6 @@ def submit(self, session): srid=int(df.crs.srs.replace("EPSG:", "")) ) - # TODO: instrument name logic here? d = self._add_entry(session, row) # session.bulk_save_objects(objects) does not resolve # foreign keys, DO NOT USE IT @@ -248,9 +254,12 @@ def _add_campaign_observation(self, session, df): # Add instrument instrument_name = self._get_first_check_unique(grouped_df, 'instrument') # Map the instrument name if we have a mapping for it - instrument_name = self.MEASUREMENT_NAMES.get( - instrument_name.lower(), instrument_name - ) + if pd.isna(instrument_name): + instrument_name = None + if instrument_name: + instrument_name = self.MEASUREMENT_NAMES.get( + instrument_name.lower(), instrument_name + ) instrument = self._check_or_add_object( session, Instrument, dict( name=instrument_name, @@ -271,13 +280,14 @@ def _add_campaign_observation(self, session, df): ) ) - # Check name is unique - self._get_first_check_unique(df, "name") + # Check name is unique because we are adding ONE + # campaign observation here + self._get_first_check_unique(grouped_df, "name") # Get the measurement name measurement_name = self._observation_name_from_row(grouped_df.iloc[0]) # Add doi - doi_string = self._get_first_check_unique(df, "doi") + doi_string = self._get_first_check_unique(grouped_df, "doi") if doi_string is not None: doi = self._check_or_add_object( session, DOI, dict(doi=doi_string) @@ -286,7 +296,7 @@ def _add_campaign_observation(self, session, df): doi = None # pass in campaign campaign_name = self._get_first_check_unique( - df, "campaign" + grouped_df, "campaign" ) or self._campaign_name if campaign_name is None: raise DataValidationError("Campaign cannot be None") @@ -295,7 +305,7 @@ def _add_campaign_observation(self, session, df): ) # add observer observer_name = self._get_first_check_unique( - df, "observer" + grouped_df, "observer" ) or self._observer observer_name = observer_name or "unknown" observer = self._check_or_add_object( @@ -318,7 +328,6 @@ def _add_campaign_observation(self, session, df): ), object_kwargs=dict( name=measurement_name, - # TODO: we lose out on row-based comments here description=description, date=date_obj, instrument=instrument, diff --git a/tests/data/pit_summary_points.csv b/tests/data/pit_summary_points.csv new file mode 100644 index 0000000..7be9dbd --- /dev/null +++ b/tests/data/pit_summary_points.csv @@ -0,0 +1,13 @@ +Location,Site,PitID,Date/Local Standard Time,UTM Zone,Easting (m),Northing (m),Latitude (deg),Longitude (deg),Density Mean (kg/m^3),SWE (mm),HS (cm),Flag +American River Basin,Caples Lake,CAAMCL_20191220_1300,2019-12-20T13:00,10N,757216,4288787,38.71033054555811,-120.04186927254749,278.0,333.5,120.0,"BDG, MW" +American River Basin,Caples Lake,CAAMCL_20200131_1215,2020-01-31T12:15,10N,757220,4288788,38.71033838194462,-120.0418229560188,329.5,446.5,135.0,MW +American River Basin,Caples Lake,CAAMCL_20200214_1200,2020-02-14T12:00,10N,757218,4288787,38.71032996387011,-120.04184629988718,359.5,442.5,123.0,STLay +American River Basin,Caples Lake,CAAMCL_20200221_1200,2020-02-21T12:00,10N,757217,4288780,38.710267256343286,-120.04186038464364,364.0,424.0,117.0,"MW, STLay" +American River Basin,Caples Lake,CAAMCL_20200228_1130,2020-02-28T11:30,10N,757215,4288778,38.71024983849251,-120.04188409968448,396.5,475.5,120.0,STLay +American River Basin,Caples Lake,CAAMCL_20200306_1145,2020-03-06T11:45,10N,757216,4288779,38.71025854741857,-120.04187224216548,403.5,479.5,119.0,"TDG, MW" +American River Basin,Caples Lake,CAAMCL_20200313_1030,2020-03-13T10:30,10N,757214,4288777,38.71024112956513,-120.04189595720058,435.5,434.5,100.0,"TDG, MW, STCom, STLay" +East River,Forest 12,COER12_20200226_1242,2020-02-26T12:42,13N,328520,4310833,38.929673072842384,-106.97828661618968,267.0,245.5,92.0,BDG +East River,Forest 12,COER12_20200428_1400,2020-04-28T14:00,13N,328526,4310840,38.929737288240055,-106.97821918710338,356.5,257.0,72.0, +East River,Forest 12,COER12_20200428_1415,2020-04-28T14:15,13N,328530,4310840,38.92973807010048,-106.97817306650516,368.0,367.0,100.0, +East River,Forest 12,COER12_20200428_1445,2020-04-28T14:45,13N,328528,4310844,38.92977370350157,-106.97819712779332,356.0,356.5,100.0,STCom +East River,Forest 12,COER12_20200512_1030,2020-05-12T10:30,13N,328519,4310837,38.92970890169526,-106.9782991473662,393.0,196.0,50.0, \ No newline at end of file diff --git a/tests/points/test_summary_pits.py b/tests/points/test_summary_pits.py new file mode 100644 index 0000000..fe34b7c --- /dev/null +++ b/tests/points/test_summary_pits.py @@ -0,0 +1,110 @@ +from datetime import datetime, timezone, date + +import pytest +from geoalchemy2 import WKTElement +from snowexsql.db import db_session_with_credentials +from snowexsql.tables import PointData, DOI, Campaign, Instrument, \ + MeasurementType, PointObservation +from snowexsql.tables.campaign_observation import CampaignObservation + +from snowex_db.upload.points import PointDataCSV +from tests.points._base import PointBaseTesting + + +class TestSummaryPits(PointBaseTesting): + """ + Test the summary csvs for a collection of pits + """ + + kwargs = { + 'timezone': 'MST', + 'doi': "some_point_pit_doi", + "row_based_timezone": True, # row based timezone + "derived": True, + "instrument_map": { + "depth": "manual", + "swe": "manual", + "density": "cutter", + } + } + UploaderClass = PointDataCSV + TableClass = PointData + + @pytest.fixture(scope="class") + def uploaded_file(self, db, data_dir): + """ + NOTE - this is part of the _modified file that we create + in the upload script, NOT the original file + """ + self.upload_file(str(data_dir.joinpath("pit_summary_points.csv"))) + + def filter_measurement_type(self, session, measurement_type, query=None): + if query is None: + query = session.query(self.TableClass) + + query = query.join( + self.TableClass.observation + ).join( + PointObservation.measurement_type + ).filter(MeasurementType.name == measurement_type) + return query + + @pytest.mark.parametrize( + "table, attribute, expected_value", [ + (Campaign, "name", "Grand Mesa"), + (Instrument, "name", "mesa"), + (Instrument, "model", "Mesa2_1"), + (MeasurementType, "name", ['depth']), + (MeasurementType, "units", ['cm']), + (MeasurementType, "derived", [True]), + (DOI, "doi", "some_point_doi"), + (CampaignObservation, "name", "example_point_name_M2Mesa2_1_depth"), + (PointData, "geom", + WKTElement('POINT (-108.13515 39.03045)', srid=4326) + ), + (PointObservation, "date", date(2020, 2, 4)), + ] + ) + def test_metadata(self, table, attribute, expected_value, uploaded_file): + self._check_metadata(table, attribute, expected_value) + + @pytest.mark.parametrize( + "data_name, attribute_to_check, filter_attribute, filter_value, expected", [ + ('depth', 'value', 'value', 94.0, [94]), + ('depth', 'units', 'value', 94.0, ['cm']), + ('depth', 'datetime', 'value', 94.0, [datetime(2020, 1, 28, 18, 48, tzinfo=timezone.utc)]), + ] + ) + def test_value( + self, data_name, attribute_to_check, + filter_attribute, filter_value, expected, uploaded_file + ): + self.check_value( + data_name, attribute_to_check, + filter_attribute, filter_value, expected, + ) + + @pytest.mark.parametrize( + "data_name, expected", [ + ("depth", 10) + ] + ) + def test_count(self, data_name, expected, uploaded_file): + n = self.check_count(data_name) + assert n == expected + + @pytest.mark.parametrize( + "data_name, attribute_to_count, expected", [ + ("depth", "value", 9), + ("depth", "units", 1) + ] + ) + def test_unique_count(self, data_name, attribute_to_count, expected, uploaded_file): + self.check_unique_count( + data_name, attribute_to_count, expected + ) + + def test_unique_types(self, uploaded_file): + with db_session_with_credentials() as (engine, session): + records = session.query(PointObservation.measurement_type).unique() + assert len(records) == 3 From 72b1f7a345f993e348d1d93f633e8b0423548747 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Wed, 2 Jul 2025 09:43:14 -0600 Subject: [PATCH 05/25] Finish testing pit summary files --- snowex_db/point_data.py | 1 + tests/points/test_summary_pits.py | 36 ++++++++++++++++++------------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/snowex_db/point_data.py b/snowex_db/point_data.py index 13093da..8190fb2 100644 --- a/snowex_db/point_data.py +++ b/snowex_db/point_data.py @@ -249,6 +249,7 @@ def _read_csv( meta_parser.primary_variables.entries["UTM_ZONE"], meta_parser.primary_variables.entries["NAME"], meta_parser.primary_variables.entries["CAMPAIGN"], + meta_parser.primary_variables.entries["COMMENTS"], ] shared_columns = [ diff --git a/tests/points/test_summary_pits.py b/tests/points/test_summary_pits.py index fe34b7c..d1068a6 100644 --- a/tests/points/test_summary_pits.py +++ b/tests/points/test_summary_pits.py @@ -25,6 +25,7 @@ class TestSummaryPits(PointBaseTesting): "depth": "manual", "swe": "manual", "density": "cutter", + "comments": "unknown" } } UploaderClass = PointDataCSV @@ -51,18 +52,18 @@ def filter_measurement_type(self, session, measurement_type, query=None): @pytest.mark.parametrize( "table, attribute, expected_value", [ - (Campaign, "name", "Grand Mesa"), - (Instrument, "name", "mesa"), - (Instrument, "model", "Mesa2_1"), - (MeasurementType, "name", ['depth']), - (MeasurementType, "units", ['cm']), - (MeasurementType, "derived", [True]), - (DOI, "doi", "some_point_doi"), - (CampaignObservation, "name", "example_point_name_M2Mesa2_1_depth"), + (Campaign, "name", "American River Basin"), + (Instrument, "name", "cutter"), + (Instrument, "model", None), + (MeasurementType, "name", ['density', 'swe', 'depth']), + (MeasurementType, "units", ['kg/m^3', 'mm', 'cm']), + (MeasurementType, "derived", [True, True, True]), + (DOI, "doi", "some_point_pit_doi"), + (CampaignObservation, "name", "CAAMCL_20191220_1300_cutter_density"), (PointData, "geom", - WKTElement('POINT (-108.13515 39.03045)', srid=4326) + WKTElement('POINT (-120.04186927254749 38.71033054555811)', srid=4326) ), - (PointObservation, "date", date(2020, 2, 4)), + (PointObservation, "date", date(2019, 12, 20)), ] ) def test_metadata(self, table, attribute, expected_value, uploaded_file): @@ -70,9 +71,9 @@ def test_metadata(self, table, attribute, expected_value, uploaded_file): @pytest.mark.parametrize( "data_name, attribute_to_check, filter_attribute, filter_value, expected", [ - ('depth', 'value', 'value', 94.0, [94]), - ('depth', 'units', 'value', 94.0, ['cm']), - ('depth', 'datetime', 'value', 94.0, [datetime(2020, 1, 28, 18, 48, tzinfo=timezone.utc)]), + ('depth', 'value', 'value', 117.0, [117.0]), + ('depth', 'units', 'value', 117.0, ['cm']), + ('depth', 'datetime', 'value', 117.0, [datetime(2020, 2, 21, 20, 00, tzinfo=timezone.utc)]), ] ) def test_value( @@ -86,7 +87,7 @@ def test_value( @pytest.mark.parametrize( "data_name, expected", [ - ("depth", 10) + ("depth", 12) ] ) def test_count(self, data_name, expected, uploaded_file): @@ -105,6 +106,11 @@ def test_unique_count(self, data_name, attribute_to_count, expected, uploaded_fi ) def test_unique_types(self, uploaded_file): + """ + Test number of unique measurement types + """ with db_session_with_credentials() as (engine, session): - records = session.query(PointObservation.measurement_type).unique() + records = session.query( + MeasurementType.name + ).distinct().all() assert len(records) == 3 From 35579061b79746be3802eb1bfbb8286b12ae34d5 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Wed, 2 Jul 2025 10:51:49 -0600 Subject: [PATCH 06/25] Upload script --- scripts/upload/add_pits_bulk_properties.py | 27 +++++++++++----------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/scripts/upload/add_pits_bulk_properties.py b/scripts/upload/add_pits_bulk_properties.py index bed5120..00d9e30 100644 --- a/scripts/upload/add_pits_bulk_properties.py +++ b/scripts/upload/add_pits_bulk_properties.py @@ -9,16 +9,14 @@ import pandas as pd -from snowex_db.upload import PointDataCSV -from snowex_db import db_session +from snowexsql.db import db_session_with_credentials +from snowex_db.upload.points import PointDataCSV def main(): """ Add bulk SWE, Depth, Density for 2020 and 2021 timeseires pits """ - db_name = 'localhost/snowex' - debug = True # Point to the downloaded data from data_dir = abspath('../download/data/SNOWEX/') @@ -35,14 +33,18 @@ def main(): }, # Preliminary data from 2023 Alask pits { + # TODO: update this "DOI": "preliminary_alaska_pits", "path": "../SNEX23_preliminary/Data/SnowEx23_SnowPits_AKIOP_Summary_SWE_v01.csv" } ] + # start a db session + # look through the pit summary files for info in path_details: doi = info["DOI"] file_path = join(data_dir, info["path"]) # Read csv and dump new one without the extra header lines + # that make parsing not possible df = pd.read_csv( file_path, skiprows=list(range(32)) + [33] @@ -61,16 +63,15 @@ def main(): df.to_csv(new_name, index=False) # Submit SWE file data as point data - with db_session( - db_name, credentials='credentials.json' - ) as (session, engine): - pcsv = PointDataCSV( - new_name, doi=doi, debug=debug, - depth_is_metadata=False, - row_based_crs=True, - row_based_timezone=True + with db_session_with_credentials() as (_engine, session): + u = PointDataCSV( + new_name, + doi=doi, + row_based_timezone=True, + derived=True ) - pcsv.submit(session) + + u.submit(session) if __name__ == '__main__': From 50e72ac5a5071f3c320de25d3a32e46760651acd Mon Sep 17 00:00:00 2001 From: Micah Sandusky <32111103+micah-prime@users.noreply.github.com> Date: Tue, 15 Jul 2025 10:41:46 -0600 Subject: [PATCH 07/25] Update to new file reading structure in insitupy (#73) * starting to make changes for insitupy >0.4 * Getting closer to having this running * I think my variable overrides logic isn't getting applied fully * Files parse, but I'm missing some columns * Point tests passing * I think this is just held up on merging the comments change now * insitupy version * Address PR comments * Doc string --- pyproject.toml | 2 +- snowex_db/metadata.py | 10 ++- snowex_db/point_data.py | 174 ++++++++++++++++++------------------ snowex_db/point_metadata.py | 20 +++-- snowex_db/profile_data.py | 21 +---- snowex_db/upload/layers.py | 9 +- snowex_db/upload/points.py | 7 +- 7 files changed, 127 insertions(+), 116 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 039dca5..e6cd6e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ classifiers = [ dependencies = [ "snowexsql==0.6.0rc1", "snowmicropyn", - "insitupy~=0.3.0", + "insitupy~=0.4", "boto3<1.24", "rasterio<1.4", "timezonefinder<7", diff --git a/snowex_db/metadata.py b/snowex_db/metadata.py index a939d30..db05ac6 100644 --- a/snowex_db/metadata.py +++ b/snowex_db/metadata.py @@ -8,6 +8,7 @@ from insitupy.io.metadata import MetaDataParser from insitupy.profiles.metadata import ProfileMetaData +from insitupy.campaigns.snowex.snowex_metadata import SnowExMetaDataParser from snowexsql.db import get_table_attributes from snowexsql.tables import Site @@ -119,12 +120,12 @@ class SnowExProfileMetadata(ProfileMetaData): wind: Union[str, None] = None -class ExtendedSnowExMetadataParser(MetaDataParser): +class ExtendedSnowExMetadataParser(SnowExMetaDataParser): """ Extend the parser to update the parsing function """ - def parse(self): + def parse(self, filename: str): """ Parse the file and return a metadata object. We can override these methods as needed to parse the different @@ -132,12 +133,15 @@ def parse(self): This populates self.rough_obj + Args: + filename: Path to the file from which to parse metadata + Returns: (metadata object, column list, position of header in file) """ ( meta_lines, columns, columns_map, header_position - ) = self.find_header_info(self._fname) + ) = self.find_header_info(filename) self._rough_obj = self._preparse_meta(meta_lines) # Create a standard metadata object metadata = SnowExProfileMetadata( diff --git a/snowex_db/point_data.py b/snowex_db/point_data.py index 8190fb2..7e0b10b 100644 --- a/snowex_db/point_data.py +++ b/snowex_db/point_data.py @@ -1,12 +1,14 @@ import logging from pathlib import Path from typing import List + +from insitupy.io.metadata import MetaDataParser from timezonefinder import TimezoneFinder import numpy as np import pandas as pd import geopandas as gpd from insitupy.campaigns.snowex import SnowExProfileData -from insitupy.io.dates import DateManager +from insitupy.io.dates import DateTimeManager from insitupy.io.locations import LocationManager from insitupy.io.yaml_codes import YamlCodes @@ -21,45 +23,24 @@ class SnowExPointData(MeasurementData): OUT_TIMEZONE = "UTC" - DEFAULT_METADATA_VARIABLE_FILES = SnowExProfileData.DEFAULT_METADATA_VARIABLE_FILES - DEFAULT_PRIMARY_VARIABLE_FILES = MeasurementData.DEFAULT_PRIMARY_VARIABLE_FILES + [ - Path(__file__).parent.joinpath( - "./point_primary_variable_overrides.yaml" - ) - ] + META_PARSER = PointSnowExMetadataParser def __init__( - self, input_df: pd.DataFrame, metadata: ProfileMetaData, - variable: MeasurementDescription, - original_file=None, meta_parser=None, allow_map_failure=False, + self, variable: MeasurementDescription = None, + meta_parser: MetaDataParser = None, row_based_timezone=False, timezone=None ): """ - Take df of layered data (SMP, pit, etc) Args: - input_df: dataframe of data - Should include depth and optional bottom depth - Should include sample or sample_a, sample_b, etc - metadata: ProfileMetaData object - variable: description of variable - original_file: optional track original file - meta_parser: MetaDataParser object. This will hold our variables - map and units map - allow_map_failures: if a mapping fails, warn us and use the - original string (default False) + See MeasurementData.__init__ row_based_timezone: does each row have a unique timezone implied timezone: input timezone for the whole file """ self._row_based_timezone = row_based_timezone self._in_timezone = timezone - super().__init__( - input_df, metadata, variable, - original_file=original_file, - meta_parser=meta_parser, - allow_map_failure=allow_map_failure - ) + super().__init__(variable, meta_parser) @staticmethod def read_csv_dataframe(profile_filename, columns, header_position): @@ -129,12 +110,13 @@ def _get_datetime(self, row): str_date = str( row[YamlCodes.DATE_TIME].replace('T', '-') ) + datetime = pd.to_datetime(str_date) if datetime is None: - datetime = DateManager.handle_separate_datetime(row) + datetime = DateTimeManager.handle_separate_datetime(row) - result = DateManager.adjust_timezone( + result = DateTimeManager.adjust_timezone( datetime, in_timezone=tz, out_timezone=self.OUT_TIMEZONE @@ -146,41 +128,58 @@ def _get_datetime(self, row): raise e return result - def _format_df(self, input_df): + def _format_df(self): """ Format the incoming df with the column headers and other info we want This will filter to a single measurement as well as the expected shared columns like depth """ - self._set_column_mappings(input_df) + self._set_column_mappings() + + # If the variable is real (not -1), check columns + if self.variable.code != "-1": + # Verify the sample column exists and rename to variable + self._check_sample_columns() + + columns = self._df.columns.tolist() - # Verify the sample column exists and rename to variable - df = self._check_sample_columns(input_df) + # If we do not have a geometry column, we need to parse + # the raw df, otherwise we assume this has been done already, + # likely on the first read of the file # Get the campaign name - if "campaign" not in df.columns: - df["campaign"] = df.get(YamlCodes.SITE_NAME) + if "campaign" not in self._df.columns: + self._df["campaign"] = self._df.get(YamlCodes.SITE_NAME) # TODO: How do we speed this up? # campaign should be very quick with a df level logic # but the other ones will take morelogic # parse the location - df[["latitude", "longitude"]] = df.apply( + self._df[["latitude", "longitude"]] = self._df.apply( self._get_location, axis=1, result_type="expand" ) - # Parse the datetime - df["datetime"] = df.apply(self._get_datetime, axis=1, result_type="expand") + # If the datetime isn't already parsed, parse it + if ( + "datetime" in self._df.columns.tolist() + and pd.api.types.is_datetime64_any_dtype( + self._df["datetime"] + ) + ): + LOG.debug("not parsing date") + else: + # Parse the datetime + self._df["datetime"] = self._df.apply( + self._get_datetime, axis=1, result_type="expand" + ) location = gpd.points_from_xy( - df["longitude"], df["latitude"] + self._df["longitude"], self._df["latitude"] ) - df = df.drop(columns=["longitude", "latitude"]) + # self._df = self._df.drop(columns=["longitude", "latitude"]) - df = gpd.GeoDataFrame( - df, geometry=location + self._df = gpd.GeoDataFrame( + self._df, geometry=location ).set_crs("EPSG:4326") - df = df.replace(-9999, np.NaN) - - return df + self._df = self._df.replace(-9999, np.NaN) class PointDataCollection: @@ -203,17 +202,12 @@ def series(self) -> List[SnowExPointData]: @classmethod def _read_csv( - cls, fname, columns, column_mapping, header_pos, - metadata: ProfileMetaData, meta_parser: PointSnowExMetadataParser, + cls, fname, meta_parser: PointSnowExMetadataParser, timezone=None, row_based_timezone=False ) -> List[SnowExPointData]: """ Args: fname: path to csv - columns: columns for dataframe - column_mapping: mapping of column name to variable description - header_pos: skiprows for pd.read_csv - metadata: metadata for each object meta_parser: parser for the metadata timezone: input timezone row_based_timezone: is the timezone row based? @@ -222,14 +216,16 @@ def _read_csv( a list of ProfileData objects """ + # parse the file for metadata before parsing the individual + # variables + all_file = cls.DATA_CLASS( + variable=None, # we do not have a variable yet + meta_parser=meta_parser, + timezone=timezone, row_based_timezone=row_based_timezone + ) + all_file.from_csv(fname) + result = [] - if columns is None and header_pos is None: - LOG.warning(f"File {fname} is empty of rows") - df = pd.DataFrame() - else: - df = cls.DATA_CLASS.read_csv_dataframe( - fname, columns, header_pos, - ) shared_column_options = [ # TODO: could we make this a 'shared' option in the definition @@ -253,33 +249,48 @@ def _read_csv( ] shared_columns = [ - c for c, v in column_mapping.items() + c for c, v in all_file.meta_columns_map.items() if v in shared_column_options ] variable_columns = [ - c for c in column_mapping.keys() if c not in shared_columns + c for c in all_file.meta_columns_map.keys() if c not in shared_columns + ] + # Filter out ignore columns + variable_columns = [ + v for v in variable_columns + if all_file.meta_columns_map[v].code != "ignore" ] # Create an object for each measurement for column in variable_columns: - target_df = df.loc[:, shared_columns + [column]] - result.append(cls.DATA_CLASS( - target_df, metadata, - column_mapping[column], # variable is a MeasurementDescription - original_file=fname, + points = cls.DATA_CLASS( + variable=all_file.meta_columns_map[column], meta_parser=meta_parser, timezone=timezone, row_based_timezone=row_based_timezone - )) - - return result + ) + # IMPORTANT - Metadata needs to be set before assigning the + # dataframe as information from the metadata is used to format_df + # the information + points.metadata = all_file.metadata + df_columns = all_file.df.columns.tolist() + # The df setter filters some columns, so adjust our shared columns + df_shared_columns = [ + c for c in shared_columns if c in df_columns + ] + # run the whole file through the df setter + points.df = all_file.df.loc[:, df_shared_columns + [column]].copy() + # -------- + result.append(points) + + return result, all_file.metadata @classmethod def from_csv( cls, fname, timezone="US/Mountain", header_sep=",", site_id=None, campaign_name=None, allow_map_failure=False, units_map=None, row_based_timezone=False, - metadata_variable_files=None, - primary_variable_files=None, + metadata_variable_file=None, + primary_variable_file=None, ): """ Find all variables in a single csv file @@ -292,36 +303,27 @@ def from_csv( allow_map_failure: allow metadata and column unknowns units_map: units map for the metadata row_based_timezone: is the timezone row based - metadata_variable_files: list of files to override the metadata + metadata_variable_file: list of files to override the metadata variables - primary_variable_files: list of files to override the + primary_variable_file: list of files to override the primary variables Returns: This class with a collection of profiles and metadata """ - primary_variables = ExtendableVariables( - primary_variable_files or cls.DATA_CLASS.DEFAULT_PRIMARY_VARIABLE_FILES - ) - metadata_variables = ExtendableVariables( - metadata_variable_files or cls.DATA_CLASS.DEFAULT_METADATA_VARIABLE_FILES, - ) # parse multiple files and create an iterable of ProfileData meta_parser = PointSnowExMetadataParser( - fname, timezone, primary_variables, metadata_variables, + timezone, primary_variable_file, metadata_variable_file, header_sep=header_sep, _id=site_id, campaign_name=campaign_name, allow_map_failures=allow_map_failure, - units_map=units_map + units_map=units_map, ) - # Parse the metadata and column info - metadata, columns, columns_map, header_pos = meta_parser.parse() + # read in the actual data - profiles = cls._read_csv( - fname, columns, columns_map, header_pos, metadata, - meta_parser, + profiles, metadata = cls._read_csv( + fname, meta_parser, timezone=timezone, row_based_timezone=row_based_timezone ) - # ignore profiles with the name 'ignore' profiles = [ p for p in profiles if diff --git a/snowex_db/point_metadata.py b/snowex_db/point_metadata.py index 61378c8..73d7c0a 100644 --- a/snowex_db/point_metadata.py +++ b/snowex_db/point_metadata.py @@ -1,5 +1,7 @@ import logging +from pathlib import Path +from insitupy.campaigns.snowex.snowex_metadata import SnowExMetaDataParser from insitupy.io.metadata import MetaDataParser from insitupy.profiles.metadata import ProfileMetaData @@ -10,8 +12,9 @@ class PointSnowExMetadataParser(MetaDataParser): """ Extend the parser to update the extended varaibles """ + DEFAULT_METADATA_VARIABLE_FILES = SnowExMetaDataParser.DEFAULT_METADATA_VARIABLE_FILES - def find_header_info(self, filename=None): + def find_header_info(self, filename): """ Read in all site details file for a pit If the filename has the word site in it then we read everything in the file. Otherwise, we use this @@ -28,7 +31,6 @@ def find_header_info(self, filename=None): **header_pos** - Index of the columns header for skiprows in read_csv """ - filename = filename or self._fname filename = str(filename) with open(filename, encoding='latin') as fp: lines = fp.readlines() @@ -62,7 +64,7 @@ def find_header_info(self, filename=None): return str_data, columns, columns_map, header_pos - def parse(self): + def parse(self, filename: str): """ Parse the file and return a metadata object. We can override these methods as needed to parse the different @@ -70,12 +72,18 @@ def parse(self): This populates self.rough_obj + Args: + filename: Path to the file from which to parse metadata + Returns: - (None, column list, position of header in file) + ( + Metadata or None, column list, column map, + position of header in file + ) """ ( meta_lines, columns, columns_map, header_position - ) = self.find_header_info(self._fname) + ) = self.find_header_info(filename) self._rough_obj = self._preparse_meta(meta_lines) # We do not have header metadata for point files if not self.rough_obj: @@ -99,4 +107,4 @@ def parse(self): flags=self.parse_flags(), observers=self.parse_observers() ) - return metadata, columns, columns_map, header_position \ No newline at end of file + return metadata, columns, columns_map, header_position diff --git a/snowex_db/profile_data.py b/snowex_db/profile_data.py index 5289e15..7d0927b 100644 --- a/snowex_db/profile_data.py +++ b/snowex_db/profile_data.py @@ -13,35 +13,20 @@ class ExtendedSnowexProfileData(SnowExProfileData): META_PARSER = ExtendedSnowExMetadataParser - DEFAULT_METADATA_VARIABLE_FILES = ( - SnowExProfileData.DEFAULT_METADATA_VARIABLE_FILES - ) + [ - Path(__file__).parent.joinpath( - "./metadata_variable_overrides.yaml" - ) - ] - DEFAULT_PRIMARY_VARIABLE_FILES = ( - SnowExProfileData.DEFAULT_PRIMARY_VARIABLE_FILES) + [ - Path(__file__).parent.joinpath( - "./profile_primary_variable_overrides.yaml" - ) - ] def __init__( - self, input_df: pd.DataFrame, - metadata: ProfileMetaData, + self, variable: MeasurementDescription, - meta_parser: MetaDataParser, **kwargs + meta_parser: MetaDataParser ): # Tricky, this needs to happen before super init self._comments_column = meta_parser.primary_variables.entries[ "COMMENTS"] - super().__init__(input_df, metadata, variable, meta_parser, **kwargs) + super().__init__(variable, meta_parser) def shared_column_options(self): return self._depth_columns + [self._comments_column] class ExtendedSnowExProfileDataCollection(SnowExProfileDataCollection): - META_PARSER = ExtendedSnowExMetadataParser PROFILE_DATA_CLASS = ExtendedSnowexProfileData diff --git a/snowex_db/upload/layers.py b/snowex_db/upload/layers.py index 909e664..1d82e97 100644 --- a/snowex_db/upload/layers.py +++ b/snowex_db/upload/layers.py @@ -2,6 +2,7 @@ Module for classes that upload single files to the database. """ import time +from pathlib import Path from typing import List import pandas as pd @@ -78,7 +79,13 @@ def _read(self, profile_filename) -> ExtendedSnowExProfileDataCollection: timezone=self._timezone, header_sep=self._header_sep, site_id=self._id, - campaign_name=self._campaign_name + campaign_name=self._campaign_name, + metadata_variable_file=Path(__file__).parent.joinpath( + "../metadata_variable_overrides.yaml" + ), + primary_variable_file=Path(__file__).parent.joinpath( + "../profile_primary_variable_overrides.yaml" + ), ) except pd.errors.ParserError as e: LOG.error(e) diff --git a/snowex_db/upload/points.py b/snowex_db/upload/points.py index 191e0ea..61bbcfb 100644 --- a/snowex_db/upload/points.py +++ b/snowex_db/upload/points.py @@ -1,6 +1,8 @@ """ Module for classes that upload single files to the database. """ +from pathlib import Path + import pandas as pd import geopandas as gpd import logging @@ -106,7 +108,10 @@ def _read(self, filename, in_timezone=None): header_sep=self._header_sep, site_id=self._id, campaign_name=self._campaign_name, units_map=self.UNITS_MAP, - row_based_timezone=self._row_based_tz + row_based_timezone=self._row_based_tz, + primary_variable_file=Path(__file__).parent.joinpath( + "../point_primary_variable_overrides.yaml" + ) ) except pd.errors.ParserError as e: LOG.error(e) From e4efd8d9f73a1db6335f2ea357a07e35d95dec0b Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Tue, 15 Jul 2025 11:03:51 -0600 Subject: [PATCH 08/25] dealing with stash --- snowex_db/point_data.py | 1 + snowex_db/point_primary_variable_overrides.yaml | 12 +++++++----- snowex_db/upload/points.py | 8 ++++++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/snowex_db/point_data.py b/snowex_db/point_data.py index 7e0b10b..5eeb801 100644 --- a/snowex_db/point_data.py +++ b/snowex_db/point_data.py @@ -246,6 +246,7 @@ def _read_csv( meta_parser.primary_variables.entries["NAME"], meta_parser.primary_variables.entries["CAMPAIGN"], meta_parser.primary_variables.entries["COMMENTS"], + meta_parser.primary_variables.entries["FLAGS"], ] shared_columns = [ diff --git a/snowex_db/point_primary_variable_overrides.yaml b/snowex_db/point_primary_variable_overrides.yaml index 2fea758..e70f4fc 100644 --- a/snowex_db/point_primary_variable_overrides.yaml +++ b/snowex_db/point_primary_variable_overrides.yaml @@ -241,10 +241,12 @@ NAME: - pitid match_on_code: true COMMENTS: - auto_remap: false code: comments description: Comments - map_from: - - comments - - flag - match_on_code: true \ No newline at end of file + match_on_code: true +FLAGS: + code: flags + description: measurement flag + map_from: + - flag + match_on_code: true \ No newline at end of file diff --git a/snowex_db/upload/points.py b/snowex_db/upload/points.py index 61bbcfb..020a113 100644 --- a/snowex_db/upload/points.py +++ b/snowex_db/upload/points.py @@ -318,9 +318,13 @@ def _add_campaign_observation(self, session, df): ) description = None if ["comments"] in grouped_df.columns.values: - description = self._get_first_check_unique( + description = (description or "") + self._get_first_check_unique( grouped_df, "comments" - ), + ) + if ["flags"] in grouped_df.columns.values: + description = (description or "") + self._get_first_check_unique( + grouped_df, "flags" + ) date_obj = self._get_first_check_unique(grouped_df, "date") observation = self._check_or_add_object( From 0e486baefac0e3654814c11bc9f8ba41bf397f1a Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Tue, 15 Jul 2025 11:17:46 -0600 Subject: [PATCH 09/25] use insitupy parse noneg --- snowex_db/upload/layers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/snowex_db/upload/layers.py b/snowex_db/upload/layers.py index dcd5bfc..27904e7 100644 --- a/snowex_db/upload/layers.py +++ b/snowex_db/upload/layers.py @@ -9,6 +9,7 @@ import pandas as pd from geoalchemy2 import WKTElement +from insitupy.io.strings import StringManager from insitupy.campaigns.snowex import SnowExProfileData from snowexsql.tables import ( Campaign, DOI, Instrument, LayerData, MeasurementType, Observer, Site @@ -17,7 +18,6 @@ from .batch import BatchBase from ..metadata import SnowExProfileMetadata from ..profile_data import ExtendedSnowExProfileDataCollection -from ..string_management import parse_none from ..utilities import get_logger LOG = logging.getLogger("snowex_db.upload.layers") @@ -135,7 +135,7 @@ def build_data(self, profile: SnowExProfileData) -> gpd.GeoDataFrame: # Manage nans and nones for c in df.columns: - df[c] = df[c].apply(lambda x: parse_none(x)) + df[c] = df[c].apply(lambda x: StringManager.parse_none(x)) df['value'] = df[variable.code].astype(str) if 'units' not in df.columns: From 1b91416598ee605221bda2580393f489640117cc Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Tue, 15 Jul 2025 11:24:44 -0600 Subject: [PATCH 10/25] working on tests --- tests/helpers.py | 2 +- tests/points/test_depth.py | 2 +- tests/points/test_gpr_bsu.py | 4 ++-- tests/points/test_perimiter_depth.py | 2 +- tests/points/test_summary_pits.py | 7 +++++-- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index 028d9cb..1a5aa71 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -13,7 +13,7 @@ class WithUploadedFile: def upload_file(self, session, filename): u = self.UploaderClass( - session=session, filename=filename, **self.kwargs + session, filename, **self.kwargs ) u.submit() diff --git a/tests/points/test_depth.py b/tests/points/test_depth.py index ffcfe48..4225805 100644 --- a/tests/points/test_depth.py +++ b/tests/points/test_depth.py @@ -28,7 +28,7 @@ class TestDepth(PointBaseTesting): @pytest.fixture(scope="class") def uploaded_file(self, session, data_dir): self.upload_file( - filename=str(data_dir.joinpath("depths.csv")), session=session + session, str(data_dir.joinpath("depths.csv")), ) def filter_measurement_type(self, session, measurement_type, query=None): diff --git a/tests/points/test_gpr_bsu.py b/tests/points/test_gpr_bsu.py index 1582af1..c41ed43 100644 --- a/tests/points/test_gpr_bsu.py +++ b/tests/points/test_gpr_bsu.py @@ -27,8 +27,8 @@ class TestGPR(PointBaseTesting): TableClass = PointData @pytest.fixture(scope="class") - def uploaded_file(self, db, data_dir): - self.upload_file(str(data_dir.joinpath("bsu_gpr.csv"))) + def uploaded_file(self, session, data_dir): + self.upload_file(session, str(data_dir.joinpath("bsu_gpr.csv"))) def filter_measurement_type(self, session, measurement_type, query=None): if query is None: diff --git a/tests/points/test_perimiter_depth.py b/tests/points/test_perimiter_depth.py index 870dbf9..7b7c6b1 100644 --- a/tests/points/test_perimiter_depth.py +++ b/tests/points/test_perimiter_depth.py @@ -31,7 +31,7 @@ class TestPerimeterDepth(PointBaseTesting): @pytest.fixture(scope="class") def uploaded_file(self, session, data_dir): self.upload_file( - filename=str(data_dir.joinpath("perimeters.csv")), session=session + session, str(data_dir.joinpath("perimeters.csv")) ) @pytest.mark.parametrize( diff --git a/tests/points/test_summary_pits.py b/tests/points/test_summary_pits.py index d1068a6..e9cfda6 100644 --- a/tests/points/test_summary_pits.py +++ b/tests/points/test_summary_pits.py @@ -32,12 +32,15 @@ class TestSummaryPits(PointBaseTesting): TableClass = PointData @pytest.fixture(scope="class") - def uploaded_file(self, db, data_dir): + def uploaded_file(self, session, data_dir): """ NOTE - this is part of the _modified file that we create in the upload script, NOT the original file """ - self.upload_file(str(data_dir.joinpath("pit_summary_points.csv"))) + self.upload_file( + session, + str(data_dir.joinpath("pit_summary_points.csv")) + ) def filter_measurement_type(self, session, measurement_type, query=None): if query is None: From 58e4692c0d68aa69a31af83de46850985a00edf6 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Tue, 15 Jul 2025 11:31:06 -0600 Subject: [PATCH 11/25] value is float in points --- snowex_db/upload/points.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snowex_db/upload/points.py b/snowex_db/upload/points.py index c205e93..67f6614 100644 --- a/snowex_db/upload/points.py +++ b/snowex_db/upload/points.py @@ -152,7 +152,7 @@ def build_data(self, series: SnowExPointData) -> gpd.GeoDataFrame: # Manage nans and nones for c in df.columns: df[c] = df[c].apply(lambda x: StringManager.parse_none(x)) - df['value'] = df[variable.code].astype(str) + df['value'] = df[variable.code].astype(float) if 'units' not in df.columns: unit_str = series.units_map.get(variable.code) From 6686a81ecb952c4c474fce71765c0fdd57ed253f Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Tue, 15 Jul 2025 12:09:18 -0600 Subject: [PATCH 12/25] fix layer data tests --- snowex_db/profile_data.py | 6 ------ snowex_db/upload/layers.py | 11 +++++++---- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/snowex_db/profile_data.py b/snowex_db/profile_data.py index 7d0927b..3f4ee3a 100644 --- a/snowex_db/profile_data.py +++ b/snowex_db/profile_data.py @@ -19,14 +19,8 @@ def __init__( variable: MeasurementDescription, meta_parser: MetaDataParser ): - # Tricky, this needs to happen before super init - self._comments_column = meta_parser.primary_variables.entries[ - "COMMENTS"] super().__init__(variable, meta_parser) - def shared_column_options(self): - return self._depth_columns + [self._comments_column] - class ExtendedSnowExProfileDataCollection(SnowExProfileDataCollection): PROFILE_DATA_CLASS = ExtendedSnowexProfileData diff --git a/snowex_db/upload/layers.py b/snowex_db/upload/layers.py index 27904e7..1b527f8 100644 --- a/snowex_db/upload/layers.py +++ b/snowex_db/upload/layers.py @@ -122,11 +122,14 @@ def build_data(self, profile: SnowExProfileData) -> gpd.GeoDataFrame: Returns: df: Dataframe ready for submission """ + if profile.df is not None: + df = profile.df.copy() + if df.empty: + LOG.debug("df is empty, returning") + return df + else: + return pd.DataFrame() - df = profile.df.copy() - if df.empty: - LOG.debug("df is empty, returning") - return df metadata = profile.metadata variable = profile.variable From d98c0207a3547b33edbf71564bc9cc16d3faeae7 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Tue, 15 Jul 2025 12:11:01 -0600 Subject: [PATCH 13/25] take advantage of github actions while we're coding --- .github/workflows/main.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d15acc2..9dba8a0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,7 +7,9 @@ on: push: branches: [ main ] pull_request: - branches: [ main ] + branches: + - main + - api_upload_update workflow_dispatch: jobs: @@ -15,7 +17,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8, 3.9, "3.10"] + python-version: [3.9, '3.10', 3.11, 3.12] services: @@ -53,7 +55,7 @@ jobs: pytest -s tests/ # Run coverage only once - - if: ${{ matrix.python-version == '3.9'}} + - if: ${{ matrix.python-version == '3.10'}} name: Get Coverage for badge run: | # Run coverage save the results From d1bd7b79899c97c06f6431e0f7d547194baf76aa Mon Sep 17 00:00:00 2001 From: Joachim Meyer Date: Thu, 24 Jul 2025 09:45:41 -0600 Subject: [PATCH 14/25] Upload - Simplify upload batch check for empty dataframe Also return a GeoDataframe to match method return signature. --- snowex_db/upload/layers.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/snowex_db/upload/layers.py b/snowex_db/upload/layers.py index 1b527f8..d0b4a37 100644 --- a/snowex_db/upload/layers.py +++ b/snowex_db/upload/layers.py @@ -122,17 +122,16 @@ def build_data(self, profile: SnowExProfileData) -> gpd.GeoDataFrame: Returns: df: Dataframe ready for submission """ - if profile.df is not None: - df = profile.df.copy() - if df.empty: - LOG.debug("df is empty, returning") - return df - else: - return pd.DataFrame() + + if profile.df is None: + LOG.debug("df is empty, returning") + return gpd.GeoDataFrame() metadata = profile.metadata variable = profile.variable + df = profile.df.copy() + # The type of measurement df['type'] = [variable.code] * len(df) From 2792beb0aafd01b41f5b4198eca5090bea9f586a Mon Sep 17 00:00:00 2001 From: Joachim Meyer Date: Thu, 24 Jul 2025 09:47:03 -0600 Subject: [PATCH 15/25] Code QC - Update a few doc strings and method return signatures. --- snowex_db/metadata.py | 3 ++- snowex_db/point_metadata.py | 13 ++++++------- snowex_db/upload/layers.py | 5 ++--- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/snowex_db/metadata.py b/snowex_db/metadata.py index 060fd22..eff169c 100644 --- a/snowex_db/metadata.py +++ b/snowex_db/metadata.py @@ -125,7 +125,8 @@ class ExtendedSnowExMetadataParser(SnowExMetaDataParser): Extend the parser to update the parsing function """ - def parse(self, filename: str): + def parse(self, filename: str) \ + -> Tuple[SnowExProfileMetadata, list, dict, int]: """ Parse the file and return a metadata object. We can override these methods as needed to parse the different diff --git a/snowex_db/point_metadata.py b/snowex_db/point_metadata.py index 73d7c0a..ba3e006 100644 --- a/snowex_db/point_metadata.py +++ b/snowex_db/point_metadata.py @@ -10,7 +10,7 @@ class PointSnowExMetadataParser(MetaDataParser): """ - Extend the parser to update the extended varaibles + Extend the parser to update the extended variables """ DEFAULT_METADATA_VARIABLE_FILES = SnowExMetaDataParser.DEFAULT_METADATA_VARIABLE_FILES @@ -64,7 +64,8 @@ def find_header_info(self, filename): return str_data, columns, columns_map, header_pos - def parse(self, filename: str): + def parse(self, filename: str) -> ( + Tuple)[Union[ProfileMetaData | None], list, dict, int]: """ Parse the file and return a metadata object. We can override these methods as needed to parse the different @@ -73,13 +74,11 @@ def parse(self, filename: str): This populates self.rough_obj Args: - filename: Path to the file from which to parse metadata + filename: (str) Full path to the file with the header info to parse Returns: - ( - Metadata or None, column list, column map, - position of header in file - ) + Tuple: + metadata object or None, column list, position of header in file """ ( meta_lines, columns, columns_map, header_position diff --git a/snowex_db/upload/layers.py b/snowex_db/upload/layers.py index d0b4a37..74798fe 100644 --- a/snowex_db/upload/layers.py +++ b/snowex_db/upload/layers.py @@ -49,8 +49,7 @@ def __init__( timezone (str): The timezone used, default is "US/Mountain". kwargs: Additional optional keyword arguments related to the profile. doi (str): Digital Object Identifier - instrument (str): Name of the instrument used - collection. + instrument (str): Name of the instrument used in the collection. header_sep (str): Delimiter for separating values in the header. Default is ','. id (str): Identifier for the profile data file. @@ -94,7 +93,7 @@ def _read(self) -> ExtendedSnowExProfileDataCollection: """ try: return ExtendedSnowExProfileDataCollection.from_csv( - self.filename, + filename=self.filename, timezone=self._timezone, header_sep=self._header_sep, site_id=self._id, From 446f3bf4ac7ce4b3c458c211ddd9fd7ff9d70888 Mon Sep 17 00:00:00 2001 From: Joachim Meyer Date: Thu, 24 Jul 2025 09:47:34 -0600 Subject: [PATCH 16/25] Code QC - Organize imports and remove unused variable. --- snowex_db/metadata.py | 10 +++++++--- snowex_db/point_data.py | 13 +++++-------- snowex_db/point_metadata.py | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/snowex_db/metadata.py b/snowex_db/metadata.py index eff169c..1a61852 100644 --- a/snowex_db/metadata.py +++ b/snowex_db/metadata.py @@ -3,16 +3,20 @@ to describing data. """ import logging +import pandas as pd +import pytz + from dataclasses import dataclass -from typing import Union +from typing import Tuple, Union -from insitupy.io.metadata import MetaDataParser from insitupy.profiles.metadata import ProfileMetaData from insitupy.campaigns.snowex.snowex_metadata import SnowExMetaDataParser from snowexsql.db import get_table_attributes from snowexsql.tables import Site -from .interpretation import * +from .interpretation import ( + manage_degree_values, convert_cardinal_to_degree, add_date_time_keys +) from .projection import add_geom, reproject_point_in_dict from .string_management import * from .utilities import assign_default_kwargs, get_logger diff --git a/snowex_db/point_data.py b/snowex_db/point_data.py index 5eeb801..58eddb8 100644 --- a/snowex_db/point_data.py +++ b/snowex_db/point_data.py @@ -1,20 +1,17 @@ import logging -from pathlib import Path from typing import List -from insitupy.io.metadata import MetaDataParser -from timezonefinder import TimezoneFinder +import geopandas as gpd import numpy as np import pandas as pd -import geopandas as gpd -from insitupy.campaigns.snowex import SnowExProfileData from insitupy.io.dates import DateTimeManager from insitupy.io.locations import LocationManager +from insitupy.io.metadata import MetaDataParser from insitupy.io.yaml_codes import YamlCodes - from insitupy.profiles.base import MeasurementData from insitupy.profiles.metadata import ProfileMetaData -from insitupy.variables import MeasurementDescription, ExtendableVariables +from insitupy.variables import MeasurementDescription +from timezonefinder import TimezoneFinder from .point_metadata import PointSnowExMetadataParser @@ -78,7 +75,7 @@ def _get_location(self, row): """ try: lat, lon, *_ = LocationManager.parse(row) - except ValueError as e: + except ValueError: if self.metadata is not None: LOG.warning( f"Row {row.name} does not have a valid location. " diff --git a/snowex_db/point_metadata.py b/snowex_db/point_metadata.py index ba3e006..d1be2d8 100644 --- a/snowex_db/point_metadata.py +++ b/snowex_db/point_metadata.py @@ -1,5 +1,5 @@ import logging -from pathlib import Path +from typing import Tuple, Union from insitupy.campaigns.snowex.snowex_metadata import SnowExMetaDataParser from insitupy.io.metadata import MetaDataParser From cb7cd938571aabf31141e1c7fdd811f194bd1397 Mon Sep 17 00:00:00 2001 From: Joachim Meyer Date: Thu, 24 Jul 2025 09:48:19 -0600 Subject: [PATCH 17/25] ProfileData - Remove overwrite of super init method. Call to super did not change anything. --- snowex_db/profile_data.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/snowex_db/profile_data.py b/snowex_db/profile_data.py index 3f4ee3a..b7c0579 100644 --- a/snowex_db/profile_data.py +++ b/snowex_db/profile_data.py @@ -1,12 +1,4 @@ -from pathlib import Path - -import pandas as pd -from insitupy.campaigns.snowex import ( - SnowExProfileData, SnowExProfileDataCollection -) -from insitupy.io.metadata import MetaDataParser -from insitupy.profiles.metadata import ProfileMetaData -from insitupy.variables import MeasurementDescription +from insitupy.campaigns.snowex import SnowExProfileData, SnowExProfileDataCollection from .metadata import ExtendedSnowExMetadataParser @@ -14,13 +6,6 @@ class ExtendedSnowexProfileData(SnowExProfileData): META_PARSER = ExtendedSnowExMetadataParser - def __init__( - self, - variable: MeasurementDescription, - meta_parser: MetaDataParser - ): - super().__init__(variable, meta_parser) - class ExtendedSnowExProfileDataCollection(SnowExProfileDataCollection): PROFILE_DATA_CLASS = ExtendedSnowexProfileData From 1026cd7ddcd15aa7ba2d689a18927eb4c1c82bbb Mon Sep 17 00:00:00 2001 From: Joachim Meyer Date: Fri, 25 Jul 2025 13:32:35 -0600 Subject: [PATCH 18/25] Tests - Skip metadata suite. To be removed soon --- tests/test_metadata.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index cde2102..c235bf1 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -1,7 +1,9 @@ """ Test all things from the metadata.py file """ +import datetime from os.path import abspath, dirname, join + import numpy as np import pandas as pd import pytest @@ -23,6 +25,7 @@ } +@pytest.mark.skip class DataHeaderTestBase: depth_is_metadata = True kwargs = {'in_timezone': 'US/Mountain'} From da246f4782af63bb2a108851089962cfd2975b9c Mon Sep 17 00:00:00 2001 From: Joachim Meyer Date: Fri, 25 Jul 2025 17:22:02 -0600 Subject: [PATCH 19/25] Tests - Rename file from poll to pole depth. --- tests/points/{test_poll_depth.py => test_pole_depth.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/points/{test_poll_depth.py => test_pole_depth.py} (100%) diff --git a/tests/points/test_poll_depth.py b/tests/points/test_pole_depth.py similarity index 100% rename from tests/points/test_poll_depth.py rename to tests/points/test_pole_depth.py From 88984745c1e1ec16f731a7053f97021b1a3ed892 Mon Sep 17 00:00:00 2001 From: Joachim Meyer Date: Fri, 25 Jul 2025 17:40:50 -0600 Subject: [PATCH 20/25] PointMetadata - Change inheritance to SnowExMetaDataParser Removes the need to declare the constant already set in that super class --- snowex_db/point_metadata.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/snowex_db/point_metadata.py b/snowex_db/point_metadata.py index d1be2d8..d242447 100644 --- a/snowex_db/point_metadata.py +++ b/snowex_db/point_metadata.py @@ -2,17 +2,15 @@ from typing import Tuple, Union from insitupy.campaigns.snowex.snowex_metadata import SnowExMetaDataParser -from insitupy.io.metadata import MetaDataParser from insitupy.profiles.metadata import ProfileMetaData LOG = logging.getLogger() -class PointSnowExMetadataParser(MetaDataParser): +class PointSnowExMetadataParser(SnowExMetaDataParser): """ Extend the parser to update the extended variables """ - DEFAULT_METADATA_VARIABLE_FILES = SnowExMetaDataParser.DEFAULT_METADATA_VARIABLE_FILES def find_header_info(self, filename): """ From 976c7b1ac7e0f7ee9a56aeac6acb347a004c1818 Mon Sep 17 00:00:00 2001 From: Joachim Meyer Date: Fri, 25 Jul 2025 17:43:54 -0600 Subject: [PATCH 21/25] Upload - Points - Compact handling of instrument, measurement, and type Instrument mapping was already done at the dataframe setup and allows to skip this step at the record creation point. Other changes combine multiple lines into one. --- snowex_db/upload/points.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/snowex_db/upload/points.py b/snowex_db/upload/points.py index b43fd42..b395d04 100644 --- a/snowex_db/upload/points.py +++ b/snowex_db/upload/points.py @@ -186,7 +186,7 @@ def build_data(self, series: SnowExPointData) -> gpd.GeoDataFrame: # Map the measurement names or default to original df["instrument"] = df['instrument'].map( - lambda x: self.MEASUREMENT_NAMES.get(x, x) + lambda x: self.MEASUREMENT_NAMES.get(x.lower(), x) ) return df @@ -264,29 +264,18 @@ def _add_campaign_observation(self, df): ): # Process each unique combination of keys (key) and its corresponding group (grouped_df) # Add instrument - instrument_name = self._get_first_check_unique(grouped_df, 'instrument') - # Map the instrument name if we have a mapping for it - if pd.isna(instrument_name): - instrument_name = None - if instrument_name: - instrument_name = self.MEASUREMENT_NAMES.get( - instrument_name.lower(), instrument_name - ) instrument = self._check_or_add_object( self._session, Instrument, dict( - name=instrument_name, + name=self._get_first_check_unique(grouped_df, 'instrument'), model=self._get_first_check_unique(grouped_df, 'instrument_model') ) ) # Add measurement type - measurement_type = self._get_first_check_unique( - grouped_df, "type" - ) measurement_obj = self._check_or_add_object( # Add units and 'derived' flag for the measurement self._session, MeasurementType, dict( - name=measurement_type, + name=self._get_first_check_unique(grouped_df, "type"), units=self._get_first_check_unique(grouped_df, "units"), derived=self._derived ) From 0bce765893385063023a0c1e36c9266b8f240c66 Mon Sep 17 00:00:00 2001 From: Joachim Meyer Date: Fri, 25 Jul 2025 17:44:52 -0600 Subject: [PATCH 22/25] Upload - Points - Change dict access to use get() Row dictionary does not always have these present on the dataframe. --- snowex_db/upload/points.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/snowex_db/upload/points.py b/snowex_db/upload/points.py index b395d04..350a147 100644 --- a/snowex_db/upload/points.py +++ b/snowex_db/upload/points.py @@ -373,8 +373,8 @@ def _add_entry(self, row: dict): datetime=row["datetime"], # Arguments from kwargs geom=row['geometry'], - version_number=row['version_number'], - elevation=row['elevation'], + version_number=row.get('version_number', None), + elevation=row.get('elevation', None), equipment=row['instrument'] ) From 859540f493f997ca3b79e24678b89f7aee34c1f6 Mon Sep 17 00:00:00 2001 From: Joachim Meyer Date: Fri, 25 Jul 2025 17:47:46 -0600 Subject: [PATCH 23/25] Upload - Points - Use PitID as instead of Name mapping. Only use the "name" attribute if it is explicitly passed as a kwarg. Otherwise use the PitID parsed from the row header. --- snowex_db/point_data.py | 25 ++++++++++++------------- snowex_db/upload/points.py | 26 ++++++++++++++++++-------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/snowex_db/point_data.py b/snowex_db/point_data.py index ef9589c..bf29726 100644 --- a/snowex_db/point_data.py +++ b/snowex_db/point_data.py @@ -226,25 +226,24 @@ def _read_csv( shared_column_options = [ # TODO: could we make this a 'shared' option in the definition - meta_parser.primary_variables.entries["INSTRUMENT"], + meta_parser.primary_variables.entries["CAMPAIGN"], + meta_parser.primary_variables.entries["COMMENTS"], meta_parser.primary_variables.entries["DATE"], - meta_parser.primary_variables.entries["TIME"], meta_parser.primary_variables.entries["DATETIME"], - meta_parser.primary_variables.entries["UTCDOY"], - meta_parser.primary_variables.entries["UTCTOD"], - meta_parser.primary_variables.entries["UTCYEAR"], - meta_parser.primary_variables.entries["LATITUDE"], - meta_parser.primary_variables.entries["LONGITUDE"], meta_parser.primary_variables.entries["EASTING"], - meta_parser.primary_variables.entries["NORTHING"], meta_parser.primary_variables.entries["ELEVATION"], - meta_parser.primary_variables.entries["INSTRUMENT_MODEL"], - meta_parser.primary_variables.entries["UTM_ZONE"], - meta_parser.primary_variables.entries["NAME"], - meta_parser.primary_variables.entries["CAMPAIGN"], - meta_parser.primary_variables.entries["COMMENTS"], meta_parser.primary_variables.entries["FLAGS"], + meta_parser.primary_variables.entries["INSTRUMENT"], + meta_parser.primary_variables.entries["INSTRUMENT_MODEL"], + meta_parser.primary_variables.entries["LATITUDE"], + meta_parser.primary_variables.entries["LONGITUDE"], + meta_parser.primary_variables.entries["NORTHING"], meta_parser.primary_variables.entries["PIT_ID"], + meta_parser.primary_variables.entries["TIME"], + meta_parser.primary_variables.entries["UTCDOY"], + meta_parser.primary_variables.entries["UTCTOD"], + meta_parser.primary_variables.entries["UTCYEAR"], + meta_parser.primary_variables.entries["UTM_ZONE"], meta_parser.primary_variables.entries["VERSION_NUMBER"], ] diff --git a/snowex_db/upload/points.py b/snowex_db/upload/points.py index 350a147..33c1144 100644 --- a/snowex_db/upload/points.py +++ b/snowex_db/upload/points.py @@ -229,13 +229,17 @@ def submit(self): ) def _observation_name_from_row(self, row): - value = f"{row['name']}_{row['instrument']}" + name = row.get('name') or row.get('pit_id') + value = f"{name}_{row['instrument']}" + if row.get('instrument_model'): - value += row['instrument_model'] + value += '_' + row['instrument_model'] + # Add the type of measurement # This is necessary because the GPR returns multiple variables if row.get('type'): value += "_" + row['type'] + return value def _get_first_check_unique(self, df, key): @@ -244,10 +248,12 @@ def _get_first_check_unique(self, df, key): it is unique. If not, raise a DataValidationError """ unique_values = df[key].unique() + if len(unique_values) > 1: raise DataValidationError( f"Multiple values for {key} found: {unique_values}" ) + return unique_values[0] def _add_campaign_observation(self, df): @@ -257,12 +263,14 @@ def _add_campaign_observation(self, df): """ df["date"] = pd.to_datetime(df["datetime"]).dt.date - # Group by our observation keys to add into the database - for keys, grouped_df in df.groupby( - ['instrument', 'instrument_model', 'name', 'type', 'date'], - dropna=False - ): - # Process each unique combination of keys (key) and its corresponding group (grouped_df) + + # Group by our observation keys to add records uniquely into the database + base_groups = ['instrument', 'instrument_model', 'name', 'type', 'date'] + if 'pit_id' in df.columns: + base_groups.append('pit_id') + + # Process each unique combination of keys (key) and its corresponding group (grouped_df) + for keys, grouped_df in df.groupby(base_groups, dropna=False): # Add instrument instrument = self._check_or_add_object( self._session, Instrument, dict( @@ -284,6 +292,8 @@ def _add_campaign_observation(self, df): # Check name is unique because we are adding ONE # campaign observation here self._get_first_check_unique(grouped_df, "name") + if 'pit_id' in grouped_df.columns: + self._get_first_check_unique(grouped_df, "pit_id") # Get the measurement name measurement_name = self._observation_name_from_row(grouped_df.iloc[0]) From 1ecc5396ed7d268cab755a8f7bb577e614cd6dfa Mon Sep 17 00:00:00 2001 From: Joachim Meyer Date: Fri, 25 Jul 2025 17:51:17 -0600 Subject: [PATCH 24/25] Upload - Points - Code and comment formatting --- snowex_db/upload/points.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/snowex_db/upload/points.py b/snowex_db/upload/points.py index 33c1144..5656691 100644 --- a/snowex_db/upload/points.py +++ b/snowex_db/upload/points.py @@ -305,16 +305,15 @@ def _add_campaign_observation(self, df): ) else: doi = None - # pass in campaign - campaign_name = self._get_first_check_unique( - grouped_df, "campaign" - ) or self._campaign_name + # Add campaign + campaign_name = self._get_first_check_unique(grouped_df, "campaign") \ + or self._campaign_name if campaign_name is None: raise DataValidationError("Campaign cannot be None") campaign = self._check_or_add_object( self._session, Campaign, dict(name=campaign_name) ) - # add observer + # Add observer observer_name = self._get_first_check_unique( grouped_df, "observer" ) or self._observer @@ -322,6 +321,7 @@ def _add_campaign_observation(self, df): observer = self._check_or_add_object( self._session, Observer, dict(name=observer_name) ) + # Construct description string description = None if ["comments"] in grouped_df.columns.values: description = (description or "") + self._get_first_check_unique( From f1ac4823fad6b306388237c10a7e5d79da108b2b Mon Sep 17 00:00:00 2001 From: Joachim Meyer Date: Fri, 25 Jul 2025 17:52:05 -0600 Subject: [PATCH 25/25] Tests - Points - Expand cases to look for more records and attributes --- tests/points/test_depth.py | 62 ++++++++++++++++++++++++++++----- tests/points/test_pole_depth.py | 61 ++++++++++++++++++++++++++++---- 2 files changed, 107 insertions(+), 16 deletions(-) diff --git a/tests/points/test_depth.py b/tests/points/test_depth.py index 4225805..8dec361 100644 --- a/tests/points/test_depth.py +++ b/tests/points/test_depth.py @@ -42,25 +42,69 @@ def filter_measurement_type(self, session, measurement_type, query=None): ).filter(MeasurementType.name == measurement_type) return query + @pytest.mark.usefixtures("uploaded_file") + def test_measurement_type(self, session): + record = self.get_records(session, MeasurementType, "name", "depth") + assert len(record) == 1 + record = record[0] + assert record.units == 'cm' + assert record.derived is False + + @pytest.mark.usefixtures("uploaded_file") + @pytest.mark.parametrize( + "name, model", [ + ("mesa", "Mesa2_1"), + ("magnaprobe", "CRREL_B"), + ("pit ruler", None), + ] + ) + def test_instrument(self, name, model, session): + record = self.get_records(session, Instrument, "name", name) + assert len(record) == 1 + record = record[0] + assert record.model == model + + @pytest.mark.usefixtures("uploaded_file") + @pytest.mark.parametrize( + "name, count", + [ + ("example_point_name_magnaprobe_CRREL_B_depth", 1), + ("example_point_name_mesa_Mesa2_1_depth", 1), + # We have three different dates + ("example_point_name_pit ruler_depth", 3) + ], + ) + def test_campaign_observation(self, name, count, session): + names = self.get_records(session, CampaignObservation, "name", name) + assert len(names) == count + + @pytest.mark.usefixtures("uploaded_file") + @pytest.mark.parametrize( + "date, count", + [ + (date(2020, 1, 28), 1), + (date(2020, 2, 4), 1), + (date(2020, 2, 11), 1), + (date(2020, 1, 30), 1), + (date(2020, 2, 5), 1), + ], + ) + def test_point_observation(self, date, count, session): + record = self.get_records(session, PointObservation, "date", date) + assert len(record) == count + @pytest.mark.parametrize( "table, attribute, expected_value", [ (Campaign, "name", "Grand Mesa"), - (Instrument, "name", "mesa"), - (Instrument, "model", "Mesa2_1"), - (MeasurementType, "name", ['depth']), - (MeasurementType, "units", ['cm']), - (MeasurementType, "derived", [False]), (DOI, "doi", "some_point_doi"), - (CampaignObservation, "name", "example_point_name_M2Mesa2_1_depth"), (PointData, "geom", WKTElement('POINT (-108.13515 39.03045)', srid=4326) - ), - (PointObservation, "date", date(2020, 2, 4)), + ), ] ) def test_metadata(self, table, attribute, expected_value, uploaded_file): self._check_metadata(table, attribute, expected_value) - + @pytest.mark.parametrize( "data_name, attribute_to_check, filter_attribute, filter_value, expected", [ ('depth', 'value', 'value', 94.0, [94]), diff --git a/tests/points/test_pole_depth.py b/tests/points/test_pole_depth.py index c176b7e..02fe0ad 100644 --- a/tests/points/test_pole_depth.py +++ b/tests/points/test_pole_depth.py @@ -33,20 +33,67 @@ def uploaded_file(self, session, data_dir): filename=str(data_dir.joinpath("pole_depths.csv")), session=session ) + @pytest.mark.usefixtures("uploaded_file") + def test_measurement_type(self, session): + record = self.get_records(session, MeasurementType, "name", "depth") + assert len(record) == 1 + record = record[0] + assert record.units == 'cm' + assert record.derived is False + + @pytest.mark.usefixtures("uploaded_file") + @pytest.mark.parametrize("model", ["W1B", "E9B", "E8A", "E6A"]) + def test_instrument(self, model, session): + record = self.get_records(session, Instrument, "model", model) + assert len(record) == 1 + record = record[0] + assert record.name == "camera" + + @pytest.mark.usefixtures("uploaded_file") + @pytest.mark.parametrize( + "date", + [ + date(2019, 11, 27), + date(2019, 12, 7), + date(2019, 12, 31), + date(2020, 2, 1), + date(2019, 10, 28), + date(2019, 11, 28), + date(2019, 12, 14), + date(2019, 11, 29), + date(2020, 2, 27), + date(2020, 4, 7), + date(2020, 5, 22), + date(2020, 1, 27), + date(2020, 3, 14), + date(2020, 5, 3), + ], + ) + def test_point_observation(self, date, session): + record = self.get_records(session, PointObservation, "date", date) + assert len(record) == 1 + + @pytest.mark.usefixtures("uploaded_file") + @pytest.mark.parametrize( + "name, count", + [ + ("example_pole_point_name_camera_E6A_depth", 4), + ("example_pole_point_name_camera_E8A_depth", 3), + ("example_pole_point_name_camera_E9B_depth", 4), + ("example_pole_point_name_camera_W1B_depth", 3), + ], + ) + def test_campaign_observation(self, name, count, session): + names = self.get_records(session, CampaignObservation, "name", name) + assert len(names) == count + @pytest.mark.parametrize( "table, attribute, expected_value", [ (Campaign, "name", "Grand Mesa"), - (Instrument, "name", "camera"), - (Instrument, "model", "E6A"), - (MeasurementType, "name", ['depth']), - (MeasurementType, "units", ['cm']), - (MeasurementType, "derived", [False]), (DOI, "doi", "some_point_doi_poles"), - (CampaignObservation, "name", "example_pole_point_name_cameraE6A_depth"), (PointData, "geom", WKTElement('POINT (-108.184794 39.008078)', srid=4326) ), - (PointObservation, "date", date(2019, 11, 27)), ] ) def test_metadata(self, table, attribute, expected_value, uploaded_file):