diff --git a/cdisc_rules_engine/check_operators/helpers.py b/cdisc_rules_engine/check_operators/helpers.py index 0816380b0..7c787b298 100644 --- a/cdisc_rules_engine/check_operators/helpers.py +++ b/cdisc_rules_engine/check_operators/helpers.py @@ -334,6 +334,26 @@ def _truncate_by_precision( ) +def format_date_preserving_precision(original_str: str) -> str: + precision = detect_datetime_precision(original_str) + if precision is None: + return "" + dt = truncate_datetime_to_precision(original_str, precision) + + if precision >= DatePrecision.second: + return dt.strftime("%Y-%m-%dT%H:%M:%S") + elif precision == DatePrecision.minute: + return dt.strftime("%Y-%m-%dT%H:%M") + elif precision == DatePrecision.hour: + return dt.strftime("%Y-%m-%dT%H") + elif precision == DatePrecision.day: + return dt.strftime("%Y-%m-%d") + elif precision == DatePrecision.month: + return dt.strftime("%Y-%m") + else: # year + return dt.strftime("%Y") + + def _compare_with_inferred_precision( operator_func, target: str, diff --git a/cdisc_rules_engine/operations/max_date.py b/cdisc_rules_engine/operations/max_date.py index 73d3fda88..bbefddfe2 100644 --- a/cdisc_rules_engine/operations/max_date.py +++ b/cdisc_rules_engine/operations/max_date.py @@ -1,26 +1,44 @@ import pandas as pd from cdisc_rules_engine.operations.base_operation import BaseOperation +from cdisc_rules_engine.check_operators.helpers import format_date_preserving_precision class MaxDate(BaseOperation): def _execute_operation(self): + original = self.params.dataframe[self.params.target] + data = pd.to_datetime(original, format="ISO8601") + if not self.params.grouping: - data = pd.to_datetime(self.params.dataframe[self.params.target]) - max_date = data.max() - if isinstance(max_date, pd._libs.tslibs.nattype.NaTType): + if data.isna().all(): result = "" else: - result = max_date.isoformat() + max_idx = data.idxmax() + result = format_date_preserving_precision(original.loc[max_idx]) + return pd.Series(result, index=self.evaluation_dataset.index) + + grouping_cols = self.params.grouping + if isinstance(grouping_cols, str): + grouping_cols = [grouping_cols] + + group_keys = [self.params.dataframe[col] for col in grouping_cols] + idx_of_max = data.groupby(group_keys).apply( + lambda s: s.idxmax() if s.notna().any() else pd.NA + ) + max_dates = idx_of_max.apply( + lambda idx: ( + "" + if pd.isna(idx) + else format_date_preserving_precision(original.loc[idx]) + ) + ) + if len(grouping_cols) == 1: + lookup_keys = self.evaluation_dataset[grouping_cols[0]] else: - result = self.params.dataframe.groupby( - self.params.grouping, as_index=False, group_keys=False - ).max() - if isinstance(result, pd.Series): - result = result.apply(lambda x: x.isoformat() if pd.notna(x) else "") - elif isinstance(result, pd.DataFrame): - for col in result.columns: - if pd.api.types.is_datetime64_any_dtype(result[col]): - result[col] = result[col].apply( - lambda x: x.isoformat() if pd.notna(x) else "" - ) + lookup_keys = pd.Series( + list(zip(*[self.evaluation_dataset[c] for c in grouping_cols])), + index=self.evaluation_dataset.index, + ) + + result = lookup_keys.map(max_dates).fillna("") + result.index = self.evaluation_dataset.index return result diff --git a/cdisc_rules_engine/operations/min_date.py b/cdisc_rules_engine/operations/min_date.py index 106a916cc..ee0e26aea 100644 --- a/cdisc_rules_engine/operations/min_date.py +++ b/cdisc_rules_engine/operations/min_date.py @@ -1,18 +1,45 @@ import pandas as pd from cdisc_rules_engine.operations.base_operation import BaseOperation +from cdisc_rules_engine.check_operators.helpers import format_date_preserving_precision class MinDate(BaseOperation): def _execute_operation(self): + original = self.params.dataframe[self.params.target] + data = pd.to_datetime(original, format="ISO8601") + if not self.params.grouping: - data = pd.to_datetime(self.params.dataframe[self.params.target]) - min_date = data.min() - if isinstance(min_date, pd._libs.tslibs.nattype.NaTType): + if data.isna().all(): result = "" else: - result = min_date.isoformat() + min_idx = data.idxmin() + result = format_date_preserving_precision(original.loc[min_idx]) + return pd.Series(result, index=self.evaluation_dataset.index) + + grouping_cols = self.params.grouping + if isinstance(grouping_cols, str): + grouping_cols = [grouping_cols] + + group_keys = [self.params.dataframe[col] for col in grouping_cols] + idx_of_min = data.groupby(group_keys).apply( + lambda s: s.idxmin() if s.notna().any() else pd.NA + ) + min_dates = idx_of_min.apply( + lambda idx: ( + "" + if pd.isna(idx) + else format_date_preserving_precision(original.loc[idx]) + ) + ) + + if len(grouping_cols) == 1: + lookup_keys = self.evaluation_dataset[grouping_cols[0]] else: - result = self.params.dataframe.groupby( - self.params.grouping, as_index=False - ).min() + lookup_keys = pd.Series( + list(zip(*[self.evaluation_dataset[c] for c in grouping_cols])), + index=self.evaluation_dataset.index, + ) + + result = lookup_keys.map(min_dates).fillna("") + result.index = self.evaluation_dataset.index return result diff --git a/tests/unit/test_operations/test_max_date.py b/tests/unit/test_operations/test_max_date.py index 50eaad04b..735fafe89 100644 --- a/tests/unit/test_operations/test_max_date.py +++ b/tests/unit/test_operations/test_max_date.py @@ -3,7 +3,7 @@ from cdisc_rules_engine.models.dataset.pandas_dataset import PandasDataset from cdisc_rules_engine.operations.max_date import MaxDate from cdisc_rules_engine.models.operation_params import OperationParams -import pandas as pd +from cdisc_rules_engine.check_operators.helpers import format_date_preserving_precision import pytest from cdisc_rules_engine.services.cache.cache_service_factory import CacheServiceFactory @@ -17,14 +17,14 @@ [ ( {"dates": ["2001-01-01", "", "2022-01-05"]}, - pd.to_datetime("2022-01-05").isoformat(), + format_date_preserving_precision("2022-01-05"), PandasDataset, None, ), ({"dates": [None, None]}, "", PandasDataset, None), ( {"dates": ["2001-01-01", "", "2022-01-05"]}, - pd.to_datetime("2022-01-05").isoformat(), + format_date_preserving_precision("2022-01-05"), DaskDataset, None, ), @@ -93,6 +93,88 @@ DaskDataset, ["USUBJID"], ), + ( + { + "dates": [ + "2025-10-10", + "2025-10-15", + "2025-12-02", + "2025-12-11", + "", + "", + ], + "USUBJID": ["00002", "00002", "00003", "00003", "00004", "00004"], + }, + PandasDataset.from_records( + [ + { + "dates": "2025-10-10", + "USUBJID": "00002", + "operation_id": "2025-10-15", + }, + { + "dates": "2025-10-15", + "USUBJID": "00002", + "operation_id": "2025-10-15", + }, + { + "dates": "2025-12-02", + "USUBJID": "00003", + "operation_id": "2025-12-11", + }, + { + "dates": "2025-12-11", + "USUBJID": "00003", + "operation_id": "2025-12-11", + }, + {"dates": "", "USUBJID": "00004", "operation_id": ""}, + {"dates": "", "USUBJID": "00004", "operation_id": ""}, + ] + ), + PandasDataset, + ["USUBJID"], + ), + ( + { + "dates": [ + "2025-10-10", + "2025-10-15", + "2025-12-02", + "2025-12-11", + "", + "", + ], + "USUBJID": ["00002", "00002", "00003", "00003", "00004", "00004"], + }, + DaskDataset.from_records( + [ + { + "dates": "2025-10-10", + "USUBJID": "00002", + "operation_id": "2025-10-15", + }, + { + "dates": "2025-10-15", + "USUBJID": "00002", + "operation_id": "2025-10-15", + }, + { + "dates": "2025-12-02", + "USUBJID": "00003", + "operation_id": "2025-12-11", + }, + { + "dates": "2025-12-11", + "USUBJID": "00003", + "operation_id": "2025-12-11", + }, + {"dates": "", "USUBJID": "00004", "operation_id": ""}, + {"dates": "", "USUBJID": "00004", "operation_id": ""}, + ] + ), + DaskDataset, + ["USUBJID"], + ), ], ) def test_max_date( diff --git a/tests/unit/test_operations/test_min_date.py b/tests/unit/test_operations/test_min_date.py index f15b3ad1f..e5626be0c 100644 --- a/tests/unit/test_operations/test_min_date.py +++ b/tests/unit/test_operations/test_min_date.py @@ -3,7 +3,7 @@ from cdisc_rules_engine.models.dataset.pandas_dataset import PandasDataset from cdisc_rules_engine.operations.min_date import MinDate from cdisc_rules_engine.models.operation_params import OperationParams -import pandas as pd +from cdisc_rules_engine.check_operators.helpers import format_date_preserving_precision import pytest from cdisc_rules_engine.services.cache.cache_service_factory import CacheServiceFactory @@ -13,31 +13,192 @@ @pytest.mark.parametrize( - "data, expected, dataset_type", + "data, expected, dataset_type, grouping", [ ( {"dates": ["2001-01-01", "", "2022-01-01"]}, - pd.to_datetime("2001-01-01").isoformat(), + format_date_preserving_precision("2001-01-01"), DaskDataset, + None, ), - ({"dates": [None, None]}, "", DaskDataset), + ({"dates": [None, None]}, "", DaskDataset, None), ( {"dates": ["2001-01-01", "", "2022-01-01"]}, - pd.to_datetime("2001-01-01").isoformat(), + format_date_preserving_precision("2001-01-01"), PandasDataset, + None, + ), + ({"dates": [None, None]}, "", PandasDataset, None), + ( + { + "dates": ["2025-10-10", "2025-10-15", "2025-12-02", "2025-12-11"], + "USUBJID": ["00002", "00002", "00003", "00003"], + }, + PandasDataset.from_records( + [ + { + "dates": "2025-10-10", + "USUBJID": "00002", + "operation_id": "2025-10-10", + }, + { + "dates": "2025-10-15", + "USUBJID": "00002", + "operation_id": "2025-10-10", + }, + { + "dates": "2025-12-02", + "USUBJID": "00003", + "operation_id": "2025-12-02", + }, + { + "dates": "2025-12-11", + "USUBJID": "00003", + "operation_id": "2025-12-02", + }, + ] + ), + PandasDataset, + ["USUBJID"], + ), + ( + { + "dates": ["2025-10-10", "2025-10-15", "2025-12-02", "2025-12-11"], + "USUBJID": ["00002", "00002", "00003", "00003"], + }, + DaskDataset.from_records( + [ + { + "dates": "2025-10-10", + "USUBJID": "00002", + "operation_id": "2025-10-10", + }, + { + "dates": "2025-10-15", + "USUBJID": "00002", + "operation_id": "2025-10-10", + }, + { + "dates": "2025-12-02", + "USUBJID": "00003", + "operation_id": "2025-12-02", + }, + { + "dates": "2025-12-11", + "USUBJID": "00003", + "operation_id": "2025-12-02", + }, + ] + ), + DaskDataset, + ["USUBJID"], + ), + ( + { + "dates": [ + "2025-10-10", + "2025-10-15", + "2025-12-02", + "2025-12-11", + "", + "", + ], + "USUBJID": ["00002", "00002", "00003", "00003", "00004", "00004"], + }, + PandasDataset.from_records( + [ + { + "dates": "2025-10-10", + "USUBJID": "00002", + "operation_id": "2025-10-10", + }, + { + "dates": "2025-10-15", + "USUBJID": "00002", + "operation_id": "2025-10-10", + }, + { + "dates": "2025-12-02", + "USUBJID": "00003", + "operation_id": "2025-12-02", + }, + { + "dates": "2025-12-11", + "USUBJID": "00003", + "operation_id": "2025-12-02", + }, + {"dates": "", "USUBJID": "00004", "operation_id": ""}, + {"dates": "", "USUBJID": "00004", "operation_id": ""}, + ] + ), + PandasDataset, + ["USUBJID"], + ), + ( + { + "dates": [ + "2025-10-10", + "2025-10-15", + "2025-12-02", + "2025-12-11", + "", + "", + ], + "USUBJID": ["00002", "00002", "00003", "00003", "00004", "00004"], + }, + DaskDataset.from_records( + [ + { + "dates": "2025-10-10", + "USUBJID": "00002", + "operation_id": "2025-10-10", + }, + { + "dates": "2025-10-15", + "USUBJID": "00002", + "operation_id": "2025-10-10", + }, + { + "dates": "2025-12-02", + "USUBJID": "00003", + "operation_id": "2025-12-02", + }, + { + "dates": "2025-12-11", + "USUBJID": "00003", + "operation_id": "2025-12-02", + }, + {"dates": "", "USUBJID": "00004", "operation_id": ""}, + {"dates": "", "USUBJID": "00004", "operation_id": ""}, + ] + ), + DaskDataset, + ["USUBJID"], ), - ({"dates": [None, None]}, "", PandasDataset), ], ) -def test_minimum(data, expected, operation_params: OperationParams, dataset_type): +def test_minimum( + data, + expected, + dataset_type, + grouping: str | None, + operation_params: OperationParams, +): config = ConfigService() cache = CacheServiceFactory(config).get_cache_service() data_service = DataServiceFactory(config, cache).get_data_service() operation_params.dataframe = dataset_type.from_dict(data) operation_params.target = "dates" + operation_params.grouping = grouping result = MinDate( operation_params, dataset_type.from_dict(data), cache, data_service ).execute() assert operation_params.operation_id in result - for val in result[operation_params.operation_id]: - assert val == expected + + if isinstance(expected, PandasDataset) and dataset_type is PandasDataset: + assert result.data.equals(expected.data) + elif isinstance(expected, DaskDataset) and dataset_type is DaskDataset: + assert expected.equals(result) + else: + for val in result[operation_params.operation_id]: + assert val == expected