diff --git a/faircode/cli.py b/faircode/cli.py index 1825b5a..e49375a 100644 --- a/faircode/cli.py +++ b/faircode/cli.py @@ -3,6 +3,8 @@ faircode profile data.csv faircode profile data.tsv faircode profile data.xlsx + faircode profile data.json + faircode profile data.parquet faircode profile data.csv --json faircode profile data.csv --html report.html faircode compare train.csv prod.csv diff --git a/faircode/loaders.py b/faircode/loaders.py index 38e0527..320a078 100644 --- a/faircode/loaders.py +++ b/faircode/loaders.py @@ -34,6 +34,21 @@ def read_table(path: str) -> pd.DataFrame: if suffix == ".csv": return pd.read_csv(path) + + if suffix == ".json": + try: + return pd.read_json(path) + except ValueError: + return pd.read_json(path, orient="split") + + if suffix == ".parquet": + try: + return pd.read_parquet(path) + except ImportError as exc: + raise RuntimeError( + "reading .parquet files requires the 'pyarrow' package " + "(install with: pip install faircode[parquet])" + ) from exc with open(path, "r", encoding="utf-8", errors="replace", newline="") as fh: sample = fh.read(SNIFF_SAMPLE_BYTES) diff --git a/tests/test_loaders.py b/tests/test_loaders.py index 74d1d5d..b6eca3d 100644 --- a/tests/test_loaders.py +++ b/tests/test_loaders.py @@ -16,6 +16,11 @@ reason="optional 'excel' extra not installed", ) +requires_pyarrow = pytest.mark.skipif( + importlib.util.find_spec("pyarrow") is None, + reason="optional 'parquet' extra not installed", +) + ROWS = { "patient_id": [1, 2, 3, 4], @@ -61,6 +66,24 @@ def test_read_table_csv(tmp_path): assert list(df.columns) == ["patient_id", "sex", "age"] +def test_read_table_json_records(tmp_path): + path = tmp_path / "data.json" + pd.DataFrame(ROWS).to_json(path, orient="records") + df = read_table(str(path)) + assert list(df.columns) == ["patient_id", "sex", "age"] + assert len(df) == 4 + + +def test_read_table_json_split(tmp_path): + path = tmp_path / "data.json" + pd.DataFrame(ROWS).to_json(path, orient="split") + + df = read_table(str(path)) + + assert list(df.columns) == ["patient_id", "sex", "age"] + assert len(df) == 4 + + @requires_openpyxl def test_read_table_xlsx(tmp_path): path = tmp_path / "data.xlsx" @@ -70,6 +93,15 @@ def test_read_table_xlsx(tmp_path): assert len(df) == 4 +@requires_pyarrow +def test_read_table_parquet(tmp_path): + path = tmp_path / "data.parquet" + pd.DataFrame(ROWS).to_parquet(path, index=False) + df = read_table(str(path)) + assert list(df.columns) == ["patient_id", "sex", "age"] + assert len(df) == 4 + + def test_read_table_unknown_extension_sniffs_tabs(tmp_path): path = tmp_path / "data.txt" _write_csv(path, sep="\t") @@ -99,3 +131,29 @@ def test_xlsx_and_csv_profile_identically(tmp_path): result_csv = profile(read_table(str(csv_path))) result_xlsx = profile(read_table(str(xlsx_path))) assert result_csv == result_xlsx + +def test_json_and_csv_profile_identically(tmp_path): + csv_path = tmp_path / "data.csv" + json_path = tmp_path / "data.json" + + _write_csv(csv_path, sep=",") + pd.DataFrame(ROWS).to_json(json_path, orient="records") + + result_csv = profile(read_table(str(csv_path))) + result_json = profile(read_table(str(json_path))) + + assert result_csv == result_json + + +@requires_pyarrow +def test_parquet_and_csv_profile_identically(tmp_path): + csv_path = tmp_path / "data.csv" + parquet_path = tmp_path / "data.parquet" + + _write_csv(csv_path, sep=",") + pd.DataFrame(ROWS).to_parquet(parquet_path, index=False) + + result_csv = profile(read_table(str(csv_path))) + result_parquet = profile(read_table(str(parquet_path))) + + assert result_csv == result_parquet