Add json parquet loader

faircode/cli.py

@@ -3,6 +3,8 @@
     faircode profile data.csv
     faircode profile data.tsv
     faircode profile data.xlsx
+    faircode profile data.json
+    faircode profile data.parquet
     faircode profile data.csv --json
     faircode profile data.csv --html report.html
     faircode compare train.csv prod.csv

faircode/loaders.py

@@ -34,6 +34,21 @@ def read_table(path: str) -> pd.DataFrame:
 
     if suffix == ".csv":
         return pd.read_csv(path)
+
+    if suffix == ".json":
+        try:
+            return pd.read_json(path)
+        except ValueError:
+            return pd.read_json(path, orient="split")
+
+    if suffix == ".parquet":
+        try:
+            return pd.read_parquet(path)
+        except ImportError as exc:
+            raise RuntimeError(
+                "reading .parquet files requires the 'pyarrow' package "
+                "(install with: pip install faircode[parquet])"
+            ) from exc
 
     with open(path, "r", encoding="utf-8", errors="replace", newline="") as fh:
         sample = fh.read(SNIFF_SAMPLE_BYTES)

tests/test_loaders.py

@@ -16,6 +16,11 @@
     reason="optional 'excel' extra not installed",
 )
 
+requires_pyarrow = pytest.mark.skipif(
+    importlib.util.find_spec("pyarrow") is None,
+    reason="optional 'parquet' extra not installed",
+)
+
 
 ROWS = {
     "patient_id": [1, 2, 3, 4],
@@ -61,6 +66,24 @@ def test_read_table_csv(tmp_path):
     assert list(df.columns) == ["patient_id", "sex", "age"]
 
 
+def test_read_table_json_records(tmp_path):
+    path = tmp_path / "data.json"
+    pd.DataFrame(ROWS).to_json(path, orient="records")
+    df = read_table(str(path))
+    assert list(df.columns) == ["patient_id", "sex", "age"]
+    assert len(df) == 4
+
+
+def test_read_table_json_split(tmp_path):
+    path = tmp_path / "data.json"
+    pd.DataFrame(ROWS).to_json(path, orient="split")
+
+    df = read_table(str(path))
+
+    assert list(df.columns) == ["patient_id", "sex", "age"]
+    assert len(df) == 4
+
+
 @requires_openpyxl
 def test_read_table_xlsx(tmp_path):
     path = tmp_path / "data.xlsx"
@@ -70,6 +93,15 @@ def test_read_table_xlsx(tmp_path):
     assert len(df) == 4
 
 
+@requires_pyarrow
+def test_read_table_parquet(tmp_path):
+    path = tmp_path / "data.parquet"
+    pd.DataFrame(ROWS).to_parquet(path, index=False)
+    df = read_table(str(path))
+    assert list(df.columns) == ["patient_id", "sex", "age"]
+    assert len(df) == 4
+
+
 def test_read_table_unknown_extension_sniffs_tabs(tmp_path):
     path = tmp_path / "data.txt"
     _write_csv(path, sep="\t")
@@ -99,3 +131,29 @@ def test_xlsx_and_csv_profile_identically(tmp_path):
     result_csv = profile(read_table(str(csv_path)))
     result_xlsx = profile(read_table(str(xlsx_path)))
     assert result_csv == result_xlsx
+
+def test_json_and_csv_profile_identically(tmp_path):
+    csv_path = tmp_path / "data.csv"
+    json_path = tmp_path / "data.json"
+
+    _write_csv(csv_path, sep=",")
+    pd.DataFrame(ROWS).to_json(json_path, orient="records")
+
+    result_csv = profile(read_table(str(csv_path)))
+    result_json = profile(read_table(str(json_path)))
+
+    assert result_csv == result_json
+
+
+@requires_pyarrow
+def test_parquet_and_csv_profile_identically(tmp_path):
+    csv_path = tmp_path / "data.csv"
+    parquet_path = tmp_path / "data.parquet"
+
+    _write_csv(csv_path, sep=",")
+    pd.DataFrame(ROWS).to_parquet(parquet_path, index=False)
+
+    result_csv = profile(read_table(str(csv_path)))
+    result_parquet = profile(read_table(str(parquet_path)))
+
+    assert result_csv == result_parquet