diff --git a/.github/workflows/all_checks.yml b/.github/workflows/all_checks.yml index ef24e312..e68a865c 100644 --- a/.github/workflows/all_checks.yml +++ b/.github/workflows/all_checks.yml @@ -38,8 +38,17 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.py }} + # Make the companion PR versions available to tox before publishing to PyPI. + # Remove this step after ruranges 0.2.8 and gtfreader 0.3.1 are released. + - name: Build backend PR wheels + if: github.event_name == 'pull_request' + run: | + python -m pip wheel --no-deps --wheel-dir "${{ runner.temp }}/backend-wheels" \ + "git+https://github.com/pyranges/ruranges_py.git@f56d1520e52f34cd68da8908aa6208ab64956f81" \ + "git+https://github.com/pyranges/gtfreader.git@9a5b3d8eb7b30743aea02fc7211586290171d68c" + echo "PIP_FIND_LINKS=${{ runner.temp }}/backend-wheels" >> "$GITHUB_ENV" - name: Install tox - run: python -m pip install tox-gh>=1.2 + run: python -m pip install 'tox-gh>=1.2' - name: Setup test suite run: tox -vv --notest - name: Run test suite diff --git a/.github/workflows/build_wheels_and_upload_to_pypi.yml b/.github/workflows/build_wheels_and_upload_to_pypi.yml index 5d4f4f88..508b0bee 100644 --- a/.github/workflows/build_wheels_and_upload_to_pypi.yml +++ b/.github/workflows/build_wheels_and_upload_to_pypi.yml @@ -1,8 +1,5 @@ name: Build and upload to PyPI -env: - CIBW_SKIP: "cp36-*" - on: workflow_dispatch: diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 49f205e1..6ce9cc9a 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +# 1.5.1 (25.10.26) +- Fix the readers under pandas 3.1, which infers the categories of a `"category"` column in `read_csv`: a chromosome named `1` came back as the integer `1` (and `01` merged with it). BED, GFF3, peak and signal files are now read as text and made categorical after; GTF needs gtfreader >= 0.3.1 +- Fix `ValueError`/`TypeError: The given array is not contiguous` from most range operations on frames whose columns are strided views (pandas >= 3.1 can return them; coordinates past int32 always reached the kernels uncopied); requires ruranges >= 0.2.8 +- Doctests and the unit tests pass on pandas 3.1 + # 1.5.0 (14.10.26) Most of this release is by Piotr Balwierz (@balwierz), and the parallel readers by Marco Mariotti (@marco-mariotti). diff --git a/docs/how_to_columns.rst b/docs/how_to_columns.rst index 27d3d421..b900648f 100644 --- a/docs/how_to_columns.rst +++ b/docs/how_to_columns.rst @@ -30,7 +30,7 @@ A single PyRanges column (which are pandas Series) can be extracted through the PyRanges with 20 rows, 6 columns, and 1 index columns. Contains 15 chromosomes and 2 strands. - >>> gr.Chromosome.head() + >>> gr.Chromosome.head() # doctest: +NORMALIZE_WHITESPACE 0 chr8 1 chr7 2 chr5 @@ -51,7 +51,7 @@ A single PyRanges column (which are pandas Series) can be extracted through the The ``gr[column_name]`` syntax also extracts a column from a PyRanges object: - >>> gr['Chromosome'].head() + >>> gr['Chromosome'].head() # doctest: +NORMALIZE_WHITESPACE 0 chr8 1 chr7 2 chr5 diff --git a/pyproject.toml b/pyproject.toml index cd2a3062..80f180c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "pyranges1" -version = "1.5.0" +version = "1.5.1" description = "GenomicRanges for Python." requires-python = ">=3.12.0" readme = "README.md" @@ -30,7 +30,7 @@ classifiers = [ "Topic :: Scientific/Engineering", ] keywords = ["bioinformatics", "genomicranges", "genomics"] -dependencies = ["pandas", "tabulate", "natsort", "ruranges>=0.1.8", "gtfreader>=0.3.0"] +dependencies = ["pandas", "tabulate", "natsort", "ruranges>=0.2.8", "gtfreader>=0.3.1"] [project.optional-dependencies] # pyarrow parses on every core instead of one: read_bed is 10-18x faster on @@ -108,7 +108,9 @@ python = 3.12 = py312 3.13 = py313 +# Allow CI to supply wheels built from companion backend PRs. [testenv] +passenv = PIP_FIND_LINKS basepython = py312: python3.12 py313: python3.13 @@ -131,8 +133,8 @@ deps = pytest ty == 0.0.87 joblib - gtfreader >= 0.3.0 - ruranges >= 0.1.8 + gtfreader >= 0.3.1 + ruranges >= 0.2.8 # The readers run every test twice, once on each parser. Without pyarrow # here the fast half silently skips, which is how a broken fast path ships. pyarrow diff --git a/pyranges1/methods/signal_formats.py b/pyranges1/methods/signal_formats.py index 38fbaa98..d01ce547 100644 --- a/pyranges1/methods/signal_formats.py +++ b/pyranges1/methods/signal_formats.py @@ -46,7 +46,7 @@ def _read_bedgraph(f: "str | Path", nrows: int | None) -> "PyRanges": nrows=nrows, skiprows=skip, compression="gzip" if compressed else "infer", - dtype={CHROM_COL: "category"}, + dtype={CHROM_COL: str}, ) df[VALUE_COL] = df[VALUE_COL].astype("float64") _as_sorted_categorical(df, (CHROM_COL,)) diff --git a/pyranges1/readers.py b/pyranges1/readers.py index 31c7b5d8..78905820 100644 --- a/pyranges1/readers.py +++ b/pyranges1/readers.py @@ -268,7 +268,7 @@ def read_bed( if df is None: df = pd.read_csv( path, - dtype={"Chromosome": "category", "Strand": "category"}, + dtype={"Chromosome": str, "Strand": str}, nrows=nrows, header=header, names=names, @@ -919,7 +919,9 @@ def read_gff3( return read_gff3(local, nrows=nrows) path = Path(f) - dtypes: Mapping = {"Chromosome": "category", "Feature": "category", "Strand": "category"} + # Read as str and categorise after: since pandas 3.1, read_csv infers the + # categories of a "category" column, so chromosome 1 would become the int 1. + dtypes: Mapping = {"Chromosome": str, "Feature": str, "Strand": str} names = ["Chromosome", "Source", "Feature", "Start", "End", "Score", "Strand", "Frame", "Attribute"] chunksize = int(1e5) @@ -1942,8 +1944,9 @@ def _read_peaks( skiprows=skip, comment="#", compression="gzip" if compressed else "infer", - dtype={"Chromosome": "category", "Strand": "category"}, + dtype={"Chromosome": str, "Strand": str}, ) + _as_sorted_categorical(df, ("Chromosome", "Strand")) return ensure_pyranges(df) diff --git a/tests/unit/test_readers.py b/tests/unit/test_readers.py index a56e5a96..474ba11f 100644 --- a/tests/unit/test_readers.py +++ b/tests/unit/test_readers.py @@ -441,16 +441,22 @@ def test_read_gtf_dtype_contract(mode): @pytest.fixture def no_pyarrow_import(monkeypatch): - """Make `import pyarrow` fail, as it does when pyarrow is not installed.""" + """Make pyranges1's `import pyarrow` fail, as it does when pyarrow is not installed. + + Only pyranges1's own imports are blocked: pandas >= 3.1 decides once, at + import time, that pyarrow is available and then imports it inside read_csv, + which a global block would break although no real install could get there. + """ import builtins real_import = builtins.__import__ - def fake_import(name, *args, **kwargs): - if name.startswith("pyarrow"): + def fake_import(name, globals=None, *args, **kwargs): # noqa: A002 + caller = (globals or {}).get("__name__", "") + if name.startswith("pyarrow") and caller.startswith("pyranges1"): msg = "No module named 'pyarrow'" raise ImportError(msg) - return real_import(name, *args, **kwargs) + return real_import(name, globals, *args, **kwargs) monkeypatch.setattr(builtins, "__import__", fake_import) diff --git a/tests/unit/test_strided_columns.py b/tests/unit/test_strided_columns.py new file mode 100644 index 00000000..c38057bc --- /dev/null +++ b/tests/unit/test_strided_columns.py @@ -0,0 +1,55 @@ +"""Operations accept columns that are strided views of a 2-D block. + +pandas can hand out such columns from `Series.values` (a frame built from a 2-D +array without copying, and since pandas 3.1 some join and concat results). The +Rust kernels need contiguous input; coordinates past int32 are passed through +uncast, so nothing on the way copies them. +""" + +import numpy as np +import pandas as pd +import pytest + +import pyranges1 as pr + +BIG = 2**40 + + +def ranges(rows: list[list[int]], *, strided: bool) -> pr.PyRanges: + block = np.array(rows, dtype=np.int64) + BIG + if strided: + df = pd.DataFrame(block, columns=["Start", "End"], copy=False) + else: + df = pd.DataFrame({"Start": block[:, 0].copy(), "End": block[:, 1].copy()}) + assert df["Start"].to_numpy().flags.c_contiguous is not strided + df.insert(0, "Chromosome", "chr1") + df["Strand"] = ["+", "-", "+"] + return pr.PyRanges(df) + + +A = [[1, 5], [3, 9], [20, 30]] +B = [[2, 4], [25, 26], [40, 41]] + + +@pytest.mark.parametrize( + "operation", + [ + lambda a, b: a.overlap(b), + lambda a, b: a.join_overlaps(b), + lambda a, b: a.nearest_ranges(b), + lambda a, b: a.subtract_overlaps(b), + lambda a, b: a.count_overlaps(b), + lambda a, b: a.merge_overlaps(), + lambda a, b: a.cluster_overlaps(), + lambda a, b: a.extend_ranges(ext=1), + lambda a, b: a.window_ranges(2), + lambda a, b: a.tile_ranges(2), + lambda a, b: a.sort_ranges(), + lambda a, b: a.max_disjoint_overlaps(), + lambda a, b: a.complement_ranges(), + ], +) +def test_operations_accept_strided_columns(operation) -> None: + contiguous = operation(ranges(A, strided=False), ranges(B, strided=False)) + result = operation(ranges(A, strided=True), ranges(B, strided=True)) + pd.testing.assert_frame_equal(pd.DataFrame(result), pd.DataFrame(contiguous))