diff --git a/.github/workflows/reusable-publish-package.yml b/.github/workflows/reusable-publish-package.yml index 654d12e..02efa48 100644 --- a/.github/workflows/reusable-publish-package.yml +++ b/.github/workflows/reusable-publish-package.yml @@ -39,7 +39,8 @@ jobs: - name: Get wheel artifacts uses: actions/download-artifact@v4 with: - name: pyjpt-wheel + pattern: pyjpt-wheel-* + merge-multiple: true path: dist/ # ---------------------------------------------------------------------------------------------------------------- diff --git a/.github/workflows/reusable-test-and-build.yml b/.github/workflows/reusable-test-and-build.yml index 234ccb6..6dfaa96 100644 --- a/.github/workflows/reusable-test-and-build.yml +++ b/.github/workflows/reusable-test-and-build.yml @@ -5,6 +5,8 @@ on: branches: - main + pull_request: + # ------------------------------------------------------------------------------------------------------------------ workflow_call: @@ -28,14 +30,14 @@ defaults: # -------------------------------------------------------------------------------------------------------------------- concurrency: - group: 'jpt-building-and-deployment' + group: 'jpt-building-and-deployment-${{ github.ref }}' cancel-in-progress: true # -------------------------------------------------------------------------------------------------------------------- jobs: test-and-build: - name: Build and Test the Python distribution + name: Build and Test the Python distribution (Linux) runs-on: ubuntu-22.04 steps: @@ -61,7 +63,9 @@ jobs: # ---------------------------------------------------------------------------------------------------------------- - name: Install dependencies - run: pip install -e ".[dev]" + # CI-only constraints until the code base supports them: pandas 3 (str dtype / + # copy-on-write) and plotly >= 6.1 (needs kaleido >= 1, while kaleido < 1 is pinned). + run: pip install -e ".[dev]" "pandas<3" "plotly<6.1" # ---------------------------------------------------------------------------------------------------------------- @@ -103,5 +107,80 @@ jobs: id: wheel_url uses: actions/upload-artifact@v4 with: - name: pyjpt-wheel + name: pyjpt-wheel-linux path: dist-raw/*manylinux*.whl + + # -------------------------------------------------------------------------------------------------------------------- + + test-and-build-windows: + name: Build and Test the Python distribution (Windows) + runs-on: windows-latest + + steps: + + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # ---------------------------------------------------------------------------------------------------------------- + + - name: Setup python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + + # ---------------------------------------------------------------------------------------------------------------- + + - name: Setup Graphviz + uses: ts-graphviz/setup-graphviz@v2 + + # ---------------------------------------------------------------------------------------------------------------- + + # Compiles the Cython extensions with MSVC. + - name: Install dependencies + # CI-only constraints until the code base supports them: pandas 3 (str dtype / + # copy-on-write) and plotly >= 6.1 (needs kaleido >= 1, while kaleido < 1 is pinned). + run: pip install -e ".[dev]" "pandas<3" "plotly<6.1" + + # ---------------------------------------------------------------------------------------------------------------- + + # Verbose output and a faulthandler watchdog: if the suite hangs, dump the stacks of all + # threads after 20 minutes and exit, instead of blocking until the 6 hour job limit. + - name: Run Tests + working-directory: tests + timeout-minutes: 30 + env: + MPLBACKEND: Agg + run: >- + python -c "import faulthandler, unittest; + faulthandler.dump_traceback_later(1200, exit=True); + unittest.main(module=None, argv=['unittest', 'discover', '-v', '-s', 'test', '-t', '.'])" + + # ---------------------------------------------------------------------------------------------------------------- + + # cibuildwheel expects build selectors such as "cp311-*" instead of the "cp311-cp311" + # tags used by the manylinux build above. + - name: Translate Python versions for cibuildwheel + run: | + versions="${{ inputs.python-versions || 'cp311-cp311' }}" + echo "CIBW_BUILD=$(echo "$versions" | sed -E 's/(cp[0-9]+)-cp[0-9]+/\1-*/g')" >> "$GITHUB_ENV" + + - name: Build binary wheels + uses: pypa/cibuildwheel@v3.4.1 + with: + output-dir: dist-raw + env: + CIBW_ARCHS_WINDOWS: AMD64 + CIBW_BUILD_VERBOSITY: 1 + CIBW_TEST_COMMAND: >- + python -c "import jpt, jpt.trees, jpt.learning.impurity; print(jpt.__version__)" + + # ---------------------------------------------------------------------------------------------------------------- + + - name: Make wheel available + uses: actions/upload-artifact@v4 + with: + name: pyjpt-wheel-windows + path: dist-raw/*.whl diff --git a/pyproject.toml b/pyproject.toml index a84ffdb..d2b916b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,13 @@ dev = [ "pyjpt[doc]" ] matplotlib = ["matplotlib", "graphviz>=0.20.1"] -plotly = ["plotly", "kaleido>=0.1.0,<1.0", "graphviz>=0.20.1"] +# kaleido 0.2.x hangs indefinitely on Windows when exporting images; 0.1.0.post1 is the known-good release there. +plotly = [ + "plotly", + "kaleido>=0.1.0,<1.0; sys_platform != 'win32'", + "kaleido==0.1.0.post1; sys_platform == 'win32'", + "graphviz>=0.20.1" +] seq = ["fglib>=0.2.3", "factorgraph>=0.0.3"] mlflow = ["mlflow>=2.5.0"] diff --git a/setup.py b/setup.py index f4b1766..f887923 100644 --- a/setup.py +++ b/setup.py @@ -27,13 +27,16 @@ "jpt/learning/impurity/impurity.pyx", ] - os.environ['CPATH'] = numpy.get_include() - ext_modules = cythonize( [os.path.join(basedir, f) for f in pyxfiles], language='c++', include_path=[numpy.get_include()] ) + + # Pass the numpy headers to the C++ compiler explicitly. The CPATH environment + # variable is only honored by GCC/Clang, not by MSVC on Windows. + for ext in ext_modules: + ext.include_dirs.append(numpy.get_include()) else: ext_modules = [] diff --git a/src/jpt/base/intervals/intset.pyx b/src/jpt/base/intervals/intset.pyx index 4d77fbe..b8f9cd8 100644 --- a/src/jpt/base/intervals/intset.pyx +++ b/src/jpt/base/intervals/intset.pyx @@ -277,7 +277,7 @@ cdef class IntSet(Interval): ) @staticmethod - def from_list(list l: List[int]) -> IntSet: + def from_list(l: list) -> IntSet: if len(l) != 2: raise ValueError( 'List must contain exactly two values (or ellipses "...") that specify the interval bounds' @@ -289,7 +289,7 @@ cdef class IntSet(Interval): # noinspection PyTypeChecker @staticmethod - def from_set(set numbers: Set[int]) -> NumberSet: + def from_set(numbers: set) -> NumberSet: return UnionSet([ IntSet(n, n) for n in numbers ]).simplify(keep_type=False) diff --git a/src/jpt/base/utils/multicore.py b/src/jpt/base/utils/multicore.py index 164a8c8..7199e7e 100644 --- a/src/jpt/base/utils/multicore.py +++ b/src/jpt/base/utils/multicore.py @@ -107,6 +107,21 @@ def worker( # ---------------------------------------------------------------------------------------------------------------------- +def workers_inherit_state() -> bool: + """ + Whether worker processes inherit the parent's module state, such as the ``threading.local`` + objects that the learning and inference code fills in before creating a pool. + + This only holds for the ``fork`` start method. With ``spawn`` (the only method on Windows, + and the default on macOS), workers start with fresh modules, so pools that rely on inherited + state must fall back to :class:`DummyPool`. + """ + return mp.get_context().get_start_method() == 'fork' + + +# ---------------------------------------------------------------------------------------------------------------------- + + class DummyPool: def __init__( diff --git a/src/jpt/distributions/univariate/distribution.py b/src/jpt/distributions/univariate/distribution.py index 2c86797..4bc273f 100644 --- a/src/jpt/distributions/univariate/distribution.py +++ b/src/jpt/distributions/univariate/distribution.py @@ -1,4 +1,5 @@ '''© Copyright 2021, Mareike Picklum, Daniel Nyga.''' +import tempfile import numbers from collections.abc import Hashable from typing import Set, Iterable, Type, Any, Dict, Union @@ -213,7 +214,7 @@ def plot( engine: str, title: str = None, fname: str = None, - directory: str = '/tmp', + directory: str = tempfile.gettempdir(), view: bool = False, **kwargs ) -> Any: diff --git a/src/jpt/inference/likelihood.py b/src/jpt/inference/likelihood.py index 83141a0..a3535da 100644 --- a/src/jpt/inference/likelihood.py +++ b/src/jpt/inference/likelihood.py @@ -15,7 +15,7 @@ from tqdm import tqdm from jpt.trees import JPT -from jpt.base.utils.multicore import Pool, DummyPool +from jpt.base.utils.multicore import Pool, DummyPool, workers_inherit_state from jpt.variables import ValueAssignment, Variable @@ -152,7 +152,8 @@ def parallel_likelihood( n_processes = ifnone(multicore, mp.cpu_count()) chunksize = max(1, int(data.shape[0] / n_processes / 2)) - if multicore == 0: + # Workers rely on state inherited from the parent via ``_locals`` (fork only). + if multicore == 0 or not workers_inherit_state(): PoolCls = DummyPool else: PoolCls = Pool diff --git a/src/jpt/learning/c45.py b/src/jpt/learning/c45.py index 56aebc4..e756ff9 100644 --- a/src/jpt/learning/c45.py +++ b/src/jpt/learning/c45.py @@ -20,7 +20,7 @@ from .preprocessing import preprocess_data from ..base.functions import PiecewiseFunction from ..base.intervals import ContinuousSet, INC, EXC, IntSet, Interval -from ..base.utils.multicore import DummyPool +from ..base.utils.multicore import DummyPool, workers_inherit_state from ..base.utils import _write_error from ..distributions import Distribution from ..distributions.qpd import QuantileDistribution @@ -434,8 +434,13 @@ def learn( indices[0] = 0 np.cumsum(indices, out=indices) - _locals.data = np.ascontiguousarray(_data.values) - _locals.indices = Array(c.c_long, indices.shape[0]) + # The Cython impurity code needs a writable, C-contiguous buffer. Since pandas 3 + # (copy-on-write), ``DataFrame.values`` may be read-only, which ``ascontiguousarray`` + # would pass through unchanged; ``np.require`` copies only when needed. + _locals.data = np.require(_data.values, requirements=['C_CONTIGUOUS', 'WRITEABLE']) + # Use a fixed 64-bit type: the buffer is read back as ``np.int64`` in ``c45split``, + # while ``c_long`` is only 32 bits wide on Windows. + _locals.indices = Array(c.c_int64, indices.shape[0]) _locals.indices[:] = indices # ---- Split validation mask ---- @@ -479,7 +484,9 @@ def learn( file=sys.stderr ) - if multicore == 0: + # The workers read the training data and the tree from the module-level ``_locals``, + # which only fork-started processes inherit. Elsewhere (Windows, macOS) learn in-process. + if multicore == 0 or not workers_inherit_state(): PoolCls = DummyPool else: PoolCls = Pool diff --git a/src/jpt/learning/preprocessing.py b/src/jpt/learning/preprocessing.py index 5385cd5..c792ed7 100644 --- a/src/jpt/learning/preprocessing.py +++ b/src/jpt/learning/preprocessing.py @@ -11,7 +11,7 @@ from dnutils import mapstr, ifnone from tqdm import tqdm -from ..base.utils.multicore import Pool, DummyPool +from ..base.utils.multicore import Pool, DummyPool, workers_inherit_state from ..trees import JPT _locals = threading.local() @@ -185,7 +185,8 @@ def preprocess_data( n_processes = ifnone(multicore, os.cpu_count()) - if multicore == 0: + # Workers read the data frame from the module-level ``_locals`` (inherited with fork only). + if multicore == 0 or not workers_inherit_state(): PoolCls = DummyPool else: PoolCls = Pool diff --git a/src/jpt/plotting/engines/matplotlib_engine.py b/src/jpt/plotting/engines/matplotlib_engine.py index 0a1702f..187c154 100644 --- a/src/jpt/plotting/engines/matplotlib_engine.py +++ b/src/jpt/plotting/engines/matplotlib_engine.py @@ -1,3 +1,4 @@ +import tempfile import os import re from pathlib import Path @@ -66,7 +67,7 @@ def plot_multinomial( dist: Any, title: str = None, fname: str = None, - directory: str = '/tmp', + directory: str = tempfile.gettempdir(), pdf: bool = False, view: bool = False, horizontal: bool = False, @@ -161,7 +162,7 @@ def plot_numeric( title: Union[str, bool] = None, fname: str = None, xlabel: str = 'value', - directory: str = '/tmp', + directory: str = tempfile.gettempdir(), view: bool = False, pdf: bool = False, **kwargs @@ -254,7 +255,7 @@ def plot_integer( dist: Any, title: str = None, fname: str = None, - directory: str = '/tmp', + directory: str = tempfile.gettempdir(), pdf: bool = False, view: bool = False, horizontal: bool = False, @@ -343,7 +344,7 @@ def plot_multivariate( dist: Any, title: str = None, fname: str = None, - directory: str = '/tmp', + directory: str = tempfile.gettempdir(), pdf: bool = False, view: bool = False, **kwargs @@ -407,7 +408,7 @@ def plot_gaussian( xlabel: str = '$x$', ylabel: str = r'$p(x|\mu,\sigma)$', dim: int = 3, - directory: str = '/tmp', + directory: str = tempfile.gettempdir(), view: bool = False, pdf: bool = False, **kwargs diff --git a/src/jpt/plotting/engines/plotly_engine.py b/src/jpt/plotting/engines/plotly_engine.py index bb0336a..142fce2 100644 --- a/src/jpt/plotting/engines/plotly_engine.py +++ b/src/jpt/plotting/engines/plotly_engine.py @@ -1,3 +1,4 @@ +import tempfile import os from typing import Any, Union @@ -19,7 +20,7 @@ def plot_multinomial( dist: Any, title: str = None, fname: str = None, - directory: str = '/tmp', + directory: str = tempfile.gettempdir(), view: bool = False, horizontal: bool = False, max_values: int = None, @@ -151,7 +152,7 @@ def plot_numeric( title: Union[str, bool] = None, fname: str = None, xlabel: str = 'value', - directory: str = '/tmp', + directory: str = tempfile.gettempdir(), view: bool = False, color: str = 'rgb(15,21,110)', fill: str = None, @@ -304,7 +305,7 @@ def plot_integer( dist: Any, title: str = None, fname: str = None, - directory: str = '/tmp', + directory: str = tempfile.gettempdir(), view: bool = False, horizontal: bool = False, max_values: int = None, @@ -423,7 +424,7 @@ def plot_multivariate( dist: Any, title: str = None, fname: str = None, - directory: str = '/tmp', + directory: str = tempfile.gettempdir(), pdf: bool = False, view: bool = False, **kwargs @@ -471,7 +472,7 @@ def plot_gaussian( ylabel: str = None, dim: int = 3, precision: int = 2, - directory: str = '/tmp', + directory: str = tempfile.gettempdir(), view: bool = False, color: str = 'rgb(15,21,110)', fill: str = None, diff --git a/src/jpt/plotting/engines/rendering.py b/src/jpt/plotting/engines/rendering.py index 11746a3..246b0be 100644 --- a/src/jpt/plotting/engines/rendering.py +++ b/src/jpt/plotting/engines/rendering.py @@ -1,3 +1,4 @@ +import tempfile from typing import Any, Union # engines @@ -31,7 +32,7 @@ def plot_multinomial( dist: Any, title: str = None, fname: str = None, - directory: str = '/tmp', + directory: str = tempfile.gettempdir(), view: bool = False, **kwargs ): @@ -43,7 +44,7 @@ def plot_numeric( title: Union[str, bool] = None, fname: str = None, xlabel: str = 'value', - directory: str = '/tmp', + directory: str = tempfile.gettempdir(), view: bool = False, **kwargs ): @@ -54,7 +55,7 @@ def plot_integer( dist: Any, title: str = None, fname: str = None, - directory: str = '/tmp', + directory: str = tempfile.gettempdir(), view: bool = False, **kwargs ): @@ -65,7 +66,7 @@ def plot_multivariate( dist: Any, title: str = None, fname: str = None, - directory: str = '/tmp', + directory: str = tempfile.gettempdir(), pdf: bool = False, view: bool = False, **kwargs @@ -77,7 +78,7 @@ def plot_gaussian( dist: Any, title: Union[str, bool] = None, fname: str = None, - directory: str = '/tmp', + directory: str = tempfile.gettempdir(), view: bool = False, **kwargs ): diff --git a/src/jpt/plotting/jpt.py b/src/jpt/plotting/jpt.py index 11fba7d..d2e49ee 100644 --- a/src/jpt/plotting/jpt.py +++ b/src/jpt/plotting/jpt.py @@ -18,6 +18,7 @@ from .engines.rendering import MATPLOTLIB, PLOTLY, DistributionRendering from ..trees import JPT from ..base.constants import green, orange +from ..base.utils.multicore import DummyPool, workers_inherit_state from ..trees import DecisionNode from ..variables import Variable @@ -264,8 +265,8 @@ def plot( _locals.jpt = self.jpt - # create nodes - pool = Pool() + # create nodes; the workers read ``_locals.jpt``, which only fork-started processes inherit + pool = Pool() if workers_inherit_state() else DummyPool() if self.verbose: progress = tqdm(total=len(self.jpt.leaves))