Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/reusable-publish-package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ jobs:
- name: Get wheel artifacts
uses: actions/download-artifact@v4
with:
name: pyjpt-wheel
pattern: pyjpt-wheel-*
merge-multiple: true
path: dist/

# ----------------------------------------------------------------------------------------------------------------
Expand Down
87 changes: 83 additions & 4 deletions .github/workflows/reusable-test-and-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ on:
branches:
- main

pull_request:

# ------------------------------------------------------------------------------------------------------------------

workflow_call:
Expand All @@ -28,14 +30,14 @@ defaults:
# --------------------------------------------------------------------------------------------------------------------

concurrency:
group: 'jpt-building-and-deployment'
group: 'jpt-building-and-deployment-${{ github.ref }}'
cancel-in-progress: true

# --------------------------------------------------------------------------------------------------------------------

jobs:
test-and-build:
name: Build and Test the Python distribution
name: Build and Test the Python distribution (Linux)
runs-on: ubuntu-22.04

steps:
Expand All @@ -61,7 +63,9 @@ jobs:
# ----------------------------------------------------------------------------------------------------------------

- name: Install dependencies
run: pip install -e ".[dev]"
# CI-only constraints until the code base supports them: pandas 3 (str dtype /
# copy-on-write) and plotly >= 6.1 (needs kaleido >= 1, while kaleido < 1 is pinned).
run: pip install -e ".[dev]" "pandas<3" "plotly<6.1"

# ----------------------------------------------------------------------------------------------------------------

Expand Down Expand Up @@ -103,5 +107,80 @@ jobs:
id: wheel_url
uses: actions/upload-artifact@v4
with:
name: pyjpt-wheel
name: pyjpt-wheel-linux
path: dist-raw/*manylinux*.whl

# --------------------------------------------------------------------------------------------------------------------

test-and-build-windows:
name: Build and Test the Python distribution (Windows)
runs-on: windows-latest

steps:

- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0

# ----------------------------------------------------------------------------------------------------------------

- name: Setup python
uses: actions/setup-python@v5
with:
python-version: "3.11"
cache: pip

# ----------------------------------------------------------------------------------------------------------------

- name: Setup Graphviz
uses: ts-graphviz/setup-graphviz@v2

# ----------------------------------------------------------------------------------------------------------------

# Compiles the Cython extensions with MSVC.
- name: Install dependencies
# CI-only constraints until the code base supports them: pandas 3 (str dtype /
# copy-on-write) and plotly >= 6.1 (needs kaleido >= 1, while kaleido < 1 is pinned).
run: pip install -e ".[dev]" "pandas<3" "plotly<6.1"

# ----------------------------------------------------------------------------------------------------------------

# Verbose output and a faulthandler watchdog: if the suite hangs, dump the stacks of all
# threads after 20 minutes and exit, instead of blocking until the 6 hour job limit.
- name: Run Tests
working-directory: tests
timeout-minutes: 30
env:
MPLBACKEND: Agg
run: >-
python -c "import faulthandler, unittest;
faulthandler.dump_traceback_later(1200, exit=True);
unittest.main(module=None, argv=['unittest', 'discover', '-v', '-s', 'test', '-t', '.'])"

# ----------------------------------------------------------------------------------------------------------------

# cibuildwheel expects build selectors such as "cp311-*" instead of the "cp311-cp311"
# tags used by the manylinux build above.
- name: Translate Python versions for cibuildwheel
run: |
versions="${{ inputs.python-versions || 'cp311-cp311' }}"
echo "CIBW_BUILD=$(echo "$versions" | sed -E 's/(cp[0-9]+)-cp[0-9]+/\1-*/g')" >> "$GITHUB_ENV"

- name: Build binary wheels
uses: pypa/cibuildwheel@v3.4.1
with:
output-dir: dist-raw
env:
CIBW_ARCHS_WINDOWS: AMD64
CIBW_BUILD_VERBOSITY: 1
CIBW_TEST_COMMAND: >-
python -c "import jpt, jpt.trees, jpt.learning.impurity; print(jpt.__version__)"

# ----------------------------------------------------------------------------------------------------------------

- name: Make wheel available
uses: actions/upload-artifact@v4
with:
name: pyjpt-wheel-windows
path: dist-raw/*.whl
8 changes: 7 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,13 @@ dev = [
"pyjpt[doc]"
]
matplotlib = ["matplotlib", "graphviz>=0.20.1"]
plotly = ["plotly", "kaleido>=0.1.0,<1.0", "graphviz>=0.20.1"]
# kaleido 0.2.x hangs indefinitely on Windows when exporting images; 0.1.0.post1 is the known-good release there.
plotly = [
"plotly",
"kaleido>=0.1.0,<1.0; sys_platform != 'win32'",
"kaleido==0.1.0.post1; sys_platform == 'win32'",
"graphviz>=0.20.1"
]
seq = ["fglib>=0.2.3", "factorgraph>=0.0.3"]
mlflow = ["mlflow>=2.5.0"]

Expand Down
7 changes: 5 additions & 2 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,16 @@
"jpt/learning/impurity/impurity.pyx",
]

os.environ['CPATH'] = numpy.get_include()

ext_modules = cythonize(
[os.path.join(basedir, f) for f in pyxfiles],
language='c++',
include_path=[numpy.get_include()]
)

# Pass the numpy headers to the C++ compiler explicitly. The CPATH environment
# variable is only honored by GCC/Clang, not by MSVC on Windows.
for ext in ext_modules:
ext.include_dirs.append(numpy.get_include())
else:
ext_modules = []

Expand Down
4 changes: 2 additions & 2 deletions src/jpt/base/intervals/intset.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@ cdef class IntSet(Interval):
)

@staticmethod
def from_list(list l: List[int]) -> IntSet:
def from_list(l: list) -> IntSet:
if len(l) != 2:
raise ValueError(
'List must contain exactly two values (or ellipses "...") that specify the interval bounds'
Expand All @@ -289,7 +289,7 @@ cdef class IntSet(Interval):

# noinspection PyTypeChecker
@staticmethod
def from_set(set numbers: Set[int]) -> NumberSet:
def from_set(numbers: set) -> NumberSet:
return UnionSet([
IntSet(n, n) for n in numbers
]).simplify(keep_type=False)
Expand Down
15 changes: 15 additions & 0 deletions src/jpt/base/utils/multicore.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,21 @@ def worker(
# ----------------------------------------------------------------------------------------------------------------------


def workers_inherit_state() -> bool:
"""
Whether worker processes inherit the parent's module state, such as the ``threading.local``
objects that the learning and inference code fills in before creating a pool.

This only holds for the ``fork`` start method. With ``spawn`` (the only method on Windows,
and the default on macOS), workers start with fresh modules, so pools that rely on inherited
state must fall back to :class:`DummyPool`.
"""
return mp.get_context().get_start_method() == 'fork'


# ----------------------------------------------------------------------------------------------------------------------


class DummyPool:

def __init__(
Expand Down
3 changes: 2 additions & 1 deletion src/jpt/distributions/univariate/distribution.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
'''© Copyright 2021, Mareike Picklum, Daniel Nyga.'''
import tempfile
import numbers
from collections.abc import Hashable
from typing import Set, Iterable, Type, Any, Dict, Union
Expand Down Expand Up @@ -213,7 +214,7 @@ def plot(
engine: str,
title: str = None,
fname: str = None,
directory: str = '/tmp',
directory: str = tempfile.gettempdir(),
view: bool = False,
**kwargs
) -> Any:
Expand Down
5 changes: 3 additions & 2 deletions src/jpt/inference/likelihood.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from tqdm import tqdm

from jpt.trees import JPT
from jpt.base.utils.multicore import Pool, DummyPool
from jpt.base.utils.multicore import Pool, DummyPool, workers_inherit_state
from jpt.variables import ValueAssignment, Variable


Expand Down Expand Up @@ -152,7 +152,8 @@ def parallel_likelihood(
n_processes = ifnone(multicore, mp.cpu_count())
chunksize = max(1, int(data.shape[0] / n_processes / 2))

if multicore == 0:
# Workers rely on state inherited from the parent via ``_locals`` (fork only).
if multicore == 0 or not workers_inherit_state():
PoolCls = DummyPool
else:
PoolCls = Pool
Expand Down
15 changes: 11 additions & 4 deletions src/jpt/learning/c45.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from .preprocessing import preprocess_data
from ..base.functions import PiecewiseFunction
from ..base.intervals import ContinuousSet, INC, EXC, IntSet, Interval
from ..base.utils.multicore import DummyPool
from ..base.utils.multicore import DummyPool, workers_inherit_state
from ..base.utils import _write_error
from ..distributions import Distribution
from ..distributions.qpd import QuantileDistribution
Expand Down Expand Up @@ -434,8 +434,13 @@ def learn(
indices[0] = 0
np.cumsum(indices, out=indices)

_locals.data = np.ascontiguousarray(_data.values)
_locals.indices = Array(c.c_long, indices.shape[0])
# The Cython impurity code needs a writable, C-contiguous buffer. Since pandas 3
# (copy-on-write), ``DataFrame.values`` may be read-only, which ``ascontiguousarray``
# would pass through unchanged; ``np.require`` copies only when needed.
_locals.data = np.require(_data.values, requirements=['C_CONTIGUOUS', 'WRITEABLE'])
# Use a fixed 64-bit type: the buffer is read back as ``np.int64`` in ``c45split``,
# while ``c_long`` is only 32 bits wide on Windows.
_locals.indices = Array(c.c_int64, indices.shape[0])
_locals.indices[:] = indices

# ---- Split validation mask ----
Expand Down Expand Up @@ -479,7 +484,9 @@ def learn(
file=sys.stderr
)

if multicore == 0:
# The workers read the training data and the tree from the module-level ``_locals``,
# which only fork-started processes inherit. Elsewhere (Windows, macOS) learn in-process.
if multicore == 0 or not workers_inherit_state():
PoolCls = DummyPool
else:
PoolCls = Pool
Expand Down
5 changes: 3 additions & 2 deletions src/jpt/learning/preprocessing.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from dnutils import mapstr, ifnone
from tqdm import tqdm

from ..base.utils.multicore import Pool, DummyPool
from ..base.utils.multicore import Pool, DummyPool, workers_inherit_state
from ..trees import JPT

_locals = threading.local()
Expand Down Expand Up @@ -185,7 +185,8 @@ def preprocess_data(

n_processes = ifnone(multicore, os.cpu_count())

if multicore == 0:
# Workers read the data frame from the module-level ``_locals`` (inherited with fork only).
if multicore == 0 or not workers_inherit_state():
PoolCls = DummyPool
else:
PoolCls = Pool
Expand Down
11 changes: 6 additions & 5 deletions src/jpt/plotting/engines/matplotlib_engine.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import tempfile
import os
import re
from pathlib import Path
Expand Down Expand Up @@ -66,7 +67,7 @@ def plot_multinomial(
dist: Any,
title: str = None,
fname: str = None,
directory: str = '/tmp',
directory: str = tempfile.gettempdir(),
pdf: bool = False,
view: bool = False,
horizontal: bool = False,
Expand Down Expand Up @@ -161,7 +162,7 @@ def plot_numeric(
title: Union[str, bool] = None,
fname: str = None,
xlabel: str = 'value',
directory: str = '/tmp',
directory: str = tempfile.gettempdir(),
view: bool = False,
pdf: bool = False,
**kwargs
Expand Down Expand Up @@ -254,7 +255,7 @@ def plot_integer(
dist: Any,
title: str = None,
fname: str = None,
directory: str = '/tmp',
directory: str = tempfile.gettempdir(),
pdf: bool = False,
view: bool = False,
horizontal: bool = False,
Expand Down Expand Up @@ -343,7 +344,7 @@ def plot_multivariate(
dist: Any,
title: str = None,
fname: str = None,
directory: str = '/tmp',
directory: str = tempfile.gettempdir(),
pdf: bool = False,
view: bool = False,
**kwargs
Expand Down Expand Up @@ -407,7 +408,7 @@ def plot_gaussian(
xlabel: str = '$x$',
ylabel: str = r'$p(x|\mu,\sigma)$',
dim: int = 3,
directory: str = '/tmp',
directory: str = tempfile.gettempdir(),
view: bool = False,
pdf: bool = False,
**kwargs
Expand Down
11 changes: 6 additions & 5 deletions src/jpt/plotting/engines/plotly_engine.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import tempfile
import os
from typing import Any, Union

Expand All @@ -19,7 +20,7 @@ def plot_multinomial(
dist: Any,
title: str = None,
fname: str = None,
directory: str = '/tmp',
directory: str = tempfile.gettempdir(),
view: bool = False,
horizontal: bool = False,
max_values: int = None,
Expand Down Expand Up @@ -151,7 +152,7 @@ def plot_numeric(
title: Union[str, bool] = None,
fname: str = None,
xlabel: str = 'value',
directory: str = '/tmp',
directory: str = tempfile.gettempdir(),
view: bool = False,
color: str = 'rgb(15,21,110)',
fill: str = None,
Expand Down Expand Up @@ -304,7 +305,7 @@ def plot_integer(
dist: Any,
title: str = None,
fname: str = None,
directory: str = '/tmp',
directory: str = tempfile.gettempdir(),
view: bool = False,
horizontal: bool = False,
max_values: int = None,
Expand Down Expand Up @@ -423,7 +424,7 @@ def plot_multivariate(
dist: Any,
title: str = None,
fname: str = None,
directory: str = '/tmp',
directory: str = tempfile.gettempdir(),
pdf: bool = False,
view: bool = False,
**kwargs
Expand Down Expand Up @@ -471,7 +472,7 @@ def plot_gaussian(
ylabel: str = None,
dim: int = 3,
precision: int = 2,
directory: str = '/tmp',
directory: str = tempfile.gettempdir(),
view: bool = False,
color: str = 'rgb(15,21,110)',
fill: str = None,
Expand Down
Loading
Loading