From b52564ea9690fa1c3aa8e1fb2a667d4efab53860 Mon Sep 17 00:00:00 2001 From: ptajvar Date: Mon, 15 Jun 2026 17:11:14 +0200 Subject: [PATCH] increase worker runtime and limit duckdb threads when called in a parallelized process. --- src/pixelator/pna/graph/cycle_analysis.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/pixelator/pna/graph/cycle_analysis.py b/src/pixelator/pna/graph/cycle_analysis.py index ec8f7920..89b671e7 100644 --- a/src/pixelator/pna/graph/cycle_analysis.py +++ b/src/pixelator/pna/graph/cycle_analysis.py @@ -214,7 +214,7 @@ def get_passing_edges(self): def process_component(comp_name, edgelist_path, tmpdir): """Process a single component to remove edges not participating in cycles.""" - with connect_duckdb() as con: + with connect_duckdb(config={"threads": "1"}) as con: con.execute( """ CREATE TEMP TABLE comp_edgelist AS @@ -352,7 +352,13 @@ def remove_no_cycle_edges( logger.info( f"Processing {len(components)} components in parallel using {n_threads} threads." ) - n_removed_edges_list = Parallel(n_jobs=n_threads)( + # Keep workers alive longer to avoid worker-restart warnings + # during long component processing batches. + n_removed_edges_list = Parallel( + n_jobs=n_threads, + backend="loky", + backend_kwargs={"timeout": 1800}, + )( delayed(process_component)(comp_name, input_edgelist_path, tmpdir) for comp_name in components )