From fc85020457654c8d43830d54635fa30933b96730 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:22:22 +0300 Subject: [PATCH 01/36] UI polish: settings button in the far bottom-right corner, margin from the right splitter, scrollbars without the dotted page background --- phyml_ui/main_window.py | 4 ++-- phyml_ui/themes.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/phyml_ui/main_window.py b/phyml_ui/main_window.py index 559569e..5b1152a 100644 --- a/phyml_ui/main_window.py +++ b/phyml_ui/main_window.py @@ -250,7 +250,7 @@ def _build_middle_pane(self): middle = QWidget() v = QVBoxLayout(middle) - v.setContentsMargins(0, 0, 0, 0) + v.setContentsMargins(0, 0, 8, 0) # breathing room from the right splitter v.setSpacing(8) v.addWidget(QLabel(_t("input_fasta"))) @@ -406,8 +406,8 @@ def _build_status_bar(self): self.status_bar.addWidget(self.cores_label) self.status_bar.addWidget(self.cpu_sys_label) self.status_bar.addWidget(self.cpu_sys_bar, 1) # stretch: fills all free space - self.status_bar.addPermanentWidget(self.settings_btn) self.status_bar.addPermanentWidget(self.cpu_run_label) + self.status_bar.addPermanentWidget(self.settings_btn) # far bottom-right corner self.cpu_timer = QTimer(self) self.cpu_timer.setInterval(1000) diff --git a/phyml_ui/themes.py b/phyml_ui/themes.py index 6b36b93..f55f885 100644 --- a/phyml_ui/themes.py +++ b/phyml_ui/themes.py @@ -120,11 +120,15 @@ def system_accent(): QListWidget::item:selected {{ background: {accent}; color: white; }} QListView::item:hover, QListWidget::item:hover {{ background: {hover}; }} QScrollBar:vertical {{ background: {bg}; width: 10px; margin: 0; }} -QScrollBar::handle:vertical {{ background: {border}; border-radius: 5px; min-height: 24px; }} +QScrollBar::handle:vertical {{ background: {border}; border: none; border-radius: 5px; min-height: 24px; }} QScrollBar::handle:vertical:hover {{ background: {accent}; }} QScrollBar:horizontal {{ background: {bg}; height: 10px; }} -QScrollBar::handle:horizontal {{ background: {border}; border-radius: 5px; min-width: 24px; }} +QScrollBar::handle:horizontal {{ background: {border}; border: none; border-radius: 5px; min-width: 24px; }} +QScrollBar::handle:horizontal:hover {{ background: {accent}; }} QScrollBar::add-line, QScrollBar::sub-line {{ height: 0; width: 0; }} +QScrollBar::add-page, QScrollBar::sub-page {{ background: none; }} +QScrollBar::add-page:vertical, QScrollBar::sub-page:vertical, +QScrollBar::add-page:horizontal, QScrollBar::sub-page:horizontal {{ background: none; }} QTableWidget {{ background: {base}; border: 1px solid {border}; gridline-color: {border}; alternate-background-color: {alt}; }} QHeaderView::section {{ background: {button}; border: none; padding: 4px; font-weight: 600; }} From 61a90721a6651fa5ce89f98841bd361fd2140234 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:27:18 +0300 Subject: [PATCH 02/36] Tree viewer stage 2: custom renderer, layouts, supports, collapse, outgroup, consensus, RF - custom matplotlib renderer replaces Bio.Phylo.draw: three layouts (rectangular / slanted / circular with rings), leaf labels, italic support labels, light grid and a scale bar - support threshold slider: branches >= threshold are accented, weaker ones fade; supports toggle; grid toggle - double-click on a clade region collapses it to a single display leaf and back - outgroup combo reroots the tree on the fly - 'Consensus' button builds a majority-rule consensus from a *_phyml_boot_trees.txt file (Bio.Phylo.Consensus) - 'Compare' button computes Robinson-Foulds distance (raw + normalized) between two Newick files with canonical split representation --- locales.json | 34 +++ phyml_ui/tree_viewer.py | 518 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 505 insertions(+), 47 deletions(-) diff --git a/locales.json b/locales.json index 9e74375..8c5feb4 100644 --- a/locales.json +++ b/locales.json @@ -126,6 +126,23 @@ "tree_pan": "Pan", "tree_export": "Export…", "tree_tool_label": "Tool:", + "tree_layout": "Layout:", + "tree_layout_rect": "Rectangular", + "tree_layout_slant": "Slanted", + "tree_layout_circ": "Circular", + "tree_supports": "Supports", + "tree_threshold": "threshold: {}%", + "tree_grid": "Grid", + "tree_outgroup": "Outgroup:", + "tree_outgroup_none": "(none)", + "btn_consensus": "Consensus…", + "btn_compare": "Compare…", + "consensus_from": "Majority consensus ({} replicates)", + "tree_pick1": "First tree (Newick)", + "tree_pick2": "Second tree (Newick)", + "rf_title": "Topology comparison (Robinson–Foulds)", + "rf_result": "RF distance: {} ({:.0%} normalized).\n\n0 — topologies are identical; 100% — no common splits.", + "tree_empty": "No tree yet — run an analysis", "log_filter_all": "All", "log_filter_error": "Errors", "log_filter_success": "Success", @@ -274,6 +291,23 @@ "tree_pan": "Перемещение", "tree_export": "Экспорт…", "tree_tool_label": "Инструмент:", + "tree_layout": "Раскладка:", + "tree_layout_rect": "Прямоугольная", + "tree_layout_slant": "Сланцевая", + "tree_layout_circ": "Круговая", + "tree_supports": "Поддержки", + "tree_threshold": "порог: {}%", + "tree_grid": "Сетка", + "tree_outgroup": "Outgroup:", + "tree_outgroup_none": "(нет)", + "btn_consensus": "Консенсус…", + "btn_compare": "Сравнить…", + "consensus_from": "Консенсус большинства ({} реплик)", + "tree_pick1": "Первое дерево (Newick)", + "tree_pick2": "Второе дерево (Newick)", + "rf_title": "Сравнение топологий (Robinson–Foulds)", + "rf_result": "RF-расстояние: {} ({:.0%} нормализованное).\n\n0 — топологии идентичны; 100% — нет общих ветвей.", + "tree_empty": "Дерева пока нет — запустите анализ", "log_filter_all": "Все", "log_filter_error": "Ошибки", "log_filter_success": "Успех", diff --git a/phyml_ui/tree_viewer.py b/phyml_ui/tree_viewer.py index af17007..933f8b3 100644 --- a/phyml_ui/tree_viewer.py +++ b/phyml_ui/tree_viewer.py @@ -1,3 +1,4 @@ +import math import os import matplotlib @@ -5,34 +6,82 @@ from matplotlib.backends.backend_qtagg import FigureCanvasQTAgg as FigureCanvas from matplotlib.backends.backend_qtagg import NavigationToolbar2QT from matplotlib.figure import Figure +from matplotlib.collections import LineCollection from PySide6.QtWidgets import (QVBoxLayout, QWidget, QDialog, QHBoxLayout, - QToolButton, QLabel, QFileDialog) + QToolButton, QLabel, QFileDialog, QComboBox, + QCheckBox, QSlider, QMessageBox) +from PySide6.QtCore import Qt from PySide6.QtGui import QIcon from Bio import Phylo +from Bio.Phylo.Consensus import majority_consensus from phyml_core.i18n import _t from phyml_core.utils import resource_path +# ---------- pure helpers (unit-testable) ---------- + +def canonical_split(clade, all_names, anchor): + """One-sided split representation: the side NOT containing the anchor.""" + side = frozenset(t.name for t in clade.get_terminals()) + return side if anchor not in side else frozenset(all_names - side) + + +def tree_splits(tree): + """All non-trivial splits of an unrooted tree, canonically represented.""" + names = {t.name for t in tree.get_terminals()} + anchor = min(names) + splits = set() + for cl in tree.get_nonterminals(): + side = canonical_split(cl, names, anchor) + if 1 < len(side) < len(names) - 1: + splits.add(side) + return splits + + +def rf_distance(tree_a, tree_b): + """Robinson–Foulds distance and its normalized value [0..1].""" + a, b = tree_splits(tree_a), tree_splits(tree_b) + raw = len(a ^ b) // 2 + n = len(tree_a.get_terminals()) + norm = min(1.0, raw / (n - 3)) if n > 3 else 0.0 + return raw, norm + + # --- Embeddable tree visualization widget --- class TreeViewerWidget(QWidget): - """Tree render with an explicit tool bar: the active tool (zoom / pan) - is a checked button plus a text indicator, instead of the stock - matplotlib toolbar whose state is invisible.""" + """Custom tree renderer with three layouts, support threshold + highlighting, clade collapsing, outgroup rerooting, consensus and + RF comparison. Zoom/pan tool state is visible on the buttons.""" + + LAYOUTS = ('rect', 'slanted', 'circle') def __init__(self, parent=None): super().__init__(parent) self.figure = Figure() + self.figure.patch.set_alpha(0.0) self.canvas = FigureCanvas(self.figure) - # hidden stock toolbar drives the actual pan/zoom/home logic - self.nav = NavigationToolbar2QT(self.canvas, self) + self.nav = NavigationToolbar2QT(self.canvas, self) # hidden: drives zoom/pan self.nav.setVisible(False) self.ax = self.figure.add_subplot(111) + self.tree = None + self.tree_title = "" self.tree_path = "" + self.layout = 'rect' + self.show_supports = True + self.threshold = 0 # 0 = highlighting off + self.show_grid = True + self.outgroup = None + self._collapsed = set() # frozensets of terminal names + self._coords = None # (tree, coords) of the last render + self._leaf_order = [] layout = QVBoxLayout(self) - bar = QHBoxLayout() + layout.setContentsMargins(0, 0, 0, 0) + layout.setSpacing(4) + + tools = QHBoxLayout() self.btn_home = QToolButton(); self.btn_home.setText(_t("tree_home")) self.btn_zoom = QToolButton(); self.btn_zoom.setText(_t("tree_zoom")) self.btn_zoom.setCheckable(True) @@ -41,24 +90,62 @@ def __init__(self, parent=None): self.btn_save = QToolButton(); self.btn_save.setText(_t("tree_export")) self.mode_label = QLabel("—") self.mode_label.setStyleSheet('font-weight: 600;') - - # no QButtonGroup: matplotlib allows exactly one active mode at a - # time, and _sync_tools() enforces that on the buttons — an exclusive - # group would veto programmatically unchecking the pressed button self.btn_home.clicked.connect(self._home) self.btn_zoom.clicked.connect(lambda: (self.nav.zoom(), self._sync_tools())) self.btn_pan.clicked.connect(lambda: (self.nav.pan(), self._sync_tools())) self.btn_save.clicked.connect(self._export) + for w in (self.btn_home, self.btn_zoom, self.btn_pan, self.btn_save): + tools.addWidget(w) + tools.addStretch(1) + tools.addWidget(QLabel(_t("tree_tool_label"))) + tools.addWidget(self.mode_label) + layout.addLayout(tools) + + opts = QHBoxLayout() + self.layout_combo = QComboBox() + self.layout_combo.addItem(_t("tree_layout_rect"), 'rect') + self.layout_combo.addItem(_t("tree_layout_slant"), 'slanted') + self.layout_combo.addItem(_t("tree_layout_circ"), 'circle') + self.layout_combo.currentIndexChanged.connect( + lambda _: self.set_layout(self.layout_combo.currentData())) + self.chk_supports = QCheckBox(_t("tree_supports")) + self.chk_supports.setChecked(True) + self.chk_supports.toggled.connect(self._rerender) + self.threshold_slider = QSlider(Qt.Horizontal) + self.threshold_slider.setRange(0, 100) + self.threshold_slider.setValue(0) + self.threshold_slider.setMaximumWidth(120) + self.threshold_slider.valueChanged.connect(self._threshold_changed) + self.threshold_label = QLabel(_t("tree_threshold").format(0)) + self.chk_grid = QCheckBox(_t("tree_grid")) + self.chk_grid.setChecked(True) + self.chk_grid.toggled.connect(self._rerender) + self.outgroup_combo = QComboBox() + self.outgroup_combo.setMinimumWidth(110) + self.outgroup_combo.currentIndexChanged.connect(self._outgroup_changed) + self.btn_consensus = QToolButton(); self.btn_consensus.setText(_t("btn_consensus")) + self.btn_consensus.clicked.connect(self._consensus) + self.btn_compare = QToolButton(); self.btn_compare.setText(_t("btn_compare")) + self.btn_compare.clicked.connect(self._compare) + + opts.addWidget(QLabel(_t("tree_layout"))) + opts.addWidget(self.layout_combo) + opts.addWidget(self.chk_supports) + opts.addWidget(self.threshold_label) + opts.addWidget(self.threshold_slider) + opts.addWidget(self.chk_grid) + opts.addStretch(1) + opts.addWidget(QLabel(_t("tree_outgroup"))) + opts.addWidget(self.outgroup_combo) + opts.addWidget(self.btn_consensus) + opts.addWidget(self.btn_compare) + layout.addLayout(opts) - bar.addWidget(self.btn_home) - bar.addWidget(self.btn_zoom) - bar.addWidget(self.btn_pan) - bar.addWidget(self.btn_save) - bar.addStretch(1) - bar.addWidget(QLabel(_t("tree_tool_label"))) - bar.addWidget(self.mode_label) - layout.addLayout(bar) layout.addWidget(self.canvas, 1) + self.canvas.mpl_connect('button_press_event', self._on_click) + self._draw_empty_hint() + + # ---------- toolbar plumbing ---------- def _sync_tools(self): mode = self.nav.mode or '' @@ -77,7 +164,8 @@ def _home(self): def _export(self): path, sel = QFileDialog.getSaveFileName( - self, _t("tree_export"), os.path.splitext(os.path.basename(self.tree_path or 'tree'))[0], + self, _t("tree_export"), + os.path.splitext(os.path.basename(self.tree_path or 'tree'))[0], "PNG (*.png);;SVG (*.svg);;PDF (*.pdf)") if not path: return @@ -86,39 +174,375 @@ def _export(self): path += '.' + ext try: self.figure.savefig(path, dpi=200, bbox_inches='tight', - facecolor=self.figure.get_facecolor()) + facecolor='white') except Exception as e: - from PySide6.QtWidgets import QMessageBox QMessageBox.warning(self, _t("tree_export"), str(e)) + # ---------- options handlers ---------- + + def set_layout(self, name): + if name in self.LAYOUTS: + self.layout = name + self._rerender() + + def _threshold_changed(self, value): + self.threshold = value + self.threshold_label.setText(_t("tree_threshold").format(value)) + self._rerender() + + def _outgroup_changed(self): + data = self.outgroup_combo.currentData() + self.outgroup = data if data else None + self._rerender() + + # ---------- data loading ---------- + def load_tree(self, tree_path): - """(Re)render the tree; on failure shows the error inside the canvas.""" - self.tree_path = tree_path - self.ax.clear() try: - tree = Phylo.read(tree_path, "newick") - def format_label(node): - if node.is_terminal(): - return node.name - else: - if node.name: - try: - return str(round(float(node.name))) - except ValueError: - return node.name - return "" - - matplotlib.rcParams.update({'font.size': 9}) - Phylo.draw(tree, axes=self.ax, do_show=False, label_func=format_label) - self.ax.set_title(_t("ml_tree"), pad=15, fontsize=12) - self.figure.tight_layout() - self.canvas.draw() - matplotlib.rcParams.update({'font.size': 10}) + self.tree = Phylo.read(tree_path, "newick") + self.tree_path = tree_path + self.tree_title = os.path.basename(tree_path) + self._collapsed.clear() + self._fill_outgroup_combo() + self._rerender() self._home() except Exception as e: - self.ax.text(0.5, 0.5, _t("tree_error").format(e), - ha="center", va="center", color="red", fontsize=12) - self.canvas.draw() + self._draw_error(e) + + def load_tree_object(self, tree, title): + self.tree = tree + self.tree_path = "" + self.tree_title = title + self._collapsed.clear() + self._fill_outgroup_combo() + self._rerender() + self._home() + + def _fill_outgroup_combo(self): + self.outgroup_combo.blockSignals(True) + self.outgroup_combo.clear() + self.outgroup_combo.addItem(_t("tree_outgroup_none"), None) + if self.tree is not None: + for t in sorted(self.tree.get_terminals(), key=lambda x: x.name or ''): + self.outgroup_combo.addItem(t.name, t.name) + self.outgroup_combo.blockSignals(False) + self.outgroup = None + + # ---------- consensus & RF ---------- + + def _consensus(self): + start_dir = os.path.dirname(self.tree_path) if self.tree_path else "" + path, _ = QFileDialog.getOpenFileName( + self, _t("btn_consensus"), start_dir, "Bootstrap trees (*_boot_trees.txt);;Newick (*.txt *.tree *.nwk);;All files (*.*)") + if not path: + return + try: + trees = list(Phylo.parse(path, 'newick')) + if not trees: + raise ValueError("no trees in file") + cons = majority_consensus(trees) + self.load_tree_object(cons, _t("consensus_from").format(len(trees))) + except Exception as e: + QMessageBox.warning(self, _t("btn_consensus"), str(e)) + + def _compare(self): + a_path, _ = QFileDialog.getOpenFileName(self, _t("tree_pick1"), "", "Newick (*.txt *.tree *.nwk);;All files (*.*)") + if not a_path: + return + b_path, _ = QFileDialog.getOpenFileName(self, _t("tree_pick2"), os.path.dirname(a_path), "Newick (*.txt *.tree *.nwk);;All files (*.*)") + if not b_path: + return + try: + raw, norm = rf_distance(Phylo.read(a_path, 'newick'), Phylo.read(b_path, 'newick')) + QMessageBox.information(self, _t("rf_title"), _t("rf_result").format(raw, int(round(norm * 100)))) + except Exception as e: + QMessageBox.warning(self, _t("rf_title"), str(e)) + + # ---------- collapsing ---------- + + def collapse_clade(self, names): + """Toggle collapse for the clade spanned exactly by these names.""" + key = frozenset(names) + if key in self._collapsed: + self._collapsed.discard(key) + else: + self._collapsed.add(key) + self._rerender() + + def _on_click(self, event): + if event.dblclick and event.inaxes == self.ax and self._coords: + clade_names = self._clade_at(event.xdata, event.ydata) + if clade_names and 1 < len(clade_names) < len(self._leaf_order): + self.collapse_clade(clade_names) + + # ---------- layout computation ---------- + + def _prep_tree(self): + t = self.tree + if self.outgroup: + try: + t = self.tree.copy() + t.root_with_outgroup(self.outgroup) + except Exception: + t = self.tree + return t + + def _compute_layout(self, tree): + """Assign (x, y) or (r, angle) to every displayed node. + Collapsed clades become single display leaves.""" + leaves = tree.get_terminals() + n = len(leaves) + + def is_collapsed(cl): + return frozenset(t.name for t in cl.get_terminals()) in self._collapsed + + display_leaves = [] + def walk(cl): + if cl.is_terminal() or is_collapsed(cl): + display_leaves.append(cl) + return + for c in cl.clades: + walk(c) + walk(tree.root) + + y_of = {} + for i, cl in enumerate(display_leaves): + y_of[id(cl)] = (n - 1 - i) if self.layout != 'circle' else None + # circular: angle per display leaf + m = len(display_leaves) + ang_of = {id(cl): (i + 0.5) / m * 2 * 3.141592653589793 + for i, cl in enumerate(display_leaves)} + + def depth(cl, parent_d): + return parent_d + (cl.branch_length or 0.0) + + coords = {} # id(clade) -> (x, y) or (r, ang) + max_x = [0.0] + + def assign(cl, pd): + x = depth(cl, pd) + max_x[0] = max(max_x[0], x) + if id(cl) in y_of or id(cl) in ang_of: + if self.layout == 'circle': + coords[id(cl)] = (x, ang_of[id(cl)]) + else: + coords[id(cl)] = (x, y_of[id(cl)]) + return coords[id(cl)] + child_pos = [assign(c, x) for c in cl.clades] + if self.layout == 'circle': + r = max(p[0] for p in child_pos) + ang = sum(p[1] for p in child_pos) / len(child_pos) + # average angle over the shortest arc + angs = sorted(p[1] for p in child_pos) + best, best_span = None, 1e9 + for i, a0 in enumerate(angs): + a1 = angs[(i - 1) % len(angs)] # predecessor going backwards + span = (a0 - a1) % (2 * 3.141592653589793) + if span < best_span: + best_span, best = span, (a1 + span / 2) % (2 * 3.141592653589793) + coords[id(cl)] = (r, best) + else: + x_c = max(p[0] for p in child_pos) + y_c = sum(p[1] for p in child_pos) / len(child_pos) + coords[id(cl)] = (x_c, y_c) + return coords[id(cl)] + + assign(tree.root, 0.0) + self._leaf_order = [cl.name or '' for cl in display_leaves] + return coords, max_x[0], display_leaves + + # ---------- rendering ---------- + + def _rerender(self): + if self.tree is None: + return + try: + self._render() + except Exception as e: + self._draw_error(e) + + def _render(self): + self.ax.clear() + self._coords = None + tree = self._prep_tree() + coords, max_x, display_leaves = self._compute_layout(tree) + self._coords = (tree, coords) + accent = '#1E9E6A' + text = '#424242' + faint = '#B9BEC4' + + def support_of(cl): + if cl.confidence is not None: + return float(cl.confidence) + try: + return float(cl.name) + except (TypeError, ValueError): + return None + + def edge_color(cl): + s = support_of(cl) + if self.threshold > 0 and s is not None: + return accent if s >= self.threshold else faint + return text + + segments, colors = [], [] + circ = self.layout == 'circle' + + def draw_edges(cl, parent_pos): + x, a = coords[id(cl)] + if circ: + cx, cy = x * math.cos(a), x * math.sin(a) + segments.append([parent_pos, (cx, cy)]) + colors.append(edge_color(cl)) + for c in cl.clades: + draw_edges(c, (cx, cy)) + else: + x, y = coords[id(cl)] + px, py = parent_pos + if self.layout == 'rect': + segments.append([(px, py), (px, y)]) # elbow at parent's x + segments.append([(px, y), (x, y)]) + else: + segments.append([(px, py), (x, y)]) + colors.append(edge_color(cl)) + for c in cl.clades: + draw_edges(c, (x, y)) + + root = tree.root + rx, ra = coords[id(root)] + if circ: + cx, cy = rx * math.cos(ra), rx * math.sin(ra) + segments.append([(0.0, 0.0), (cx, cy)]) + colors.append(edge_color(root)) + for c in root.clades: + draw_edges(c, (cx, cy)) + elif self.layout == 'rect': + segments.append([(0.0, ra), (rx, ra)]) + colors.append(edge_color(root)) + for c in root.clades: + draw_edges(c, (rx, ra)) + else: # slanted: children radiate from the origin point + for c in root.clades: + draw_edges(c, (0.0, ra)) + + lc = LineCollection(segments, colors=colors, linewidths=1.4, zorder=2) + self.ax.add_collection(lc) + + # leaf labels + for cl in display_leaves: + x, a = coords[id(cl)] + if circ: + lx, ly = (x + max_x * 0.02) * math.cos(a), (x + max_x * 0.02) * math.sin(a) + ha = 'left' if math.cos(a) >= 0 else 'right' + self.ax.text(lx, ly, cl.name or '', fontsize=8, color=text, + ha=ha, va='center', zorder=3) + else: + self.ax.text(x + max_x * 0.01, a, cl.name or '', fontsize=8, + color=text, va='center', zorder=3) + + # support labels on internal nodes + if self.show_supports: + for cl in tree.get_nonterminals(): + if id(cl) not in coords: + continue + s = support_of(cl) + if s is None: + continue + x, a = coords[id(cl)] + if circ: + lx, ly = x * math.cos(a), x * math.sin(a) + else: + lx, ly = x, a + color = accent if (self.threshold > 0 and s >= self.threshold) else text + self.ax.text(lx, ly, f"{s:.0f}" if s > 1 else f"{s:.2f}", + fontsize=7, color=color, style='italic', + ha='center', va='bottom', zorder=3) + + # scale bar (rect/slanted) or subtle rings (circular) + if circ: + step = 10 ** (len(str(int(max_x))) - 1) if max_x >= 1 else 0.01 + ring = step + while ring < max_x * 1.05: + self.ax.add_patch(_ring(ring)) + ring += step + else: + bar = max_x * 0.15 if max_x > 0 else 0.0 + ys = [p[1] for p in coords.values()] + y0 = min(ys) - (max(ys) - min(ys)) * 0.06 - 1 + self.ax.plot([0, bar], [y0, y0], color=text, lw=1.2, zorder=3) + self.ax.text(bar / 2, y0 - 0.4, f"{bar:.3g}", fontsize=7, + color=text, ha='center', zorder=3) + + if self.show_grid and not circ: + self.ax.grid(True, linestyle=':', linewidth=0.5, alpha=0.3, zorder=0) + self.ax.set_axis_off() + if circ: + lim = max_x * 1.3 + self.ax.set_xlim(-lim, lim); self.ax.set_ylim(-lim, lim) + self.ax.set_aspect('equal') + else: + xs = [p[0] for p in coords.values()] + ys = [p[1] for p in coords.values()] + self.ax.set_xlim(min(xs) - 0.05, max(xs) * 1.25 + 0.1) + self.ax.set_ylim(min(ys) - 1.5, max(ys) + 0.8) + self.ax.set_title(self.tree_title, fontsize=10, pad=8) + self.canvas.draw_idle() + + def _clade_at(self, x, y): + """Innermost displayed clade whose bbox contains the point.""" + if not self._coords: + return None + tree, coords = self._coords + best = None + + def to_xy(p): + if self.layout == 'circle': + return (p[0] * math.cos(p[1]), p[0] * math.sin(p[1])) + return (p[0], p[1]) + + def walk(cl): + nonlocal best + if cl.is_terminal(): + return + xs, ys = [], [] + def collect(c): + if id(c) in coords: + px, py = to_xy(coords[id(c)]) + xs.append(px); ys.append(py) + for c2 in c.clades: + collect(c2) + collect(cl) + if xs and min(xs) - 1e-9 <= x <= max(xs) + 1e-9 and min(ys) - 1e-9 <= y <= max(ys) + 1e-9: + best = cl + for c in cl.clades: + walk(c) + walk(tree.root) + if best is None: + return None + return frozenset(t.name for t in best.get_terminals()) + + # ---------- empty / error states ---------- + + def _draw_empty_hint(self): + self.ax.clear() + self.ax.set_axis_off() + self.ax.text(0.5, 0.5, _t("tree_empty"), ha='center', va='center', + fontsize=12, color='#9AA0A6', transform=self.ax.transAxes) + self.canvas.draw_idle() + + def _draw_error(self, e): + self.ax.clear() + self.ax.set_axis_off() + self.ax.text(0.5, 0.5, _t("tree_error").format(e), ha='center', + va='center', color='red', fontsize=12, + transform=self.ax.transAxes) + self.canvas.draw_idle() + + +def _ring(radius): + import matplotlib.patches as mpatches + return mpatches.Circle((0, 0), radius, fill=False, + linestyle=':', linewidth=0.5, color='#B9BEC4') # --- Kept for backward compatibility: modal dialog wrapping the widget --- @@ -129,6 +553,6 @@ def __init__(self, tree_path, parent=None): self.setWindowIcon(QIcon(resource_path('icon.png'))) self.resize(1200, 800) layout = QVBoxLayout(self) - self.viewer = TreeViewerWidget(self) + self.viewer = TreeViewerWidget() self.viewer.load_tree(tree_path) layout.addWidget(self.viewer) From 7695cafea8dea0070aa47007d43bac5ef9f031e9 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:31:54 +0300 Subject: [PATCH 03/36] Stage 3: run history, parameter presets, batch queue with retry, open folder - worker emits per-file status (pending/running/ok/fail); a new Queue tab lists the batch with live markers, and 'Retry failed files' restarts only the failed inputs - run history (phyml_history.json, capped at 200): timestamp, input, OK count, model, duration, tree path; double-click reloads the tree; panel placeholder replaced by the real list - presets: built-in Quick screening (NNI+aLRT) and Publication (GTR+BEST+bootstrap 1000) plus user presets saved from the current parameters (double-click the save row; right-click a user preset to delete); stored in the config - 'Folder' status-bar button opens the output directory after a run --- .gitignore | 1 + locales.json | 24 ++++ phyml_core/pipeline.py | 7 ++ phyml_ui/main_window.py | 255 ++++++++++++++++++++++++++++++++++++---- 4 files changed, 264 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index 150e29f..2a35062 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ phymlgui v1.1/ tools/ testrun/ ROADMAP.md +phyml_history.json release_body.md release.json release_resp.json diff --git a/locales.json b/locales.json index 8c5feb4..0566c91 100644 --- a/locales.json +++ b/locales.json @@ -143,6 +143,18 @@ "rf_title": "Topology comparison (Robinson–Foulds)", "rf_result": "RF distance: {} ({:.0%} normalized).\n\n0 — topologies are identical; 100% — no common splits.", "tree_empty": "No tree yet — run an analysis", + "tab_queue": "Queue", + "btn_retry_failed": "Retry failed files", + "btn_open_dir": "Folder", + "hist_entry": "{} | {} | {}/{} OK", + "hist_no_tree": "[!] Tree file from that run is gone", + "preset_builtin_screen": "Quick screening (NNI + aLRT)", + "preset_builtin_pub": "Publication (GTR + BEST + bootstrap 1000)", + "preset_save_current": "Save current as preset…", + "preset_prompt": "Preset name:", + "preset_applied": "[+] Preset applied: {}", + "preset_saved": "[+] Preset saved: {}", + "preset_delete": "Delete preset", "log_filter_all": "All", "log_filter_error": "Errors", "log_filter_success": "Success", @@ -308,6 +320,18 @@ "rf_title": "Сравнение топологий (Robinson–Foulds)", "rf_result": "RF-расстояние: {} ({:.0%} нормализованное).\n\n0 — топологии идентичны; 100% — нет общих ветвей.", "tree_empty": "Дерева пока нет — запустите анализ", + "tab_queue": "Очередь", + "btn_retry_failed": "Повторить неудавшиеся", + "btn_open_dir": "Папка", + "hist_entry": "{} | {} | {}/{} OK", + "hist_no_tree": "[!] Файл дерева того запуска не найден", + "preset_builtin_screen": "Быстрый скрининг (NNI + aLRT)", + "preset_builtin_pub": "Публикация (GTR + BEST + бутстреп 1000)", + "preset_save_current": "Сохранить текущие как пресет…", + "preset_prompt": "Имя пресета:", + "preset_applied": "[+] Применён пресет: {}", + "preset_saved": "[+] Пресет сохранён: {}", + "preset_delete": "Удалить пресет", "log_filter_all": "Все", "log_filter_error": "Ошибки", "log_filter_success": "Успех", diff --git a/phyml_core/pipeline.py b/phyml_core/pipeline.py index a64c296..8326dc0 100644 --- a/phyml_core/pipeline.py +++ b/phyml_core/pipeline.py @@ -32,6 +32,7 @@ def grab(pattern): # --- Background Worker Thread --- class PipelineWorker(QThread): log_signal = Signal(str) + file_status_signal = Signal(str, str) # (file name, pending|running|ok|fail) finished_signal = Signal(bool, str) def __init__(self, config, fasta_files): @@ -49,12 +50,18 @@ def run(self): batch = len(self.fasta_files) > 1 if batch: self.log_signal.emit(_t("batch_found").format(len(self.fasta_files))) + for i, fasta in enumerate(self.fasta_files, 1): + if self._is_cancelled: + break + self.file_status_signal.emit(os.path.basename(fasta), 'pending') for i, fasta in enumerate(self.fasta_files, 1): if self._is_cancelled: break if batch: self.log_signal.emit(_t("batch_file_header").format(i, len(self.fasta_files), os.path.basename(fasta))) + self.file_status_signal.emit(os.path.basename(fasta), 'running') ok, tree, stats = self._process_one(fasta) + self.file_status_signal.emit(os.path.basename(fasta), 'ok' if ok else 'fail') row = {'file': os.path.basename(fasta), 'status': 'OK' if ok else 'FAIL', 'tree': tree if ok else ''} row.update({k: stats.get(k, '') for k in ('model', 'taxa', 'lnL', 'gamma', 'tstv', 'time')}) self.results.append(row) diff --git a/phyml_ui/main_window.py b/phyml_ui/main_window.py index 5b1152a..d420b51 100644 --- a/phyml_ui/main_window.py +++ b/phyml_ui/main_window.py @@ -2,6 +2,7 @@ import json import os import shutil +from datetime import datetime from html import escape from PySide6.QtWidgets import (QWidget, QVBoxLayout, QHBoxLayout, @@ -9,9 +10,10 @@ QFileDialog, QMessageBox, QTabWidget, QCheckBox, QGroupBox, QSizePolicy, QLabel, QStatusBar, QProgressBar, QSplitter, QToolBox, - QListWidget, QListWidgetItem, QTextEdit, - QTableWidget, QTableWidgetItem, QHeaderView, - QScrollArea, QDialog, QDialogButtonBox) + QToolButton, QListWidget, QListWidgetItem, + QTextEdit, QTableWidget, QTableWidgetItem, + QHeaderView, QScrollArea, QDialog, + QDialogButtonBox, QMenu, QInputDialog) from PySide6.QtCore import QTimer, Qt from PySide6.QtGui import QIcon, QFont, QShortcut, QKeySequence @@ -23,6 +25,25 @@ from phyml_ui.themes import resolve, build_qss, role_colors, ACCENTS CONFIG_FILE = 'phyml_config.json' +HISTORY_FILE = 'phyml_history.json' + +# parameter subset persisted in presets (no paths, no display options) +_PRESET_KEYS = ('datatype', 'model', 'support_method', 'bootstrap_replicates', + 'freq', 'tstv', 'pinv', 'classes', 'gamma', 'search', + 'optimize', 'parsimony') + +_BUILTIN_PRESETS = { + 'preset_builtin_screen': { + 'datatype': 'nt', 'model': 'HKY85', 'support_method': '-4', + 'bootstrap_replicates': '100', 'freq': 'default', 'tstv': 'e', + 'pinv': '0.0', 'classes': '4', 'gamma': 'e', 'search': 'NNI', + 'optimize': 'tlr', 'parsimony': False}, + 'preset_builtin_pub': { + 'datatype': 'nt', 'model': 'GTR', 'support_method': 'bootstrap', + 'bootstrap_replicates': '1000', 'freq': 'default', 'tstv': 'e', + 'pinv': 'e', 'classes': '4', 'gamma': 'e', 'search': 'BEST', + 'optimize': 'tlr', 'parsimony': False}, +} class LogConsole(QWidget): @@ -193,6 +214,10 @@ def __init__(self): self._theme_setting = 'system' self._accent_setting = 'green' self._recent = [] + self._user_presets = {} + self._failed_files = [] + self._run_started = None + self._last_out_dir = None self.phys_cores, self.logi_cores = detect_cores() self.initUI() self.load_config() @@ -228,15 +253,13 @@ def _build_nav_panel(self): self.panel.addItem(self.recent_list, _t("panel_recent")) self.presets_list = QListWidget() - hint = QListWidgetItem(_t("panel_hint_presets")) - hint.setFlags(hint.flags() & ~Qt.ItemIsEnabled) - self.presets_list.addItem(hint) + self.presets_list.itemDoubleClicked.connect(self._preset_activated) + self.presets_list.setContextMenuPolicy(Qt.CustomContextMenu) + self.presets_list.customContextMenuRequested.connect(self._preset_menu) self.panel.addItem(self.presets_list, _t("panel_presets")) self.history_list = QListWidget() - hint = QListWidgetItem(_t("panel_hint_history")) - hint.setFlags(hint.flags() & ~Qt.ItemIsEnabled) - self.history_list.addItem(hint) + self.history_list.itemDoubleClicked.connect(self._open_history_entry) self.panel.addItem(self.history_list, _t("panel_history")) return self.panel @@ -382,6 +405,18 @@ def _build_right_pane(self): self.summary_table.horizontalHeader().setSectionResizeMode(col, QHeaderView.ResizeToContents) self.result_tabs.addTab(self.summary_table, _t("tab_summary")) + queue_tab = QWidget() + queue_lay = QVBoxLayout(queue_tab) + self.queue_list = QListWidget() + self.queue_list.setAlternatingRowColors(True) + self.retry_btn = QToolButton() + self.retry_btn.setText(_t("btn_retry_failed")) + self.retry_btn.setVisible(False) + self.retry_btn.clicked.connect(self._retry_failed) + queue_lay.addWidget(self.queue_list, 1) + queue_lay.addWidget(self.retry_btn) + self.result_tabs.addTab(queue_tab, _t("tab_queue")) + split = QSplitter(Qt.Vertical) split.addWidget(self.tree_view) split.addWidget(self.result_tabs) @@ -401,12 +436,17 @@ def _build_status_bar(self): self.cpu_sys_bar.setTextVisible(True) self.cpu_sys_bar.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Preferred) self.cpu_run_label = QLabel(_t("cpu_analysis_idle")) + self.open_dir_btn = QToolButton() + self.open_dir_btn.setText(_t("btn_open_dir")) + self.open_dir_btn.setEnabled(False) + self.open_dir_btn.clicked.connect(self._open_output_dir) self.settings_btn = QPushButton(_t("btn_settings")) self.settings_btn.clicked.connect(self.open_settings) self.status_bar.addWidget(self.cores_label) self.status_bar.addWidget(self.cpu_sys_label) self.status_bar.addWidget(self.cpu_sys_bar, 1) # stretch: fills all free space self.status_bar.addPermanentWidget(self.cpu_run_label) + self.status_bar.addPermanentWidget(self.open_dir_btn) self.status_bar.addPermanentWidget(self.settings_btn) # far bottom-right corner self.cpu_timer = QTimer(self) @@ -533,6 +573,7 @@ def get_current_config(self): "theme": self._theme_setting, "accent": self._accent_setting, "recent": self._recent, + "presets": self._user_presets, "window": { "w": self.width(), "h": self.height(), "splitter": self.splitter.sizes(), @@ -577,6 +618,9 @@ def load_config(self): self.chk_show_tree.setChecked(config.get("show_tree", True)) self._recent = [p for p in config.get("recent", []) if os.path.exists(p)] self._refresh_recent() + self._user_presets = dict(config.get("presets", {})) + self._refresh_presets() + self._refresh_history() self.apply_theme(config.get("theme", "system"), config.get("accent", "green")) win = config.get("window") or {} @@ -589,39 +633,50 @@ def load_config(self): self.toggle_bootstrap_input() self.log_console.append(_t("sys_load")) except Exception as e: self.log_console.append(_t("err_load").format(e)) + # lists must be populated even without a config file + self._refresh_presets() + self._refresh_history() def set_combo_by_data(self, combo, data_val): idx = combo.findData(data_val) if idx >= 0: combo.setCurrentIndex(idx) - def start_pipeline(self): - target = self.fasta_path_input.text() or self.fasta_file - if not target or not os.path.exists(target): - QMessageBox.warning(self, _t("err_box_title"), _t("err_box_msg")); return - if os.path.isdir(target): - # skip *_aligned.fasta: those are this app's own MUSCLE outputs - files = sorted(f for f in os.listdir(target) - if f.lower().endswith(('.fasta', '.fas', '.fa')) - and not f.lower().endswith('_aligned.fasta')) - if not files: - QMessageBox.warning(self, _t("err_box_title"), _t("batch_empty")); return - files = [os.path.join(target, f) for f in files] + def start_pipeline(self, files_override=None): + if files_override is None: + target = self.fasta_path_input.text() or self.fasta_file + if not target or not os.path.exists(target): + QMessageBox.warning(self, _t("err_box_title"), _t("err_box_msg")); return + if os.path.isdir(target): + # skip *_aligned.fasta: those are this app's own MUSCLE outputs + files = sorted(f for f in os.listdir(target) + if f.lower().endswith(('.fasta', '.fas', '.fa')) + and not f.lower().endswith('_aligned.fasta')) + if not files: + QMessageBox.warning(self, _t("err_box_title"), _t("batch_empty")); return + files = [os.path.join(target, f) for f in files] + else: + files = [target] else: - files = [target] + files = list(files_override) if (is_mpi_binary(self.phyml_path_input.text()) and not self.chk_use_mpi.isChecked() and self.cb_support_method.currentData() == "bootstrap"): QMessageBox.warning(self, _t("mpi_boot_title"), _t("mpi_boot_msg")); return self._batch_mode = len(files) > 1 - self._remember_recent(target) + self._remember_recent(target if files_override is None else os.path.dirname(files[0])) if not self._is_loading: self.auto_save() self.btn_run.setEnabled(False); self.btn_cancel.setEnabled(True); self.log_console.clear() self.result_tabs.setCurrentWidget(self.log_console) + self._populate_queue(files) + if self._batch_mode: + self.result_tabs.setCurrentWidget(self.queue_list.parentWidget()) self._cpu_samples = [] self._mon_procs = {} + self._run_started = datetime.now() config = self.get_current_config() self.worker = PipelineWorker(config, files) self.worker.log_signal.connect(self.log_console.append) + self.worker.file_status_signal.connect(self._on_file_status) self.worker.finished_signal.connect(self.pipeline_finished) self.worker.start() @@ -639,9 +694,163 @@ def pipeline_finished(self, success, tree_path): self._populate_stats() self._populate_summary() self._save_summary_csv() + n_ok = sum(1 for r in getattr(self.worker, 'results', []) if r['status'] == 'OK') + n_all = len(getattr(self.worker, 'results', [])) + self._append_history(n_ok, n_all, tree_path) + self._failed_files = [f for f, r in zip(getattr(self.worker, 'fasta_files', []), + getattr(self.worker, 'results', [])) + if r.get('status') == 'FAIL'] + self.retry_btn.setVisible(bool(self._failed_files)) + target = self.fasta_path_input.text() or self.fasta_file + if target and os.path.isdir(target): + self._last_out_dir = target + elif getattr(self.worker, 'fasta_files', None): + self._last_out_dir = os.path.dirname(os.path.abspath(self.worker.fasta_files[0])) + self.open_dir_btn.setEnabled(self._last_out_dir is not None) if success and tree_path and os.path.exists(tree_path) and self.chk_show_tree.isChecked(): self.tree_view.load_tree(tree_path) + # ---------- queue ---------- + + def _populate_queue(self, files): + self.queue_list.clear() + for f in files: + self.queue_list.addItem(os.path.basename(f) + ' …') + + def _on_file_status(self, name, status): + marker = {'pending': '…', 'running': '▶', 'ok': '✓', 'fail': '✗'}.get(status, '?') + for i in range(self.queue_list.count()): + item = self.queue_list.item(i) + if item.text().split(' ')[0] == name: + item.setText(f'{name} {marker}') + return + + def _retry_failed(self): + if self._failed_files: + self.start_pipeline(files_override=self._failed_files) + + def _open_output_dir(self): + if self._last_out_dir and os.path.isdir(self._last_out_dir): + os.startfile(self._last_out_dir) + + # ---------- history ---------- + + def _load_history(self): + try: + with open(HISTORY_FILE, 'r', encoding='utf-8') as f: + return json.load(f) + except (OSError, ValueError): + return [] + + def _append_history(self, n_ok, n_all, tree_path): + duration = '' + if self._run_started: + duration = f'{(datetime.now() - self._run_started).total_seconds():.0f}s' + rec = { + 'ts': datetime.now().strftime('%Y-%m-%d %H:%M'), + 'input': os.path.basename(self.fasta_path_input.text() or (self.worker.fasta_files[0] if getattr(self.worker, 'fasta_files', None) else '')), + 'ok': n_ok, 'n': n_all, 'time': duration, + 'model': self.cb_model.currentData() or '', + 'tree': tree_path if tree_path else '', + } + hist = self._load_history() + hist.insert(0, rec) + try: + with open(HISTORY_FILE, 'w', encoding='utf-8') as f: + json.dump(hist[:200], f, indent=2, ensure_ascii=False) + except OSError: + pass + self._refresh_history() + + def _refresh_history(self): + self.history_list.clear() + for rec in self._load_history()[:50]: + self.history_list.addItem(_t('hist_entry').format( + rec.get('ts', ''), rec.get('input', ''), rec.get('ok', 0), rec.get('n', 0))) + + def _open_history_entry(self, item): + row = self.history_list.row(item) + hist = self._load_history()[:50] + if row >= len(hist): + return + tree = hist[row].get('tree', '') + if tree and os.path.exists(tree): + self.tree_view.load_tree(tree) + else: + self.log_console.append(_t('hist_no_tree')) + + # ---------- presets ---------- + + def _refresh_presets(self): + self.presets_list.clear() + save_item = QListWidgetItem(_t('preset_save_current')) + save_item.setData(Qt.UserRole, '__save__') + self.presets_list.addItem(save_item) + for label_key in _BUILTIN_PRESETS: + it = QListWidgetItem(_t(label_key)) + it.setData(Qt.UserRole, ('builtin', label_key)) + self.presets_list.addItem(it) + for name in self._user_presets: + it = QListWidgetItem(name) + it.setData(Qt.UserRole, ('user', name)) + self.presets_list.addItem(it) + + def _preset_activated(self, item): + data = item.data(Qt.UserRole) + if data == '__save__': + self._save_preset() + return + kind, key = data + params = _BUILTIN_PRESETS[key] if kind == 'builtin' else self._user_presets.get(key, {}) + self._apply_preset(params) + self.log_console.append(_t('preset_applied').format(item.text())) + + def _apply_preset(self, params): + self._is_loading = True + try: + self.set_combo_by_data(self.cb_datatype, params.get('datatype', 'nt')) + self.update_models() + self.set_combo_by_data(self.cb_model, params.get('model', 'HKY85')) + self.set_combo_by_data(self.cb_support_method, params.get('support_method', 'bootstrap')) + self.set_combo_by_data(self.cb_freq, params.get('freq', 'default')) + self.set_combo_by_data(self.cb_search, params.get('search', 'NNI')) + self.set_combo_by_data(self.cb_optimize, params.get('optimize', 'tlr')) + self.le_bootstrap_replicates.setText(params.get('bootstrap_replicates', '100')) + self.le_tstv.setText(params.get('tstv', 'e')) + self.le_pinv.setText(params.get('pinv', '0.0')) + self.le_classes.setText(params.get('classes', '4')) + self.le_gamma.setText(params.get('gamma', 'e')) + self.chk_parsimony.setChecked(params.get('parsimony', False)) + finally: + self._is_loading = False + self.toggle_bootstrap_input() + self.auto_save() + + def _save_preset(self): + name, ok = QInputDialog.getText(self, _t('preset_save_current'), _t('preset_prompt')) + if not ok or not name.strip(): + return + name = name.strip() + cfg = self.get_current_config() + self._user_presets[name] = {k: cfg.get(k) for k in _PRESET_KEYS} + self.auto_save() + self._refresh_presets() + self.log_console.append(_t('preset_saved').format(name)) + + def _preset_menu(self, pos): + item = self.presets_list.itemAt(pos) + if item is None: + return + data = item.data(Qt.UserRole) + if not (isinstance(data, tuple) and data[0] == 'user'): + return + menu = QMenu(self) + act = menu.addAction(_t('preset_delete')) + if menu.exec(self.presets_list.mapToGlobal(pos)) == act: + self._user_presets.pop(data[1], None) + self.auto_save() + self._refresh_presets() + def _populate_stats(self): rows = [r for r in self.worker.results if r['status'] == 'OK'] if not rows: From 03c4df246da6aa2a63ee85d404e5cdf5bb36645a Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:37:27 +0300 Subject: [PATCH 04/36] Stage 4: IQ-TREE engine with ModelFinder, alignment info, XLSX export - engine selection in Settings (PhyML / IQ-TREE); IQ-TREE branch in the pipeline: -m MFP or explicit model, -T threads, --prefix, ultrafast bootstrap (-B) or aLRT (--alrt), per-file .iqtree/.treefile outputs parsed into the same stats/summary rows; ModelFinder's best-fit model lands in the log line, stats tab and CSV/XLSX - engine-aware model lists (MFP + common +G4 models for IQ-TREE) - quick alignment summary under the FASTA field: taxa count, site range, gap percentage - summary is additionally written as phymlgui_summary.xlsx (openpyxl, optional dependency added to requirements) --- locales.json | 16 +++++++ phyml_core/pipeline.py | 75 +++++++++++++++++++++++++++++++- phyml_ui/main_window.py | 94 ++++++++++++++++++++++++++++++++++++++--- requirements.txt | 1 + 4 files changed, 180 insertions(+), 6 deletions(-) diff --git a/locales.json b/locales.json index 0566c91..9a31bcd 100644 --- a/locales.json +++ b/locales.json @@ -155,6 +155,14 @@ "preset_applied": "[+] Preset applied: {}", "preset_saved": "[+] Preset saved: {}", "preset_delete": "Delete preset", + "settings_engine_box": "Engine", + "settings_engine": "Analysis engine:", + "engine_phyml": "PhyML (MPI-capable)", + "engine_iqtree": "IQ-TREE (ModelFinder, UFBoot)", + "path_iqtree": "Path to IQ-TREE (iqtree2.exe):", + "err_iqtree": "[ERROR] Invalid path to IQ-TREE.", + "fasta_info": "{} taxa | {}-{} sites | gaps {}%", + "xlsx_saved": "[+] Excel workbook saved: {}", "log_filter_all": "All", "log_filter_error": "Errors", "log_filter_success": "Success", @@ -332,6 +340,14 @@ "preset_applied": "[+] Применён пресет: {}", "preset_saved": "[+] Пресет сохранён: {}", "preset_delete": "Удалить пресет", + "settings_engine_box": "Движок", + "settings_engine": "Движок анализа:", + "engine_phyml": "PhyML (с поддержкой MPI)", + "engine_iqtree": "IQ-TREE (ModelFinder, UFBoot)", + "path_iqtree": "Путь к IQ-TREE (iqtree2.exe):", + "err_iqtree": "[ОШИБКА] Некорректный путь к IQ-TREE.", + "fasta_info": "{} такс | {}-{} сайтов | гэпы {}%", + "xlsx_saved": "[+] Excel-файл сохранён: {}", "log_filter_all": "Все", "log_filter_error": "Ошибки", "log_filter_success": "Успех", diff --git a/phyml_core/pipeline.py b/phyml_core/pipeline.py index 8326dc0..d505fdd 100644 --- a/phyml_core/pipeline.py +++ b/phyml_core/pipeline.py @@ -29,6 +29,26 @@ def grab(pattern): } +def parse_iqtree_stats(stats_path): + """Parse key values from an IQ-TREE .iqtree report into a dict.""" + try: + with open(stats_path, encoding='utf-8', errors='replace') as f: + content = f.read() + except OSError: + return {} + def grab(pattern): + m = re.search(pattern, content) + return m.group(1) if m else "" + return { + 'model': grab(r'Best-fit model according to \w+:\s*(\S+)') or grab(r'Model of substitution:\s*(\S+)'), + 'taxa': grab(r'Input data:\s*(\d+) sequences'), + 'lnL': grab(r'Log-likelihood of the tree:\s*(-?[\d.]+)'), + 'gamma': grab(r'Gamma shape alpha:\s*([\d.]+)'), + 'tstv': '', + 'time': (grab(r'Total wall-clock time used:\s*([\d.]+)') + ' s').strip(), + } + + # --- Background Worker Thread --- class PipelineWorker(QThread): log_signal = Signal(str) @@ -148,7 +168,18 @@ def _process_one(self, current_fasta): if self._is_cancelled: return False, "", {} - # 3. Run PhyML (MPI support added here) + # 3. Run the configured engine + engine = self.config.get('engine', 'phyml') + if engine == 'iqtree': + return self._run_iqtree(phylip_path, creation_flags) + return self._run_phyml(phylip_path, creation_flags) + + except Exception as e: + self.log_signal.emit(_t("err_crit").format(str(e))) + return False, "", "" + + def _run_phyml(self, phylip_path, creation_flags): + try: phyml_exe = self.config.get("phyml_path", "") if not phyml_exe or not os.path.exists(phyml_exe): self.log_signal.emit(_t("err_phyml")); return False, "", {} @@ -207,6 +238,48 @@ def _process_one(self, current_fasta): self.log_signal.emit(_t("err_crit").format(str(e))) return False, "", {} + def _run_iqtree(self, phylip_path, creation_flags): + try: + iqtree_exe = self.config.get("iqtree_path", "") + if not iqtree_exe or not os.path.exists(iqtree_exe): + self.log_signal.emit(_t("err_iqtree")); return False, "", {} + + prefix = phylip_path.rsplit('.', 1)[0] + '_iqtree' + cmd_iqtree = [iqtree_exe, '-s', phylip_path, '--prefix', prefix, + '-redo', '-m', self.config.get("model", "MFP"), + '-T', self.config.get("mpi_np", "2")] + support_method = self.config.get("support_method", "bootstrap") + if support_method == "bootstrap": + cmd_iqtree.extend(['-B', self.config.get("bootstrap_replicates", "1000")]) + elif support_method.startswith('-'): + cmd_iqtree.extend(['--alrt', '1000']) # aLRT branch support + + self.log_signal.emit(_t("run_phyml").format(' '.join(cmd_iqtree))) + self.active_process = subprocess.Popen(cmd_iqtree, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, + creationflags=creation_flags) + for line in iter(self.active_process.stdout.readline, ''): + if line: self.log_signal.emit(line.strip()) + if self._is_cancelled: break + self.active_process.stdout.close() + self.active_process.wait() + + if self._is_cancelled: + self.log_signal.emit(_t("cancel_analysis")) + return False, "", {} + tree_path = prefix + '.treefile' + if self.active_process.returncode != 0 or not os.path.exists(tree_path): + self.log_signal.emit(_t("err_crit_phyml").format(self.active_process.returncode)) + return False, "", {} + self.log_signal.emit(_t("success_pipe")) + stats = parse_iqtree_stats(prefix + '.iqtree') + self.log_signal.emit(_t("stats_summary").format( + *(stats.get(k) or "n/a" for k in ('model', 'taxa', 'lnL', 'gamma', 'tstv', 'time')))) + return True, tree_path, stats + except Exception as e: + self.log_signal.emit(_t("err_crit").format(str(e))) + return False, "", {} + def cancel(self): self._is_cancelled = True if self.active_process and self.active_process.poll() is None: diff --git a/phyml_ui/main_window.py b/phyml_ui/main_window.py index d420b51..f248318 100644 --- a/phyml_ui/main_window.py +++ b/phyml_ui/main_window.py @@ -144,6 +144,22 @@ def __init__(self, app_window, parent=None): self.setWindowIcon(QIcon(resource_path('icon.png'))) self.resize(640, 340) root = QVBoxLayout(self) + engine_box = QGroupBox(_t("settings_engine_box")) + eform = QFormLayout(engine_box) + self.engine_combo = QComboBox() + self.engine_combo.addItem(_t("engine_phyml"), "phyml") + self.engine_combo.addItem(_t("engine_iqtree"), "iqtree") + idx = self.engine_combo.findData(app_window.engine_setting) + self.engine_combo.setCurrentIndex(max(idx, 0)) + eform.addRow(_t("settings_engine"), self.engine_combo) + self.iqtree_edit = QLineEdit(app_window.iqtree_path_input.text()) + iq_btn = QPushButton(_t("btn_browse")) + iq_btn.clicked.connect(lambda: self._browse(self.iqtree_edit)) + irow = QHBoxLayout(); irow.addWidget(self.iqtree_edit); irow.addWidget(iq_btn) + iwrap = QWidget(); iwrap.setLayout(irow) + eform.addRow(_t("path_iqtree"), iwrap) + root.addWidget(engine_box) + paths_box = QGroupBox(_t("settings_paths")) form = QFormLayout(paths_box) self.edits = [] @@ -193,6 +209,11 @@ def accept(self): self.app_window.muscle_path_input] for edit, source in zip(self.edits, inputs): source.setText(edit.text()) # programmatic: triggers autosave & MPI detect + self.app_window.iqtree_path_input.setText(self.iqtree_edit.text()) + engine = self.engine_combo.currentData() + if engine != self.app_window.engine_setting: + self.app_window.engine_setting = engine + self.app_window.update_models() self.app_window.apply_theme(self.theme_combo.currentData(), self.accent_combo.currentData()) if not self.app_window._is_loading: @@ -213,6 +234,7 @@ def __init__(self): self._batch_mode = False self._theme_setting = 'system' self._accent_setting = 'green' + self.engine_setting = 'phyml' self._recent = [] self._user_presets = {} self._failed_files = [] @@ -270,6 +292,7 @@ def _build_middle_pane(self): self.phyml_path_input.textChanged.connect(self.on_phyml_path_changed) self.mpirun_path_input = QLineEdit() self.muscle_path_input = QLineEdit() + self.iqtree_path_input = QLineEdit() middle = QWidget() v = QVBoxLayout(middle) @@ -285,6 +308,10 @@ def _build_middle_pane(self): row = QHBoxLayout(); row.addWidget(self.fasta_path_input, 1) row.addWidget(btn_fasta); row.addWidget(btn_fasta_dir) v.addLayout(row) + self.fasta_info_label = QLabel(' ') + self.fasta_info_label.setStyleSheet('color: #9AA0A6;') + self.fasta_path_input.textChanged.connect(self._update_fasta_info) + v.addWidget(self.fasta_info_label) scroll = QScrollArea() scroll.setWidgetResizable(True) @@ -522,12 +549,43 @@ def toggle_bootstrap_input(self): def update_models(self): self.cb_model.clear() - dt = self.cb_datatype.currentData() - models = ["HKY85", "JC69", "K80", "F81", "F84", "TN93", "GTR", "custom"] if dt == "nt" else \ - ["LG", "WAG", "JTT", "MtREV", "Dayhoff", "DCMut", "RtREV", "CPREV", "VT", "Blosum62", "MtMam", "MtArt", "HIVw", "HIVb", "custom"] + if self.engine_setting == 'iqtree': + models = ["MFP", "GTR+F+G4", "HKY2+F+G4", "TIM2+F+G4", "TVM+F+G4"] \ + if self.cb_datatype.currentData() == "nt" else \ + ["MFP", "LG+G4", "WAG+G4", "JTT+G4", "mtREV+G4"] + else: + models = ["HKY85", "JC69", "K80", "F81", "F84", "TN93", "GTR", "custom"] if self.cb_datatype.currentData() == "nt" else \ + ["LG", "WAG", "JTT", "MtREV", "Dayhoff", "DCMut", "RtREV", "CPREV", "VT", "Blosum62", "MtMam", "MtArt", "HIVw", "HIVb", "custom"] for m in models: self.cb_model.addItem(m, m) if not self._is_loading: self.auto_save() + def _update_fasta_info(self, path): + """Quick alignment summary under the FASTA field.""" + try: + if not path or not os.path.exists(path): + self.fasta_info_label.setText(' ') + return + if os.path.isdir(path): + n = sum(1 for f in os.listdir(path) + if f.lower().endswith(('.fasta', '.fas', '.fa')) + and not f.lower().endswith('_aligned.fasta')) + self.fasta_info_label.setText(_t('fasta_info').format(n, '—', '—', '—')) + return + from Bio import SeqIO + lengths = [len(r.seq) for r in SeqIO.parse(path, 'fasta')] + if not lengths: + self.fasta_info_label.setText(' ') + return + gaps = 0 + total = sum(lengths) + for r in SeqIO.parse(path, 'fasta'): + gaps += str(r.seq).count('-') + pct = round(gaps / total * 100, 1) if total else 0 + self.fasta_info_label.setText( + _t('fasta_info').format(len(lengths), min(lengths), max(lengths), pct)) + except Exception: + self.fasta_info_label.setText(' ') + def browse_fasta(self): path, _ = QFileDialog.getOpenFileName(self, _t("file_fasta"), "", "FASTA Files (*.fasta *.fas *.fa)") if path: @@ -552,6 +610,8 @@ def trigger_autosave(self, *args): def get_current_config(self): return { + "engine": self.engine_setting, + "iqtree_path": self.iqtree_path_input.text(), "phyml_path": self.phyml_path_input.text(), "mpirun_path": self.mpirun_path_input.text(), "muscle_path": self.muscle_path_input.text(), @@ -594,6 +654,8 @@ def load_config(self): try: with open(CONFIG_FILE, 'r', encoding='utf-8') as f: config = json.load(f) + self.engine_setting = config.get("engine", "phyml") + self.iqtree_path_input.setText(config.get("iqtree_path", "")) self.phyml_path_input.setText(config.get("phyml_path", "")) mpirun = config.get("mpirun_path", "mpiexec") if mpirun in ("mpirun", "mpiexec"): @@ -658,7 +720,10 @@ def start_pipeline(self, files_override=None): files = [target] else: files = list(files_override) - if (is_mpi_binary(self.phyml_path_input.text()) + if self.engine_setting == 'iqtree': + if not self.iqtree_path_input.text() or not os.path.exists(self.iqtree_path_input.text()): + QMessageBox.warning(self, _t("err_box_title"), _t("err_iqtree")); return + elif (is_mpi_binary(self.phyml_path_input.text()) and not self.chk_use_mpi.isChecked() and self.cb_support_method.currentData() == "bootstrap"): QMessageBox.warning(self, _t("mpi_boot_title"), _t("mpi_boot_msg")); return @@ -750,7 +815,8 @@ def _append_history(self, n_ok, n_all, tree_path): 'ts': datetime.now().strftime('%Y-%m-%d %H:%M'), 'input': os.path.basename(self.fasta_path_input.text() or (self.worker.fasta_files[0] if getattr(self.worker, 'fasta_files', None) else '')), 'ok': n_ok, 'n': n_all, 'time': duration, - 'model': self.cb_model.currentData() or '', + 'model': (self.cb_model.currentData() or ''), + 'engine': self.engine_setting, 'tree': tree_path if tree_path else '', } hist = self._load_history() @@ -889,6 +955,24 @@ def _save_summary_csv(self): self.log_console.append(_t("summary_saved").format(out_path)) except OSError as e: self.log_console.append(_t("err_summary_save").format(e)) + # optional Excel workbook with the same data (needs openpyxl) + try: + from openpyxl import Workbook + wb = Workbook() + ws = wb.active + ws.title = 'Summary' + ws.append([_t("sum_col_" + c) for c in fields[:-1]]) + for r in self.worker.results: + ws.append([r.get(c) or '' for c in fields[:-1]]) + for col, width in zip('ABCDEFGHI', (28, 8, 16, 8, 12, 10, 10, 12)): + ws.column_dimensions[col].width = width + xlsx_path = out_path[:-4] + '.xlsx' + wb.save(xlsx_path) + self.log_console.append(_t("xlsx_saved").format(xlsx_path)) + except ImportError: + pass + except Exception as e: + self.log_console.append(_t("err_summary_save").format(e)) def on_phyml_path_changed(self, path): if is_mpi_binary(path): diff --git a/requirements.txt b/requirements.txt index 948cfb4..a0738ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ PySide6>=6.5 matplotlib>=3.5 biopython>=1.81 psutil>=5.9 +openpyxl>=3.1 From bbe1087f5070c0d5621e5f080656cb307da9305f Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:44:11 +0300 Subject: [PATCH 05/36] Stage 5: tests in the repository, CI workflow, headless CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - test suites moved into tests/ (smoke + e2e + gui logic + batch + tree viewer + iqtree); suites needing local analysis binaries print SKIP and exit 0 when absent, so the set is CI-friendly; scratch data stays in the gitignored testrun/ - tests/test_smoke.py: imports/i18n integrity, theme resolution, stats parsers, RF/splits math, log roles, offscreen main window — no binaries required - .github/workflows/tests.yml: windows-latest smoke job on push/PR - headless CLI: phymlgui.py --file/--batch [--config]; same pipeline, same i18n (locales.json now also resolved relative to the package so CLI works from any working directory) - README updated for engines, viewer features, history/presets, CLI --- .github/workflows/tests.yml | 19 ++++ README.md | 34 ++++--- phyml_core/cli.py | 57 +++++++++++ phyml_core/i18n.py | 4 + phymlgui.py | 10 +- tests/README.md | 18 ++++ tests/fetch_real_data.py | 88 +++++++++++++++++ tests/run_tests.py | 187 ++++++++++++++++++++++++++++++++++++ tests/test_batch.py | 112 +++++++++++++++++++++ tests/test_gui_logic.py | 156 ++++++++++++++++++++++++++++++ tests/test_iqtree.py | 88 +++++++++++++++++ tests/test_smoke.py | 86 +++++++++++++++++ tests/test_tree_viewer.py | 96 ++++++++++++++++++ 13 files changed, 938 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/tests.yml create mode 100644 phyml_core/cli.py create mode 100644 tests/README.md create mode 100644 tests/fetch_real_data.py create mode 100644 tests/run_tests.py create mode 100644 tests/test_batch.py create mode 100644 tests/test_gui_logic.py create mode 100644 tests/test_iqtree.py create mode 100644 tests/test_smoke.py create mode 100644 tests/test_tree_viewer.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..2b70824 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,19 @@ +name: tests + +on: + push: + branches: [master] + pull_request: + +jobs: + smoke: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install -r requirements.txt + - name: Smoke tests (no analysis binaries) + run: python tests/test_smoke.py diff --git a/README.md b/README.md index 22c067a..bd40f1e 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,16 @@ **phymlGUI** is a graphical interface for Windows that automates the construction of phylogenetic trees using the Maximum Likelihood method. ### 🛠 Core Capabilities -* **Integrated Pipeline:** Automatically detects unaligned sequences and runs **MUSCLE** before passing data to **PhyML**. +* **Two Engines:** **PhyML** (serial or MPI) and **IQ-TREE** (automatic model selection with ModelFinder, ultrafast bootstrap) — switchable in Settings. +* **Integrated Pipeline:** Automatically detects unaligned sequences and runs **MUSCLE** before passing data to the engine. * **Parallel Computing (MPI):** Supports execution via `mpirun`/`mpiexec` for multi-core processing; MPI binaries are detected automatically and misconfigured bootstrap runs are blocked with an explanation. -* **Batch Mode:** Process a whole folder of FASTA files in one run — per-file log sections, a final N/M summary and an automatic `phymlgui_summary.csv` report. +* **Batch Mode:** Process a whole folder of FASTA files in one run — a live per-file queue with statuses, retry of failed files, an N/M summary and automatic `phymlgui_summary.csv` / `.xlsx` reports. * **Hardware Awareness:** Physical/logical core detection (the `np` field is pre-filled), plus a live CPU status bar showing both system load and the utilization of the analysis process tree. -* **Run Statistics:** Model, taxa count, log-likelihood, gamma shape, Ts/Tv and run time are printed in the log after every analysis. -* **Data Sanitization:** Automatically cleans sequence IDs and alignments from characters that cause PhyML to crash (dots, colons, brackets, spaces). -* **Format Conversion:** Converts `FASTA` to `PHYLIP-relaxed` format, supporting long sequence names. -* **Visualization:** Built-in viewer for Newick trees with support for branch labels and image export. -* **Configuration Persistence:** Automatic saving and loading of all parameters and file paths via `phyml_config.json`. +* **Tree Viewer:** Rectangular / slanted / circular layouts, support-value threshold highlighting, clade collapsing, outgroup rerooting, majority-rule consensus from bootstrap trees, Robinson–Foulds topology comparison and PNG/SVG/PDF export. +* **Run History and Presets:** every run is recorded (double-click reloads its tree); built-in and user parameter presets. +* **Headless CLI:** `python phymlgui.py --file data.fasta` / `--batch folder/` runs the same pipeline without the GUI. +* **Data Sanitization / Conversion:** cleans sequence IDs, converts `FASTA` to `PHYLIP-relaxed`. +* **Themes:** light/dark (can follow the Windows system theme) with a selectable accent color. ### ⚙️ Requirements * **Executables:** `phyml.exe` (or MPI version) and `muscle.exe`. @@ -33,15 +34,16 @@ **phymlGUI** — графический интерфейс для Windows, автоматизирующий построение филогенетических деревьев методом максимального правдоподобия. ### 🛠 Основные возможности -* **Интегрированный пайплайн:** Автоматическое определение невыровненных последовательностей и запуск **MUSCLE** перед началом работы **PhyML**. -* **Параллельные вычисления (MPI):** Поддержка запуска через `mpirun`/`mpiexec`; MPI-бинарники определяются автоматически, а некорректная конфигурация бутстрепа блокируется с пояснением. -* **Пакетный режим:** Обработка целой папки FASTA-файлов за один запуск — секции лога по каждому файлу, итоговая сводка N/M и автоматический отчёт `phymlgui_summary.csv`. -* **Аппаратная осведомлённость:** Определение физических/логических ядер (поле `np` заполняется автоматически) и живая строка состояния с загрузкой CPU — как системы, так и дерева процессов анализа. -* **Статистика прогона:** Модель, число таксонов, лог-правдоподобие, параметр гаммы, Ts/Tv и время выводятся в лог после каждого анализа. -* **Санитарная очистка данных:** Автоматическая замена символов в ID и последовательностях, вызывающих ошибки в PhyML (точки, двоеточия, скобки, пробелы). -* **Конвертация форматов:** Перевод из `FASTA` в `PHYLIP-relaxed` с поддержкой длинных названий последовательностей. -* **Визуализация:** Встроенный просмотрщик деревьев в формате Newick с отображением поддержек ветвей и возможностью сохранения графики. -* **Сохранение состояния:** Автоматическое сохранение всех настроек и путей в файл `phyml_config.json`. +* **Два движка:** **PhyML** (последовательно или MPI) и **IQ-TREE** (автоподбор модели ModelFinder, ультрабыстрый бутстреп) — переключение в настройках. +* **Интегрированный пайплайн:** Автоматическое определение невыровненных последовательностей и запуск **MUSCLE** перед анализом. +* **Параллельные вычисления (MPI):** Запуск через `mpirun`/`mpiexec`; MPI-бинарники определяются автоматически, некорректная конфигурация бутстрепа блокируется с пояснением. +* **Пакетный режим:** Обработка папки FASTA-файлов — живая очередь со статусами, повтор неудавшихся, сводка N/M и автоматические отчёты `phymlgui_summary.csv` / `.xlsx`. +* **Аппаратная осведомлённость:** Определение физических/логических ядер и живая строка состояния с загрузкой CPU — системы и дерева процессов анализа. +* **Просмотрщик деревьев:** Прямоугольная/сланцевая/круговая раскладки, подсветка поддержек по порогу, сворачивание клад, рерутинг по outgroup, консенсус большинства из бутстреп-деревьев, сравнение топологий по Робинсону–Фаулдсу, экспорт PNG/SVG/PDF. +* **История и пресеты:** каждый запуск записывается (двойной клик загружает его дерево); встроенные и пользовательские пресеты параметров. +* **Консольный режим:** `python phymlgui.py --file data.fasta` / `--batch папка/` — тот же пайплайн без GUI. +* **Очистка и конвертация:** санитарная очистка ID, перевод `FASTA` → `PHYLIP-relaxed`. +* **Темы:** светлая/тёмная (могут следовать системной Windows) с выбором акцентного цвета. ### ⚙️ Требования * **Программные файлы:** `phyml.exe` (или MPI-версия) и `muscle.exe`. diff --git a/phyml_core/cli.py b/phyml_core/cli.py new file mode 100644 index 0000000..356a433 --- /dev/null +++ b/phyml_core/cli.py @@ -0,0 +1,57 @@ +"""Headless CLI: run the same pipeline without the GUI. + + python phymlgui.py --file input.fasta + python phymlgui.py --batch folder/ --config preset.json +""" +import argparse +import json +import os +import sys + +from PySide6.QtCore import QCoreApplication + + +def main(argv=None): + parser = argparse.ArgumentParser(prog='phymlgui', description='phymlGUI headless pipeline') + parser.add_argument('--file', help='single FASTA file') + parser.add_argument('--batch', help='folder with FASTA files') + parser.add_argument('--config', default='phyml_config.json', + help='config JSON (defaults to phyml_config.json)') + args = parser.parse_args(argv) + + if not args.file and not args.batch: + parser.error('specify --file or --batch') + + from phyml_core.pipeline import PipelineWorker + from phyml_core.i18n import _t + + config = {} + if os.path.exists(args.config): + with open(args.config, encoding='utf-8') as f: + config = json.load(f) + + if args.batch: + files = sorted(os.path.join(args.batch, f) for f in os.listdir(args.batch) + if f.lower().endswith(('.fasta', '.fas', '.fa')) + and not f.lower().endswith('_aligned.fasta')) + if not files: + print(_t("batch_empty")) + return 2 + else: + files = [args.file] + + app = QCoreApplication.instance() or QCoreApplication([]) + worker = PipelineWorker(config, files) + worker.log_signal.connect(print) + result = {} + worker.finished_signal.connect(lambda ok, tree: result.update(ok=ok, tree=tree)) + worker.run() # synchronous, no event loop needed + + print('OK' if result.get('ok') else 'FAILED') + if result.get('tree'): + print('tree:', result['tree']) + return 0 if result.get('ok') else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/phyml_core/i18n.py b/phyml_core/i18n.py index 53ff071..4e7fe3c 100644 --- a/phyml_core/i18n.py +++ b/phyml_core/i18n.py @@ -18,6 +18,10 @@ def detect_language(): def load_translations(): """ Load localization strings from external JSON file """ locales_path = resource_path('locales.json') + if not os.path.exists(locales_path): + # fallback: next to the package root (CLI runs from any cwd) + locales_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'locales.json') if not os.path.exists(locales_path): print(f"Warning: {locales_path} not found. Fallback to empty dictionary.") return {"en": {}, "ru": {}} diff --git a/phymlgui.py b/phymlgui.py index 55e3576..3e0b882 100644 --- a/phymlgui.py +++ b/phymlgui.py @@ -10,8 +10,16 @@ from phyml_ui.main_window import PhyMLApp, CONFIG_FILE # noqa: F401 from phyml_ui.tree_viewer import TreeViewerDialog # noqa: F401 -if __name__ == '__main__': + +def main(): + if len(sys.argv) > 1: + from phyml_core.cli import main as cli_main + sys.exit(cli_main()) app = QApplication(sys.argv) window = PhyMLApp() window.show() sys.exit(app.exec()) + + +if __name__ == '__main__': + main() diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..9043f04 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,18 @@ +# phymlGUI test suites + +Standalone scripts, runnable on any machine with the Python dependencies: + + python tests/test_smoke.py # no analysis binaries needed (CI-friendly) + python tests/run_tests.py # end-to-end pipeline, needs local binaries + python tests/test_gui_logic.py # GUI behaviour + live CPU monitoring + python tests/test_batch.py # batch mode + CSV summary + history + python tests/test_tree_viewer.py # layouts, collapse, outgroup, consensus, RF + python tests/test_iqtree.py # IQ-TREE engine (skips without the binary) + +Suites that need local analysis binaries (`phymlgui v1.1/phyml-mpi.exe`, +`tools/muscle.exe`, Microsoft MPI, optionally `tools/iqtree2/iqtree2.exe`) +print `SKIP` and exit 0 when the binaries are absent — e.g. on CI. + +Working files and generated datasets live in `testrun/` (gitignored); +`tests/fetch_real_data.py` downloads and caches real NCBI mitochondrial +CytB sequences used by the `t9` end-to-end scenario. diff --git a/tests/fetch_real_data.py b/tests/fetch_real_data.py new file mode 100644 index 0000000..2272d83 --- /dev/null +++ b/tests/fetch_real_data.py @@ -0,0 +1,88 @@ +"""Fetch real mammalian mitochondrial CytB sequences from NCBI. + +Downloads complete mitochondrial genomes via E-utilities (one request), +extracts the cytochrome-b CDS from each and writes a FASTA file used by +the t9 'real data' test scenario. The result is cached: if the output +already exists the network is not touched, so tests run offline too. + +Species set has a well-known phylogeny: + Hominidae: human, chimp, bonobo, gorilla, orangutan + Cercopithecidae: macaque (outgroup inside primates) + Rodentia: mouse, rat (outgroup to primates) +""" +import io +import os +import sys +import urllib.request + +from Bio import SeqIO + +RUN = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'testrun') +os.makedirs(RUN, exist_ok=True) +OUT_DIR = os.path.join(RUN, 'real') +OUT_FA = os.path.join(OUT_DIR, 'cytb_mammals.fasta') + +# stable RefSeq master records for complete mitochondrial genomes +ACCESSIONS = { + 'NC_012920': 'Homo_sapiens', + 'NC_001643': 'Pan_troglodytes', + 'NC_001644': 'Pan_paniscus', + 'NC_011120': 'Gorilla_gorilla', + 'NC_002083': 'Pongo_pygmaeus', + 'NC_005943': 'Macaca_mulatta', + 'NC_005089': 'Mus_musculus', + 'NC_001665': 'Rattus_norvegicus', +} + + +def fetch(): + ids = ','.join(ACCESSIONS) + url = (f'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi' + f'?db=nucleotide&id={ids}&rettype=gb&retmode=text') + req = urllib.request.Request(url, headers={'User-Agent': 'phymlGUI-testsuite/1.0'}) + with urllib.request.urlopen(req, timeout=60) as r: + return r.read().decode('utf-8', errors='replace') + + +def extract_cytb(genbank_text): + out = [] + for rec in SeqIO.parse(io.StringIO(genbank_text), 'genbank'): + name = ACCESSIONS.get(rec.id.split('.')[0], rec.id) + hit = None + for f in rec.features: + if f.type != 'CDS': + continue + gene = (f.qualifiers.get('gene') or [''])[0].lower() + product = (f.qualifiers.get('product') or [''])[0].lower() + if gene in ('cytb', 'cyb') or 'cytochrome b' in product: + hit = f + break + if hit is None: + print(f'WARN: no CytB CDS found in {rec.id} ({name})') + continue + seq = hit.extract(rec.seq) + seq = str(seq).rstrip('*') # drop terminal stop codon if annotated + out.append((name, seq)) + print(f'{rec.id:12s} {name:20s} CytB {len(seq)} bp') + return out + + +def main(): + if os.path.exists(OUT_FA): + n = sum(1 for _ in SeqIO.parse(OUT_FA, 'fasta')) + print(f'cached: {OUT_FA} ({n} records), skipping download') + return 0 + records = extract_cytb(fetch()) + if len(records) < len(ACCESSIONS): + print(f'ERROR: got {len(records)}/{len(ACCESSIONS)} records', file=sys.stderr) + return 1 + os.makedirs(OUT_DIR, exist_ok=True) + with open(OUT_FA, 'w') as f: + for name, seq in records: + f.write(f'>{name}\n{seq}\n') + print(f'written: {OUT_FA}') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/run_tests.py b/tests/run_tests.py new file mode 100644 index 0000000..73ac2d9 --- /dev/null +++ b/tests/run_tests.py @@ -0,0 +1,187 @@ +"""End-to-end tests for phymlGUI PipelineWorker using real MUSCLE + phyml-mpi. + +Runs the production pipeline code (phymlgui.PipelineWorker) on synthetic data +with a known topology: two clades A=(A1..A4), B=(B1..B4). After each run we +check the output files and that both clades come out monophyletic. + +Scenarios: + t1: unaligned input, no MPI (MUSCLE + phyml-mpi standalone) + t2: unaligned input, MPI -np2 (MUSCLE + mpiexec) + t3: aligned input, no MPI (no MUSCLE) + t4: aligned input, MPI -np2 +""" +import os, sys, random, shutil, re + +os.environ['QT_QPA_PLATFORM'] = 'offscreen' +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +import phymlgui +from Bio import Phylo, SeqIO +from Bio.Seq import Seq +from Bio.SeqRecord import SeqRecord + +RUN = os.path.join(ROOT, 'testrun') +os.makedirs(RUN, exist_ok=True) +PHYML_EXE = os.path.join(ROOT, 'phymlgui v1.1', 'phyml-mpi.exe') +MUSCLE_EXE = os.path.join(ROOT, 'tools', 'muscle.exe') +MPIEXEC_EXE = r'C:\Program Files\Microsoft MPI\Bin\mpiexec.exe' + +if not all(map(os.path.exists, (PHYML_EXE, MUSCLE_EXE, MPIEXEC_EXE))): + print('SKIP: local analysis binaries not found (phymlgui v1.1/phyml-mpi.exe, tools/muscle.exe, MS-MPI)') + sys.exit(0) + +# ---------- test data generation ---------- +def evolve(seq, rate, rng): + bases = 'ACGT' + return ''.join(b if rng.random() > rate else rng.choice(bases) for b in seq) + +def make_dataset(): + rng = random.Random(42) + ancestor = ''.join(rng.choice('ACGT') for _ in range(600)) + names, seqs_aligned, seqs_unaligned = [], [], [] + for clade, clade_rate in (('A', 0.16), ('B', 0.18)): + clade_consensus = evolve(ancestor, clade_rate, rng) + for i in range(1, 5): + s = evolve(clade_consensus, 0.03, rng) + name = f'{clade}{i}_testsp' + names.append(name) + seqs_aligned.append(s) + # unaligned variant: drop a random block from most sequences + if rng.random() < 0.8: + start = rng.randrange(50, 500) + ln = rng.randrange(8, 15) + s = s[:start] + s[start + ln:] + seqs_unaligned.append(s) + return names, seqs_aligned, seqs_unaligned + +def write_fasta(path, names, seqs): + with open(path, 'w') as f: + for n, s in zip(names, seqs): + f.write(f'>{n}\n{s}\n') + +# ---------- pipeline runner ---------- +def run_pipeline(folder, fasta_name, use_mpi, support='bootstrap', model='HKY85', search='NNI'): + wd = os.path.join(RUN, folder) + shutil.rmtree(wd, ignore_errors=True) + os.makedirs(wd, exist_ok=True) + shutil.copy(os.path.join(RUN, fasta_name), wd) + os.chdir(wd) + app = phymlgui.QApplication.instance() or phymlgui.QApplication([]) + + config = { + 'phyml_path': PHYML_EXE, 'muscle_path': MUSCLE_EXE, + 'mpirun_path': MPIEXEC_EXE, 'mpi_np': '2', 'use_mpi': use_mpi, + 'datatype': 'nt', 'model': model, + 'support_method': support, 'bootstrap_replicates': '20', + 'freq': 'default', 'tstv': 'e', 'pinv': '0.0', + 'classes': '4', 'gamma': 'e', 'search': search, + 'optimize': 'tlr', 'parsimony': False, + } + logs = [] + worker = phymlgui.PipelineWorker(config, os.path.join(wd, fasta_name)) + worker.log_signal.connect(logs.append) + worker.finished_signal.connect(lambda ok, tree: logs.append(f'__FINISHED__{ok}|{tree}')) + worker.run() # run synchronously in this thread + + os.chdir(ROOT) + return wd, logs + +def check(wd, logs, expect_alignment): + ok = any(l.startswith('__FINISHED__True') for l in logs) + assert ok, 'pipeline reported failure:\n' + '\n'.join(logs) + + aligned = [f for f in os.listdir(wd) if f.endswith('_aligned.fasta')] + assert bool(aligned) == expect_alignment, f'aligned file presence mismatch: {aligned}' + + tree_path = [f for f in os.listdir(wd) if f.endswith('_phyml_tree.txt')][0] + stats_path = [f for f in os.listdir(wd) if f.endswith('_phyml_stats.txt')][0] + tree = Phylo.read(os.path.join(wd, tree_path), 'newick') + terms = {t.name for t in tree.get_terminals()} + assert len(terms) == 8, f'expected 8 taxa, got {terms}' + + cladeA = {t for t in terms if t.startswith('A')} + cladeB = {t for t in terms if t.startswith('B')} + # Trees are UNROOTED with an arbitrary trifurcating root: the A|B split + # shows up as a node whose terminal set equals EITHER clade (whichever + # side faces away from the arbitrary root), never necessarily both. + def is_split_present(names): + return any( + not cl.is_terminal() and {t.name for t in cl.get_terminals()} == names + for cl in tree.find_clades()) + split_ok = is_split_present(cladeA) or is_split_present(cladeB) + assert split_ok, 'A|B split missing — tree topology wrong' + + stats = open(os.path.join(wd, stats_path)).read() + assert 'Log-likelihood' in stats, 'stats file lacks log-likelihood' + stats_lines = [l for l in logs if 'HKY85' in l and '|' in l] + assert stats_lines, 'stats summary line missing from worker log' + m = re.search(r'lnL[^\d-]*(-?[\d.]+)', stats_lines[0]) + lnl_file = re.search(r'Log-likelihood:\s*(-?[\d.]+)', stats).group(1) + assert m and abs(float(m.group(1)) - float(lnl_file)) < 0.01, 'lnL mismatch in stats summary' + return tree_path, split_ok + +if __name__ == '__main__': + names, sa, su = make_dataset() + write_fasta(os.path.join(RUN, 'test_prealigned.fasta'), names, sa) + write_fasta(os.path.join(RUN, 'test_unaligned.fasta'), names, su) + + # phyml-mpi cannot distribute a serial bootstrap (asserts n_proc > 1), + # so standalone runs use aLRT branch support; MPI runs use bootstrap. + scenarios = [ + ('t1_unaligned_serial_alrt', 'test_unaligned.fasta', False, True, '-4'), + ('t2_unaligned_mpi_boot', 'test_unaligned.fasta', True, True, 'bootstrap'), + ('t3_aligned_serial_alrt', 'test_prealigned.fasta', False, False, '-4'), + ('t4_aligned_mpi_boot', 'test_prealigned.fasta', True, False, 'bootstrap'), + ] + for folder, fasta, mpi, expect_aln, support in scenarios: + wd, logs = run_pipeline(folder, fasta, mpi, support) + tree, _ = check(wd, logs, expect_aln) + mpi_tag = 'MPI x2 + bootstrap' if mpi else 'serial + aLRT' + aln_tag = 'MUSCLE ran' if expect_aln else 'pre-aligned' + print(f'PASS {folder}: {mpi_tag}, {aln_tag}, tree OK ({tree})') + + # t5: serial bootstrap on the MPI build must fail gracefully (PhyML assert) + wd, logs = run_pipeline('t5_serial_boot_expected_fail', 'test_unaligned.fasta', False, 'bootstrap') + failed_ok = any(l.startswith('__FINISHED__False') for l in logs) + assert failed_ok, 'expected graceful failure for serial bootstrap on MPI build' + print('PASS t5_serial_boot_expected_fail: PhyML error surfaced in log, no crash') + + # t9: real NCBI data (mammalian mitochondrial CytB) with known phylogeny + sys.path.insert(0, RUN) + import fetch_real_data + if not os.path.exists(fetch_real_data.OUT_FA): + try: + assert fetch_real_data.main() == 0 + except Exception as e: + print(f'SKIP t9_real_ncbi: real data unavailable ({e})') + if os.path.exists(fetch_real_data.OUT_FA): + shutil.copy(fetch_real_data.OUT_FA, os.path.join(RUN, 'cytb_mammals.fasta')) + # cross-order mammalian mtDNA needs GTR+gamma and SPR search; + # HKY85+NNI scrambles the short hominoid internal branches + wd, logs = run_pipeline('t9_real_ncbi', 'cytb_mammals.fasta', True, + 'bootstrap', model='GTR', search='SPR') + ok = any(l.startswith('__FINISHED__True') for l in logs) + assert ok, 'real-data pipeline failed:\n' + '\n'.join(logs[-8:]) + tree_path = [f for f in os.listdir(wd) if f.endswith('_phyml_tree.txt')][0] + tree = Phylo.read(os.path.join(wd, tree_path), 'newick') + terms = {t.name for t in tree.get_terminals()} + rodents = {'Mus_musculus', 'Rattus_norvegicus'} + primates = {'Homo_sapiens', 'Pan_troglodytes', 'Pan_paniscus', + 'Gorilla_gorilla', 'Pongo_pygmaeus', 'Macaca_mulatta'} + assert terms == primates | rodents, f'unexpected taxa: {terms}' + def split(names): + return any(not c.is_terminal() and {t.name for t in c.get_terminals()} == names + for c in tree.find_clades()) + # rodents|primates is one split: either side may face the arbitrary root + assert split(rodents) or split(primates), 'rodents/primates split missing' + # Pan paniscus + Pan troglodytes is the strongly supported sister pair + # (~19/20). NB: Homo+Pan and the exact hominoid backbone are NOT + # recoverable from a single 1141 bp mitochondrial gene (saturated 3rd + # codon positions, long macaque branch) — a known single-gene CytB + # limitation, not a pipeline defect, so we do not assert them. + pan_pair = {'Pan_troglodytes', 'Pan_paniscus'} + assert split(pan_pair), 'Pan sister pair missing in CytB tree' + print('PASS t9_real_ncbi: real CytB tree recovers rodents/primates split and Pan pair') + + print('ALL PIPELINE TESTS PASSED') diff --git a/tests/test_batch.py b/tests/test_batch.py new file mode 100644 index 0000000..447285b --- /dev/null +++ b/tests/test_batch.py @@ -0,0 +1,112 @@ +"""Tests for the stretching CPU bar and batch mode.""" +import os, sys, time, shutil + +os.environ['QT_QPA_PLATFORM'] = 'offscreen' +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) +os.chdir(ROOT) + +import phymlgui +from PySide6.QtWidgets import QSizePolicy, QMessageBox + +app = phymlgui.QApplication([]) +if not all(os.path.exists(p) for p in ( + os.path.join(ROOT, 'phymlgui v1.1', 'phyml-mpi.exe'), + os.path.join(ROOT, 'tools', 'muscle.exe'))): + print('SKIP: local analysis binaries not found') + sys.exit(0) + +w = phymlgui.PhyMLApp() + +# 1. CPU bar expands to fill free status bar space +assert w.cpu_sys_bar.sizePolicy().horizontalPolicy() == QSizePolicy.Expanding +print('PASS status bar: cpu bar has Expanding policy') + +# 2. empty-folder guard +empty_dir = os.path.abspath(os.path.join('testrun', 't8_empty_dir')) +os.makedirs(empty_dir, exist_ok=True) +warned = [] +phymlgui.QMessageBox.warning = staticmethod(lambda *a, **k: warned.append(a) or QMessageBox.Ok) +w.fasta_path_input.setText(empty_dir) +w.start_pipeline() +assert warned, 'empty folder must trigger a warning' +assert w.worker is None or not w.worker.isRunning() +print('PASS batch guard: empty folder blocked with warning') + +# 3. batch of two files through the real worker (MPI + bootstrap) +run_dir = os.path.join('testrun', 't7_batch') +shutil.rmtree(run_dir, ignore_errors=True) +os.makedirs(run_dir, exist_ok=True) +shutil.copy('testrun/test_unaligned.fasta', os.path.join(run_dir, 'batchA.fasta')) +shutil.copy('testrun/test_prealigned.fasta', os.path.join(run_dir, 'batchB.fasta')) + +config = { + 'phyml_path': os.path.join(ROOT, 'phymlgui v1.1', 'phyml-mpi.exe'), + 'muscle_path': os.path.join(ROOT, 'tools', 'muscle.exe'), + 'mpirun_path': r'C:\Program Files\Microsoft MPI\Bin\mpiexec.exe', + 'mpi_np': '2', 'use_mpi': True, 'datatype': 'nt', 'model': 'HKY85', + 'support_method': 'bootstrap', 'bootstrap_replicates': '20', + 'freq': 'default', 'tstv': 'e', 'pinv': '0.0', 'classes': '4', + 'gamma': 'e', 'search': 'NNI', 'optimize': 'tlr', 'parsimony': False, +} +files = [os.path.join(run_dir, 'batchA.fasta'), os.path.join(run_dir, 'batchB.fasta')] +done, logs = [], [] +worker = phymlgui.PipelineWorker(config, files) +worker.log_signal.connect(logs.append) +worker.finished_signal.connect(lambda ok, tree: done.append((ok, tree))) +worker.start() +while not done: + app.processEvents() + time.sleep(0.3) + +assert done[0][0], 'batch failed:\n' + '\n'.join(logs[-10:]) +assert any('===' in l for l in logs), 'per-file headers missing' +assert any('2/2' in l for l in logs), 'batch summary missing' +assert os.path.exists(os.path.join(run_dir, 'batchA_aligned.fasta')), 'batchA not aligned' +assert os.path.exists(os.path.join(run_dir, 'batchA_aligned.phy_phyml_tree.txt')) +assert os.path.exists(os.path.join(run_dir, 'batchB.phy_phyml_tree.txt')) +from Bio import Phylo +assert len(Phylo.read(os.path.join(run_dir, 'batchB.phy_phyml_tree.txt'), 'newick').get_terminals()) == 8 +print('PASS batch: 2/2 files processed, trees for both, headers and summary logged') + +# 4. full GUI path: batch via start_pipeline writes phymlgui_summary.csv +cfg_bak = None +if os.path.exists(phymlgui.CONFIG_FILE): + import shutil as _sh + cfg_bak = phymlgui.CONFIG_FILE + '.bak' + _sh.copy(phymlgui.CONFIG_FILE, cfg_bak) +try: + w.le_bootstrap_replicates.setText('20') # user config may hold 10000+ + w.le_mpi_np.setText('2') + run_dir_abs = os.path.abspath(run_dir) + w.fasta_path_input.setText(run_dir_abs) + w.start_pipeline() + guard = 0 + while (not w.btn_run.isEnabled()) and guard < 400: + app.processEvents(); time.sleep(0.3); guard += 1 + assert w.btn_run.isEnabled(), 'pipeline did not finish through GUI path' + csv_path = os.path.join(run_dir, 'phymlgui_summary.csv') + assert os.path.exists(csv_path), 'summary CSV missing' + content = open(csv_path, encoding='utf-8-sig').read() + assert 'batchA.fasta' in content and 'batchB.fasta' in content and 'OK' in content and 'HKY85' in content + first_col = [l.split(';')[0] for l in content.splitlines()[1:] if l] + assert first_col == ['batchA.fasta', 'batchB.fasta'], f'queue leaked artifacts: {first_col}' + print('PASS csv summary: phymlgui_summary.csv written with both files and stats') + + # 5. stage-3: queue statuses, history record, open-folder enabled + assert w.queue_list.count() == 2 + statuses = [w.queue_list.item(i).text().split(' ')[-1] for i in range(2)] + assert statuses == ['✓', '✓'], statuses + import json as _json + from phyml_ui.main_window import HISTORY_FILE + records = _json.load(open(HISTORY_FILE, encoding='utf-8')) + assert records and records[0]['n'] == 2 and records[0]['ok'] == 2 + assert w.history_list.count() >= 1 + assert w.open_dir_btn.isEnabled() + assert not w.retry_btn.isVisible() or not w._failed_files + print('PASS stage3: queue statuses, history record, open-folder enabled') +finally: + if cfg_bak: + import shutil as _sh + _sh.move(cfg_bak, phymlgui.CONFIG_FILE) +print('ALL BATCH TESTS PASSED') diff --git a/tests/test_gui_logic.py b/tests/test_gui_logic.py new file mode 100644 index 0000000..5352241 --- /dev/null +++ b/tests/test_gui_logic.py @@ -0,0 +1,156 @@ +"""GUI logic tests for the new MPI/core/CPU-monitoring features.""" +import os, sys, json, time, shutil + +os.environ['QT_QPA_PLATFORM'] = 'offscreen' +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) +os.chdir(ROOT) + +import phymlgui +from PySide6.QtWidgets import QMessageBox + +# 1. locales.json: valid JSON, all new keys present in both languages +BIN_ROOT = ROOT +PHYML_OK = all(os.path.exists(p) for p in ( + os.path.join(BIN_ROOT, 'phymlgui v1.1', 'phyml-mpi.exe'), + os.path.join(BIN_ROOT, 'tools', 'muscle.exe'), + r'C:\Program Files\Microsoft MPI\Bin\mpiexec.exe')) + +loc = json.load(open('locales.json', encoding='utf-8')) +new_keys = ['cores_info', 'cpu_sys', 'cpu_analysis', 'cpu_analysis_idle', + 'cpu_analysis_done', 'mpi_detected', 'mpi_boot_title', 'mpi_boot_msg'] +for lang in ('en', 'ru'): + for k in new_keys: + assert k in loc[lang], f'{lang} missing {k}' +print('PASS locales: new keys present in en+ru') + +app = phymlgui.QApplication([]) + +# keep existing config out of the way to test defaults +had_config = os.path.exists(phymlgui.CONFIG_FILE) +if had_config: + shutil.move(phymlgui.CONFIG_FILE, phymlgui.CONFIG_FILE + '.bak') + +try: + w = phymlgui.PhyMLApp() + + # 2. np default == physical cores; status bar shows core info + assert w.le_mpi_np.text() == str(w.phys_cores), w.le_mpi_np.text() + assert w.phys_cores == 8 and w.logi_cores == 16, (w.phys_cores, w.logi_cores) + assert '8' in w.cores_label.text() and '16' in w.cores_label.text() + print(f'PASS defaults: np={w.le_mpi_np.text()}, cores label: {w.cores_label.text()!r}') + + # 3. MPI binary autodetection + w.chk_use_mpi.setChecked(False) + w.phyml_path_input.setText(r'D:\tools\phyml.exe') + assert not w.chk_use_mpi.isChecked(), 'plain phyml.exe must not enable MPI' + w.phyml_path_input.setText(r'D:\phyml\phymlgui v1.1\phyml-mpi.exe') + assert w.chk_use_mpi.isChecked(), 'phyml-mpi.exe must auto-enable MPI' + log_text = w.log_console.toPlainText() + assert 'MPI' in log_text, 'detection message missing from log' + print('PASS mpi autodetect: checkbox auto-enabled + logged') + + # 4. validation: MPI binary + MPI off + bootstrap -> blocked with warning + w.chk_use_mpi.setChecked(False) + w.set_combo_by_data(w.cb_support_method, 'bootstrap') + w.fasta_file = os.path.join('testrun', 'test_unaligned.fasta') + warned = [] + phymlgui.QMessageBox.warning = staticmethod(lambda *a, **k: warned.append(a) or QMessageBox.Ok) + w.start_pipeline() + assert warned, 'warning box must be shown' + assert w.btn_run.isEnabled(), 'Run must stay enabled after blocked start' + assert w.worker is None or not w.worker.isRunning() + print('PASS validation: serial bootstrap on MPI build blocked with warning') + + # 5. CPU monitor during a real MPI run + run_dir = os.path.join('testrun', 't6_monitor') + shutil.rmtree(run_dir, ignore_errors=True) + os.makedirs(run_dir, exist_ok=True) + shutil.copy(w.fasta_file, run_dir) + fasta = os.path.join(run_dir, 'test_unaligned.fasta') + + config = { + 'phyml_path': os.path.join(ROOT, 'phymlgui v1.1', 'phyml-mpi.exe'), + 'muscle_path': os.path.join(ROOT, 'tools', 'muscle.exe'), + 'mpirun_path': r'C:\Program Files\Microsoft MPI\Bin\mpiexec.exe', + 'mpi_np': '2', 'use_mpi': True, 'datatype': 'nt', 'model': 'HKY85', + 'support_method': 'bootstrap', 'bootstrap_replicates': '20', + 'freq': 'default', 'tstv': 'e', 'pinv': '0.0', 'classes': '4', + 'gamma': 'e', 'search': 'NNI', 'optimize': 'tlr', 'parsimony': False, + } + os.chdir(run_dir) + fasta = os.path.abspath('test_unaligned.fasta') + w._cpu_samples = [] + worker = phymlgui.PipelineWorker(config, fasta) + w.worker = worker + done, wlogs = [], [] + worker.log_signal.connect(wlogs.append) + worker.finished_signal.connect(lambda ok, tree: done.append((ok, tree))) + worker.start() + ticks = 0 + while not done and ticks < 300: + app.processEvents() + time.sleep(0.3) + w.update_cpu_status() + ticks += 1 + os.chdir(ROOT) + assert done, 'worker did not finish' + assert done[0][0], 'pipeline failed:\n' + '\n'.join(wlogs[-10:]) + assert w._cpu_samples, 'no CPU samples collected' + peak = max(w._cpu_samples) + assert peak > 50, f'analysis CPU unexpectedly low: peak {peak}' + print(f'PASS cpu monitor: {len(w._cpu_samples)} samples, peak {peak:.0f}% ' + f'(~{peak/100:.1f} cores), final label: {w.cpu_run_label.text()!r}') + + # 6. new UI: themes, results tabs, structured log, recent list, cpu timer + from phyml_ui.themes import resolve, build_qss + w.apply_theme('dark', 'blue') + theme, accent = resolve('dark', 'blue') + assert theme == 'dark' and accent == '#2D7DD2' + assert app.styleSheet() == build_qss(theme, accent) + assert w.log_console._role_colors['out'] == '#E6E8EB', 'phyml output must be themed' + assert w.log_console._role_colors['system'] == '#2D7DD2', 'system lines use accent' + assert w.cpu_timer.isActive(), 'CPU monitor timer must be running' + w.apply_theme('system', 'system') + assert w.result_tabs.count() == 4, 'expected 4 result tabs (log/stats/summary/queue)' + # presets: built-ins listed, apply changes support method, user preset saves + assert w.presets_list.count() >= 3 + texts = [w.presets_list.item(i).text() for i in range(w.presets_list.count())] + screen_row = next(i for i, t in enumerate(texts) if 'скрининг' in t.lower() or 'screening' in t.lower()) + w._preset_activated(w.presets_list.item(screen_row)) + assert w.cb_support_method.currentData() == '-4', 'screening preset must select aLRT' + w._user_presets['my-test'] = {k: w.get_current_config()[k] for k in ('model',)} + w._refresh_presets() + assert any(w.presets_list.item(i).text() == 'my-test' for i in range(w.presets_list.count())) + w.tree_view.btn_zoom.click() + assert w.tree_view.btn_zoom.isChecked() and w.tree_view.mode_label.text() == phymlgui._t('tree_zoom') + w.tree_view.btn_pan.click() + assert w.tree_view.btn_pan.isChecked() and not w.tree_view.btn_zoom.isChecked() + w.tree_view.btn_pan.click() + assert not w.tree_view.btn_pan.isChecked() + from phyml_ui.main_window import SettingsDialog + d = SettingsDialog(w, w) + assert d.edits[0].text() == w.phyml_path_input.text() + d.edits[0].setText('D:/some/phyml-mpi.exe') + d.accept() + assert w.phyml_path_input.text() == 'D:/some/phyml-mpi.exe' + assert w.chk_use_mpi.isChecked(), 'settings edit must re-trigger MPI autodetect' + w.log_console.filter_box.setCurrentIndex(0) + w.log_console.clear() + w.log_console.append('[ERROR] test failure line') + w.log_console.append('plain phyml output line') + assert '[ERROR] test failure line' in w.log_console.toPlainText() + assert 'plain phyml output line' in w.log_console.toPlainText() + w.set_combo_by_data(w.log_console.filter_box, 'error') + assert '[ERROR]' in w.log_console.view.toPlainText() + assert 'plain phyml output line' not in w.log_console.view.toPlainText() + w.set_combo_by_data(w.log_console.filter_box, 'all') + assert 'plain phyml output line' in w.log_console.view.toPlainText() + w._remember_recent('X:/a.fasta'); w._remember_recent('X:/b.fasta') + w._remember_recent('X:/a.fasta') + assert w.recent_list.count() == 2 and w._recent[0] == 'X:/a.fasta' + print('PASS new UI: theme toggle, 4 result tabs, log filter, recent list') + print('ALL GUI LOGIC TESTS PASSED') +finally: + if had_config: + shutil.move(phymlgui.CONFIG_FILE + '.bak', phymlgui.CONFIG_FILE) diff --git a/tests/test_iqtree.py b/tests/test_iqtree.py new file mode 100644 index 0000000..4ec2b94 --- /dev/null +++ b/tests/test_iqtree.py @@ -0,0 +1,88 @@ +"""Tests for the IQ-TREE engine, ModelFinder and the XLSX summary.""" +import os, sys, time, shutil, re, json + +os.environ['QT_QPA_PLATFORM'] = 'offscreen' +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) +os.chdir(ROOT) + +import phymlgui +from PySide6.QtWidgets import QMessageBox + +IQTREE_EXE = os.path.join(ROOT, 'tools', 'iqtree2', 'iqtree2.exe') +if not os.path.exists(IQTREE_EXE): + print('SKIP: IQ-TREE binary not available') + sys.exit(0) + +app = phymlgui.QApplication([]) + +# 1. worker-level: MFP + ultrafast bootstrap on pre-aligned synthetic data +run_dir = os.path.join('testrun', 't10_iqtree') +shutil.rmtree(run_dir, ignore_errors=True) +os.makedirs(run_dir, exist_ok=True) +shutil.copy('testrun/test_prealigned.fasta', run_dir) +os.chdir(run_dir) + +config = { + 'engine': 'iqtree', 'iqtree_path': IQTREE_EXE, + 'phyml_path': os.path.join(ROOT, 'phymlgui v1.1', 'phyml-mpi.exe'), + 'muscle_path': os.path.join(ROOT, 'tools', 'muscle.exe'), + 'use_mpi': False, 'mpi_np': '2', 'datatype': 'nt', 'model': 'MFP', + 'support_method': 'bootstrap', 'bootstrap_replicates': '1000', + 'freq': 'default', 'tstv': 'e', 'pinv': '0.0', + 'classes': '4', 'gamma': 'e', 'search': 'NNI', 'optimize': 'tlr', 'parsimony': False, +} +logs, done = [], [] +worker = phymlgui.PipelineWorker(config, os.path.abspath('test_prealigned.fasta')) +worker.log_signal.connect(logs.append) +worker.finished_signal.connect(lambda ok, tree: done.append((ok, tree))) +worker.run() +os.chdir(ROOT) + +assert done[0][0], 'iqtree pipeline failed:\n' + '\n'.join(logs[-8:]) +tree_rel = os.path.relpath(done[0][1], ROOT) +assert os.path.exists(done[0][1]), done[0][1] +from Bio import Phylo +tree = Phylo.read(done[0][1], 'newick') +assert len(tree.get_terminals()) == 8 +row = worker.results[0] +assert row['model'] and row['model'] != 'n/a', row +iq_report = open(os.path.join(run_dir, 'test_prealigned_iqtree.iqtree'), encoding='utf-8').read() +lnl_file = re.search(r'Log-likelihood of the tree:\s*(-?[\d.]+)', iq_report).group(1) +assert row['lnL'] == lnl_file, (row['lnL'], lnl_file) +print(f'PASS iqtree engine: MFP picked {row["model"]}, lnL {row["lnL"]}, 8 taxa') + +# 2. GUI: engine validation, model list, full run + xlsx summary +cfg_bak = phymlgui.CONFIG_FILE + '.bak' +had_cfg = os.path.exists(phymlgui.CONFIG_FILE) +if had_cfg: + shutil.copy(phymlgui.CONFIG_FILE, cfg_bak) +try: + w = phymlgui.PhyMLApp() + warned = [] + phymlgui.QMessageBox.warning = staticmethod(lambda *a, **k: warned.append(a) or QMessageBox.Ok) + w.engine_setting = 'iqtree' + w.update_models() + assert w.cb_model.currentData() == 'MFP' + w.iqtree_path_input.setText('') # no binary -> blocked with warning + w.fasta_path_input.setText(os.path.abspath(os.path.join(run_dir, 'test_prealigned.fasta'))) + w.start_pipeline() + assert warned and (w.worker is None or not w.worker.isRunning()) + + w.iqtree_path_input.setText(IQTREE_EXE) + w.le_bootstrap_replicates.setText('100') + w.start_pipeline() + guard = 0 + while not w.btn_run.isEnabled() and guard < 400: + app.processEvents(); time.sleep(0.3); guard += 1 + assert w.btn_run.isEnabled(), 'GUI iqtree run did not finish' + xlsx = os.path.join(run_dir, 'phymlgui_summary.xlsx') + assert os.path.exists(xlsx), 'xlsx summary missing' + from openpyxl import load_workbook + wb = load_workbook(xlsx) + assert wb.active.max_row == 2 + print('PASS iqtree GUI: validation, model list, run + xlsx summary') +finally: + if had_cfg: + shutil.move(cfg_bak, phymlgui.CONFIG_FILE) +print('ALL IQ-TREE TESTS PASSED') diff --git a/tests/test_smoke.py b/tests/test_smoke.py new file mode 100644 index 0000000..388d3f5 --- /dev/null +++ b/tests/test_smoke.py @@ -0,0 +1,86 @@ +"""Smoke test: runs everywhere (CI included), needs no analysis binaries.""" +import json +import os +import sys + +os.environ['QT_QPA_PLATFORM'] = 'offscreen' +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +# 1. module imports and i18n integrity +import phymlgui +from phyml_core.pipeline import parse_stats, parse_iqtree_stats +from phyml_ui.tree_viewer import rf_distance, tree_splits +from phyml_ui.themes import resolve, build_qss, role_colors, ACCENTS, system_theme +from phyml_ui.main_window import LogConsole, CONFIG_FILE + +loc = json.load(open(os.path.join(ROOT, 'locales.json'), encoding='utf-8')) +assert set(loc) >= {'en', 'ru'} +assert len(loc['en']) == len(loc['ru']), 'locale key sets diverge' +print(f'PASS imports and i18n: {len(loc["en"])} keys in both languages') + +# 2. theme resolution and dynamic QSS +theme, accent = resolve('dark', 'blue') +assert theme == 'dark' and accent == ACCENTS['blue'] +qss = build_qss(theme, accent) +assert accent in qss and 'QPushButton#btn_run' in qss +assert role_colors(theme, accent)['out'] == '#E6E8EB' +assert system_theme() in ('light', 'dark') +print('PASS themes: resolve, build_qss, role colors') + +# 3. stats parsers on synthetic content (no binaries needed) +import io +import tempfile +from Bio import Phylo + +tmp = tempfile.mkdtemp() +phy_stats = os.path.join(tmp, 'x_phyml_stats.txt') +open(phy_stats, 'w', encoding='utf-8').write( + '. Model of nucleotides substitution: \tHKY85\n' + '. Number of taxa: \t\t8\n' + '. Log-likelihood: \t\t-1811.15467\n' + ' - Gamma shape parameter: \t881.743\n' + '. transition/transversion ratio: \t0.980317\n' + '. Time used:\t\t\t0h0m0s (0 seconds)\n') +s = parse_stats(phy_stats) +assert s['model'] == 'HKY85' and s['taxa'] == '8' and s['lnL'] == '-1811.15467' +assert s['gamma'] == '881.743' and s['tstv'] == '0.980317' + +iq = os.path.join(tmp, 'x.iqtree') +open(iq, 'w', encoding='utf-8').write( + 'Input data: 8 sequences with 600 nucleotide sites\n' + 'Best-fit model according to BIC: JC\n' + 'Log-likelihood of the tree: -1828.7117 (s.e. 50.96)\n' + 'Total wall-clock time used: 0.7434 seconds (0h:0m:0s)\n') +s2 = parse_iqtree_stats(iq) +assert s2['model'] == 'JC' and s2['taxa'] == '8' and s2['lnL'] == '-1828.7117' +print('PASS stats parsers: phyml and iqtree formats') + +# 4. RF distance / splits on newick strings +t1 = Phylo.read(io.StringIO('(A,(B,(C,D)));'), 'newick') +t2 = Phylo.read(io.StringIO('(A,(C,(B,D)));'), 'newick') +assert rf_distance(t1, t1) == (0, 0.0) +assert rf_distance(t1, t2) == (1, 1.0) +assert len(tree_splits(t1)) == 1 +print('PASS rf distance and splits') + +# 5. log classification +app = phymlgui.QApplication([]) +log = LogConsole() +assert log._classify('[ERROR] bad') == 'error' +assert log._classify('[+] done') == 'success' +assert log._classify('[*] system line') == 'system' +assert log._classify('. PhyML raw output') == 'out' +log.append('[ERROR] bad') +log.append('raw') +assert '[ERROR] bad' in log.toPlainText() and 'raw' in log.toPlainText() +print('PASS log console roles') + +# 6. main window builds offscreen +w = phymlgui.PhyMLApp() +assert w.result_tabs.count() == 4 +assert w.presets_list.count() >= 3 +assert w.cpu_timer.isActive() +print('PASS main window builds offscreen') + +print('ALL SMOKE TESTS PASSED') diff --git a/tests/test_tree_viewer.py b/tests/test_tree_viewer.py new file mode 100644 index 0000000..e317e3b --- /dev/null +++ b/tests/test_tree_viewer.py @@ -0,0 +1,96 @@ +"""Tests for the stage-2 tree viewer: layouts, collapse, outgroup, consensus, RF.""" +import io +import os +import sys + +os.environ['QT_QPA_PLATFORM'] = 'offscreen' +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) +os.chdir(ROOT) + +import phymlgui +from phyml_ui.tree_viewer import TreeViewerWidget, rf_distance +from Bio import Phylo + +# 1. RF distance: identical -> 0, swapped quartet -> max +t1 = Phylo.read(io.StringIO('(A,(B,(C,D)));'), 'newick') +t2 = Phylo.read(io.StringIO('(A,(C,(B,D)));'), 'newick') +assert rf_distance(t1, t1) == (0, 0.0) +raw, norm = rf_distance(t1, t2) +assert raw == 1 and norm == 1.0, (raw, norm) +print('PASS rf: identical=0, quartet swap=1 (100%)') + +app = phymlgui.QApplication([]) +if not os.path.isdir('testrun') or not any( + os.path.exists(os.path.join('testrun', d, f)) + for d in os.listdir('testrun') if os.path.isdir(os.path.join('testrun', d)) + for f in os.listdir(os.path.join('testrun', d)) if f.endswith('_phyml_tree.txt')): + print('SKIP: no rendered trees available; run tests/run_tests.py first') + sys.exit(0) + +w = TreeViewerWidget() + +# real tree with supports from the e2e runs +tree_file = None +for d in ('t9_real_ncbi', 't2_unaligned_mpi_boot'): + p = os.path.join('testrun', d) + if os.path.isdir(p): + for f in os.listdir(p): + if f.endswith('_phyml_tree.txt'): + tree_file = os.path.join(p, f) + break + if tree_file: + break +assert tree_file, 'no e2e tree available; run run_tests.py first' + +w.load_tree(tree_file) +assert len(w._leaf_order) == 8 + +# 2. all three layouts render without errors +for layout in ('rect', 'slanted', 'circle'): + w.set_layout(layout) + assert w.layout == layout +print('PASS layouts: rect / slanted / circle render') + +# 3. support threshold + grid toggles do not break rendering +w.threshold_slider.setValue(70) +assert w.threshold == 70 +w.chk_grid.setChecked(False) +w.chk_supports.setChecked(False) +w.chk_supports.setChecked(True) +print('PASS options: threshold, grid, supports') + +# 4. collapse: first real internal clade folds into one display leaf +src = Phylo.read(tree_file, 'newick') +candidates = [cl for cl in src.get_nonterminals() + if 2 <= len(cl.get_terminals()) < len(src.get_terminals())] +assert candidates, 'no internal clades to collapse' +target = frozenset(t.name for t in candidates[0].get_terminals()) +w.set_layout('rect') +before = len(w._leaf_order) +w.collapse_clade(target) +assert len(w._leaf_order) == before - len(target) + 1, (before, len(w._leaf_order)) +w.collapse_clade(target) +assert len(w._leaf_order) == before +print(f'PASS collapse: clade of {len(target)} taxa folds to one leaf and back') + +# 5. outgroup reroot renders +w.outgroup_combo.setCurrentIndex(3) # some taxon +assert w.outgroup is not None +w.outgroup_combo.setCurrentIndex(0) +print('PASS outgroup: reroot and reset') + +# 6. consensus from a bootstrap trees file +boot = tree_file.replace('_phyml_tree.txt', '_phyml_boot_trees.txt') +if os.path.exists(boot): + trees = list(Phylo.parse(boot, 'newick')) + w.load_tree_object(Phylo, 'unused') if False else None + from Bio.Phylo.Consensus import majority_consensus + cons = majority_consensus(trees) + w.load_tree_object(cons, 'consensus-test') + assert len(w._leaf_order) == 8 + print(f'PASS consensus: majority over {len(trees)} replicate trees') +else: + print('SKIP consensus: no *_boot_trees.txt next to the tree') + +print('ALL TREE VIEWER TESTS PASSED') From a631dd7e4201151e0a79a05e9cf4abd6d8f7e18f Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sat, 24 Oct 2026 17:47:39 +0300 Subject: [PATCH 06/36] Fix crash on Run click and margin from the left splitter in the tree pane - QPushButton.clicked(checked) leaked the bool into start_pipeline's files_override parameter after stage 3 (list(False) TypeError); connect via a no-arg lambda and guard bool defensively; the batch GUI test now drives the real button click so this path stays covered - the tree pane gets a left margin so the viewer tool rows no longer stick to the splitter handle --- phyml_ui/main_window.py | 6 ++++-- tests/test_batch.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/phyml_ui/main_window.py b/phyml_ui/main_window.py index f248318..cd5f5d7 100644 --- a/phyml_ui/main_window.py +++ b/phyml_ui/main_window.py @@ -386,7 +386,7 @@ def _build_middle_pane(self): control_row = QHBoxLayout() self.btn_run = QPushButton(_t("btn_run")) self.btn_run.setObjectName('btn_run') - self.btn_run.clicked.connect(self.start_pipeline) + self.btn_run.clicked.connect(lambda: self.start_pipeline()) # clicked(bool) must not leak into files_override self.btn_cancel = QPushButton(_t("btn_cancel")) self.btn_cancel.setObjectName('btn_cancel') self.btn_cancel.setEnabled(False) @@ -404,7 +404,7 @@ def _build_right_pane(self): """Big tree viewer on top, Log/Stats/Summary tabs underneath.""" right = QWidget() v = QVBoxLayout(right) - v.setContentsMargins(0, 0, 0, 0) + v.setContentsMargins(8, 0, 0, 0) # breathing room from the left splitter self.tree_view = TreeViewerWidget() self.result_tabs = QTabWidget() @@ -704,6 +704,8 @@ def set_combo_by_data(self, combo, data_val): if idx >= 0: combo.setCurrentIndex(idx) def start_pipeline(self, files_override=None): + if isinstance(files_override, bool): # defensive: Qt clicked(checked) + files_override = None if files_override is None: target = self.fasta_path_input.text() or self.fasta_file if not target or not os.path.exists(target): diff --git a/tests/test_batch.py b/tests/test_batch.py index 447285b..df7157e 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -80,7 +80,7 @@ w.le_mpi_np.setText('2') run_dir_abs = os.path.abspath(run_dir) w.fasta_path_input.setText(run_dir_abs) - w.start_pipeline() + w.btn_run.click() # real signal path: clicked(bool) must not break start_pipeline guard = 0 while (not w.btn_run.isEnabled()) and guard < 400: app.processEvents(); time.sleep(0.3); guard += 1 From bb7d1bb97c8e55b7eea4afd68e087c04040e70f5 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:29:47 +0300 Subject: [PATCH 07/36] Rework tree rendering: real distance grid, non-overlapping supports, theme-aware canvas - distance gridlines are now drawn manually at nice 1-2-5 intervals with numeric labels and an axis caption (ax.grid is hidden by axis_off, which is why the grid never appeared); circular layout gets rings - support values moved to branch midpoints with rounded background badges and collision nudging that also keeps clear of taxon labels - all paddings are proportional to tree size (the old absolute +0.5 produced the huge empty right half on small-distance trees); leaf labels got a wider gap from branch tips - grid/support checkboxes actually update the render state now (they were connected straight to _rerender without refreshing the fields) - branches, labels, badges and the figure background follow the app theme via TreeViewerWidget.set_colors, wired from apply_theme; exports use the themed background instead of hardcoded white --- locales.json | 2 + phyml_ui/main_window.py | 4 +- phyml_ui/tree_viewer.py | 167 ++++++++++++++++++++++++-------------- tests/test_tree_viewer.py | 16 ++++ 4 files changed, 128 insertions(+), 61 deletions(-) diff --git a/locales.json b/locales.json index 9a31bcd..f34bb33 100644 --- a/locales.json +++ b/locales.json @@ -133,6 +133,7 @@ "tree_supports": "Supports", "tree_threshold": "threshold: {}%", "tree_grid": "Grid", + "tree_axis_label": "substitutions per site", "tree_outgroup": "Outgroup:", "tree_outgroup_none": "(none)", "btn_consensus": "Consensus…", @@ -318,6 +319,7 @@ "tree_supports": "Поддержки", "tree_threshold": "порог: {}%", "tree_grid": "Сетка", + "tree_axis_label": "замены на сайт", "tree_outgroup": "Outgroup:", "tree_outgroup_none": "(нет)", "btn_consensus": "Консенсус…", diff --git a/phyml_ui/main_window.py b/phyml_ui/main_window.py index cd5f5d7..092fd65 100644 --- a/phyml_ui/main_window.py +++ b/phyml_ui/main_window.py @@ -22,7 +22,7 @@ from phyml_core.utils import psutil, detect_cores, is_mpi_binary, resource_path from phyml_core.pipeline import PipelineWorker from phyml_ui.tree_viewer import TreeViewerWidget -from phyml_ui.themes import resolve, build_qss, role_colors, ACCENTS +from phyml_ui.themes import resolve, build_qss, role_colors, ACCENTS, PALETTES CONFIG_FILE = 'phyml_config.json' HISTORY_FILE = 'phyml_history.json' @@ -513,6 +513,8 @@ def apply_theme(self, theme_setting=None, accent_setting=None): theme, accent = resolve(self._theme_setting, self._accent_setting) QApplication.instance().setStyleSheet(build_qss(theme, accent)) self.log_console.set_role_colors(role_colors(theme, accent)) + pal = PALETTES[theme] + self.tree_view.set_colors(pal['text'], accent, pal['base']) def _refresh_recent(self): self.recent_list.clear() diff --git a/phyml_ui/tree_viewer.py b/phyml_ui/tree_viewer.py index 933f8b3..be43747 100644 --- a/phyml_ui/tree_viewer.py +++ b/phyml_ui/tree_viewer.py @@ -60,7 +60,7 @@ class TreeViewerWidget(QWidget): def __init__(self, parent=None): super().__init__(parent) self.figure = Figure() - self.figure.patch.set_alpha(0.0) + self.figure.patch.set_facecolor('#FFFFFF') self.canvas = FigureCanvas(self.figure) self.nav = NavigationToolbar2QT(self.canvas, self) # hidden: drives zoom/pan self.nav.setVisible(False) @@ -76,6 +76,9 @@ def __init__(self, parent=None): self._collapsed = set() # frozensets of terminal names self._coords = None # (tree, coords) of the last render self._leaf_order = [] + self._text = '#424242' # branch / label color, set per theme + self._accent = '#1E9E6A' + self._bg = '#FFFFFF' layout = QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) @@ -110,7 +113,8 @@ def __init__(self, parent=None): lambda _: self.set_layout(self.layout_combo.currentData())) self.chk_supports = QCheckBox(_t("tree_supports")) self.chk_supports.setChecked(True) - self.chk_supports.toggled.connect(self._rerender) + self.chk_supports.toggled.connect( + lambda on: (setattr(self, 'show_supports', on), self._rerender())) self.threshold_slider = QSlider(Qt.Horizontal) self.threshold_slider.setRange(0, 100) self.threshold_slider.setValue(0) @@ -119,7 +123,8 @@ def __init__(self, parent=None): self.threshold_label = QLabel(_t("tree_threshold").format(0)) self.chk_grid = QCheckBox(_t("tree_grid")) self.chk_grid.setChecked(True) - self.chk_grid.toggled.connect(self._rerender) + self.chk_grid.toggled.connect( + lambda on: (setattr(self, 'show_grid', on), self._rerender())) self.outgroup_combo = QComboBox() self.outgroup_combo.setMinimumWidth(110) self.outgroup_combo.currentIndexChanged.connect(self._outgroup_changed) @@ -174,10 +179,16 @@ def _export(self): path += '.' + ext try: self.figure.savefig(path, dpi=200, bbox_inches='tight', - facecolor='white') + facecolor=self._bg) except Exception as e: QMessageBox.warning(self, _t("tree_export"), str(e)) + def set_colors(self, text, accent, bg): + """Theme-aware rendering colors; re-renders the current tree.""" + self._text, self._accent, self._bg = text, accent, bg + self.figure.patch.set_facecolor(bg) + self._rerender() + # ---------- options handlers ---------- def set_layout(self, name): @@ -361,15 +372,33 @@ def _rerender(self): except Exception as e: self._draw_error(e) + @staticmethod + def _nice_ticks(vmax): + """1-2-5 progression ticks covering (0, vmax] with ~5 steps.""" + if vmax <= 0: + return [] + raw = vmax / 5.0 + mag = 10 ** math.floor(math.log10(raw)) if raw > 0 else 1 + step = 10 * mag + for m in (1, 2, 5): + if m * mag >= raw: + step = m * mag + break + ticks, t = [], step + while t < vmax * 1.001: + ticks.append(t) + t += step + return ticks + def _render(self): self.ax.clear() self._coords = None tree = self._prep_tree() coords, max_x, display_leaves = self._compute_layout(tree) self._coords = (tree, coords) - accent = '#1E9E6A' - text = '#424242' - faint = '#B9BEC4' + text, accent, bg = self._text, self._accent, self._bg + faint = '#9AA0A6' + grid_c = '#AEB4BB' def support_of(cl): if cl.confidence is not None: @@ -379,13 +408,16 @@ def support_of(cl): except (TypeError, ValueError): return None + def fmt_support(s): + return f"{s:.0f}" if s >= 1 else f"{s:.2f}" + def edge_color(cl): s = support_of(cl) if self.threshold > 0 and s is not None: return accent if s >= self.threshold else faint return text - segments, colors = [], [] + segments, colors, support_pos = [], [], [] circ = self.layout == 'circle' def draw_edges(cl, parent_pos): @@ -394,6 +426,9 @@ def draw_edges(cl, parent_pos): cx, cy = x * math.cos(a), x * math.sin(a) segments.append([parent_pos, (cx, cy)]) colors.append(edge_color(cl)) + if not cl.is_terminal(): + mx, my = (parent_pos[0] + cx) / 2, (parent_pos[1] + cy) / 2 + support_pos.append((mx, my, cl)) for c in cl.clades: draw_edges(c, (cx, cy)) else: @@ -402,9 +437,14 @@ def draw_edges(cl, parent_pos): if self.layout == 'rect': segments.append([(px, py), (px, y)]) # elbow at parent's x segments.append([(px, y), (x, y)]) + colors.append(edge_color(cl)); colors.append(edge_color(cl)) + if not cl.is_terminal(): + support_pos.append(((px + x) / 2, y, cl)) else: segments.append([(px, py), (x, y)]) - colors.append(edge_color(cl)) + colors.append(edge_color(cl)) + if not cl.is_terminal(): + support_pos.append(((px + x) / 2, (py + y) / 2, cl)) for c in cl.clades: draw_edges(c, (x, y)) @@ -425,67 +465,80 @@ def draw_edges(cl, parent_pos): for c in root.clades: draw_edges(c, (0.0, ra)) - lc = LineCollection(segments, colors=colors, linewidths=1.4, zorder=2) - self.ax.add_collection(lc) - - # leaf labels + # ---- distance grid (drawn manually: set_axis_off hides ax.grid) ---- + ticks = self._nice_ticks(max_x) + ys = [p[1] for p in coords.values()] + if circ: + lim = max_x * 1.18 + y_min, y_max = -lim, lim + else: + y_min, y_max = min(ys), max(ys) + if self.show_grid: + import matplotlib.patches as mpatches + for t in ticks: + if circ: + self.ax.add_patch(mpatches.Circle( + (0, 0), t, fill=False, linestyle=':', linewidth=0.7, + color=grid_c, alpha=0.85, zorder=0)) + self.ax.text(t, max_x * 0.03, f"{t:g}", fontsize=6.5, + color=faint, ha='left', va='bottom', zorder=1) + else: + self.ax.plot([t, t], [y_min, y_max + 0.6], linestyle=':', + linewidth=0.7, color=grid_c, alpha=0.9, zorder=0) + self.ax.text(t, y_min - 0.35, f"{t:g}", fontsize=7, + color=faint, ha='center', va='top', zorder=1) + if not circ and ticks: + self.ax.text(max_x / 2, y_min - 0.9, _t("tree_axis_label"), + fontsize=7, color=faint, ha='center', va='top') + + self.ax.add_collection(LineCollection(segments, colors=colors, + linewidths=1.5, zorder=2)) + + # ---- leaf labels ---- + leaf_anchors = [] for cl in display_leaves: x, a = coords[id(cl)] if circ: - lx, ly = (x + max_x * 0.02) * math.cos(a), (x + max_x * 0.02) * math.sin(a) + lx, ly = (x + max_x * 0.025) * math.cos(a), (x + max_x * 0.025) * math.sin(a) ha = 'left' if math.cos(a) >= 0 else 'right' self.ax.text(lx, ly, cl.name or '', fontsize=8, color=text, - ha=ha, va='center', zorder=3) + ha=ha, va='center', zorder=4) else: - self.ax.text(x + max_x * 0.01, a, cl.name or '', fontsize=8, - color=text, va='center', zorder=3) + lx = x + max_x * 0.012 + self.ax.text(lx, a, cl.name or '', fontsize=8.5, + color=text, va='center', zorder=4) + leaf_anchors.append((lx, a)) - # support labels on internal nodes + # ---- support labels at branch midpoints, nudged apart ---- if self.show_supports: - for cl in tree.get_nonterminals(): - if id(cl) not in coords: - continue + placed = list(leaf_anchors) # never overlap a taxon label either + for x, y, cl in sorted(support_pos, key=lambda p: -p[1]): s = support_of(cl) if s is None: continue - x, a = coords[id(cl)] - if circ: - lx, ly = x * math.cos(a), x * math.sin(a) - else: - lx, ly = x, a - color = accent if (self.threshold > 0 and s >= self.threshold) else text - self.ax.text(lx, ly, f"{s:.0f}" if s > 1 else f"{s:.2f}", - fontsize=7, color=color, style='italic', - ha='center', va='bottom', zorder=3) - - # scale bar (rect/slanted) or subtle rings (circular) - if circ: - step = 10 ** (len(str(int(max_x))) - 1) if max_x >= 1 else 0.01 - ring = step - while ring < max_x * 1.05: - self.ax.add_patch(_ring(ring)) - ring += step - else: - bar = max_x * 0.15 if max_x > 0 else 0.0 - ys = [p[1] for p in coords.values()] - y0 = min(ys) - (max(ys) - min(ys)) * 0.06 - 1 - self.ax.plot([0, bar], [y0, y0], color=text, lw=1.2, zorder=3) - self.ax.text(bar / 2, y0 - 0.4, f"{bar:.3g}", fontsize=7, - color=text, ha='center', zorder=3) - - if self.show_grid and not circ: - self.ax.grid(True, linestyle=':', linewidth=0.5, alpha=0.3, zorder=0) + yy = y + for px_, py_ in placed: + if abs(px_ - x) < max_x * 0.25 and abs(py_ - yy) < 0.5: + yy = py_ + 0.5 + placed.append((x, yy)) + strong = self.threshold > 0 and s >= self.threshold + self.ax.text(x, yy + 0.14, fmt_support(s), fontsize=7, + color=accent if strong else text, style='italic', + ha='center', va='bottom', zorder=5, + bbox=dict(boxstyle='round,pad=0.12', fc=bg, + ec='none', alpha=0.85)) + + # ---- limits with proportional padding ---- self.ax.set_axis_off() if circ: - lim = max_x * 1.3 self.ax.set_xlim(-lim, lim); self.ax.set_ylim(-lim, lim) self.ax.set_aspect('equal') else: - xs = [p[0] for p in coords.values()] - ys = [p[1] for p in coords.values()] - self.ax.set_xlim(min(xs) - 0.05, max(xs) * 1.25 + 0.1) - self.ax.set_ylim(min(ys) - 1.5, max(ys) + 0.8) - self.ax.set_title(self.tree_title, fontsize=10, pad=8) + pad_x = max_x * 0.45 if display_leaves else max_x * 0.05 + self.ax.set_xlim(-max_x * 0.02, max_x + pad_x) + self.ax.set_ylim(y_min - 1.2, y_max + 0.9) + self.ax.set_title(self.tree_title, fontsize=10, pad=10, color=text) + self.figure.tight_layout() self.canvas.draw_idle() def _clade_at(self, x, y): @@ -539,12 +592,6 @@ def _draw_error(self, e): self.canvas.draw_idle() -def _ring(radius): - import matplotlib.patches as mpatches - return mpatches.Circle((0, 0), radius, fill=False, - linestyle=':', linewidth=0.5, color='#B9BEC4') - - # --- Kept for backward compatibility: modal dialog wrapping the widget --- class TreeViewerDialog(QDialog): def __init__(self, tree_path, parent=None): diff --git a/tests/test_tree_viewer.py b/tests/test_tree_viewer.py index e317e3b..215fa6c 100644 --- a/tests/test_tree_viewer.py +++ b/tests/test_tree_viewer.py @@ -52,6 +52,22 @@ assert w.layout == layout print('PASS layouts: rect / slanted / circle render') +# 2b. manual distance grid is actually drawn (ax.grid dies with axis_off) +w.set_layout('rect') +from matplotlib.lines import Line2D +grid_on = sum(1 for l in w.ax.lines if isinstance(l, Line2D)) +w.chk_grid.setChecked(False) +grid_off = sum(1 for l in w.ax.lines if isinstance(l, Line2D)) +w.chk_grid.setChecked(True) +assert grid_on >= 3 and grid_off < grid_on, (grid_on, grid_off) +print(f'PASS grid: {grid_on} distance gridlines drawn, toggleable') + +# 2c. theme-aware colors recolor the render +w.set_colors('#E6E8EB', '#37C894', '#26282D') +assert w._bg == '#26282D' and w.figure.patch.get_facecolor()[:3] != (1, 1, 1) +w.set_colors('#424242', '#1E9E6A', '#FFFFFF') +print('PASS theme colors: branches/background follow the app theme') + # 3. support threshold + grid toggles do not break rendering w.threshold_slider.setValue(70) assert w.threshold == 70 From 55e6b2065b250c76748be873890856d93ee492a3 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:46:04 +0300 Subject: [PATCH 08/36] Fix tree direction: internal nodes sit at their own cumulative depth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Placing internal nodes at their deepest descendant's x made elbows overrun short children, so branches left of the root (e.g. rodents hanging off the basal trifurcation) drew leftward. Every node now uses its own cumulative branch-length depth in all three layouts — verified programmatically (0 of 14 horizontal segments go left) and visually. --- phyml_ui/tree_viewer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/phyml_ui/tree_viewer.py b/phyml_ui/tree_viewer.py index be43747..d5f384d 100644 --- a/phyml_ui/tree_viewer.py +++ b/phyml_ui/tree_viewer.py @@ -341,7 +341,7 @@ def assign(cl, pd): return coords[id(cl)] child_pos = [assign(c, x) for c in cl.clades] if self.layout == 'circle': - r = max(p[0] for p in child_pos) + # radius = the node's own cumulative depth (rightward growth) ang = sum(p[1] for p in child_pos) / len(child_pos) # average angle over the shortest arc angs = sorted(p[1] for p in child_pos) @@ -351,11 +351,11 @@ def assign(cl, pd): span = (a0 - a1) % (2 * 3.141592653589793) if span < best_span: best_span, best = span, (a1 + span / 2) % (2 * 3.141592653589793) - coords[id(cl)] = (r, best) + coords[id(cl)] = (x, best) else: - x_c = max(p[0] for p in child_pos) + # x = the node's own cumulative depth: every branch grows rightward y_c = sum(p[1] for p in child_pos) / len(child_pos) - coords[id(cl)] = (x_c, y_c) + coords[id(cl)] = (x, y_c) return coords[id(cl)] assign(tree.root, 0.0) From 40ebc159b2a4f90b89db14ccfe800452d57b897f Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:07:44 +0300 Subject: [PATCH 09/36] Fix empty-input worker crash, IQ-TREE UFBoot/model errors, circular polytomy angles - _process_one's outer except returned "" as stats, so run() died on stats.get() for an empty/invalid FASTA: the worker thread crashed, finished_signal never fired and the GUI stayed stuck with Run disabled - IQ-TREE received the PhyML-sized replicate count (default 100) as -B, which UFBoot rejects ("#replicates must be >= 1000"); the value is now raised to 1000 via ufboot_replicates() with a log notice (en/ru) - the IQ-TREE nucleotide model list offered "HKY2+F+G4", which IQ-TREE does not know; replaced with HKY+F+G4 - circular layout placed multifurcating nodes at the middle of the smallest gap between children instead of the middle of the arc that covers them (the circle minus the widest gap) Regression tests: empty FASTA and ufboot_replicates in the smoke suite (CI), 100-replicate run + every listed nt/aa model accepted by the real binary in test_iqtree, polytomy angle in test_tree_viewer. The IQ-TREE GUI test now also asserts the run status, not only the xlsx presence. Co-Authored-By: Claude Opus 5.5 --- locales.json | 2 ++ phyml_core/pipeline.py | 20 ++++++++++++++++-- phyml_ui/main_window.py | 2 +- phyml_ui/tree_viewer.py | 21 ++++++++++--------- tests/test_iqtree.py | 44 +++++++++++++++++++++++++++++++++++++++ tests/test_smoke.py | 24 +++++++++++++++++++++ tests/test_tree_viewer.py | 12 +++++++++++ 7 files changed, 112 insertions(+), 13 deletions(-) diff --git a/locales.json b/locales.json index f34bb33..759f753 100644 --- a/locales.json +++ b/locales.json @@ -162,6 +162,7 @@ "engine_iqtree": "IQ-TREE (ModelFinder, UFBoot)", "path_iqtree": "Path to IQ-TREE (iqtree2.exe):", "err_iqtree": "[ERROR] Invalid path to IQ-TREE.", + "iqtree_ufboot_min": "[*] IQ-TREE ultrafast bootstrap needs at least 1000 replicates: using {} instead of {}.", "fasta_info": "{} taxa | {}-{} sites | gaps {}%", "xlsx_saved": "[+] Excel workbook saved: {}", "log_filter_all": "All", @@ -348,6 +349,7 @@ "engine_iqtree": "IQ-TREE (ModelFinder, UFBoot)", "path_iqtree": "Путь к IQ-TREE (iqtree2.exe):", "err_iqtree": "[ОШИБКА] Некорректный путь к IQ-TREE.", + "iqtree_ufboot_min": "[*] Ультрабыстрому бутстрепу IQ-TREE нужно не меньше 1000 реплик: используется {} вместо {}.", "fasta_info": "{} такс | {}-{} сайтов | гэпы {}%", "xlsx_saved": "[+] Excel-файл сохранён: {}", "log_filter_all": "Все", diff --git a/phyml_core/pipeline.py b/phyml_core/pipeline.py index d505fdd..1f94066 100644 --- a/phyml_core/pipeline.py +++ b/phyml_core/pipeline.py @@ -49,6 +49,18 @@ def grab(pattern): } +UFBOOT_MIN_REPLICATES = 1000 + + +def ufboot_replicates(value): + """IQ-TREE -B value: UFBoot refuses fewer than 1000 replicates.""" + try: + n = int(str(value).strip()) + except ValueError: + n = 0 + return str(max(n, UFBOOT_MIN_REPLICATES)) + + # --- Background Worker Thread --- class PipelineWorker(QThread): log_signal = Signal(str) @@ -176,7 +188,7 @@ def _process_one(self, current_fasta): except Exception as e: self.log_signal.emit(_t("err_crit").format(str(e))) - return False, "", "" + return False, "", {} def _run_phyml(self, phylip_path, creation_flags): try: @@ -250,7 +262,11 @@ def _run_iqtree(self, phylip_path, creation_flags): '-T', self.config.get("mpi_np", "2")] support_method = self.config.get("support_method", "bootstrap") if support_method == "bootstrap": - cmd_iqtree.extend(['-B', self.config.get("bootstrap_replicates", "1000")]) + requested = self.config.get("bootstrap_replicates", "1000") + replicates = ufboot_replicates(requested) + if replicates != str(requested).strip(): + self.log_signal.emit(_t("iqtree_ufboot_min").format(replicates, requested)) + cmd_iqtree.extend(['-B', replicates]) elif support_method.startswith('-'): cmd_iqtree.extend(['--alrt', '1000']) # aLRT branch support diff --git a/phyml_ui/main_window.py b/phyml_ui/main_window.py index 092fd65..a8becfb 100644 --- a/phyml_ui/main_window.py +++ b/phyml_ui/main_window.py @@ -552,7 +552,7 @@ def toggle_bootstrap_input(self): def update_models(self): self.cb_model.clear() if self.engine_setting == 'iqtree': - models = ["MFP", "GTR+F+G4", "HKY2+F+G4", "TIM2+F+G4", "TVM+F+G4"] \ + models = ["MFP", "GTR+F+G4", "HKY+F+G4", "TIM2+F+G4", "TVM+F+G4"] \ if self.cb_datatype.currentData() == "nt" else \ ["MFP", "LG+G4", "WAG+G4", "JTT+G4", "mtREV+G4"] else: diff --git a/phyml_ui/tree_viewer.py b/phyml_ui/tree_viewer.py index d5f384d..a64ed0b 100644 --- a/phyml_ui/tree_viewer.py +++ b/phyml_ui/tree_viewer.py @@ -341,17 +341,18 @@ def assign(cl, pd): return coords[id(cl)] child_pos = [assign(c, x) for c in cl.clades] if self.layout == 'circle': - # radius = the node's own cumulative depth (rightward growth) - ang = sum(p[1] for p in child_pos) / len(child_pos) - # average angle over the shortest arc + # radius = the node's own cumulative depth (rightward growth); + # angle = middle of the shortest arc covering all children, + # i.e. the full circle minus the widest gap between neighbours + two_pi = 2 * math.pi angs = sorted(p[1] for p in child_pos) - best, best_span = None, 1e9 - for i, a0 in enumerate(angs): - a1 = angs[(i - 1) % len(angs)] # predecessor going backwards - span = (a0 - a1) % (2 * 3.141592653589793) - if span < best_span: - best_span, best = span, (a1 + span / 2) % (2 * 3.141592653589793) - coords[id(cl)] = (x, best) + if len(angs) == 1: + mid = angs[0] + else: + gap, start = max(((angs[i] - angs[i - 1]) % two_pi, angs[i]) + for i in range(len(angs))) + mid = (start + (two_pi - gap) / 2) % two_pi + coords[id(cl)] = (x, mid) else: # x = the node's own cumulative depth: every branch grows rightward y_c = sum(p[1] for p in child_pos) / len(child_pos) diff --git a/tests/test_iqtree.py b/tests/test_iqtree.py index 4ec2b94..3ecc483 100644 --- a/tests/test_iqtree.py +++ b/tests/test_iqtree.py @@ -52,6 +52,19 @@ assert row['lnL'] == lnl_file, (row['lnL'], lnl_file) print(f'PASS iqtree engine: MFP picked {row["model"]}, lnL {row["lnL"]}, 8 taxa') +# 1b. PhyML-sized replicate count (100) must not break UFBoot (needs >= 1000) +os.chdir(run_dir) +logs, done = [], [] +worker = phymlgui.PipelineWorker(dict(config, bootstrap_replicates='100'), + os.path.abspath('test_prealigned.fasta')) +worker.log_signal.connect(logs.append) +worker.finished_signal.connect(lambda ok, tree: done.append((ok, tree))) +worker.run() +os.chdir(ROOT) +assert done[0][0], 'iqtree with 100 replicates failed:\n' + '\n'.join(logs[-8:]) +assert any(' -B 1000' in l for l in logs), 'UFBoot replicates were not raised to 1000' +print('PASS iqtree ufboot: 100 replicates raised to 1000, run succeeds') + # 2. GUI: engine validation, model list, full run + xlsx summary cfg_bak = phymlgui.CONFIG_FILE + '.bak' had_cfg = os.path.exists(phymlgui.CONFIG_FILE) @@ -70,18 +83,49 @@ assert warned and (w.worker is None or not w.worker.isRunning()) w.iqtree_path_input.setText(IQTREE_EXE) + w.set_combo_by_data(w.cb_support_method, 'bootstrap') w.le_bootstrap_replicates.setText('100') w.start_pipeline() guard = 0 while not w.btn_run.isEnabled() and guard < 400: app.processEvents(); time.sleep(0.3); guard += 1 assert w.btn_run.isEnabled(), 'GUI iqtree run did not finish' + assert w.worker.results[0]['status'] == 'OK', \ + 'GUI iqtree run failed:\n' + w.log_console.toPlainText()[-800:] xlsx = os.path.join(run_dir, 'phymlgui_summary.xlsx') assert os.path.exists(xlsx), 'xlsx summary missing' from openpyxl import load_workbook wb = load_workbook(xlsx) assert wb.active.max_row == 2 print('PASS iqtree GUI: validation, model list, run + xlsx summary') + + # 3. every model offered in the IQ-TREE model list is accepted by IQ-TREE + import random + import subprocess + rnd = random.Random(7) + aa_phy = os.path.join(run_dir, 'models_aa.phy') + with open(aa_phy, 'w') as f: + base = ''.join(rnd.choice('ACDEFGHIKLMNPQRSTVWY') for _ in range(120)) + f.write('6 120\n') + for i in range(6): + seq = ''.join(c if rnd.random() > 0.15 else rnd.choice('ACDEFGHIKLMNPQRSTVWY') + for c in base) + f.write(f'P{i} {seq}\n') + alignments = {'nt': os.path.join(run_dir, 'test_prealigned.phy'), 'aa': aa_phy} + rejected = [] + for dt, phy in alignments.items(): + w.set_combo_by_data(w.cb_datatype, dt) + models = [w.cb_model.itemData(i) for i in range(w.cb_model.count())] + for m in models: + if m == 'MFP': + continue # ModelFinder is exercised by the runs above + r = subprocess.run([IQTREE_EXE, '-s', phy, '-m', m, '-n', '0', '-T', '1', '-redo', + '--prefix', os.path.join(run_dir, 'model_probe')], + capture_output=True, text=True) + if r.returncode != 0: + rejected.append(m) + assert not rejected, f'IQ-TREE rejects listed models: {rejected}' + print('PASS iqtree models: every listed nt/aa model is accepted by IQ-TREE') finally: if had_cfg: shutil.move(cfg_bak, phymlgui.CONFIG_FILE) diff --git a/tests/test_smoke.py b/tests/test_smoke.py index 388d3f5..e3dce68 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -83,4 +83,28 @@ assert w.cpu_timer.isActive() print('PASS main window builds offscreen') +# 7. a broken input fails the run cleanly instead of killing the worker thread +from phyml_core.pipeline import PipelineWorker +empty = os.path.join(tmp, 'empty.fasta') +open(empty, 'w').close() +worker = PipelineWorker({'engine': 'phyml'}, [empty]) +done = [] +worker.finished_signal.connect(lambda ok, tree: done.append((ok, tree))) +try: + worker.run() +except Exception as e: + raise AssertionError(f'worker crashed on an empty FASTA: {e!r}') +assert done == [(False, '')], done +assert worker.results[0]['status'] == 'FAIL', worker.results +print('PASS empty fasta: run fails cleanly, finished signal emitted') + +# 8. IQ-TREE UFBoot refuses < 1000 replicates: the PhyML-sized value is raised +from phyml_core.pipeline import ufboot_replicates +assert ufboot_replicates('100') == '1000' +assert ufboot_replicates('1000') == '1000' +assert ufboot_replicates('5000') == '5000' +assert ufboot_replicates('') == '1000' +assert ufboot_replicates('abc') == '1000' +print('PASS ufboot replicates: raised to the IQ-TREE minimum of 1000') + print('ALL SMOKE TESTS PASSED') diff --git a/tests/test_tree_viewer.py b/tests/test_tree_viewer.py index 215fa6c..46a2306 100644 --- a/tests/test_tree_viewer.py +++ b/tests/test_tree_viewer.py @@ -21,6 +21,18 @@ print('PASS rf: identical=0, quartet swap=1 (100%)') app = phymlgui.QApplication([]) + +# 1b. circular layout: a polytomy node sits in the middle of its children's arc +w0 = TreeViewerWidget() +w0.load_tree_object(Phylo.read(io.StringIO('((A:1,B:1,C:1,D:1):1,E:1,F:1);'), 'newick'), 'polytomy') +w0.set_layout('circle') +tree0, coords0 = w0._coords +node = tree0.root.clades[0] +child_angles = [coords0[id(c)][1] for c in node.clades] +expected = (min(child_angles) + max(child_angles)) / 2 +assert abs(coords0[id(node)][1] - expected) < 1e-9, (coords0[id(node)][1], expected) +print('PASS circle layout: polytomy node centered on its children arc') + if not os.path.isdir('testrun') or not any( os.path.exists(os.path.join('testrun', d, f)) for d in os.listdir('testrun') if os.path.isdir(os.path.join('testrun', d)) From 997c869bf4d34efbdbdfae8867c26fcff8d66a50 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sun, 25 Oct 2026 00:10:10 +0300 Subject: [PATCH 10/36] Add Foundation design spec and RAxML-NG Windows build spike Design (docs/superpowers/specs/2026-10-25-foundation-design.md), approved section by section: Qt-free phyml_core (paths, settings, history, presets, alignment, launchers, engines, runner), thin phyml_ui shell, data stored next to the exe with %APPDATA% fallback and one-time legacy migration, pytest + pytest-qt with binary markers, CI with pinned IQ-TREE/MUSCLE, Python >= 3.11. Spike (docs/superpowers/spikes/2026-10-24-raxml-ng-windows/): RAxML-NG 2.0.3 builds natively with MSYS2 UCRT64 using 4 configure flags and a 167-line patch; the result matches the official Linux binary (identical bootstrap replicates, start trees and fixed-tree likelihoods, RF 0 on all ML searches). Fixes include an RNG normalization segfault, aligned alloc/free heap corruption and 15-bit rand() seeds that produced duplicate bootstrap replicates on Windows. Co-Authored-By: Claude Opus 5.5 --- .../specs/2026-10-25-foundation-design.md | 293 +++++++++++ .../2026-10-24-raxml-ng-windows/README.md | 73 +++ .../raxml-ng-2.0.3-windows.patch | 470 ++++++++++++++++++ 3 files changed, 836 insertions(+) create mode 100644 docs/superpowers/specs/2026-10-25-foundation-design.md create mode 100644 docs/superpowers/spikes/2026-10-24-raxml-ng-windows/README.md create mode 100644 docs/superpowers/spikes/2026-10-24-raxml-ng-windows/raxml-ng-2.0.3-windows.patch diff --git a/docs/superpowers/specs/2026-10-25-foundation-design.md b/docs/superpowers/specs/2026-10-25-foundation-design.md new file mode 100644 index 0000000..e413756 --- /dev/null +++ b/docs/superpowers/specs/2026-10-25-foundation-design.md @@ -0,0 +1,293 @@ +# Подпроект «Фундамент» — дизайн + +Дата: 2026-10-25 · Статус: на ревью · Связано: [спайк RAxML-NG под Windows](../spikes/2026-10-24-raxml-ng-windows/README.md) + +## 1. Контекст и цель + +Раунд доработок phymlGUI разбит на четыре подпроекта в порядке +**Фундамент → (Наука ∥ Рисунки) → Релиз**. «Фундамент» идёт первым: все +новые функции «Науки» (повтор прогона из истории, таблица сравнения, +третий движок) и «Релиза» (exe со встроенными бинарниками) должны лечь на +код, который тестируется без окна и не зависит от текущей папки. + +**Сказано пользователем:** делаем все направления, релиз в конце; подход A +(ядро без Qt); конфиг рядом с exe; полный переход на pytest; CI с IQ-TREE и +MUSCLE; RAxML-NG в этом раунде, собранный нативно под Windows (спайк +успешен); Python ≥ 3.11. + +**Допущения:** пользователи — коллеги и студенты на Windows 10/11; +интерфейс остаётся двуязычным (en/ru); видимое поведение окна в этом +подпроекте не меняется, кроме мест, явно перечисленных ниже. + +**Проблемы, которые закрывает подпроект:** + +- `phyml_ui/main_window.py` (~1000 строк) смешивает виджеты, конфиг, + историю, пресеты, проверки и запуск; +- конфиг и история пишутся относительно текущей папки — у exe, запущенного + ярлыком, «теряются» настройки; +- ядро (`pipeline.py`) зависит от Qt-сигналов, поэтому и CLI тянет PySide6; +- выбор движка — ветвление `if engine == ...` в нескольких местах; +- тесты — скрипты, которые двигают настоящий `phyml_config.json`, содержат + машинно-зависимые проверки (8 ядер / 16 потоков) и в CI запускаются + только частично (просмотрщик деревьев там пропускается). + +## 2. Архитектура + +Правило зависимостей: **`phyml_core` не импортирует PySide6** (проверяется +тестом). `phyml_ui` зависит от `phyml_core`, но не наоборот. + +``` +phyml_core/ — без Qt + paths.py AppPaths: папка приложения, папка данных, bin/, ресурсы + settings.py схема конфига с умолчаниями, атомарные загрузка/сохранение, миграция + history.py HistoryStore + presets.py встроенные и пользовательские пресеты + alignment.py чтение FASTA, проверка выравнивания, очистка ID, PHYLIP, сводка + launchers.py Native, Mpi + engines/ + base.py Engine, Issue, EngineOutputs + phyml.py PhyML + iqtree.py IqTree + __init__.py ENGINES = {'phyml': PhyML(), 'iqtree': IqTree()} + runner.py PipelineRunner, FileResult, RunResult + cli.py headless-режим поверх runner + i18n.py, utils.py без изменений по смыслу + +phyml_ui/ + main_window.py только сборка окна и связи между частями (цель ≤ ~350 строк) + params_panel.py группы параметров ⇄ словарь params + nav_panel.py недавние / пресеты / история + log_console.py структурированный журнал (уровни приходят из ядра) + settings_dialog.py + status_bar.py CPU-монитор, кнопка папки результатов, настройки + worker.py QThread-адаптер над PipelineRunner + tree_viewer.py, themes.py без изменений (просмотрщик — в подпроекте «Рисунки») + +phymlgui.py только точка входа; реэкспорты «для совместимости» удаляются +``` + +## 3. Движки и способы запуска + +```python +# launchers.py +class Native: wrap(argv) -> argv +class Mpi: __init__(mpiexec: str, np: int); wrap(argv) -> [mpiexec, '-np', str(np), *argv] + +# engines/base.py +@dataclass(frozen=True) +class Issue: + level: Literal['error', 'warning'] + key: str # ключ локализации + args: tuple = () + +@dataclass(frozen=True) +class EngineOutputs: + tree: str + stats: str + boot_trees: str | None + log: str | None + +class Engine(ABC): + key: str # 'phyml' | 'iqtree' | позже 'raxml' + label_key: str + def models(self, datatype: str) -> list[str] + def validate(self, params: dict, paths: AppPaths) -> list[Issue] + def normalize(self, params: dict) -> tuple[dict, list[Issue]] + def executable(self, params: dict, paths: AppPaths) -> str + def command(self, exe: str, phylip: str, params: dict) -> list[str] # чистая функция + def outputs(self, phylip: str) -> EngineOutputs + def parse_stats(self, outputs: EngineOutputs) -> dict[str, str] # model, taxa, lnL, gamma, tstv, time + def launcher(self, params: dict) -> Native | Mpi +``` + +- Параметры остаются **плоским словарём** с теми же ключами, что в + нынешнем конфиге, — сохранённые настройки и пресеты не ломаются. +- `validate` — единственный источник правил: «бутстреп на MPI-сборке + PhyML без MPI» (error), «нечисловое число реплик/ядер/pinv» (error), + «нет исполняемого файла» (error), «UFBoot < 1000» (warning) и т. п. +- `normalize` подгоняет параметры с заметкой в журнал (UFBoot → 1000, + сегодняшняя `ufboot_replicates` переезжает сюда). +- `executable`: путь из конфига; если он пуст — бинарник из `bin/` рядом с + приложением. +- Сообщение об ошибке процесса подставляет имя движка (сейчас при ошибке + IQ-TREE пишется «PhyML завершил работу с кодом …»). +- RAxML-NG добавляется в «Науке» одним классом `engines/raxml.py`; + исполняемый файл — нативная сборка по рецепту спайка, запуск `Native`. + WSL-запуск не нужен. + +## 4. Хранение данных + +**`AppPaths`** + +- `app_dir`: у собранного exe — `dirname(sys.executable)`, из исходников — + корень репозитория; +- `data_dir`: `$PHYMLGUI_DATA_DIR`, если задана; иначе `app_dir`, если в неё + можно писать (проба созданием временного файла); иначе + `%APPDATA%\phymlGUI` (создаётся); +- `bin_dir`: `app_dir/bin`; +- `resource(name)`: ресурсы сборки (`sys._MEIPASS`) с откатом на `app_dir`; +- `to_stored(path)` / `from_stored(path)`: пути внутри `app_dir` хранятся + относительными, остальные — абсолютными. + +**`settings.py`** + +- `DEFAULTS` — все ключи нынешнего конфига + `schema_version: 2`; +- `load(path) -> dict`: умолчания ← файл; неизвестные ключи сохраняются; + битый JSON → умолчания + ошибка в журнал (файл не перезаписывается до + первого успешного сохранения, копия кладётся в `*.broken`); +- `save(path, cfg) -> None`: временный файл + `os.replace`; исключение + `OSError` пробрасывается вызывающему, интерфейс пишет его в журнал; +- `migrate_legacy(paths, cwd)`: если в `data_dir` нет конфига, а в `cwd` + лежат `phyml_config.json` / `phyml_history.json` — **копирует** их, + возвращает список скопированного для журнала. Однократно. + +**`HistoryStore(path, limit=200)`** + +- `load() -> list[dict]` (новые сверху; повреждённый файл → пустой список + + ошибка в журнал), `append(record)`, `get(index)`; +- запись: `ts`, `inputs` (пути), `engine`, `params` (полный словарь), + `ok`/`n`, `duration_s`, `files` (по файлу: статус, дерево, статистика); +- старые записи без `params`/`files` читаются (поля по умолчанию). + +**`presets.py`**: `BUILTIN` (как сейчас + ключ `engine`), пользовательские +пресеты живут в конфиге; `apply(preset, params) -> dict` переключает и движок. + +## 5. Пайплайн и ошибки + +```python +@dataclass +class FileResult: + file: str + status: Literal['ok', 'fail', 'cancelled'] + tree: str = '' + stats: dict = field(default_factory=dict) + error: str = '' + +@dataclass +class RunResult: + files: list[FileResult] + cancelled: bool + # ok_count, success — вычисляемые свойства + +class PipelineRunner: + def __init__(self, params: dict, files: list[str], paths: AppPaths, + on_log: Callable[[str, str], None] | None = None, # (level, text); None — молча + on_status: Callable[[str, str], None] | None = None) # (file name, status) + def run(self) -> RunResult # никогда не бросает исключений + def cancel(self) -> None # потокобезопасно +``` + +Шаги для каждого файла: + +1. `alignment.read_fasta` — пустой/битый файл → `fail` с понятным текстом; +2. если длины различаются — MUSCLE (синтаксис v5, при неудаче v3) через + `Native`; +3. очистка ID → **проверка коллизий** (разные ID дали одинаковое имя → + `fail` со списком конфликтов) → PHYLIP-relaxed; +4. `engine.validate(params, paths)` (error → `fail`, warning → журнал) → `engine.normalize` + → `launcher.wrap(engine.command(...))` → построчный вывод в журнал → + проверка кода возврата и наличия `outputs.tree` → `engine.parse_stats`. + +Гарантии: + +- каждый файл обёрнут в `try/except` → `FileResult(status='fail', error=...)`; + `run()` в целом — `try/finally`, итог возвращается всегда; +- `cancel()` ставит флаг и завершает **дерево процессов** текущего шага + (через `psutil`, если доступен; иначе `terminate()` корневого процесса); +- уровни журнала (`error` / `success` / `system` / `out`) задаёт ядро; + `LogConsole` больше не угадывает их по префиксам строк; +- проверка перед запуском в окне вызывает тот же `engine.validate` и + показывает проблемы до старта (ad-hoc проверки в `start_pipeline` + удаляются). + +`phyml_ui/worker.py`: `QThread`, который запускает `PipelineRunner.run()` и +переводит колбэки в сигналы `log(str, str)`, `status(str, str)`, +`finished(object)`; `cancel()` проксируется. + +CLI: `python phymlgui.py --file X | --batch DIR [--config PRESET.json]`; +работает без импорта PySide6; коды выхода: 0 — все файлы `ok`, 1 — есть +`fail`/`cancelled`, 2 — ошибка аргументов или входа. + +## 6. Интерфейс + +Окно собирается из модулей раздела 2; внешний вид и раскладка не +меняются. Видимые изменения только такие: + +- конфиг и история — в `data_dir` (при первом запуске — сообщение о + миграции в журнале); +- проблемы параметров показываются до запуска (раньше часть из них + всплывала только в журнале); +- журнал раскрашивается по уровням из ядра; +- при отмене завершаются и дочерние процессы MPI. + +## 7. Тесты и CI + +- `requirements-dev.txt`: `pytest`, `pytest-qt`; +- `tests/unit/` (ядро, без Qt), `tests/gui/` (`qtbot`, offscreen), + `tests/e2e/` (реальные бинарники), `tests/data/` (маленькие деревья с + поддержками, файл бутстреп-деревьев, FASTA, образцы статистики); +- `conftest.py`: фикстура `data_dir` (временная папка через + `PHYMLGUI_DATA_DIR`), поиск бинарников (`PHYMLGUI_PHYML`, `PHYMLGUI_IQTREE`, + `PHYMLGUI_MUSCLE`, `PHYMLGUI_RAXML`, затем `bin/`, затем нынешние локальные + пути), маркеры `needs_phyml`, `needs_mpi`, `needs_iqtree`, `needs_muscle` + с автоматическим пропуском и причиной; +- все нынешние проверки переносятся один к одному (включая регрессионные + тесты от 2026-10-24); машинно-зависимые утверждения заменяются + сравнением с `detect_cores()`; старые скрипты-наборы удаляются, а + загрузчик реальных данных `fetch_real_data.py` переезжает в `tests/e2e/` + как вспомогательный модуль; +- новые тесты: ядро не импортирует PySide6; CLI работает без PySide6; + `run()` не бросает исключений и при падении движка-заглушки окно + возвращается в исходное состояние; отмена убивает дочерний процесс; + коллизия ID → понятная ошибка; миграция старого конфига; относительные + пути; повреждённые конфиг/история; +- CI (`windows-latest`, Python 3.11 и 3.13): скачивает официальные IQ-TREE и + MUSCLE (версии и SHA256 — в `packaging/third_party.json`, его же потом + использует «Релиз»), кэширует, запускает `pytest`; PhyML+MPI пропускаются; +- README: Python ≥ 3.11, где лежат настройки, как запускать тесты. + +## 8. Вне рамок подпроекта + +- класс движка RAxML-NG, таблица сравнения прогонов, повтор из истории, + предпросмотр выравнивания — «Наука»; +- любые изменения просмотрщика деревьев — «Рисунки»; +- сборка exe, бинарники в `bin/`, сборка RAxML-NG в CI, проверка обновлений, + лицензии в релизе — «Релиз»; +- изменение внешнего вида окна. + +## 9. Критерии готовности + +1. `phyml_core` не импортирует PySide6; CLI работает в окружении без PySide6. +2. `main_window.py` ≤ ~350 строк и содержит только сборку окна и связи. +3. Все прежние проверки и новые тесты из раздела 7 проходят: локально — + полностью (с PhyML/MPI/IQ-TREE/MUSCLE), в CI — всё, кроме `needs_phyml` + и `needs_mpi`. +4. Тестовый прогон не изменяет `phyml_config.json` / `phyml_history.json` + пользователя (проверяется сравнением хешей до и после). +5. При запуске из любой текущей папки настройки читаются из `data_dir`; + старые настройки из текущей папки переносятся один раз. +6. Ручная проверка в окне: одиночный прогон, пакетный режим, отмена при + MPI-прогоне (в диспетчере задач не остаётся `phyml-mpi.exe`), пресеты, + история, смена движка. + +## 10. Порядок перехода (без «большого взрыва») + +Каждый шаг оставляет приложение рабочим, а тесты — зелёными: + +1. pytest-инфраструктура и перенос текущих проверок как есть; +2. `paths` + `settings` + `history` + `presets` (окно переходит на них); +3. `launchers` + `engines` (PhyML, IQ-TREE) с тестами команд; +4. `alignment` + `runner` + `worker`; CLI на runner; +5. разрезание `main_window.py` на модули интерфейса; +6. CI с IQ-TREE/MUSCLE, обновление README. + +## 11. Риски + +- **Перенос тестов меняет то, что они проверяют.** Смягчение: шаг 1 — + перенос без изменения логики, сверка числа проверок до и после. +- **Скрытые зависимости окна от порядка сигналов** (автосохранение во время + загрузки конфига, `_is_loading`). Смягчение: GUI-тесты на загрузку + конфига, применение пресета и смену движка до разрезания окна. +- **Разные версии IQ-TREE локально и в CI.** Смягчение: версия в CI + фиксируется в `packaging/third_party.json`; локальный бинарник не + обязан совпадать, тесты не сравнивают числа между машинами. diff --git a/docs/superpowers/spikes/2026-10-24-raxml-ng-windows/README.md b/docs/superpowers/spikes/2026-10-24-raxml-ng-windows/README.md new file mode 100644 index 0000000..578807a --- /dev/null +++ b/docs/superpowers/spikes/2026-10-24-raxml-ng-windows/README.md @@ -0,0 +1,73 @@ +# Spike: native Windows build of RAxML-NG 2.0.3 + +**Question.** Can RAxML-NG be built as a self-contained native `raxml-ng.exe` +for Windows x64, producing the same results as the official Linux binary? + +**Answer: yes.** With 4 configure flags and a 167-line patch (11 files), the +build is self-contained (imports only `KERNEL32` + the Universal CRT that ships +with Windows 10/11) and reproduces the official Linux binary bit for bit on +everything driven by the RNG, and to 1e-4 log-units after numerical +optimization. The exe built during the spike is **throwaway**; the deliverable +is the recipe + patch below, to be reproduced in CI (release sub-project). + +Upstream has no Windows binaries (Linux/macOS + source only; Windows users are +pointed to WSL), so this is the only way to ship RAxML-NG in a +"unzip and run" phymlGUI release. + +## Recipe + +- Sources: `raxml-ng_v2.0.3_source.zip` from the official GitHub release, + SHA256 `7cd3fa6b1de373777d688032c9ec6731078b67e6a812d7a1d779295d016843d2` +- Toolchain: MSYS2 **UCRT64** — `mingw-w64-ucrt-x86_64-{gcc,cmake,ninja}` + (tested with GCC 16.2.0, CMake 4.4.4, Ninja 1.13.2) +- Patch: `git apply -p1 raxml-ng-2.0.3-windows.patch` (from the source root) +- Configure + build: + +```bash +cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DSTATIC_BUILD=ON -DPORTABLE_BUILD=ON -DCORAX_BUILD_PORTABLE_ARCH=x86-64 \ + -DUSE_MPI=OFF "-DCMAKE_CXX_FLAGS=-D_USE_MATH_DEFINES -D_GNU_SOURCE" +cmake --build build -j +``` + +## What had to change, and why + +| # | Symptom | Root cause | Fix | +|---|---|---|---| +| 1 | `-march=` with empty value | portable build needs `CORAX_BUILD_PORTABLE_ARCH`; its default is commented out | configure flag `x86-64` (SIMD kernels still dispatched at runtime) | +| 2 | `thread-local variable declared as dllexport` | `CORAX_EXPORT` is `dllexport` on Windows even for the static lib | export only when `CORAX_SHARED` is defined | +| 3 | `M_PI` undeclared | MinGW hides math constants in strict C++ mode | `-D_USE_MATH_DEFINES` | +| 4 | `asprintf` undeclared | g++ predefines `_GNU_SOURCE` on Linux only | `-D_GNU_SOURCE` | +| 5 | `std::max(unsigned long, size_t)` | LLP64: `long` is 32-bit on Windows | `std::max` | +| 6 | `sys/resource.h`, `sysconf`, `sysinfo`, `realpath`, `posix_memalign`, `u_int32_t` missing | POSIX-only APIs in `sysutil.cpp` | Win32 equivalents (`GetProcessTimes`, `GetProcessMemoryInfo`, `GlobalMemoryStatusEx`, `_fullpath`, `_aligned_malloc`); `uint32_t` | +| 7 | RAM/peak memory silently truncated to 32 bits | `unsigned long` return type (feeds thread/worker limits) | `unsigned long long` | +| 8 | **Segfault** building the first parsimony tree | `corax_random_r` yields [0, 2^31-1] but was normalized by `RAND_MAX` (32767 on Windows) → out-of-bounds index in the Fisher–Yates shuffle | normalize by 2^31-1 | +| 9 | **Heap corruption** (`c0000374`) on exit of a tree search | `_aligned_malloc` memory freed with `free()` (legal on POSIX, not on Windows) — 5 sites; plus 1 reverse case (`malloc` → `corax_aligned_free`) | matching free in all 6 places (found by a scripted audit of all 84 aligned allocations) | +| 10 | **16 duplicate replicates out of 1000** in bootstrap; results differ from Linux for the same `--seed` | per-tree/per-replicate seeds come from `rand()`, a 15-bit LCG on Windows | `raxml_rand()`/`raxml_srand()`: glibc-identical generator from coraxlib on Windows, plain `rand`/`srand` elsewhere | + +Fixes 8–10 are silent-correctness issues: a build that merely compiles would +crash, corrupt memory, or quietly produce non-independent bootstrap replicates. + +## Verification against the official Linux binary (WSL, same arguments, `--seed 42`) + +| Check | Result | +|---|---| +| ML search, synthetic nt (8 taxa) | lnL identical, RF 0 | +| ML search, real CytB mammals | RF 0, \|ΔlnL\| = 9e-5 (libm differences during optimization) | +| ML search, synthetic protein (LG+G) | lnL identical, RF 0 | +| Fixed tree + fixed model, no optimization (nt and aa) | lnL identical — likelihood kernels match | +| `--all`, 50 bootstraps, 4 threads | 50/50 replicate trees identical, supports identical | +| Bootstrap MSAs, 1000 replicates | 1000/1000 identical to Linux, 0 duplicates (was 16) | +| Parsimony / random starting trees | identical | + +## Open points for the release sub-project + +- Reproduce the build in GitHub Actions (MSYS2 is preinstalled on + `windows-latest`) and keep the Linux-vs-Windows comparison as a regression job. +- Not exercised yet: partitioned analyses, checkpoint/restart, `--terrace` + (terraphast/GMP disabled), very large alignments (> 4 GB RAM). +- License: RAxML-NG is **AGPL-3.0** — the release must ship (or link to) the + exact sources plus this patch. +- The fixes are upstreamable (mostly `#ifdef _WIN32` or platform-neutral); + sending them upstream is optional and outward-facing, so it needs a separate + decision. diff --git a/docs/superpowers/spikes/2026-10-24-raxml-ng-windows/raxml-ng-2.0.3-windows.patch b/docs/superpowers/spikes/2026-10-24-raxml-ng-windows/raxml-ng-2.0.3-windows.patch new file mode 100644 index 0000000..8d64993 --- /dev/null +++ b/docs/superpowers/spikes/2026-10-24-raxml-ng-windows/raxml-ng-2.0.3-windows.patch @@ -0,0 +1,470 @@ +Binary files pristine/bin/raxml-ng.exe and dl_src/bin/raxml-ng.exe differ +diff -ruN '--exclude=src.zip' pristine/libs/coraxlib/src/corax/core/common.h dl_src/libs/coraxlib/src/corax/core/common.h +--- pristine/libs/coraxlib/src/corax/core/common.h 2025-10-31 18:41:19.000000000 +0300 ++++ dl_src/libs/coraxlib/src/corax/core/common.h 2026-10-24 19:21:29.744785700 +0300 +@@ -44,7 +44,7 @@ + #include + #endif + +-#if (defined(__WIN32__) || defined(__WIN64__)) ++#if (defined(__WIN32__) || defined(__WIN64__)) && defined(CORAX_SHARED) + #define CORAX_EXPORT __declspec(dllexport) + #else + #define CORAX_EXPORT +diff -ruN '--exclude=src.zip' pristine/libs/coraxlib/src/corax/core/partition.c dl_src/libs/coraxlib/src/corax/core/partition.c +--- pristine/libs/coraxlib/src/corax/core/partition.c 2025-10-31 18:41:19.000000000 +0300 ++++ dl_src/libs/coraxlib/src/corax/core/partition.c 2026-10-24 22:38:25.828525000 +0300 +@@ -45,7 +45,7 @@ + + if (partition->tipchars) + for (i = 0; i < partition->tips; ++i) +- corax_aligned_free(partition->tipchars[i]); ++ free(partition->tipchars[i]); + free(partition->tipchars); + + if (partition->ttlookup) corax_aligned_free(partition->ttlookup); +@@ -114,7 +114,7 @@ + free(repeats->lookup_buffer); + free(repeats->toclean_buffer); + free(repeats->id_site_buffer); +- free(repeats->bclv_buffer); ++ corax_aligned_free(repeats->bclv_buffer); + free(repeats->charmap); + free(repeats); + } +@@ -236,7 +236,7 @@ + && CORAX_HAS_CPU_FEATURE(avx_present)) + return CORAX_SUCCESS; + +- free(partition->ttlookup); ++ corax_aligned_free(partition->ttlookup); + partition->ttlookup = + corax_aligned_alloc(alloc_size * sizeof(double), partition->alignment); + if (!partition->ttlookup) +diff -ruN '--exclude=src.zip' pristine/libs/coraxlib/src/corax/core/repeats.c dl_src/libs/coraxlib/src/corax/core/repeats.c +--- pristine/libs/coraxlib/src/corax/core/repeats.c 2025-10-31 18:41:19.000000000 +0300 ++++ dl_src/libs/coraxlib/src/corax/core/repeats.c 2026-10-24 22:37:42.855902200 +0300 +@@ -221,7 +221,7 @@ + } + unsigned int sizealloc = (ids + additional_sites) * partition->states_padded + * partition->rate_cats * sizeof(double); +- free(partition->clv[tip_index]); ++ corax_aligned_free(partition->clv[tip_index]); + + partition->clv[tip_index] = + corax_aligned_alloc(sizealloc, partition->alignment); +diff -ruN '--exclude=src.zip' pristine/libs/coraxlib/src/corax/io/binary.c dl_src/libs/coraxlib/src/corax/io/binary.c +--- pristine/libs/coraxlib/src/corax/io/binary.c 2026-03-11 21:17:41.000000000 +0300 ++++ dl_src/libs/coraxlib/src/corax/io/binary.c 2026-10-24 22:37:47.461549500 +0300 +@@ -1029,7 +1029,10 @@ + if (!bin_fread(data, *size, 1, bin_file)) + { + corax_set_error(CORAX_BIN_ERROR_BINARY_IO, "Error reading data."); +- free(data); ++ if (*attributes & CORAX_BIN_ATTRIB_ALIGNED) ++ corax_aligned_free(data); ++ else ++ free(data); + return CORAX_FAILURE; + } + +diff -ruN '--exclude=src.zip' pristine/libs/coraxlib/src/corax/tree/treeinfo.c dl_src/libs/coraxlib/src/corax/tree/treeinfo.c +--- pristine/libs/coraxlib/src/corax/tree/treeinfo.c 2026-04-25 00:35:45.000000000 +0300 ++++ dl_src/libs/coraxlib/src/corax/tree/treeinfo.c 2026-10-24 22:37:44.852293600 +0300 +@@ -791,7 +791,7 @@ + + if (treeinfo->deriv_precomp[partition_index]) + { +- free(treeinfo->deriv_precomp[partition_index]); ++ corax_aligned_free(treeinfo->deriv_precomp[partition_index]); + treeinfo->deriv_precomp[partition_index] = NULL; + } + +diff -ruN '--exclude=src.zip' pristine/libs/coraxlib/src/corax/util/stepwise.c dl_src/libs/coraxlib/src/corax/util/stepwise.c +--- pristine/libs/coraxlib/src/corax/util/stepwise.c 2026-03-11 21:17:41.000000000 +0300 ++++ dl_src/libs/coraxlib/src/corax/util/stepwise.c 2026-10-24 22:35:47.977481800 +0300 +@@ -85,7 +85,9 @@ + { + int rint; + corax_random_r(buf, &rint); +- double r = ((double)rint / RAND_MAX); ++ /* corax_random_r yields [0, 2^31-1] everywhere, while RAND_MAX is ++ only 32767 on Windows: normalize by the generator's own maximum */ ++ double r = ((double)rint / 2147483647.0); + j = (unsigned int)(r * (i + 1)); + + CORAX_SWAP(x[i], x[j]); +diff -ruN '--exclude=src.zip' pristine/src/adaptive/DifficultyPredictor.cpp dl_src/src/adaptive/DifficultyPredictor.cpp +--- pristine/src/adaptive/DifficultyPredictor.cpp 2026-09-03 19:23:46.000000000 +0300 ++++ dl_src/src/adaptive/DifficultyPredictor.cpp 2026-10-24 22:46:13.171567100 +0300 +@@ -76,7 +76,7 @@ + // init seeds + uintVector seeds(n_trees); + for (unsigned int i = 0; i < n_trees; ++i) +- seeds[i] = (unsigned int) rand(); ++ seeds[i] = (unsigned int) raxml_rand(); + + for (unsigned int i = 0; i < n_trees; ++i) + _pars_tree_list.emplace_back(Tree::buildParsimony(*_parsimony_msa_ptr, seeds[i], false, &score)); +diff -ruN '--exclude=src.zip' pristine/src/adaptive/StoppingCriterion.cpp dl_src/src/adaptive/StoppingCriterion.cpp +--- pristine/src/adaptive/StoppingCriterion.cpp 2026-09-03 19:23:46.000000000 +0300 ++++ dl_src/src/adaptive/StoppingCriterion.cpp 2026-10-24 22:46:13.163049200 +0300 +@@ -391,14 +391,14 @@ + + { + ParallelContext::UniqueLock lock; +- srand(seed); ++ raxml_srand(seed); + + for(exp = 0; exp < rell_size; exp++){ + + double rell_loglh = 0; + for (i = 0; i p = find_by_site_index(site_index); + assert(p.first >= 0 && p.second >= 0); + auto p_id = (unsigned int) p.first; +@@ -410,8 +410,8 @@ + } + + for(exp = 0; exp < rell_size; exp++){ +- auto ind1 = (unsigned int) rand() % rell_size; +- auto ind2 = (unsigned int) rand() % rell_size; ++ auto ind1 = (unsigned int) raxml_rand() % rell_size; ++ auto ind2 = (unsigned int) raxml_rand() % rell_size; + logl_diff_dist[exp] = fabs(rell_dist[ind1] - rell_dist[ind2]); + } + } +diff -ruN '--exclude=src.zip' pristine/src/common.h dl_src/src/common.h +--- pristine/src/common.h 2026-05-27 16:34:30.000000000 +0300 ++++ dl_src/src/common.h 2026-10-24 22:46:05.179337100 +0300 +@@ -15,6 +15,12 @@ + #include "log.hpp" + #include "util/SystemTimer.hpp" + ++/* rand()/srand() with glibc semantics on every platform: Windows' rand() is a ++ 15-bit LCG, so per-tree and per-replicate seeds drawn from it collide ++ (duplicate bootstrap replicates) and differ from Linux for the same --seed */ ++int raxml_rand(); ++void raxml_srand(unsigned int seed); ++ + /* used to suppress compiler warnings about unused args */ + #define RAXML_UNUSED(expr) (void)(expr) + +@@ -26,8 +32,8 @@ + + double sysutil_gettime(); + void sysutil_show_rusage(); +-unsigned long sysutil_get_memused(); +-unsigned long sysutil_get_memtotal(bool ignore_errors = true); ++unsigned long long sysutil_get_memused(); ++unsigned long long sysutil_get_memtotal(bool ignore_errors = true); + + std::string sysutil_get_cpu_model(); + unsigned int sysutil_get_cpu_cores(); +diff -ruN '--exclude=src.zip' pristine/src/main.cpp dl_src/src/main.cpp +--- pristine/src/main.cpp 2026-09-03 19:23:46.000000000 +0300 ++++ dl_src/src/main.cpp 2026-10-24 22:46:13.153672200 +0300 +@@ -1314,7 +1314,7 @@ + // temp workaround: since MSA pattern compression calls rand(), it will change all random + // numbers generated afterwards. so just reset seed to the initial value to ensure that + // starting trees, BS replicates etc. are the same regardless whether pat.comp is ON or OFF +- srand(opts.random_seed); ++ raxml_srand(opts.random_seed); + } + + // if (parted_msa.part_count() > 1) +@@ -1851,7 +1851,7 @@ + // init seeds + uintVector seeds(tree_count); + for (size_t i = 0; i < tree_count; ++i) +- seeds[i] = (unsigned int) rand(); ++ seeds[i] = (unsigned int) raxml_rand(); + + auto thread_fn = std::bind(thread_start_trees, + std::ref(instance), +@@ -1946,7 +1946,7 @@ + // init seeds + uintVector seeds(st_tree_count); + for (size_t i = 0; i < st_tree_count; ++i) +- seeds[i] = (unsigned int) rand(); ++ seeds[i] = (unsigned int) raxml_rand(); + + for (size_t i = 0; i < st_tree_count; ++i) + { +@@ -2178,7 +2178,7 @@ + auto& seeds = instance.bs_seeds; + seeds.resize(num_bsreps); + for (size_t i = 0; i < seeds.size(); ++i) +- seeds[i] = (unsigned int) rand(); ++ seeds[i] = (unsigned int) raxml_rand(); + + /* generate missing parsimony trees from the original MSA */ + if (instance.opts.num_pars_trees() > instance.pars_trees.size()) +@@ -2191,7 +2191,7 @@ + { + for (size_t i = 0; i < trees_to_generate; ++i) + { +- auto tree = generate_tree(instance, StartingTree::parsimony, (unsigned int) rand()); ++ auto tree = generate_tree(instance, StartingTree::parsimony, (unsigned int) raxml_rand()); + instance.pars_trees.emplace_back(std::move(tree)); + } + } +@@ -2378,7 +2378,7 @@ + /* no starting trees -> use a first tree from pars_trees, generate if needed */ + if (instance.pars_trees.empty()) + { +- auto tree = generate_tree(instance, StartingTree::parsimony, (unsigned int) rand()); ++ auto tree = generate_tree(instance, StartingTree::parsimony, (unsigned int) raxml_rand()); + instance.pars_trees.emplace_back(std::move(tree)); + } + } +@@ -2740,7 +2740,7 @@ + + if (bs_num % opts.bootstop_interval == 0 || bs_num == bs_trees.size()) + { +- converged = bootstop_checker->converged((unsigned int) rand()); ++ converged = bootstop_checker->converged((unsigned int) raxml_rand()); + + if (print) + { +@@ -3474,7 +3474,7 @@ + + // we need at least 2 doubles for each partition AND threads to perform parallel reduction, + // and even more with FreeRates in EM mode -> so resize the buffer accordingly +- auto doubles_per_thread_part = std::max(2UL, total_freerate_category_count); ++ auto doubles_per_thread_part = std::max(2, total_freerate_category_count); + const size_t reduce_buffer_size = std::max(1024u, doubles_per_thread_part * sizeof(double) * + parted_msa.part_count() * ParallelContext::num_threads()); + +@@ -4012,8 +4012,8 @@ + check_options(instance); + + /* init template tree */ +- srand(instance.opts.random_seed); +- instance.random_tree = generate_tree(instance, StartingTree::random, (unsigned int) rand()); ++ raxml_srand(instance.opts.random_seed); ++ instance.random_tree = generate_tree(instance, StartingTree::random, (unsigned int) raxml_rand()); + + // read start trees from file to avoid re-generation + // NOTE: doesn't work for OLD constrained tree search +@@ -4050,7 +4050,7 @@ + + // TEMP WORKAROUND: here we reset random seed once again to make sure that BS replicates + // are not affected by the number of ML search starting trees that has been generated before +- srand(instance.opts.random_seed); ++ raxml_srand(instance.opts.random_seed); + + /* generate bootstrap replicates */ + generate_bootstraps(instance, cm.checkp_file()); +@@ -4246,7 +4246,7 @@ + { + // make sure all MPI ranks use the same random seed + ParallelContext::mpi_broadcast(&opts.random_seed, sizeof(unsigned int)); +- srand(opts.random_seed); ++ raxml_srand(opts.random_seed); + + logger().log_level(instance.opts.log_level); + logger().precision(instance.opts.precision); +diff -ruN '--exclude=src.zip' pristine/src/util/sysutil.cpp dl_src/src/util/sysutil.cpp +--- pristine/src/util/sysutil.cpp 2026-09-03 19:23:46.000000000 +0300 ++++ dl_src/src/util/sysutil.cpp 2026-10-24 22:46:07.639509900 +0300 +@@ -1,8 +1,10 @@ +-#if (!defined(_WIN32) && !defined(__aarch64__)) ++#if ((!defined(_WIN32) || defined(__GNUC__)) && !defined(__aarch64__)) + #include + #endif + #include ++#ifndef _WIN32 + #include ++#endif + #include + #include + #if defined __APPLE__ +@@ -20,6 +22,55 @@ + + #include "../common.h" + ++#ifdef _WIN32 ++#ifndef WIN32_LEAN_AND_MEAN ++#define WIN32_LEAN_AND_MEAN ++#endif ++#ifndef NOMINMAX ++#define NOMINMAX ++#endif ++#include ++#include ++ ++static double filetime_to_sec(const FILETIME& ft) ++{ ++ return (double)(((ULONGLONG)ft.dwHighDateTime << 32) | ft.dwLowDateTime) * 1.0e-7; ++} ++ ++#include ++ ++static std::mutex rand_mutex; ++static struct corax_random_data rand_data; ++static char rand_state[128]; /* 128 bytes = glibc's default TYPE_3 state */ ++static bool rand_seeded = false; ++ ++static void rand_seed_locked(unsigned int seed) ++{ ++ memset(&rand_data, 0, sizeof(rand_data)); ++ corax_initstate_r(seed, rand_state, sizeof(rand_state), &rand_data); ++ rand_seeded = true; ++} ++ ++void raxml_srand(unsigned int seed) ++{ ++ std::lock_guard lock(rand_mutex); ++ rand_seed_locked(seed); ++} ++ ++int raxml_rand() ++{ ++ std::lock_guard lock(rand_mutex); ++ if (!rand_seeded) ++ rand_seed_locked(1); /* as in glibc: rand() before srand() == srand(1) */ ++ int32_t r; ++ corax_random_r(&rand_data, &r); ++ return r; ++} ++#else ++int raxml_rand() { return rand(); } ++void raxml_srand(unsigned int seed) { srand(seed); } ++#endif ++ + using namespace std; + + SystemTimer systimer; +@@ -55,8 +106,13 @@ + + void * xmemalign(size_t size, size_t alignment) + { ++#ifdef _WIN32 ++ void * t = _aligned_malloc(size, alignment); ++ int err = 0; ++#else + void * t = nullptr; + int err = posix_memalign(& t, alignment, size); ++#endif + + if (err || !t) + sysutil_fatal("Unable to allocate enough memory."); +@@ -85,11 +141,18 @@ + + void sysutil_show_rusage() + { ++#ifdef _WIN32 ++ FILETIME creation, exit_time, kernel, user; ++ GetProcessTimes(GetCurrentProcess(), &creation, &exit_time, &kernel, &user); ++ fprintf(stderr, "Time: %.3fs (user)", filetime_to_sec(user)); ++ fprintf(stderr, " %.3fs (sys)", filetime_to_sec(kernel)); ++ fprintf(stderr, " Memory: %.0fMB\n", sysutil_get_memused() * 1.0e-6); ++#else + struct rusage r_usage; + getrusage(RUSAGE_SELF, & r_usage); +- ++ + fprintf(stderr, +- "Time: %.3fs (user)", ++ "Time: %.3fs (user)", + r_usage.ru_utime.tv_sec * 1.0 + (double)r_usage.ru_utime.tv_usec * 1.0e-6); + fprintf(stderr, + " %.3fs (sys)", +@@ -102,10 +165,17 @@ + /* Linux: ru_maxrss gives the size in kilobytes */ + fprintf(stderr, " Memory: %.0fMB\n", r_usage.ru_maxrss * 1.0e-3); + #endif ++#endif + } + +-unsigned long sysutil_get_memused() ++unsigned long long sysutil_get_memused() + { ++#ifdef _WIN32 ++ PROCESS_MEMORY_COUNTERS pmc; ++ if (GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc))) ++ return (unsigned long long) pmc.PeakWorkingSetSize; ++ return 0; ++#else + struct rusage r_usage; + getrusage(RUSAGE_SELF, & r_usage); + +@@ -116,11 +186,25 @@ + /* Linux: ru_maxrss gives the size in kilobytes */ + return (unsigned long)r_usage.ru_maxrss * 1024; + #endif ++#endif + } + +-unsigned long sysutil_get_memtotal(bool ignore_errors) ++unsigned long long sysutil_get_memtotal(bool ignore_errors) + { +-#if defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE) ++#if defined(_WIN32) ++ ++ MEMORYSTATUSEX status; ++ status.dwLength = sizeof(status); ++ if (!GlobalMemoryStatusEx(&status)) ++ { ++ if (ignore_errors) ++ return 0; ++ else ++ throw runtime_error("Cannot determine amount of RAM"); ++ } ++ return status.ullTotalPhys; ++ ++#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE) + + long phys_pages = sysconf(_SC_PHYS_PAGES); + long pagesize = sysconf(_SC_PAGESIZE); +@@ -178,7 +262,7 @@ + // not supported + RAXML_UNUSED(out); + RAXML_UNUSED(x); +-#elif defined(_WIN32) ++#elif defined(_WIN32) && !defined(__GNUC__) + __cpuid(out, x); + #else + __cpuid_count(x, 0, out[0], out[1], out[2], out[3]); +@@ -295,7 +379,11 @@ + + long sysutil_online_cpu_cores() + { ++#ifdef _WIN32 ++ return (long) std::thread::hardware_concurrency(); ++#else + return sysconf(_SC_NPROCESSORS_ONLN); ++#endif + } + + unsigned long sysutil_get_cpu_features() +@@ -314,7 +402,7 @@ + int32_t nIds = info[0]; + + get_cpuid(info, 0x80000000); +- u_int32_t nExIds = info[0]; ++ uint32_t nExIds = info[0]; + + // Detect Features + if (nIds >= 0x00000001) +@@ -463,7 +551,16 @@ + + std::string sysutil_realpath(const std::string& path) + { ++#ifdef _WIN32 ++ // _fullpath does not check existence: mimic realpath's ENOENT ++ char * real_path = nullptr; ++ if (sysutil_file_exists(path) || sysutil_dir_exists(path)) ++ real_path = _fullpath(nullptr, path.c_str(), 0); ++ else ++ errno = ENOENT; ++#else + char * real_path = realpath(path.c_str(), nullptr); ++#endif + if (real_path) + { + const string result(real_path); From dc907cb3c690eb12f56877d80593c7a6a376acce Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sun, 25 Oct 2026 00:10:32 +0300 Subject: [PATCH 11/36] Keep .patch files LF so they apply to upstream sources on Windows checkouts core.autocrlf would turn the RAxML-NG patch into CRLF on checkout, and a CRLF patch no longer applies to the LF upstream sources (verified). Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..721c5e1 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# patches are applied to upstream LF sources: never convert their line endings +*.patch text eol=lf From 4f10619e692a08bb5e3f2854780fc2960dc7d6d2 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sun, 25 Oct 2026 00:21:14 +0300 Subject: [PATCH 12/36] Add the Foundation implementation plan (15 tasks) Co-Authored-By: Claude Opus 5.5 --- .../plans/2026-10-25-foundation.md | 658 ++++++++++++++++++ 1 file changed, 658 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-25-foundation.md diff --git a/docs/superpowers/plans/2026-10-25-foundation.md b/docs/superpowers/plans/2026-10-25-foundation.md new file mode 100644 index 0000000..02547ea --- /dev/null +++ b/docs/superpowers/plans/2026-10-25-foundation.md @@ -0,0 +1,658 @@ +# Фундамент — план реализации + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** вынести логику phymlGUI в ядро без Qt (пути, настройки, история, пресеты, выравнивание, движки, запуск), оставить в `phyml_ui` тонкую оболочку, перевести тесты на pytest и запустить их в CI. + +**Architecture:** `phyml_core` — чистый Python, общается с интерфейсом через колбэки и dataclass-результаты; `phyml_ui` собирает окно из небольших модулей и оборачивает `PipelineRunner` в `QThread`. Переход идёт мелкими шагами: после каждой задачи приложение работает, а `python -m pytest` зелёный. + +**Tech Stack:** Python ≥ 3.11, PySide6 ≥ 6.5, Biopython, psutil, openpyxl, matplotlib; pytest ≥ 8, pytest-qt ≥ 4.4; GitHub Actions `windows-latest`. + +**Spec:** [docs/superpowers/specs/2026-10-25-foundation-design.md](../specs/2026-10-25-foundation-design.md) + +## Global Constraints + +- `phyml_core` не импортирует PySide6 — ни напрямую, ни транзитивно. +- Python ≥ 3.11; матрица CI — 3.11 и 3.13. +- Параметры прогона — плоский словарь с ключами нынешнего `phyml_config.json`; существующие конфиги и пресеты продолжают загружаться. +- Все видимые пользователю строки — через `_t(key)`; каждый новый ключ есть и в `en`, и в `ru` файла `locales.json` (тексты — в разделе «Новые строки локализации»). +- Файлы данных — `phyml_config.json` и `phyml_history.json` в `AppPaths.data_dir`; переопределение — переменная окружения `PHYMLGUI_DATA_DIR`. +- Тесты не читают и не пишут настоящие `phyml_config.json` / `phyml_history.json` пользователя. +- Внешний вид и раскладка окна не меняются. +- Работа идёт в ветке `foundation`; сообщения коммитов — английское предложение в стиле истории репозитория, в конце `Co-Authored-By: Claude Opus 5.5 `. +- Команда тестов: `python -m pytest` из корня репозитория. + +## Review Focus + +1. **Кириллица и пробелы в путях** (папка данных, FASTA, исполняемые файлы, например `D:\Мои данные\проект 1\`) — всё работает. Тесты: Task 3 (`test_to_stored_roundtrip_cyrillic`), Task 9 (`test_run_in_cyrillic_dir_with_spaces`). +2. **Старый или испорченный конфиг** (нет части ключей, числа вместо строк, `"recent": null`, неизвестные ключи) — загружается с умолчаниями и приведением типов, окно открывается. Тест: Task 3 (`test_load_fills_missing_keys_and_coerces_types`). +3. **Папка приложения недоступна для записи / текущая папка = папка данных** — откат в `%APPDATA%\phymlGUI`, миграция не копирует файл сам в себя. Тесты: Task 3 (`test_detect_falls_back_to_appdata`, `test_migrate_legacy_noop_when_cwd_is_data_dir`). +4. **Отмена во время MUSCLE и посреди пакета** — дерево процессов текущего шага завершено, остальные файлы получают `cancelled`, исключений нет. Тест: Task 9 (`test_cancel_during_alignment_marks_rest_cancelled`). +5. **FASTA с UTF-8 BOM / CRLF и полностью одинаковыми ID** — BOM и CRLF допускаются, дубликаты сообщаются как коллизия. Тесты: Task 8 (`test_read_fasta_tolerates_bom_and_crlf`, `test_find_collisions_reports_exact_duplicates`). + +--- + +## Файловая структура + +| Файл | Действие | Ответственность | +|---|---|---| +| `pytest.ini`, `requirements-dev.txt` | создать | настройки pytest, маркеры, dev-зависимости | +| `tests/conftest.py` | создать | offscreen-Qt, изоляция данных, поиск бинарников, маркеры | +| `tests/data/*` | создать | маленькие фикстуры: деревья CytB, файлы статистики | +| `tests/unit/`, `tests/gui/`, `tests/e2e/` | создать | новые наборы тестов | +| `tests/*.py` (старые скрипты) | удалить | заменены pytest-наборами | +| `phyml_core/paths.py` | создать | `AppPaths` | +| `phyml_core/settings.py` | создать | схема конфига, load/save, миграция | +| `phyml_core/history.py` | создать | `HistoryStore`, формат записи | +| `phyml_core/presets.py` | создать | встроенные и пользовательские пресеты | +| `phyml_core/alignment.py` | создать | FASTA, выравненность, очистка ID, PHYLIP, сводка | +| `phyml_core/launchers.py` | создать | `Native`, `Mpi` | +| `phyml_core/engines/{__init__,base,phyml,iqtree}.py` | создать | интерфейс движка, PhyML, IQ-TREE, реестр | +| `phyml_core/runner.py` | создать | `PipelineRunner`, `FileResult`, `RunResult`, `kill_process_tree` | +| `phyml_core/report.py` | создать | сводка CSV/XLSX (сейчас в окне) | +| `phyml_core/pipeline.py` | удалить (Task 11) | заменён `runner.py` | +| `phyml_core/cli.py` | переписать (Task 12) | CLI поверх runner, без Qt | +| `phyml_ui/worker.py` | создать | `RunWorker(QThread)` | +| `phyml_ui/log_console.py`, `settings_dialog.py`, `status_bar.py`, `params_panel.py`, `nav_panel.py` | создать | части окна | +| `phyml_ui/main_window.py` | сократить до ≤ ~350 строк | сборка окна и связи | +| `phymlgui.py` | упростить | только точка входа, Qt импортируется лениво | +| `packaging/third_party.json`, `scripts/fetch_third_party.py` | создать | закреплённые IQ-TREE/MUSCLE для CI | +| `.github/workflows/tests.yml`, `README.md`, `.gitignore` | изменить | CI, документация, `/bin/` | + +## Новые строки локализации + +Добавляются в задачах, которые их используют (указано в скобках); тексты зафиксированы здесь. + +| Ключ | en | ru | +|---|---|---| +| `settings_migrated` (T5) | `[System] Settings copied from {} to {}.` | `[Система] Настройки скопированы из {} в {}.` | +| `err_settings_broken` (T3) | `[ERROR] Settings file is damaged; defaults loaded, a copy is saved as {}.` | `[ОШИБКА] Файл настроек повреждён: загружены настройки по умолчанию, копия сохранена как {}.` | +| `err_settings_save` (T5) | `[ERROR] Cannot save settings: {}` | `[ОШИБКА] Не удалось сохранить настройки: {}` | +| `err_history_broken` (T4) | `[ERROR] Run history file is damaged and was ignored: {}` | `[ОШИБКА] Файл истории запусков повреждён и пропущен: {}` | +| `val_not_number` (T6) | `Parameter "{}" must be a positive number, got "{}".` | `Параметр «{}» должен быть положительным числом, получено «{}».` | +| `val_no_exe` (T6) | `{} executable not found: {}` | `Не найден исполняемый файл {}: {}` | +| `val_no_mpiexec` (T6) | `MPI is enabled but mpiexec was not found: {}` | `MPI включён, но mpiexec не найден: {}` | +| `err_engine_exit` (T9) | `[CRITICAL ERROR] {} exited with code {}.` | `[КРИТИЧЕСКАЯ ОШИБКА] {} завершил работу с кодом {}.` | +| `err_no_tree` (T9) | `[ERROR] {} finished but the tree file is missing: {}` | `[ОШИБКА] {} завершился, но файл дерева не найден: {}` | +| `err_id_collision` (T8) | `[ERROR] Sequence IDs collide after cleaning: {}` | `[ОШИБКА] После очистки совпали ID последовательностей: {}` | + +--- + +### Task 1: pytest-инфраструктура и перенос проверок без бинарников + +**Files:** +- Create: `pytest.ini`, `requirements-dev.txt`, `tests/conftest.py` +- Create: `tests/data/cytb_tree.nwk`, `tests/data/cytb_boot_trees.nwk`, `tests/data/x_phyml_stats.txt`, `tests/data/x.iqtree` +- Create: `tests/unit/test_i18n.py`, `tests/unit/test_stats_parsers.py`, `tests/unit/test_rf.py`, `tests/unit/test_pipeline_regressions.py` +- Create: `tests/gui/test_themes.py`, `tests/gui/test_log_console.py`, `tests/gui/test_tree_viewer.py`, `tests/gui/test_main_window_smoke.py` +- Delete: `tests/test_smoke.py`, `tests/test_tree_viewer.py` + +**Interfaces:** +- Produces (`tests/conftest.py`): `ROOT: Path`, `DATA: Path` (= `ROOT/'tests'/'data'`); `find_binary(kind: str) -> str | None` для `kind in ('phyml', 'mpiexec', 'iqtree', 'muscle', 'raxml')`: сначала переменные `PHYMLGUI_PHYML` / `PHYMLGUI_MPIEXEC` / `PHYMLGUI_IQTREE` / `PHYMLGUI_MUSCLE` / `PHYMLGUI_RAXML`, затем `ROOT/'bin'/{phyml-mpi.exe, mpiexec.exe, iqtree2.exe, muscle.exe, raxml-ng.exe}`, затем нынешние локальные пути (для `raxml` локального пути нет — он понадобится в «Науке») (`ROOT/'phymlgui v1.1'/'phyml-mpi.exe'`, `C:\Program Files\Microsoft MPI\Bin\mpiexec.exe`, `ROOT/'tools'/'iqtree2'/'iqtree2.exe'`, `ROOT/'tools'/'muscle.exe'`); фикстура `binaries -> dict[str, str | None]`; фикстура `isolated_config(tmp_path, monkeypatch)` — подменяет `phyml_ui.main_window.CONFIG_FILE` и `HISTORY_FILE` путями в `tmp_path` (временная мера, Task 5 заменит её на `data_dir`); session-autouse фикстура `guard_user_files` — SHA256 `ROOT/phyml_config.json` и `ROOT/phyml_history.json` (если есть) до сессии и `assert` равенства после. +- Produces (`pytest.ini`): маркеры `needs_phyml`, `needs_mpi`, `needs_iqtree`, `needs_muscle`, `slow`; `pytest_runtest_setup` пропускает тест с маркером `needs_X`, если `find_binary(X)` вернул `None`, с причиной `" not found (set PHYMLGUI_)"`. + +- [ ] **Step 1: Создать `requirements-dev.txt` (`pytest>=8`, `pytest-qt>=4.4`) и `pytest.ini`** + +```ini +[pytest] +testpaths = tests +qt_api = pyside6 +markers = + needs_phyml: requires a PhyML (MPI) executable + needs_mpi: requires MS-MPI mpiexec + needs_iqtree: requires an IQ-TREE executable + needs_muscle: requires a MUSCLE executable + slow: long-running end-to-end test +``` + +- [ ] **Step 2: Написать `tests/conftest.py`** + +Первая строка модуля — `os.environ.setdefault('QT_QPA_PLATFORM', 'offscreen')`, до любого импорта Qt; `sys.path.insert(0, str(ROOT))`. Остальное — по блоку Interfaces. + +- [ ] **Step 3: Положить фикстуры данных** + +```bash +mkdir -p tests/data +cp testrun/t9_real_ncbi/cytb_mammals_aligned.phy_phyml_tree.txt tests/data/cytb_tree.nwk +cp testrun/t9_real_ncbi/cytb_mammals_aligned.phy_phyml_boot_trees.txt tests/data/cytb_boot_trees.nwk +``` + +`x_phyml_stats.txt` и `x.iqtree` — ровно те строки, что сейчас пишет `tests/test_smoke.py` в разделе 3. + +- [ ] **Step 4: Перенести проверки один к одному** + +Логика проверок не меняется; меняется только источник данных (просмотрщик читает `tests/data/`, а не `testrun/`, поэтому больше не пропускается). + +| Было (PASS-строка) | Стало | +|---|---| +| smoke 1 imports and i18n | `unit/test_i18n.py::test_locales_have_same_keys` | +| smoke 2 themes | `gui/test_themes.py::test_resolve_qss_and_role_colors` | +| smoke 3 stats parsers | `unit/test_stats_parsers.py::test_parse_phyml_stats`, `::test_parse_iqtree_stats` | +| smoke 4 rf | `unit/test_rf.py::test_rf_identical_and_quartet_swap`, `::test_tree_splits_count` | +| smoke 5 log console | `gui/test_log_console.py::test_roles_and_plain_text` | +| smoke 6 main window | `gui/test_main_window_smoke.py::test_window_builds(isolated_config)` | +| smoke 7 empty fasta | `unit/test_pipeline_regressions.py::test_empty_fasta_fails_cleanly` | +| smoke 8 ufboot | `unit/test_pipeline_regressions.py::test_ufboot_replicates_raised_to_1000` | +| viewer 1b circle polytomy | `gui/test_tree_viewer.py::test_circle_polytomy_centered` | +| viewer 2 layouts | `::test_layouts_render` | +| viewer 2b grid | `::test_grid_drawn_and_toggleable` | +| viewer 2c theme colors | `::test_theme_colors` | +| viewer 3 options | `::test_threshold_grid_supports_options` | +| viewer 4 collapse | `::test_collapse_and_expand` | +| viewer 5 outgroup | `::test_outgroup_reroot_and_reset` | +| viewer 6 consensus | `::test_majority_consensus_from_boot_trees` (8 листьев, 20 деревьев из `cytb_boot_trees.nwk`) | + +Тесты окна используют фикстуру `qtbot` из pytest-qt вместо ручного `QApplication([])`. + +- [ ] **Step 5: Запустить и сверить** + +Run: `python -m pytest tests/unit tests/gui -q` +Expected: `18 passed` (16 строк таблицы; строки smoke 3 и smoke 4 дают по два теста), без skip; `guard_user_files` не падает. + +- [ ] **Step 6: Удалить `tests/test_smoke.py`, `tests/test_tree_viewer.py`; временно поменять шаг CI на `python -m pytest tests/unit tests/gui` с установкой `requirements-dev.txt`; коммит** + +```bash +git add pytest.ini requirements-dev.txt tests .github/workflows/tests.yml +git commit -m "Move binary-free test checks to pytest with fixtures in tests/data" +``` + +--- + +### Task 2: перенос e2e- и GUI-проверок, которым нужны бинарники + +**Files:** +- Create: `tests/e2e/datasets.py`, `tests/e2e/fetch_real_data.py` (перенос `tests/fetch_real_data.py` без изменения логики; кэш остаётся в `testrun/real`) +- Create: `tests/e2e/test_phyml_pipeline.py`, `tests/e2e/test_batch.py`, `tests/e2e/test_iqtree.py`, `tests/e2e/test_cpu_monitor.py`, `tests/gui/test_main_window.py` +- Delete: `tests/run_tests.py`, `tests/test_gui_logic.py`, `tests/test_batch.py`, `tests/test_iqtree.py`, `tests/fetch_real_data.py` +- Modify: `tests/README.md` + +**Interfaces:** +- Consumes: фикстуры и маркеры Task 1. +- Produces (`tests/e2e/datasets.py`): `make_dataset() -> tuple[list[str], list[str], list[str]]` (имена, выровненные, невыровненные; `random.Random(42)`, как сейчас), `write_fasta(path, names, seqs) -> None`, фикстура `synthetic(tmp_path) -> dict[str, Path]` с ключами `'aligned'`, `'unaligned'`. + +- [ ] **Step 1: Перенести проверки; рабочие папки — `tmp_path`, не `testrun/`** + +| Было | Стало | Маркеры | +|---|---|---| +| run_tests t1–t4 | `e2e/test_phyml_pipeline.py::test_scenarios[t1..t4]` (параметризация) | `needs_phyml`, `needs_muscle`, `needs_mpi` для t2/t4 | +| run_tests t5 | `::test_serial_bootstrap_on_mpi_build_fails_gracefully` | `needs_phyml`, `needs_muscle` | +| run_tests t9 | `::test_real_cytb_tree` (пропуск, если данных нет и сеть недоступна) | `needs_phyml`, `needs_mpi`, `needs_muscle`, `slow` | +| gui_logic 2 defaults | `gui/test_main_window.py::test_defaults_match_detected_cores` — **сравнение с `detect_cores()`**, без 8/16 | — | +| gui_logic 3 mpi autodetect | `::test_mpi_binary_autodetect` | — | +| gui_logic 4 validation | `::test_serial_bootstrap_on_mpi_build_blocked` | — | +| gui_logic 5 cpu monitor | `e2e/test_cpu_monitor.py::test_cpu_samples_during_mpi_run` | `needs_phyml`, `needs_mpi`, `needs_muscle` | +| gui_logic 6 new UI | `gui/test_main_window.py::test_theme_tabs_presets_log_filter_recent` | — | +| batch 1 status bar | `gui/test_main_window.py::test_cpu_bar_expands` | — | +| batch 2 empty folder | `gui/test_main_window.py::test_empty_folder_blocked` | — | +| batch 3 two files | `e2e/test_batch.py::test_two_file_batch` | `needs_phyml`, `needs_mpi`, `needs_muscle` | +| batch 4 csv via GUI | `e2e/test_batch.py::test_gui_batch_writes_summary_csv` | то же | +| batch 5 stage3 | `e2e/test_batch.py::test_queue_history_open_folder` | то же | +| iqtree 1 engine | `e2e/test_iqtree.py::test_mfp_ufboot_run` | `needs_iqtree` | +| iqtree 1b 100 reps | `::test_100_replicates_raised_to_1000` | `needs_iqtree` | +| iqtree 2 GUI | `::test_gui_validation_and_run_with_xlsx` | `needs_iqtree` | +| iqtree 3 models | `::test_every_listed_model_accepted` | `needs_iqtree` | + +Все GUI-тесты берут фикстуру `isolated_config`. + +- [ ] **Step 2: Прогнать всё локально** + +Run: `python -m pytest -q -rs` +Expected: `38 passed` (18 из Task 1 + 20 из таблицы: строка t1–t4 даёт четыре теста, остальные 16 строк — по одному), `0 skipped` на этой машине; `guard_user_files` проходит. + +- [ ] **Step 3: Удалить старые скрипты, обновить `tests/README.md` (как запускать, переменные `PHYMLGUI_*`, маркеры); коммит** + +```bash +git add tests +git commit -m "Move end-to-end and GUI suites to pytest with binary markers" +``` + +--- + +### Task 3: `AppPaths` и `settings` + +**Files:** +- Create: `phyml_core/paths.py`, `phyml_core/settings.py` +- Test: `tests/unit/test_paths.py`, `tests/unit/test_settings.py` +- Modify: `locales.json` (`err_settings_broken`) + +**Interfaces:** +- Produces (`phyml_core/paths.py`): + - `CONFIG_NAME = 'phyml_config.json'`, `HISTORY_NAME = 'phyml_history.json'` + - `default_app_dir() -> str` — `dirname(sys.executable)` при `sys.frozen`, иначе корень репозитория (родитель пакета `phyml_core`) + - `@dataclass(frozen=True) class AppPaths: app_dir: str; data_dir: str` + - свойства `bin_dir -> str` (`app_dir/bin`), `config_file -> str`, `history_file -> str` + - `resource(name: str) -> str` — `sys._MEIPASS/name`, если существует, иначе `app_dir/name` + - `to_stored(path: str) -> str` — относительный путь, если `path` внутри `app_dir`, иначе абсолютный; `''` → `''` + - `from_stored(stored: str) -> str` — относительный → `join(app_dir, stored)`; абсолютный и `''` без изменений + - `find_tool(stored: str, names: tuple[str, ...]) -> str` — `from_stored(stored)`, если файл существует; иначе первый существующий `bin_dir/name`; иначе `from_stored(stored)` (чтобы сообщение об ошибке показало настроенный путь) + - `@classmethod detect(cls, env: Mapping[str, str] | None = None, is_writable: Callable[[str], bool] | None = None, app_dir: str | None = None) -> AppPaths` — порядок: `env['PHYMLGUI_DATA_DIR']` → `app_dir`, если `is_writable(app_dir)` → `env['APPDATA']/phymlGUI`; выбранная папка создаётся; `is_writable` по умолчанию — создание и удаление временного файла. +- Produces (`phyml_core/settings.py`): + - `SCHEMA_VERSION = 2` + - `DEFAULTS: dict` — `engine 'phyml'`, `iqtree_path ''`, `phyml_path ''`, `mpirun_path 'mpiexec'`, `muscle_path ''`, `datatype 'nt'`, `model 'HKY85'`, `support_method 'bootstrap'`, `bootstrap_replicates '100'`, `use_mpi False`, `mpi_np ''`, `freq 'default'`, `tstv 'e'`, `pinv '0.0'`, `classes '4'`, `gamma 'e'`, `search 'NNI'`, `optimize 'tlr'`, `parsimony False`, `show_tree True`, `theme 'system'`, `accent 'green'`, `recent []`, `presets {}`, `window {}`, `schema_version 2` + - `PATH_KEYS = ('phyml_path', 'iqtree_path', 'muscle_path', 'mpirun_path')` + - `load(path: str, on_error: Callable[[str], None] | None = None) -> dict` — нет файла → копия `DEFAULTS`; битый JSON → копия `DEFAULTS`, исходный текст сохраняется в `path + '.broken'`, `on_error(_t('err_settings_broken').format(path + '.broken'))`; иначе умолчания ← файл с приведением типов к типу умолчания (`str` ← число через `str()`; `bool` ← только `bool`, иначе умолчание; `list`/`dict` ← только того же типа, иначе умолчание); неизвестные ключи сохраняются; `schema_version` = 2 + - `save(path: str, cfg: dict) -> None` — JSON `indent=4, ensure_ascii=False`, UTF-8, временный файл в той же папке + `os.replace`; `OSError` пробрасывается + - `to_disk(cfg: dict, paths: AppPaths) -> dict` / `from_disk(cfg: dict, paths: AppPaths) -> dict` — `to_stored`/`from_stored` для `PATH_KEYS` и элементов `recent` + - `migrate_legacy(paths: AppPaths, cwd: str) -> list[str]` — если `realpath(cwd) != realpath(data_dir)`, в `data_dir` нет `CONFIG_NAME`, а в `cwd` есть `CONFIG_NAME` и/или `HISTORY_NAME` — копирует найденные (`shutil.copy2`), возвращает их имена; иначе `[]` + +- [ ] **Step 1: Написать падающие тесты** + +```python +# tests/unit/test_paths.py +def test_detect_uses_env_override(tmp_path): + p = AppPaths.detect(env={'PHYMLGUI_DATA_DIR': str(tmp_path / 'd')}, app_dir=str(tmp_path)) + assert p.data_dir == str(tmp_path / 'd') and os.path.isdir(p.data_dir) + +def test_detect_prefers_writable_app_dir(tmp_path): + p = AppPaths.detect(env={}, is_writable=lambda d: True, app_dir=str(tmp_path)) + assert p.data_dir == str(tmp_path) + +def test_detect_falls_back_to_appdata(tmp_path): + p = AppPaths.detect(env={'APPDATA': str(tmp_path / 'ad')}, is_writable=lambda d: False, + app_dir=str(tmp_path / 'app')) + assert p.data_dir == str(tmp_path / 'ad' / 'phymlGUI') and os.path.isdir(p.data_dir) + +def test_to_stored_roundtrip_cyrillic(tmp_path): + app = tmp_path / 'Мои программы' / 'phymlGUI' + p = AppPaths(app_dir=str(app), data_dir=str(app)) + inside = str(app / 'bin' / 'muscle.exe') + assert p.to_stored(inside) == os.path.join('bin', 'muscle.exe') + assert p.from_stored(p.to_stored(inside)) == inside + outside = str(tmp_path / 'данные' / 'a.fasta') + assert p.to_stored(outside) == outside and p.to_stored('') == '' + +def test_find_tool_prefers_stored_then_bin(tmp_path) + # stored существует → он; stored не существует, bin/muscle.exe есть → bin; ничего нет → from_stored(stored) + +def test_resource_prefers_meipass(tmp_path, monkeypatch) + # sys._MEIPASS с locales.json → путь внутри _MEIPASS; без _MEIPASS → app_dir/locales.json +``` + +```python +# tests/unit/test_settings.py +def test_load_missing_file_returns_defaults(tmp_path) # == DEFAULTS, не тот же объект +def test_load_fills_missing_keys_and_coerces_types(tmp_path): + path = tmp_path / 'c.json' + path.write_text(json.dumps({'mpi_np': 8, 'recent': None, 'use_mpi': 'yes', 'custom_key': 1}), 'utf-8') + cfg = settings.load(str(path)) + assert cfg['mpi_np'] == '8' and cfg['recent'] == [] and cfg['use_mpi'] is False + assert cfg['custom_key'] == 1 and cfg['model'] == 'HKY85' and cfg['schema_version'] == 2 +def test_load_broken_json_keeps_copy_and_reports(tmp_path) # '{oops' → DEFAULTS, on_error вызван 1 раз, текст содержит '.broken', файл .broken == '{oops' +def test_save_is_atomic_and_roundtrips(tmp_path) # save → load равны; в папке только c.json +def test_save_raises_oserror_for_directory_target(tmp_path) # path — папка → pytest.raises(OSError) +def test_to_disk_from_disk_roundtrip(tmp_path) # phyml_path внутри app_dir → относительный на диске, абсолютный после from_disk +def test_migrate_legacy_copies_once(tmp_path) # оба файла скопированы, оригиналы на месте; второй вызов → [] +def test_migrate_legacy_noop_when_cwd_is_data_dir(tmp_path) # → [] и файл не тронут +``` + +- [ ] **Step 2: Run `python -m pytest tests/unit/test_paths.py tests/unit/test_settings.py -q`** — Expected: FAIL (`ModuleNotFoundError: phyml_core.paths`) +- [ ] **Step 3: Реализовать `phyml_core/paths.py` и `phyml_core/settings.py` по Interfaces; добавить ключ `err_settings_broken`** +- [ ] **Step 4: Run тот же набор и `python -m pytest -q`** — Expected: PASS, остальные тесты не изменились +- [ ] **Step 5: Коммит** — `git commit -m "Add AppPaths and settings with atomic save, type coercion and legacy migration"` + +--- + +### Task 4: `HistoryStore` и пресеты + +**Files:** +- Create: `phyml_core/history.py`, `phyml_core/presets.py` +- Test: `tests/unit/test_history.py`, `tests/unit/test_presets.py` +- Modify: `locales.json` (`err_history_broken`) + +**Interfaces:** +- Produces (`phyml_core/history.py`): + - `class HistoryStore: __init__(self, path: str, limit: int = 200)`; `load(self, on_error=None) -> list[dict]` (нет файла → `[]`; битый → `[]` и `on_error(_t('err_history_broken').format(path))`); `append(self, record: dict) -> None` (новые сверху, обрезка до `limit`, атомарная запись как в `settings.save`, `OSError` пробрасывается); `get(self, index: int) -> dict | None` + - `new_record(*, inputs: list[str], engine: str, params: dict, files: list[dict], duration_s: float, ts: datetime | None = None) -> dict` — ключи `ts` (`'%Y-%m-%d %H:%M'`), `inputs`, `engine`, `params`, `files` (элементы `{'file', 'status', 'tree', 'stats', 'error'}`), `ok` (число `status == 'ok'`), `n` (`len(files)`), `duration_s` + - `display_input(rec: dict) -> str` — новая запись: basename `inputs[0]` (+ `'/'` для папки; при нескольких файлах — basename их общей папки + `'/'`); старая: `rec.get('input', '')` + - `best_tree(rec: dict) -> str` — новая: дерево последнего файла со статусом `ok`; старая: `rec.get('tree', '')` +- Produces (`phyml_core/presets.py`): + - `PRESET_KEYS` — нынешние 12 ключей `_PRESET_KEYS` + `'engine'` + - `BUILTIN: dict[str, dict]` — `'preset_builtin_screen'` и `'preset_builtin_pub'` с нынешними значениями `_BUILTIN_PRESETS` + `'engine': 'phyml'` + - `extract(params: dict) -> dict` — только `PRESET_KEYS` + - `apply(preset: dict, params: dict) -> dict` — новый словарь: `params`, обновлённый ключами пресета из `PRESET_KEYS` (движок тоже) + +- [ ] **Step 1: Написать падающие тесты** — `test_append_keeps_newest_first_and_limit` (205 записей → 200, `load()[0]` — последняя), `test_load_corrupt_file_reports_and_returns_empty`, `test_new_record_counts_ok_and_n`, `test_old_record_display_and_tree` (`{'input': 'a.fasta', 'tree': 'x.txt', 'ok': 1, 'n': 1}` → `'a.fasta'`, `'x.txt'`), `test_new_record_best_tree_is_last_ok`, `test_extract_excludes_paths_and_display`, `test_apply_switches_engine_and_keeps_paths`, `test_builtin_screening_uses_alrt` (`BUILTIN['preset_builtin_screen']['support_method'] == '-4'`) +- [ ] **Step 2: Run `python -m pytest tests/unit/test_history.py tests/unit/test_presets.py -q`** — Expected: FAIL (модулей нет) +- [ ] **Step 3: Реализовать оба модуля; добавить ключ `err_history_broken`** +- [ ] **Step 4: Run набор и `python -m pytest -q`** — Expected: PASS +- [ ] **Step 5: Коммит** — `git commit -m "Add HistoryStore with full run params and engine-aware presets"` + +--- + +### Task 5: окно на `AppPaths` / `settings` / `HistoryStore` / `presets` + +**Files:** +- Modify: `phyml_ui/main_window.py` (удалить `CONFIG_FILE`, `HISTORY_FILE`, `_BUILTIN_PRESETS`, `_PRESET_KEYS`, `_load_history`; `load_config`/`auto_save`/`_append_history`/`_refresh_history`/`_open_history_entry`/пресеты — через новые модули) +- Modify: `phymlgui.py` (убрать реэкспорт `CONFIG_FILE`) +- Modify: `tests/conftest.py` (фикстура `data_dir` вместо `isolated_config`), все GUI/e2e-тесты, которые брали `isolated_config`; `e2e/test_batch.py::test_queue_history_open_folder` читает историю через `HistoryStore(w.paths.history_file)` вместо `HISTORY_FILE` +- Modify: `locales.json` (`settings_migrated`, `err_settings_save`) +- Test: `tests/gui/test_main_window_storage.py` + +**Interfaces:** +- Consumes: Task 3, Task 4. +- Produces: `PhyMLApp.__init__(self, paths: AppPaths | None = None)` — `None` → `AppPaths.detect()`; атрибут `self.paths`. Перед загрузкой конфига окно вызывает `settings.migrate_legacy(self.paths, os.getcwd())` и пишет `settings_migrated` в журнал для каждого скопированного файла. Ошибки `settings.save` / `HistoryStore.append` → строка `err_settings_save` в журнал. Пути инструментов и `recent` пишутся через `settings.to_disk`, читаются через `settings.from_disk`. +- Produces (`tests/conftest.py`): фикстура `data_dir(tmp_path, monkeypatch) -> Path` — `monkeypatch.setenv('PHYMLGUI_DATA_DIR', ...)` и `monkeypatch.chdir(tmp_path / 'cwd')` (пустая папка), чтобы миграция не находила настоящий конфиг в корне репозитория. + +- [ ] **Step 1: Написать падающие тесты** — `test_config_saved_in_data_dir` (сменить модель → `data_dir/phyml_config.json` содержит её; в `cwd` файла нет), `test_legacy_config_migrated_on_first_start` (в `cwd` старый конфиг с `"model": "GTR"` → окно показывает `GTR`, журнал содержит текст `settings_migrated`), `test_history_written_to_data_dir` (`w._append_history(1, 1, 'x.txt')` → `HistoryStore(paths.history_file).load()[0]['ok'] == 1`), `test_tool_path_inside_app_dir_saved_relative` (`AppPaths(app_dir=tmp/'app', data_dir=tmp/'d')`, путь PhyML `app/bin/phyml-mpi.exe` → в JSON `os.path.join('bin', 'phyml-mpi.exe')`) +- [ ] **Step 2: Run `python -m pytest tests/gui/test_main_window_storage.py -q`** — Expected: FAIL +- [ ] **Step 3: Перевести окно; заменить `isolated_config` на `data_dir` во всех тестах и удалить старую фикстуру** +- [ ] **Step 4: Run `python -m pytest -q -rs`** — Expected: всё PASS, `guard_user_files` проходит +- [ ] **Step 5: Ручная проверка** — `python phymlgui.py` из корня: настройки на месте (миграция в журнале не появляется, так как `cwd == app_dir == data_dir`); закрыть окно +- [ ] **Step 6: Коммит** — `git commit -m "Store settings and history in the data dir via AppPaths, migrate legacy files once"` + +--- + +### Task 6: способы запуска, интерфейс движка и PhyML + +**Files:** +- Create: `phyml_core/launchers.py`, `phyml_core/engines/__init__.py`, `phyml_core/engines/base.py`, `phyml_core/engines/phyml.py` +- Modify: `phyml_core/pipeline.py` (`parse_stats` → импорт `parse_phyml_stats` из `engines.phyml`), `locales.json` (`val_not_number`, `val_no_exe`, `val_no_mpiexec`) +- Test: `tests/unit/test_launchers.py`, `tests/unit/test_engine_phyml.py`; `tests/unit/test_stats_parsers.py` импортирует новые функции + +**Interfaces:** +- Produces (`phyml_core/launchers.py`): `class Native: wrap(self, argv: list[str]) -> list[str]`; `@dataclass(frozen=True) class Mpi: mpiexec: str; np: int; wrap(self, argv) -> [mpiexec, '-np', str(np), *argv]` +- Produces (`phyml_core/engines/base.py`): `Issue`, `EngineOutputs` — как в спецификации, разделы 3; `format_issue(issue: Issue) -> str` = `_t(issue.key).format(*issue.args)`; `positive_number(value: str) -> bool` (целое или дробное > 0); `class Engine(ABC)` с атрибутами `key: str`, `name: str` (для сообщений: `'PhyML'`, `'IQ-TREE'`), `label_key: str` и методами спецификации; сигнатура проверки — `validate(self, params: dict, paths: AppPaths) -> list[Issue]` +- Produces (`phyml_core/engines/phyml.py`): `parse_phyml_stats(path: str) -> dict` (бывшая `pipeline.parse_stats`), `class PhyML(Engine)`: `key='phyml'`, `name='PhyML'`, `label_key='engine_phyml'` +- Produces (`phyml_core/engines/__init__.py`): `ENGINES: dict[str, Engine] = {'phyml': PhyML()}` (IQ-TREE добавит Task 7) + +- [ ] **Step 1: Написать падающие тесты** + +```python +P = {'datatype': 'nt', 'model': 'HKY85', 'support_method': 'bootstrap', 'bootstrap_replicates': '100', + 'freq': 'default', 'tstv': 'e', 'pinv': '0.0', 'classes': '4', 'gamma': 'e', + 'search': 'NNI', 'optimize': 'tlr', 'parsimony': False} + +def test_command_matches_current_argv(): + assert PhyML().command('C:/b/phyml.exe', 'D:/d/a.phy', P) == [ + 'C:/b/phyml.exe', '-i', 'D:/d/a.phy', '-d', 'nt', '-m', 'HKY85', '-b', '100', + '-t', 'e', '-v', '0.0', '-c', '4', '-a', 'e', '-s', 'NNI', '-o', 'tlr'] + +def test_command_alrt_freq_parsimony_and_empty_values(): + argv = PhyML().command('x', 'a.phy', dict(P, support_method='-4', freq='m', parsimony=True, tstv='')) + assert argv[argv.index('-b') + 1] == '-4' and argv[argv.index('-f') + 1] == 'm' + assert '-t' not in argv and argv[-1] == '-p' + +def test_command_omits_freq_when_missing(): # dict без 'freq' → '-f' отсутствует (сейчас уходит None) +def test_launcher_mpi_and_native(): + l = PhyML().launcher(dict(P, use_mpi=True, mpirun_path='C:/MPI/mpiexec.exe', mpi_np='2')) + assert l.wrap(['x']) == ['C:/MPI/mpiexec.exe', '-np', '2', 'x'] + assert PhyML().launcher(dict(P, use_mpi=False)).wrap(['x']) == ['x'] +def test_outputs(): # tree/stats/boot_trees = 'D:/d/a.phy' + '_phyml_tree.txt' / '_phyml_stats.txt' / '_phyml_boot_trees.txt', log None +def test_validate_ok(tmp_path) # exe существует, use_mpi False, support '-4' → [] +def test_validate_missing_exe(tmp_path) # → [Issue('error', 'val_no_exe', ...)] +def test_validate_serial_bootstrap_on_mpi_build(tmp_path) # exe 'phyml-mpi.exe', use_mpi False, bootstrap → key 'mpi_boot_msg' +def test_validate_numbers(tmp_path) # bootstrap_replicates 'abc', classes '0', pinv 'x' → три 'val_not_number'; pinv 'e', gamma 'e' допустимы +def test_validate_mpiexec_missing(tmp_path) # use_mpi True, mpirun_path не существует и не находится shutil.which → 'val_no_mpiexec' +def test_executable_falls_back_to_bin(tmp_path) # phyml_path '' и app/bin/phyml-mpi.exe есть → этот путь +def test_models_unchanged() # models('nt') и models('aa') == нынешние списки update_models +``` + +- [ ] **Step 2: Run `python -m pytest tests/unit/test_launchers.py tests/unit/test_engine_phyml.py -q`** — Expected: FAIL (модулей нет) +- [ ] **Step 3: Реализовать модули; `pipeline.py` импортирует `parse_phyml_stats`; ключи локализации** +- [ ] **Step 4: Run набор и `python -m pytest -q`** — Expected: PASS +- [ ] **Step 5: Коммит** — `git commit -m "Add launchers, the Engine interface and the PhyML engine with argv tests"` + +--- + +### Task 7: движок IQ-TREE, реестр и модели из движков + +**Files:** +- Create: `phyml_core/engines/iqtree.py` +- Modify: `phyml_core/engines/__init__.py` (`ENGINES = {'phyml': PhyML(), 'iqtree': IqTree()}`), `phyml_core/pipeline.py` (импорт `parse_iqtree_stats` и `ufboot_replicates` из `engines.iqtree`), `phyml_ui/main_window.py` (`update_models` → `ENGINES[self.engine_setting].models(datatype)`) +- Test: `tests/unit/test_engine_iqtree.py`; `tests/unit/test_pipeline_regressions.py::test_ufboot_replicates_raised_to_1000` переносится сюда как `test_normalize_raises_ufboot`; `tests/e2e/test_iqtree.py::test_every_listed_model_accepted` берёт список из `ENGINES['iqtree'].models(dt)` + +**Interfaces:** +- Produces: `parse_iqtree_stats(path: str) -> dict`, `UFBOOT_MIN_REPLICATES = 1000`, `ufboot_replicates(value) -> str` (перенос без изменений), `class IqTree(Engine)`: `key='iqtree'`, `name='IQ-TREE'`, `label_key='engine_iqtree'`; префикс вывода — `phylip.rsplit('.', 1)[0] + '_iqtree'` + +- [ ] **Step 1: Написать падающие тесты** + +```python +Q = {'model': 'MFP', 'mpi_np': '2', 'support_method': 'bootstrap', 'bootstrap_replicates': '1000'} + +def test_command_ufboot(): + assert IqTree().command('C:/b/iqtree2.exe', 'D:/d/a.phy', Q) == [ + 'C:/b/iqtree2.exe', '-s', 'D:/d/a.phy', '--prefix', 'D:/d/a_iqtree', '-redo', + '-m', 'MFP', '-T', '2', '-B', '1000'] +def test_command_alrt_and_none(): # '-4' → хвост ['--alrt', '1000']; '0' → после ['-T', '2'] ничего +def test_normalize_raises_ufboot(): + p, notes = IqTree().normalize(dict(Q, bootstrap_replicates='100')) + assert p['bootstrap_replicates'] == '1000' and notes == [Issue('warning', 'iqtree_ufboot_min', ('1000', '100'))] + assert IqTree().normalize(dict(Q, bootstrap_replicates='5000')) == (dict(Q, bootstrap_replicates='5000'), []) +def test_validate(tmp_path) # нет exe → 'val_no_exe'; mpi_np 'x' → 'val_not_number'; всё верно → [] +def test_outputs() # tree 'D:/d/a_iqtree.treefile', stats '.iqtree', boot_trees None, log '.log' +def test_launcher_is_native() +def test_models_list_has_hky_not_hky2() # 'HKY+F+G4' in models('nt'), 'HKY2+F+G4' not in +def test_registry_keys() # list(ENGINES) == ['phyml', 'iqtree'] +``` + +- [ ] **Step 2: Run `python -m pytest tests/unit/test_engine_iqtree.py -q`** — Expected: FAIL +- [ ] **Step 3: Реализовать; окно берёт модели из реестра; `pipeline.py` импортирует перенесённые функции** +- [ ] **Step 4: Run `python -m pytest -q -rs`** — Expected: PASS (включая `e2e/test_iqtree.py` локально) +- [ ] **Step 5: Коммит** — `git commit -m "Add the IQ-TREE engine and an engine registry; the model list comes from engines"` + +--- + +### Task 8: `alignment.py` + +**Files:** +- Create: `phyml_core/alignment.py` +- Modify: `phyml_ui/main_window.py` (`_update_fasta_info` → `alignment.summarize`), `locales.json` (`err_id_collision`) +- Test: `tests/unit/test_alignment.py` + +**Interfaces:** +- Produces: + - `class AlignmentError(ValueError)` + - `read_fasta(path: str) -> list[SeqRecord]` — открывает с `encoding='utf-8-sig'` (BOM), CRLF допустим; пустой или без записей → `AlignmentError(_t('err_empty'))` + - `is_aligned(records: list[SeqRecord]) -> bool` + - `clean_id(raw: str) -> str` — `re.sub(r'[.|:;(),\s]', '_', raw)` (как сейчас) + - `find_collisions(ids: list[str]) -> dict[str, list[str]]` — очищенный ID → исходные, только где исходных больше одного (включая полные дубликаты) + - `write_phylip(records: list[SeqRecord], out_path: str) -> None` — `.` → `-` в последовательностях, ID через `clean_id`, формат `phylip-relaxed`; при коллизиях — `AlignmentError(_t('err_id_collision').format('; '.join(f"{', '.join(orig)} → {clean}" ...)))` + - `summarize(path: str) -> dict` — `{'n', 'min_len', 'max_len', 'gap_pct'}` (как в нынешнем `_update_fasta_info`) + - `aligned_path(fasta: str) -> str` (`rsplit('.', 1)[0] + '_aligned.fasta'`), `phylip_path(fasta: str) -> str` (`rsplit('.', 1)[0] + '.phy'`) + +- [ ] **Step 1: Написать падающие тесты** — `test_read_fasta_empty_raises`, `test_read_fasta_tolerates_bom_and_crlf` (байты `b'\xef\xbb\xbf>A\r\nACGT\r\n>B\r\nACGA\r\n'` → ID `['A', 'B']`, `str(seq) == 'ACGT'`), `test_is_aligned`, `test_clean_id` (`'Homo sapiens|x:1'` → `'Homo_sapiens_x_1'`), `test_find_collisions` (`['A.1', 'A_1', 'B']` → `{'A_1': ['A.1', 'A_1']}`), `test_find_collisions_reports_exact_duplicates` (`['X', 'X']` → `{'X': ['X', 'X']}`), `test_write_phylip_cleans_ids_and_dots` (чтение через `AlignIO` возвращает очищенные ID и `-` вместо `.`), `test_write_phylip_raises_on_collision` (текст содержит `A.1` и `A_1`), `test_summarize`, `test_path_helpers` +- [ ] **Step 2: Run `python -m pytest tests/unit/test_alignment.py -q`** — Expected: FAIL +- [ ] **Step 3: Реализовать; окно использует `summarize`; ключ `err_id_collision`** +- [ ] **Step 4: Run `python -m pytest -q`** — Expected: PASS +- [ ] **Step 5: Коммит** — `git commit -m "Add alignment helpers with BOM-tolerant reading and ID collision detection"` + +--- + +### Task 9: `PipelineRunner` + +**Files:** +- Create: `phyml_core/runner.py` +- Test: `tests/unit/fakes.py`, `tests/unit/test_runner.py`; `test_pipeline_regressions.py::test_empty_fasta_fails_cleanly` переносится сюда +- Modify: `locales.json` (`err_engine_exit`, `err_no_tree`) + +**Interfaces:** +- Consumes: `AppPaths` (T3), `Engine`/`Issue`/`format_issue`/`ENGINES` (T6–T7), `alignment` (T8). +- Produces: + - `LEVELS = ('error', 'success', 'system', 'out')`; статусы файла для `on_status`: `'pending'`, `'running'`, `'ok'`, `'fail'`, `'cancelled'` + - `FileResult`, `RunResult` — как в спецификации, раздел 5; `RunResult.ok_count -> int`, `RunResult.success -> bool` (файлы есть, все `ok`, не `cancelled`) + - `kill_process_tree(pid: int) -> None` — `psutil.Process(pid).children(recursive=True)` + сам процесс: `terminate`, ожидание 3 с, затем `kill`; без `psutil` — `os.kill(pid, signal.SIGTERM)`; исчезнувшие процессы игнорируются + - `class PipelineRunner: __init__(self, params: dict, files: list[str], paths: AppPaths, on_log=None, on_status=None, engines: Mapping[str, Engine] | None = None)`; `run(self) -> RunResult`; `cancel(self) -> None`; свойство `active_pid -> int | None` +- Поведение (порядок шагов и гарантии — спецификация, раздел 5): MUSCLE — `paths.find_tool(params['muscle_path'], ('muscle.exe',))`, аргументы v5 `['-align', in, '-output', out]`, при ненулевом коде v3 `['-in', in, '-out', out]`; вывод процессов — уровень `'out'`; предупреждения `validate`/`normalize` — `'system'`; сообщения пакета — нынешние ключи `batch_found`, `batch_file_header`, `batch_summary`; после отмены оставшиеся файлы получают `cancelled`. +- Produces (`tests/unit/fakes.py`): `FakeEngine(Engine)` с `key='fake'`, `name='Fake'`; `command` возвращает `[sys.executable, FAKE_TOOL, mode, phylip]`, где `mode` берётся из `params['fake_mode']`: `'ok'` (пишет `outputs.tree` и `outputs.stats`), `'exit3'` (код 3), `'notree'` (код 0, без дерева), `'hang'` (запускает дочерний `python -c "import time; time.sleep(60)"` и сам спит), `'raise'` (`command` бросает `RuntimeError('boom')`); `fake_muscle(tmp_path, mode) -> str` — `.cmd`-обёртка над Python-скриптом: `'copy'` (копирует вход в выход) или `'hang'`. + +- [ ] **Step 1: Написать падающие тесты** + +```python +def test_ok_run_returns_tree_and_stats(tmp_path) # статус ok, tree существует, on_status: pending → running → ok +def test_empty_fasta_fails_cleanly(tmp_path) # status 'fail', error содержит текст err_empty, run() не бросает +def test_engine_exit_code_reported(tmp_path) # 'exit3' → fail, error == _t('err_engine_exit').format('Fake', 3) +def test_missing_tree_reported(tmp_path) # 'notree' → fail, текст err_no_tree +def test_engine_exception_never_escapes(tmp_path) # 'raise' → fail, 'boom' в error, второй файл пакета обработан +def test_validate_error_stops_file_and_warning_is_logged(tmp_path) +def test_id_collision_fails_with_listing(tmp_path) # ID 'A.1' и 'A_1' → fail, оба в error +def test_unaligned_input_runs_muscle(tmp_path) # fake_muscle 'copy', длины разные → *_aligned.fasta создан +def test_cancel_kills_process_tree(tmp_path): + # 'hang': дождаться on_status 'running', запомнить active_pid и pid дочернего (psutil), + # cancel() из другого потока → run() вернулся < 10 с, status 'cancelled', оба pid не живы +def test_cancel_during_alignment_marks_rest_cancelled(tmp_path) # fake_muscle 'hang', 2 файла → оба 'cancelled' +def test_run_in_cyrillic_dir_with_spaces(tmp_path) # папка tmp_path / 'Мои данные' / 'проект 1' → ok +def test_log_levels_are_from_levels(tmp_path) # все уровни в on_log ∈ LEVELS +``` + +- [ ] **Step 2: Run `python -m pytest tests/unit/test_runner.py -q`** — Expected: FAIL (`ModuleNotFoundError: phyml_core.runner`) +- [ ] **Step 3: Реализовать `runner.py` и фейки; ключи локализации** +- [ ] **Step 4: Run `python -m pytest tests/unit/test_runner.py -q` три раза подряд, затем `python -m pytest -q`** — Expected: PASS каждый раз (тесты отмены не должны «мигать») +- [ ] **Step 5: Коммит** — `git commit -m "Add a Qt-free PipelineRunner that never raises and cancels whole process trees"` + +--- + +### Task 10: `report.py` — сводка CSV/XLSX + +**Files:** +- Create: `phyml_core/report.py` +- Test: `tests/unit/test_report.py` + +**Interfaces:** +- Consumes: `FileResult` (T9). +- Produces: `SUMMARY_COLUMNS = ('file', 'status', 'model', 'taxa', 'lnL', 'gamma', 'tstv', 'time', 'tree')`; `summary_rows(files: list[FileResult]) -> list[dict]` (`status` → `'OK'`/`'FAIL'`/`'CANCELLED'`, `file` — basename, статистика из `stats`); `write_summary(rows: list[dict], out_dir: str) -> list[str]` — `phymlgui_summary.csv` (`;`, `utf-8-sig`, заголовок — ключи колонок, как сейчас) и, если установлен openpyxl, `phymlgui_summary.xlsx` (лист `Summary`, заголовки `_t('sum_col_' + c)` без `tree`, ширины как сейчас); возвращает пути записанных файлов; `OSError` пробрасывается. + +- [ ] **Step 1: Написать падающие тесты** — `test_rows_from_results`, `test_csv_matches_current_format` (первая строка `file;status;model;taxa;lnL;gamma;tstv;time;tree`, кодировка с BOM), `test_xlsx_written_when_openpyxl_available` (`pytest.importorskip('openpyxl')`, `max_row == len(rows) + 1`) +- [ ] **Step 2: Run `python -m pytest tests/unit/test_report.py -q`** — Expected: FAIL +- [ ] **Step 3: Реализовать** +- [ ] **Step 4: Run** — Expected: PASS +- [ ] **Step 5: Коммит** — `git commit -m "Move the CSV/XLSX run summary into phyml_core.report"` + +--- + +### Task 11: `RunWorker`, окно на runner, журнал по уровням, проверка до запуска + +**Files:** +- Create: `phyml_ui/worker.py` +- Modify: `phyml_ui/main_window.py` (`start_pipeline`, `cancel_pipeline`, `pipeline_finished`, `update_cpu_status`, `LogConsole`, удаление `_save_summary_csv`) +- Delete: `phyml_core/pipeline.py`; из `phymlgui.py` — реэкспорт `PipelineWorker` +- Modify: тесты, использовавшие `PipelineWorker` или `LogConsole.append(text)` +- Test: `tests/gui/test_worker.py`, дополнения в `tests/gui/test_main_window.py` + +**Interfaces:** +- Consumes: `PipelineRunner`, `RunResult`, `ENGINES`, `format_issue`, `report.summary_rows/write_summary`, `history.new_record`. +- Produces (`phyml_ui/worker.py`): `class RunWorker(QThread)` — сигналы `log = Signal(str, str)`, `status = Signal(str, str)`, `done = Signal(object)` (`RunResult`); `__init__(self, runner: PipelineRunner)`; `run()` эмитит `done(runner.run())`; `cancel()`; свойство `active_pid` +- Produces (`LogConsole`): `append(self, level: str, text: str) -> None` (уровень обязателен, `_classify` удаляется); остальной API без изменений +- Поведение окна: `start_pipeline` вызывает `ENGINES[engine].validate(params, self.paths)`; есть `error` → `QMessageBox.warning(self, _t('err_box_title'), '\n'.join(format_issue(i) for i in errors))` и выход; нынешние проверки «нет IQ-TREE» и «бутстреп на MPI-сборке» удаляются (их делает `validate`). `pipeline_finished(result: RunResult)` заполняет статистику и сводку из `result.files`, пишет отчёт, историю (`new_record`), включает «Повторить неудавшиеся» для `fail`, грузит дерево при `result.success` и включённом «Показывать дерево». CPU-монитор берёт `self.worker.active_pid`. + +- [ ] **Step 1: Написать падающие тесты** — `gui/test_worker.py::test_worker_emits_log_status_done(qtbot, tmp_path)` (FakeEngine `'ok'` через `engines=`: `qtbot.waitSignal(w.done, timeout=10000)`, `result.success`), `gui/test_main_window.py::test_window_recovers_after_engine_exception(qtbot, data_dir)` (окно с реестром, где FakeEngine `'raise'`: после `done` кнопка Run снова доступна, журнал содержит `boom`), `::test_prerun_validation_lists_all_errors(qtbot, data_dir)` (два ошибочных параметра → в тексте предупреждения обе строки), `gui/test_log_console.py::test_levels_drive_colors_and_filter` (`append('error', 'x')` виден в фильтре «ошибки», `append('out', 'y')` — нет) +- [ ] **Step 2: Run эти тесты** — Expected: FAIL (`ModuleNotFoundError: phyml_ui.worker`) +- [ ] **Step 3: Реализовать `RunWorker`, перевести окно, удалить `pipeline.py`, обновить затронутые тесты (e2e используют `PipelineRunner` напрямую)** +- [ ] **Step 4: Run `python -m pytest -q -rs`** — Expected: PASS, включая e2e локально +- [ ] **Step 5: Ручная проверка** — одиночный прогон, пакет из двух файлов, отмена MPI-прогона (в диспетчере задач не остаётся `phyml-mpi.exe`), смена движка +- [ ] **Step 6: Коммит** — `git commit -m "Drive the window through PipelineRunner and RunWorker; log levels come from the core"` + +--- + +### Task 12: CLI без Qt и ленивая точка входа + +**Files:** +- Modify: `phyml_core/cli.py`, `phymlgui.py` (импорт PySide6 и `phyml_ui` — только в ветке GUI; реэкспорты удаляются) +- Test: `tests/unit/test_cli.py`, `tests/unit/test_core_has_no_qt.py` + +**Interfaces:** +- Consumes: `AppPaths.detect`, `settings.load`, `PipelineRunner`. +- Produces: `phyml_core.cli.main(argv: list[str] | None = None) -> int` — аргументы `--file`, `--batch`, `--config` (как сейчас); параметры = `settings.from_disk(settings.load(paths.config_file), paths)`, поверх — JSON из `--config`, если задан; пакет — те же правила отбора файлов, что в окне (`.fasta/.fas/.fa`, без `*_aligned.fasta`); лог в stdout строками `text`; коды: 0 — `result.success`, 1 — есть `fail`/`cancelled`, 2 — нет `--file`/`--batch`, пустая папка или несуществующий путь. + +- [ ] **Step 1: Написать падающие тесты** + +```python +BLOCK_QT = "import sys; sys.modules['PySide6'] = None; " + +def test_core_has_no_qt(): + code = BLOCK_QT + ("import pkgutil, importlib, phyml_core; " + "[importlib.import_module(m.name) for m in " + "pkgutil.walk_packages(phyml_core.__path__, 'phyml_core.')]") + r = subprocess.run([sys.executable, '-c', code], cwd=ROOT, capture_output=True, text=True) + assert r.returncode == 0, r.stderr + +def test_cli_without_qt_fails_cleanly_on_empty_fasta(tmp_path): # env PHYMLGUI_DATA_DIR=tmp; rc == 1; 'ImportError' not in stderr +def test_entry_point_cli_without_qt(tmp_path) # BLOCK_QT + sys.argv=['phymlgui.py','--file',empty]; import phymlgui; phymlgui.main() → SystemExit 1 +def test_cli_usage_error_exit_2() # main([]) → 2 (argparse error перехватывается) +def test_cli_empty_batch_dir_exit_2(tmp_path) +``` + +- [ ] **Step 2: Run `python -m pytest tests/unit/test_cli.py tests/unit/test_core_has_no_qt.py -q`** — Expected: FAIL (CLI сейчас тянет `QCoreApplication`) +- [ ] **Step 3: Переписать CLI и точку входа** +- [ ] **Step 4: Run `python -m pytest -q`** — Expected: PASS +- [ ] **Step 5: Коммит** — `git commit -m "Run the CLI on PipelineRunner without importing PySide6"` + +--- + +### Task 13: вынести журнал, диалог настроек и строку состояния + +**Files:** +- Create: `phyml_ui/log_console.py` (`LogConsole`), `phyml_ui/settings_dialog.py` (`SettingsDialog`), `phyml_ui/status_bar.py` +- Modify: `phyml_ui/main_window.py`, импорты в тестах + +**Interfaces:** +- Produces (`phyml_ui/status_bar.py`): `class AppStatusBar(QStatusBar)` — сигналы `open_dir_clicked`, `settings_clicked`; `track(self, pid_provider: Callable[[], int | None]) -> None` (начать сбор выборок CPU по дереву процессов), `stop_tracking(self) -> float` (вернуть среднее и показать `cpu_analysis_done`), `set_open_dir_enabled(self, on: bool)`; атрибуты `cpu_sys_bar`, `cores_label`, `cpu_run_label`, `cpu_timer`, `samples` (список выборок) — сохраняют то, что проверяют нынешние тесты +- `SettingsDialog` получает окно как сейчас; `LogConsole` — API из Task 11 + +- [ ] **Step 1: Перенести классы без изменения поведения; окно хранит `self.status = AppStatusBar(...)`** +- [ ] **Step 2: Обновить тесты на новые пути импорта и атрибуты (`w.status.cpu_sys_bar`, `w.status.samples`)** +- [ ] **Step 3: Run `python -m pytest -q -rs`** — Expected: PASS +- [ ] **Step 4: Коммит** — `git commit -m "Split LogConsole, SettingsDialog and the status bar out of main_window"` + +--- + +### Task 14: панели параметров и навигации, `main_window.py` ≤ ~350 строк + +**Files:** +- Create: `phyml_ui/params_panel.py`, `phyml_ui/nav_panel.py` +- Modify: `phyml_ui/main_window.py`, GUI-тесты +- Test: `tests/gui/test_params_panel.py`, `tests/gui/test_nav_panel.py` + +**Interfaces:** +- Produces (`params_panel.py`): `class ParamsPanel(QWidget)` — сигнал `changed`; `set_engine(self, engine: Engine) -> None` (перезаполняет модели); `params(self) -> dict` (ключи `PRESET_KEYS` без `engine`, плюс `use_mpi`, `mpi_np`); `set_params(self, params: dict) -> None` (без эмиссии `changed` на каждое поле, один `changed` в конце); виджеты остаются атрибутами с нынешними именами (`cb_model`, `cb_support_method`, `le_bootstrap_replicates`, `chk_use_mpi`, `le_mpi_np`, …) +- Produces (`nav_panel.py`): `class NavPanel(QToolBox)` — сигналы `recent_activated(str)`, `preset_activated(dict)`, `preset_save_requested()`, `preset_delete_requested(str)`, `history_activated(int)`; методы `set_recent(list[str])`, `set_presets(builtin: dict, user: dict)`, `set_history(records: list[dict])` (подписи через `history.display_input`) +- Окно: `self.params = ParamsPanel()`, `self.nav = NavPanel()`; автосохранение — по `params.changed` + +- [ ] **Step 1: Написать падающие тесты** — `test_params_roundtrip(qtbot)` (`set_params(p)`; `params() == p` для полного словаря), `test_set_params_emits_changed_once(qtbot)` (`qtbot.waitSignal`; счётчик == 1), `test_set_engine_refills_models(qtbot)` (IQ-TREE → первая модель `MFP`), `test_nav_signals(qtbot)` (двойной клик по пресету → `preset_activated` с его параметрами; по записи истории → `history_activated(0)`) +- [ ] **Step 2: Run `python -m pytest tests/gui/test_params_panel.py tests/gui/test_nav_panel.py -q`** — Expected: FAIL +- [ ] **Step 3: Реализовать панели, перевести окно, обновить GUI-тесты (`w.params.cb_model` и т. п.)** +- [ ] **Step 4: Проверить размер** — Run: `python -c "print(sum(1 for _ in open('phyml_ui/main_window.py', encoding='utf-8')))"` — Expected: ≤ 350 +- [ ] **Step 5: Run `python -m pytest -q -rs`** — Expected: PASS +- [ ] **Step 6: Ручная проверка** — критерий 6 спецификации целиком: одиночный прогон, пакет, отмена MPI, пресеты, история, смена движка; внешний вид не изменился +- [ ] **Step 7: Коммит** — `git commit -m "Split parameter and navigation panels; main_window only composes the window"` + +--- + +### Task 15: CI с закреплёнными IQ-TREE/MUSCLE, README + +**Files:** +- Create: `packaging/third_party.json`, `scripts/fetch_third_party.py` +- Modify: `.github/workflows/tests.yml`, `README.md`, `.gitignore` (`/bin/`), `tests/conftest.py` (уже ищет в `bin/` — проверить) +- Test: `tests/unit/test_fetch_third_party.py` + +**Interfaces:** +- Produces (`packaging/third_party.json`) — значения проверены 2026-10-25 (`muscle-win64.v5.3.exe` совпадает с локальным `tools/muscle.exe`): + +```json +{ + "iqtree": { + "version": "2.4.0", + "url": "https://github.com/iqtree/iqtree2/releases/download/v2.4.0/iqtree-2.4.0-Windows.zip", + "sha256": "93f488a6cfef96ef664b23fefe5be97c2c80ebcae9cd2031c0cac92b467278be", + "extract": {"iqtree-2.4.0-Windows/bin/iqtree2.exe": "iqtree2.exe", + "iqtree-2.4.0-Windows/bin/libiomp5md.dll": "libiomp5md.dll"} + }, + "muscle": { + "version": "5.3", + "url": "https://github.com/rcedgar/muscle/releases/download/v5.3/muscle-win64.v5.3.exe", + "sha256": "a86637db99deb3efa1cb6762c4e3160ca33644608270eb167e4214f1f9409711", + "save_as": "muscle.exe" + } +} +``` + +- Produces (`scripts/fetch_third_party.py`): `fetch(name: str, spec: dict, dest: str) -> list[str]` — скачивает `url` (`urllib.request`, поддерживает `file://` для тестов) во временный файл, сверяет SHA256 (несовпадение → `ValueError` с ожидаемым и фактическим хешем, в `dest` ничего не пишется), затем `extract` (zip → только перечисленные члены) или `save_as`; возвращает пути; `main(argv) -> int` с `--dest` (по умолчанию `bin`) и `--only`. + +- [ ] **Step 1: Написать падающие тесты** — `test_fetch_verifies_sha256(tmp_path)` (`file://` на локальный файл, неверный хеш → `ValueError`, `dest` пуст), `test_fetch_save_as(tmp_path)`, `test_fetch_extracts_listed_members(tmp_path)` (zip из двух файлов, в `extract` один → в `dest` только он) +- [ ] **Step 2: Run `python -m pytest tests/unit/test_fetch_third_party.py -q`** — Expected: FAIL +- [ ] **Step 3: Реализовать скрипт и `third_party.json`; добавить `/bin/` в `.gitignore`** +- [ ] **Step 4: Переписать `.github/workflows/tests.yml`** — job `test` на `windows-latest`, матрица `python-version: ['3.11', '3.13']`; шаги: checkout → setup-python → `pip install -r requirements.txt -r requirements-dev.txt` → `actions/cache` для `bin/` с ключом `third-party-${{ hashFiles('packaging/third_party.json') }}` → при промахе кэша `python scripts/fetch_third_party.py --dest bin` → `python -m pytest -q -rs` с `env: QT_QPA_PLATFORM: offscreen` +- [ ] **Step 5: Повторить шаги CI локально** — Run: `python scripts/fetch_third_party.py --dest bin` (папка в `.gitignore`; `conftest` находит бинарники в `bin/`), затем `python -m pytest -q -rs -m "not needs_phyml and not needs_mpi"` — Expected: PASS; в `-rs` нет пропусков по `needs_iqtree`/`needs_muscle` +- [ ] **Step 6: README (en/ru)** — Python ≥ 3.11; где лежат настройки (рядом с exe, иначе `%APPDATA%\phymlGUI`, переменная `PHYMLGUI_DATA_DIR`); раздел «Тесты»: `pip install -r requirements-dev.txt`, `python -m pytest`, переменные `PHYMLGUI_*` +- [ ] **Step 7: Финальная проверка** — Run: `python -m pytest -q -rs` — Expected: всё PASS, 0 skipped на этой машине; `git status` чистый, кроме коммита +- [ ] **Step 8: Коммит** — `git commit -m "Run the full pytest suite in CI with pinned IQ-TREE and MUSCLE"` +- [ ] **Step 9: Пуш ветки и проверка CI — только после явного «да» пользователя** (действие наружу) From 5e31349c50b61cdfe2038b68ee06229465632b14 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sun, 25 Oct 2026 00:45:42 +0300 Subject: [PATCH 13/36] Move binary-free test checks to pytest with fixtures in tests/data Co-Authored-By: Claude Opus 5.5 --- .github/workflows/tests.yml | 6 +- pytest.ini | 9 ++ requirements-dev.txt | 2 + tests/README.md | 6 +- tests/conftest.py | 81 ++++++++++++++++ tests/data/cytb_boot_trees.nwk | 20 ++++ tests/data/cytb_tree.nwk | 1 + tests/data/x.iqtree | 4 + tests/data/x_phyml_stats.txt | 6 ++ tests/gui/test_log_console.py | 13 +++ tests/gui/test_main_window_smoke.py | 11 +++ tests/gui/test_themes.py | 10 ++ tests/gui/test_tree_viewer.py | 99 +++++++++++++++++++ tests/test_smoke.py | 110 --------------------- tests/test_tree_viewer.py | 124 ------------------------ tests/unit/test_i18n.py | 10 ++ tests/unit/test_pipeline_regressions.py | 25 +++++ tests/unit/test_rf.py | 20 ++++ tests/unit/test_stats_parsers.py | 13 +++ 19 files changed, 331 insertions(+), 239 deletions(-) create mode 100644 pytest.ini create mode 100644 requirements-dev.txt create mode 100644 tests/conftest.py create mode 100644 tests/data/cytb_boot_trees.nwk create mode 100644 tests/data/cytb_tree.nwk create mode 100644 tests/data/x.iqtree create mode 100644 tests/data/x_phyml_stats.txt create mode 100644 tests/gui/test_log_console.py create mode 100644 tests/gui/test_main_window_smoke.py create mode 100644 tests/gui/test_themes.py create mode 100644 tests/gui/test_tree_viewer.py delete mode 100644 tests/test_smoke.py delete mode 100644 tests/test_tree_viewer.py create mode 100644 tests/unit/test_i18n.py create mode 100644 tests/unit/test_pipeline_regressions.py create mode 100644 tests/unit/test_rf.py create mode 100644 tests/unit/test_stats_parsers.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2b70824..93b54ad 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,6 +14,6 @@ jobs: with: python-version: '3.12' - name: Install dependencies - run: pip install -r requirements.txt - - name: Smoke tests (no analysis binaries) - run: python tests/test_smoke.py + run: pip install -r requirements.txt -r requirements-dev.txt + - name: Tests (no analysis binaries) + run: python -m pytest tests/unit tests/gui diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..6638658 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +testpaths = tests +qt_api = pyside6 +markers = + needs_phyml: requires a PhyML (MPI) executable + needs_mpi: requires MS-MPI mpiexec + needs_iqtree: requires an IQ-TREE executable + needs_muscle: requires a MUSCLE executable + slow: long-running end-to-end test diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..f6c9b3a --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +pytest>=8 +pytest-qt>=4.4 diff --git a/tests/README.md b/tests/README.md index 9043f04..e5bb8f9 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,12 +1,14 @@ # phymlGUI test suites +Binary-free checks run under pytest (`pip install -r requirements-dev.txt`): + + python -m pytest tests/unit tests/gui # no analysis binaries needed (CI-friendly) + Standalone scripts, runnable on any machine with the Python dependencies: - python tests/test_smoke.py # no analysis binaries needed (CI-friendly) python tests/run_tests.py # end-to-end pipeline, needs local binaries python tests/test_gui_logic.py # GUI behaviour + live CPU monitoring python tests/test_batch.py # batch mode + CSV summary + history - python tests/test_tree_viewer.py # layouts, collapse, outgroup, consensus, RF python tests/test_iqtree.py # IQ-TREE engine (skips without the binary) Suites that need local analysis binaries (`phymlgui v1.1/phyml-mpi.exe`, diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a305bff --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,81 @@ +import os +os.environ.setdefault('QT_QPA_PLATFORM', 'offscreen') + +import hashlib +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +DATA = ROOT / 'tests' / 'data' +sys.path.insert(0, str(ROOT)) + +_BINARIES = { + # kind: (env var, bin/ file name, extra local locations) + 'phyml': ('PHYMLGUI_PHYML', 'phyml-mpi.exe', + [ROOT / 'phymlgui v1.1' / 'phyml-mpi.exe']), + 'mpiexec': ('PHYMLGUI_MPIEXEC', 'mpiexec.exe', + [Path(r'C:\Program Files\Microsoft MPI\Bin\mpiexec.exe')]), + 'iqtree': ('PHYMLGUI_IQTREE', 'iqtree2.exe', + [ROOT / 'tools' / 'iqtree2' / 'iqtree2.exe']), + 'muscle': ('PHYMLGUI_MUSCLE', 'muscle.exe', + [ROOT / 'tools' / 'muscle.exe']), + 'raxml': ('PHYMLGUI_RAXML', 'raxml-ng.exe', []), +} + +# marker name -> binary kind (needs_mpi is satisfied by mpiexec) +_MARKER_KINDS = { + 'needs_phyml': 'phyml', + 'needs_mpi': 'mpiexec', + 'needs_iqtree': 'iqtree', + 'needs_muscle': 'muscle', +} + + +def find_binary(kind): + """Path to an analysis binary or None: env var, then ROOT/bin, then local installs.""" + env_var, bin_name, local = _BINARIES[kind] + candidates = [] + if os.environ.get(env_var): + candidates.append(Path(os.environ[env_var])) + candidates.append(ROOT / 'bin' / bin_name) + candidates.extend(local) + for path in candidates: + if path.is_file(): + return str(path) + return None + + +def pytest_runtest_setup(item): + for marker, kind in _MARKER_KINDS.items(): + if item.get_closest_marker(marker) and find_binary(kind) is None: + pytest.skip(f'{kind} not found (set {_BINARIES[kind][0]})') + + +@pytest.fixture(scope='session') +def binaries(): + return {kind: find_binary(kind) for kind in _BINARIES} + + +@pytest.fixture +def isolated_config(tmp_path, monkeypatch): + """Point the window's config/history files at tmp_path (interim: Task 5 uses data_dir).""" + import phyml_ui.main_window as main_window + monkeypatch.setattr(main_window, 'CONFIG_FILE', str(tmp_path / 'phyml_config.json')) + monkeypatch.setattr(main_window, 'HISTORY_FILE', str(tmp_path / 'phyml_history.json')) + return tmp_path + + +def _digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None + + +@pytest.fixture(scope='session', autouse=True) +def guard_user_files(): + """Fail the session if a test touched the user's real config or history.""" + files = [ROOT / 'phyml_config.json', ROOT / 'phyml_history.json'] + before = {f: _digest(f) for f in files} + yield + after = {f: _digest(f) for f in files} + assert after == before, 'tests modified the real user config/history files' diff --git a/tests/data/cytb_boot_trees.nwk b/tests/data/cytb_boot_trees.nwk new file mode 100644 index 0000000..e6b300c --- /dev/null +++ b/tests/data/cytb_boot_trees.nwk @@ -0,0 +1,20 @@ +(Rattus_norvegicus:0.18770731,Mus_musculus:0.12324570,(Homo_sapiens:0.04034483,((Gorilla_gorilla:0.10048860,(Pongo_pygmaeus:0.13268269,Macaca_mulatta:0.30589892)0:0.03768774)0:0.02358356,(Pan_paniscus:0.01563311,Pan_troglodytes:0.03321206)0:0.02300720)0:0.03832110)0:0.56511122); +(Mus_musculus:0.15119580,Rattus_norvegicus:0.17224063,(Gorilla_gorilla:0.04552491,((Macaca_mulatta:0.37380058,Pongo_pygmaeus:0.13627396)0:0.05396938,(Homo_sapiens:0.04786471,(Pan_paniscus:0.02713306,Pan_troglodytes:0.03619245)0:0.03471855)0:0.02303760)0:0.02268509)0:0.64487876); +(Mus_musculus:0.16787210,Rattus_norvegicus:0.16537991,(Pongo_pygmaeus:0.14168982,(Gorilla_gorilla:0.07541985,(Homo_sapiens:0.06552747,(Macaca_mulatta:0.43653221,(Pan_troglodytes:0.03270771,Pan_paniscus:0.02531593)0:0.01646027)0:0.03093322)0:0.03465620)0:0.05703642)0:0.51998779); +(Rattus_norvegicus:0.15778769,Mus_musculus:0.13444362,(Homo_sapiens:0.00000001,((Pan_paniscus:0.02528597,Pan_troglodytes:0.03608873)0:0.04062699,(Gorilla_gorilla:0.02504434,(Pongo_pygmaeus:0.12359310,Macaca_mulatta:0.36371904)0:0.02073863)0:0.04994449)0:0.05874740)0:0.59738774); +(Rattus_norvegicus:0.23601941,Mus_musculus:0.13583315,(Pongo_pygmaeus:0.09986846,((Homo_sapiens:0.06529474,(Pan_troglodytes:0.02780559,Pan_paniscus:0.02719473)0:0.03815261)0:0.02763543,(Gorilla_gorilla:0.08432875,Macaca_mulatta:0.35206055)0:0.00406725)0:0.04295501)0:0.54163639); +(((((Pan_paniscus:0.01668075,Pan_troglodytes:0.03506271)0:0.02355206,(Macaca_mulatta:0.32441779,Gorilla_gorilla:0.05577043)0:0.03760818)0:0.02861340,Homo_sapiens:0.07657604)0:0.04601437,Pongo_pygmaeus:0.11448159)0:0.58054021,Rattus_norvegicus:0.16670786,Mus_musculus:0.13599687); +(((((Pongo_pygmaeus:0.11545283,Macaca_mulatta:0.28654491)0:0.05807108,Gorilla_gorilla:0.07438339)0:0.02129741,(Pan_troglodytes:0.03437116,Pan_paniscus:0.04161595)0:0.05970256)0:0.03110581,Homo_sapiens:0.01710694)0:0.48622133,Mus_musculus:0.16800774,Rattus_norvegicus:0.12546053); +(Rattus_norvegicus:0.18612045,Mus_musculus:0.11421262,(Homo_sapiens:0.03756043,((Pan_troglodytes:0.03764546,Pan_paniscus:0.02253496)0:0.01727685,(Gorilla_gorilla:0.05651310,(Macaca_mulatta:0.34666079,Pongo_pygmaeus:0.09836627)0:0.02914529)0:0.03803908)0:0.05378318)0:0.53664531); +(Mus_musculus:0.18723642,Rattus_norvegicus:0.08206100,(Homo_sapiens:0.05884236,((Pan_troglodytes:0.03498138,Pan_paniscus:0.01315749)0:0.01441984,(Gorilla_gorilla:0.06654187,(Pongo_pygmaeus:0.10491134,Macaca_mulatta:0.34623873)0:0.06107660)0:0.01891564)0:0.02686464)0:0.38619256); +(Mus_musculus:0.17323349,Rattus_norvegicus:0.18785460,(Homo_sapiens:0.03028619,((Pan_troglodytes:0.03127831,Pan_paniscus:0.02420224)0:0.03227408,(Gorilla_gorilla:0.07564871,(Pongo_pygmaeus:0.07027105,Macaca_mulatta:0.37190303)0:0.05792108)0:0.02587411)0:0.06038335)0:0.50172789); +(Mus_musculus:0.16952846,Rattus_norvegicus:0.11577952,(Homo_sapiens:0.03782202,((Pan_troglodytes:0.02732214,Pan_paniscus:0.02338809)0:0.03557437,(Gorilla_gorilla:0.06855054,(Pongo_pygmaeus:0.14689259,Macaca_mulatta:0.36064832)0:0.01458471)0:0.03591677)0:0.03352678)0:0.58751528); +(Mus_musculus:0.14285765,Rattus_norvegicus:0.21065467,(Homo_sapiens:0.06549472,((Pan_troglodytes:0.03343488,Pan_paniscus:0.02633569)0:0.01984373,(Gorilla_gorilla:0.06225423,(Macaca_mulatta:0.36794546,Pongo_pygmaeus:0.12412600)0:0.05845648)0:0.04546145)0:0.03839232)0:0.52740025); +(Mus_musculus:0.09445958,Rattus_norvegicus:0.15193077,((Pongo_pygmaeus:0.09832768,Macaca_mulatta:0.28476238)0:0.02316242,(Gorilla_gorilla:0.07019243,(Homo_sapiens:0.08668996,(Pan_troglodytes:0.03584197,Pan_paniscus:0.02233530)0:0.04952190)0:0.02325765)0:0.00896747)0:0.44999502); +(Mus_musculus:0.16050383,Rattus_norvegicus:0.15819190,(Macaca_mulatta:0.31313780,(Pongo_pygmaeus:0.10356261,(Gorilla_gorilla:0.09377602,(Homo_sapiens:0.08708336,(Pan_paniscus:0.02693422,Pan_troglodytes:0.02874582)0:0.04408687)0:0.01755437)0:0.05010402)0:0.06092969)0:0.33434545); +(Rattus_norvegicus:0.17944973,Mus_musculus:0.14509775,(Homo_sapiens:0.03901130,((Gorilla_gorilla:0.06329922,(Macaca_mulatta:0.33070316,Pongo_pygmaeus:0.13168108)0:0.02432519)0:0.04509929,(Pan_paniscus:0.03942828,Pan_troglodytes:0.02882357)0:0.04217594)0:0.04483050)0:0.52433872); +((Gorilla_gorilla:0.06858551,(Pongo_pygmaeus:0.14156527,Macaca_mulatta:0.33823897)0:0.03841110)0:0.03040457,(Homo_sapiens:0.00839412,(Rattus_norvegicus:0.18485785,Mus_musculus:0.11520460)0:0.53810561)0:0.08624522,(Pan_paniscus:0.02843255,Pan_troglodytes:0.03547273)0:0.02872229); +(Rattus_norvegicus:0.14466255,Mus_musculus:0.19846512,(Pongo_pygmaeus:0.13500758,(Homo_sapiens:0.07691297,((Pan_paniscus:0.03494440,Pan_troglodytes:0.03235353)0:0.01985657,(Gorilla_gorilla:0.08497038,Macaca_mulatta:0.33617845)0:0.02713991)0:0.01770200)0:0.02519035)0:0.39273894); +(Mus_musculus:0.20743827,Rattus_norvegicus:0.17318983,(Homo_sapiens:0.01124678,((Pan_paniscus:0.02618029,Pan_troglodytes:0.03295157)0:0.03443510,(Gorilla_gorilla:0.06754831,(Pongo_pygmaeus:0.12012469,Macaca_mulatta:0.38841416)0:0.05831689)0:0.02983764)0:0.04696147)0:0.53572390); +(Mus_musculus:0.10254939,Rattus_norvegicus:0.17136571,(Homo_sapiens:0.01990635,((Pan_paniscus:0.03314134,Pan_troglodytes:0.03499254)0:0.03761208,(Gorilla_gorilla:0.04711548,(Pongo_pygmaeus:0.13594442,Macaca_mulatta:0.31013880)0:0.03640258)0:0.07192342)0:0.05979503)0:0.56147577); +(Rattus_norvegicus:0.10760019,Mus_musculus:0.16492639,(Pongo_pygmaeus:0.06663073,(Macaca_mulatta:0.36737855,(Gorilla_gorilla:0.09321729,(Homo_sapiens:0.07336706,(Pan_troglodytes:0.03307957,Pan_paniscus:0.02047722)0:0.04787185)0:0.02371622)0:0.02613566)0:0.05770642)0:0.52629545); diff --git a/tests/data/cytb_tree.nwk b/tests/data/cytb_tree.nwk new file mode 100644 index 0000000..e6f4fbf --- /dev/null +++ b/tests/data/cytb_tree.nwk @@ -0,0 +1 @@ +(Mus_musculus:0.15034134,Rattus_norvegicus:0.16487440,(Homo_sapiens:0.05205989,((Pan_paniscus:0.02550273,Pan_troglodytes:0.03389749)20:0.03585372,(Gorilla_gorilla:0.07433773,(Pongo_pygmaeus:0.13141843,Macaca_mulatta:0.36200494)14:0.03216865)12:0.03028294)12:0.03401688)20:0.53482002); diff --git a/tests/data/x.iqtree b/tests/data/x.iqtree new file mode 100644 index 0000000..d9ea55d --- /dev/null +++ b/tests/data/x.iqtree @@ -0,0 +1,4 @@ +Input data: 8 sequences with 600 nucleotide sites +Best-fit model according to BIC: JC +Log-likelihood of the tree: -1828.7117 (s.e. 50.96) +Total wall-clock time used: 0.7434 seconds (0h:0m:0s) diff --git a/tests/data/x_phyml_stats.txt b/tests/data/x_phyml_stats.txt new file mode 100644 index 0000000..c0605ec --- /dev/null +++ b/tests/data/x_phyml_stats.txt @@ -0,0 +1,6 @@ +. Model of nucleotides substitution: HKY85 +. Number of taxa: 8 +. Log-likelihood: -1811.15467 + - Gamma shape parameter: 881.743 +. transition/transversion ratio: 0.980317 +. Time used: 0h0m0s (0 seconds) diff --git a/tests/gui/test_log_console.py b/tests/gui/test_log_console.py new file mode 100644 index 0000000..e88115e --- /dev/null +++ b/tests/gui/test_log_console.py @@ -0,0 +1,13 @@ +from phyml_ui.main_window import LogConsole + + +def test_roles_and_plain_text(qtbot): + log = LogConsole() + qtbot.addWidget(log) + assert log._classify('[ERROR] bad') == 'error' + assert log._classify('[+] done') == 'success' + assert log._classify('[*] system line') == 'system' + assert log._classify('. PhyML raw output') == 'out' + log.append('[ERROR] bad') + log.append('raw') + assert '[ERROR] bad' in log.toPlainText() and 'raw' in log.toPlainText() diff --git a/tests/gui/test_main_window_smoke.py b/tests/gui/test_main_window_smoke.py new file mode 100644 index 0000000..1112549 --- /dev/null +++ b/tests/gui/test_main_window_smoke.py @@ -0,0 +1,11 @@ +from phyml_ui.main_window import PhyMLApp + + +def test_window_builds(isolated_config, qtbot): + # isolated_config goes first so it is torn down last: qtbot closes the + # window at teardown and closeEvent still writes the (patched) config file + w = PhyMLApp() + qtbot.addWidget(w) + assert w.result_tabs.count() == 4 + assert w.presets_list.count() >= 3 + assert w.cpu_timer.isActive() diff --git a/tests/gui/test_themes.py b/tests/gui/test_themes.py new file mode 100644 index 0000000..ddcc339 --- /dev/null +++ b/tests/gui/test_themes.py @@ -0,0 +1,10 @@ +from phyml_ui.themes import resolve, build_qss, role_colors, ACCENTS, system_theme + + +def test_resolve_qss_and_role_colors(): + theme, accent = resolve('dark', 'blue') + assert theme == 'dark' and accent == ACCENTS['blue'] + qss = build_qss(theme, accent) + assert accent in qss and 'QPushButton#btn_run' in qss + assert role_colors(theme, accent)['out'] == '#E6E8EB' + assert system_theme() in ('light', 'dark') diff --git a/tests/gui/test_tree_viewer.py b/tests/gui/test_tree_viewer.py new file mode 100644 index 0000000..f871c6d --- /dev/null +++ b/tests/gui/test_tree_viewer.py @@ -0,0 +1,99 @@ +"""Stage-2 tree viewer: layouts, grid, colors, collapse, outgroup, consensus.""" +import io + +import pytest +from Bio import Phylo +from Bio.Phylo.Consensus import majority_consensus +from matplotlib.lines import Line2D + +from conftest import DATA +from phyml_ui.tree_viewer import TreeViewerWidget + +TREE_FILE = DATA / 'cytb_tree.nwk' +BOOT_TREES_FILE = DATA / 'cytb_boot_trees.nwk' + + +@pytest.fixture +def viewer(qtbot): + """A viewer with the 8-taxon CytB tree (with supports) loaded.""" + w = TreeViewerWidget() + qtbot.addWidget(w) + w.load_tree(str(TREE_FILE)) + assert len(w._leaf_order) == 8 + return w + + +def test_circle_polytomy_centered(qtbot): + """Circular layout: a polytomy node sits in the middle of its children's arc.""" + w = TreeViewerWidget() + qtbot.addWidget(w) + w.load_tree_object( + Phylo.read(io.StringIO('((A:1,B:1,C:1,D:1):1,E:1,F:1);'), 'newick'), 'polytomy') + w.set_layout('circle') + tree, coords = w._coords + node = tree.root.clades[0] + child_angles = [coords[id(c)][1] for c in node.clades] + expected = (min(child_angles) + max(child_angles)) / 2 + assert abs(coords[id(node)][1] - expected) < 1e-9, (coords[id(node)][1], expected) + + +def test_layouts_render(viewer): + for layout in ('rect', 'slanted', 'circle'): + viewer.set_layout(layout) + assert viewer.layout == layout + + +def test_grid_drawn_and_toggleable(viewer): + """The manual distance grid is actually drawn (ax.grid dies with axis_off).""" + viewer.set_layout('rect') + grid_on = sum(1 for l in viewer.ax.lines if isinstance(l, Line2D)) + viewer.chk_grid.setChecked(False) + grid_off = sum(1 for l in viewer.ax.lines if isinstance(l, Line2D)) + viewer.chk_grid.setChecked(True) + assert grid_on >= 3 and grid_off < grid_on, (grid_on, grid_off) + + +def test_theme_colors(viewer): + """Theme-aware colors recolor the render.""" + viewer.set_colors('#E6E8EB', '#37C894', '#26282D') + assert viewer._bg == '#26282D' and viewer.figure.patch.get_facecolor()[:3] != (1, 1, 1) + viewer.set_colors('#424242', '#1E9E6A', '#FFFFFF') + + +def test_threshold_grid_supports_options(viewer): + """Support threshold + grid/support toggles do not break rendering.""" + viewer.threshold_slider.setValue(70) + assert viewer.threshold == 70 + viewer.chk_grid.setChecked(False) + viewer.chk_supports.setChecked(False) + viewer.chk_supports.setChecked(True) + + +def test_collapse_and_expand(viewer): + """The first real internal clade folds into one display leaf and back.""" + src = Phylo.read(str(TREE_FILE), 'newick') + candidates = [cl for cl in src.get_nonterminals() + if 2 <= len(cl.get_terminals()) < len(src.get_terminals())] + assert candidates, 'no internal clades to collapse' + target = frozenset(t.name for t in candidates[0].get_terminals()) + viewer.set_layout('rect') + before = len(viewer._leaf_order) + viewer.collapse_clade(target) + assert len(viewer._leaf_order) == before - len(target) + 1, (before, len(viewer._leaf_order)) + viewer.collapse_clade(target) + assert len(viewer._leaf_order) == before + + +def test_outgroup_reroot_and_reset(viewer): + viewer.outgroup_combo.setCurrentIndex(3) # some taxon + assert viewer.outgroup is not None + viewer.outgroup_combo.setCurrentIndex(0) + assert viewer.outgroup is None + + +def test_majority_consensus_from_boot_trees(viewer): + trees = list(Phylo.parse(str(BOOT_TREES_FILE), 'newick')) + assert len(trees) == 20 + cons = majority_consensus(trees) + viewer.load_tree_object(cons, 'consensus-test') + assert len(viewer._leaf_order) == 8 diff --git a/tests/test_smoke.py b/tests/test_smoke.py deleted file mode 100644 index e3dce68..0000000 --- a/tests/test_smoke.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Smoke test: runs everywhere (CI included), needs no analysis binaries.""" -import json -import os -import sys - -os.environ['QT_QPA_PLATFORM'] = 'offscreen' -ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, ROOT) - -# 1. module imports and i18n integrity -import phymlgui -from phyml_core.pipeline import parse_stats, parse_iqtree_stats -from phyml_ui.tree_viewer import rf_distance, tree_splits -from phyml_ui.themes import resolve, build_qss, role_colors, ACCENTS, system_theme -from phyml_ui.main_window import LogConsole, CONFIG_FILE - -loc = json.load(open(os.path.join(ROOT, 'locales.json'), encoding='utf-8')) -assert set(loc) >= {'en', 'ru'} -assert len(loc['en']) == len(loc['ru']), 'locale key sets diverge' -print(f'PASS imports and i18n: {len(loc["en"])} keys in both languages') - -# 2. theme resolution and dynamic QSS -theme, accent = resolve('dark', 'blue') -assert theme == 'dark' and accent == ACCENTS['blue'] -qss = build_qss(theme, accent) -assert accent in qss and 'QPushButton#btn_run' in qss -assert role_colors(theme, accent)['out'] == '#E6E8EB' -assert system_theme() in ('light', 'dark') -print('PASS themes: resolve, build_qss, role colors') - -# 3. stats parsers on synthetic content (no binaries needed) -import io -import tempfile -from Bio import Phylo - -tmp = tempfile.mkdtemp() -phy_stats = os.path.join(tmp, 'x_phyml_stats.txt') -open(phy_stats, 'w', encoding='utf-8').write( - '. Model of nucleotides substitution: \tHKY85\n' - '. Number of taxa: \t\t8\n' - '. Log-likelihood: \t\t-1811.15467\n' - ' - Gamma shape parameter: \t881.743\n' - '. transition/transversion ratio: \t0.980317\n' - '. Time used:\t\t\t0h0m0s (0 seconds)\n') -s = parse_stats(phy_stats) -assert s['model'] == 'HKY85' and s['taxa'] == '8' and s['lnL'] == '-1811.15467' -assert s['gamma'] == '881.743' and s['tstv'] == '0.980317' - -iq = os.path.join(tmp, 'x.iqtree') -open(iq, 'w', encoding='utf-8').write( - 'Input data: 8 sequences with 600 nucleotide sites\n' - 'Best-fit model according to BIC: JC\n' - 'Log-likelihood of the tree: -1828.7117 (s.e. 50.96)\n' - 'Total wall-clock time used: 0.7434 seconds (0h:0m:0s)\n') -s2 = parse_iqtree_stats(iq) -assert s2['model'] == 'JC' and s2['taxa'] == '8' and s2['lnL'] == '-1828.7117' -print('PASS stats parsers: phyml and iqtree formats') - -# 4. RF distance / splits on newick strings -t1 = Phylo.read(io.StringIO('(A,(B,(C,D)));'), 'newick') -t2 = Phylo.read(io.StringIO('(A,(C,(B,D)));'), 'newick') -assert rf_distance(t1, t1) == (0, 0.0) -assert rf_distance(t1, t2) == (1, 1.0) -assert len(tree_splits(t1)) == 1 -print('PASS rf distance and splits') - -# 5. log classification -app = phymlgui.QApplication([]) -log = LogConsole() -assert log._classify('[ERROR] bad') == 'error' -assert log._classify('[+] done') == 'success' -assert log._classify('[*] system line') == 'system' -assert log._classify('. PhyML raw output') == 'out' -log.append('[ERROR] bad') -log.append('raw') -assert '[ERROR] bad' in log.toPlainText() and 'raw' in log.toPlainText() -print('PASS log console roles') - -# 6. main window builds offscreen -w = phymlgui.PhyMLApp() -assert w.result_tabs.count() == 4 -assert w.presets_list.count() >= 3 -assert w.cpu_timer.isActive() -print('PASS main window builds offscreen') - -# 7. a broken input fails the run cleanly instead of killing the worker thread -from phyml_core.pipeline import PipelineWorker -empty = os.path.join(tmp, 'empty.fasta') -open(empty, 'w').close() -worker = PipelineWorker({'engine': 'phyml'}, [empty]) -done = [] -worker.finished_signal.connect(lambda ok, tree: done.append((ok, tree))) -try: - worker.run() -except Exception as e: - raise AssertionError(f'worker crashed on an empty FASTA: {e!r}') -assert done == [(False, '')], done -assert worker.results[0]['status'] == 'FAIL', worker.results -print('PASS empty fasta: run fails cleanly, finished signal emitted') - -# 8. IQ-TREE UFBoot refuses < 1000 replicates: the PhyML-sized value is raised -from phyml_core.pipeline import ufboot_replicates -assert ufboot_replicates('100') == '1000' -assert ufboot_replicates('1000') == '1000' -assert ufboot_replicates('5000') == '5000' -assert ufboot_replicates('') == '1000' -assert ufboot_replicates('abc') == '1000' -print('PASS ufboot replicates: raised to the IQ-TREE minimum of 1000') - -print('ALL SMOKE TESTS PASSED') diff --git a/tests/test_tree_viewer.py b/tests/test_tree_viewer.py deleted file mode 100644 index 46a2306..0000000 --- a/tests/test_tree_viewer.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Tests for the stage-2 tree viewer: layouts, collapse, outgroup, consensus, RF.""" -import io -import os -import sys - -os.environ['QT_QPA_PLATFORM'] = 'offscreen' -ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, ROOT) -os.chdir(ROOT) - -import phymlgui -from phyml_ui.tree_viewer import TreeViewerWidget, rf_distance -from Bio import Phylo - -# 1. RF distance: identical -> 0, swapped quartet -> max -t1 = Phylo.read(io.StringIO('(A,(B,(C,D)));'), 'newick') -t2 = Phylo.read(io.StringIO('(A,(C,(B,D)));'), 'newick') -assert rf_distance(t1, t1) == (0, 0.0) -raw, norm = rf_distance(t1, t2) -assert raw == 1 and norm == 1.0, (raw, norm) -print('PASS rf: identical=0, quartet swap=1 (100%)') - -app = phymlgui.QApplication([]) - -# 1b. circular layout: a polytomy node sits in the middle of its children's arc -w0 = TreeViewerWidget() -w0.load_tree_object(Phylo.read(io.StringIO('((A:1,B:1,C:1,D:1):1,E:1,F:1);'), 'newick'), 'polytomy') -w0.set_layout('circle') -tree0, coords0 = w0._coords -node = tree0.root.clades[0] -child_angles = [coords0[id(c)][1] for c in node.clades] -expected = (min(child_angles) + max(child_angles)) / 2 -assert abs(coords0[id(node)][1] - expected) < 1e-9, (coords0[id(node)][1], expected) -print('PASS circle layout: polytomy node centered on its children arc') - -if not os.path.isdir('testrun') or not any( - os.path.exists(os.path.join('testrun', d, f)) - for d in os.listdir('testrun') if os.path.isdir(os.path.join('testrun', d)) - for f in os.listdir(os.path.join('testrun', d)) if f.endswith('_phyml_tree.txt')): - print('SKIP: no rendered trees available; run tests/run_tests.py first') - sys.exit(0) - -w = TreeViewerWidget() - -# real tree with supports from the e2e runs -tree_file = None -for d in ('t9_real_ncbi', 't2_unaligned_mpi_boot'): - p = os.path.join('testrun', d) - if os.path.isdir(p): - for f in os.listdir(p): - if f.endswith('_phyml_tree.txt'): - tree_file = os.path.join(p, f) - break - if tree_file: - break -assert tree_file, 'no e2e tree available; run run_tests.py first' - -w.load_tree(tree_file) -assert len(w._leaf_order) == 8 - -# 2. all three layouts render without errors -for layout in ('rect', 'slanted', 'circle'): - w.set_layout(layout) - assert w.layout == layout -print('PASS layouts: rect / slanted / circle render') - -# 2b. manual distance grid is actually drawn (ax.grid dies with axis_off) -w.set_layout('rect') -from matplotlib.lines import Line2D -grid_on = sum(1 for l in w.ax.lines if isinstance(l, Line2D)) -w.chk_grid.setChecked(False) -grid_off = sum(1 for l in w.ax.lines if isinstance(l, Line2D)) -w.chk_grid.setChecked(True) -assert grid_on >= 3 and grid_off < grid_on, (grid_on, grid_off) -print(f'PASS grid: {grid_on} distance gridlines drawn, toggleable') - -# 2c. theme-aware colors recolor the render -w.set_colors('#E6E8EB', '#37C894', '#26282D') -assert w._bg == '#26282D' and w.figure.patch.get_facecolor()[:3] != (1, 1, 1) -w.set_colors('#424242', '#1E9E6A', '#FFFFFF') -print('PASS theme colors: branches/background follow the app theme') - -# 3. support threshold + grid toggles do not break rendering -w.threshold_slider.setValue(70) -assert w.threshold == 70 -w.chk_grid.setChecked(False) -w.chk_supports.setChecked(False) -w.chk_supports.setChecked(True) -print('PASS options: threshold, grid, supports') - -# 4. collapse: first real internal clade folds into one display leaf -src = Phylo.read(tree_file, 'newick') -candidates = [cl for cl in src.get_nonterminals() - if 2 <= len(cl.get_terminals()) < len(src.get_terminals())] -assert candidates, 'no internal clades to collapse' -target = frozenset(t.name for t in candidates[0].get_terminals()) -w.set_layout('rect') -before = len(w._leaf_order) -w.collapse_clade(target) -assert len(w._leaf_order) == before - len(target) + 1, (before, len(w._leaf_order)) -w.collapse_clade(target) -assert len(w._leaf_order) == before -print(f'PASS collapse: clade of {len(target)} taxa folds to one leaf and back') - -# 5. outgroup reroot renders -w.outgroup_combo.setCurrentIndex(3) # some taxon -assert w.outgroup is not None -w.outgroup_combo.setCurrentIndex(0) -print('PASS outgroup: reroot and reset') - -# 6. consensus from a bootstrap trees file -boot = tree_file.replace('_phyml_tree.txt', '_phyml_boot_trees.txt') -if os.path.exists(boot): - trees = list(Phylo.parse(boot, 'newick')) - w.load_tree_object(Phylo, 'unused') if False else None - from Bio.Phylo.Consensus import majority_consensus - cons = majority_consensus(trees) - w.load_tree_object(cons, 'consensus-test') - assert len(w._leaf_order) == 8 - print(f'PASS consensus: majority over {len(trees)} replicate trees') -else: - print('SKIP consensus: no *_boot_trees.txt next to the tree') - -print('ALL TREE VIEWER TESTS PASSED') diff --git a/tests/unit/test_i18n.py b/tests/unit/test_i18n.py new file mode 100644 index 0000000..3aabbe6 --- /dev/null +++ b/tests/unit/test_i18n.py @@ -0,0 +1,10 @@ +import json + +from conftest import ROOT + + +def test_locales_have_same_keys(): + loc = json.loads((ROOT / 'locales.json').read_text(encoding='utf-8')) + assert set(loc) >= {'en', 'ru'} + assert len(loc['en']) == len(loc['ru']), 'locale key sets diverge' + assert set(loc['en']) == set(loc['ru']), 'locale key sets diverge' diff --git a/tests/unit/test_pipeline_regressions.py b/tests/unit/test_pipeline_regressions.py new file mode 100644 index 0000000..1541942 --- /dev/null +++ b/tests/unit/test_pipeline_regressions.py @@ -0,0 +1,25 @@ +from phyml_core.pipeline import PipelineWorker, ufboot_replicates + + +def test_empty_fasta_fails_cleanly(qapp, tmp_path): + """A broken input fails the run cleanly instead of killing the worker thread.""" + empty = tmp_path / 'empty.fasta' + empty.write_text('') + worker = PipelineWorker({'engine': 'phyml'}, [str(empty)]) + done = [] + worker.finished_signal.connect(lambda ok, tree: done.append((ok, tree))) + try: + worker.run() + except Exception as e: + raise AssertionError(f'worker crashed on an empty FASTA: {e!r}') + assert done == [(False, '')], done + assert worker.results[0]['status'] == 'FAIL', worker.results + + +def test_ufboot_replicates_raised_to_1000(): + """IQ-TREE UFBoot refuses < 1000 replicates: the PhyML-sized value is raised.""" + assert ufboot_replicates('100') == '1000' + assert ufboot_replicates('1000') == '1000' + assert ufboot_replicates('5000') == '5000' + assert ufboot_replicates('') == '1000' + assert ufboot_replicates('abc') == '1000' diff --git a/tests/unit/test_rf.py b/tests/unit/test_rf.py new file mode 100644 index 0000000..40c01ca --- /dev/null +++ b/tests/unit/test_rf.py @@ -0,0 +1,20 @@ +import io + +from Bio import Phylo + +from phyml_ui.tree_viewer import rf_distance, tree_splits + + +def _tree(newick): + return Phylo.read(io.StringIO(newick), 'newick') + + +def test_rf_identical_and_quartet_swap(): + t1 = _tree('(A,(B,(C,D)));') + t2 = _tree('(A,(C,(B,D)));') + assert rf_distance(t1, t1) == (0, 0.0) + assert rf_distance(t1, t2) == (1, 1.0) + + +def test_tree_splits_count(): + assert len(tree_splits(_tree('(A,(B,(C,D)));'))) == 1 diff --git a/tests/unit/test_stats_parsers.py b/tests/unit/test_stats_parsers.py new file mode 100644 index 0000000..4a7702c --- /dev/null +++ b/tests/unit/test_stats_parsers.py @@ -0,0 +1,13 @@ +from conftest import DATA +from phyml_core.pipeline import parse_stats, parse_iqtree_stats + + +def test_parse_phyml_stats(): + s = parse_stats(str(DATA / 'x_phyml_stats.txt')) + assert s['model'] == 'HKY85' and s['taxa'] == '8' and s['lnL'] == '-1811.15467' + assert s['gamma'] == '881.743' and s['tstv'] == '0.980317' + + +def test_parse_iqtree_stats(): + s = parse_iqtree_stats(str(DATA / 'x.iqtree')) + assert s['model'] == 'JC' and s['taxa'] == '8' and s['lnL'] == '-1828.7117' From 6fc906dfc40118faee5f4a2b5d78ad33027bb46d Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sun, 25 Oct 2026 00:58:12 +0300 Subject: [PATCH 14/36] Move end-to-end and GUI suites to pytest with binary markers Co-Authored-By: Claude Opus 5.5 --- tests/README.md | 46 +++++-- tests/e2e/datasets.py | 68 +++++++++++ tests/{ => e2e}/fetch_real_data.py | 3 +- tests/e2e/test_batch.py | 107 +++++++++++++++++ tests/e2e/test_cpu_monitor.py | 49 ++++++++ tests/e2e/test_iqtree.py | 143 ++++++++++++++++++++++ tests/e2e/test_phyml_pipeline.py | 142 ++++++++++++++++++++++ tests/gui/test_main_window.py | 143 ++++++++++++++++++++++ tests/run_tests.py | 187 ----------------------------- tests/test_batch.py | 112 ----------------- tests/test_gui_logic.py | 156 ------------------------ tests/test_iqtree.py | 132 -------------------- 12 files changed, 687 insertions(+), 601 deletions(-) create mode 100644 tests/e2e/datasets.py rename tests/{ => e2e}/fetch_real_data.py (96%) create mode 100644 tests/e2e/test_batch.py create mode 100644 tests/e2e/test_cpu_monitor.py create mode 100644 tests/e2e/test_iqtree.py create mode 100644 tests/e2e/test_phyml_pipeline.py create mode 100644 tests/gui/test_main_window.py delete mode 100644 tests/run_tests.py delete mode 100644 tests/test_batch.py delete mode 100644 tests/test_gui_logic.py delete mode 100644 tests/test_iqtree.py diff --git a/tests/README.md b/tests/README.md index e5bb8f9..3133ca7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,20 +1,40 @@ # phymlGUI test suites -Binary-free checks run under pytest (`pip install -r requirements-dev.txt`): +All suites run under pytest (`pip install -r requirements-dev.txt`): - python -m pytest tests/unit tests/gui # no analysis binaries needed (CI-friendly) + python -m pytest -q -rs # everything; tests without their binary are skipped + python -m pytest tests/unit tests/gui # no analysis binaries needed (CI-friendly) + python -m pytest tests/e2e # real MUSCLE / PhyML / MPI / IQ-TREE runs + python -m pytest -m "not slow" # skip the long real-data scenario -Standalone scripts, runnable on any machine with the Python dependencies: +Layout: - python tests/run_tests.py # end-to-end pipeline, needs local binaries - python tests/test_gui_logic.py # GUI behaviour + live CPU monitoring - python tests/test_batch.py # batch mode + CSV summary + history - python tests/test_iqtree.py # IQ-TREE engine (skips without the binary) +- `unit/`, `gui/` - fast checks. `gui/` builds `PhyMLApp` under `QT_QPA_PLATFORM=offscreen`. +- `e2e/` - end-to-end runs of the pipeline, batch mode, CPU monitor and IQ-TREE engine. + `e2e/datasets.py` holds the synthetic two-clade dataset (fixture `synthetic`). +- `data/` - static fixtures (stats files, trees). +- `conftest.py` - binary lookup, markers, the `isolated_config` fixture and a session guard + that fails the run if a test rewrites the real `phyml_config.json` / `phyml_history.json`. -Suites that need local analysis binaries (`phymlgui v1.1/phyml-mpi.exe`, -`tools/muscle.exe`, Microsoft MPI, optionally `tools/iqtree2/iqtree2.exe`) -print `SKIP` and exit 0 when the binaries are absent — e.g. on CI. +## Binaries and markers -Working files and generated datasets live in `testrun/` (gitignored); -`tests/fetch_real_data.py` downloads and caches real NCBI mitochondrial -CytB sequences used by the `t9` end-to-end scenario. +Tests that need an analysis binary carry a marker and are skipped (not failed) when the +binary is missing, e.g. on CI: + +| Marker | Binary | Environment variable | Default locations | +|---|---|---|---| +| `needs_phyml` | PhyML (MPI build) | `PHYMLGUI_PHYML` | `bin/phyml-mpi.exe`, `phymlgui v1.1/phyml-mpi.exe` | +| `needs_mpi` | MS-MPI `mpiexec` | `PHYMLGUI_MPIEXEC` | `bin/mpiexec.exe`, `C:\Program Files\Microsoft MPI\Bin\mpiexec.exe` | +| `needs_muscle` | MUSCLE | `PHYMLGUI_MUSCLE` | `bin/muscle.exe`, `tools/muscle.exe` | +| `needs_iqtree` | IQ-TREE 2 | `PHYMLGUI_IQTREE` | `bin/iqtree2.exe`, `tools/iqtree2/iqtree2.exe` | + +`PHYMLGUI_RAXML` (RAxML-NG) is looked up the same way for the upcoming engine. +The environment variable wins over the default locations. `slow` marks long end-to-end +tests; `-rs` prints the reason for every skip. + +## Working files + +Tests work in pytest's `tmp_path` and never write to the repository. The one exception +is the real-data cache `testrun/real/cytb_mammals.fasta` (gitignored): `e2e/fetch_real_data.py` +downloads mitochondrial CytB sequences from NCBI once for `test_real_cytb_tree`, which is +skipped when the cache is absent and the network is unreachable. diff --git a/tests/e2e/datasets.py b/tests/e2e/datasets.py new file mode 100644 index 0000000..77a0a58 --- /dev/null +++ b/tests/e2e/datasets.py @@ -0,0 +1,68 @@ +"""Synthetic test data and shared configs for the end-to-end suites. + +The dataset has a known topology: two clades A=(A1..A4) and B=(B1..B4), so a +finished run can be checked for the A|B split. +""" +import random + +import pytest + + +def evolve(seq, rate, rng): + bases = 'ACGT' + return ''.join(b if rng.random() > rate else rng.choice(bases) for b in seq) + + +def make_dataset(): + """(names, aligned sequences, unaligned sequences) of the two-clade dataset.""" + rng = random.Random(42) + ancestor = ''.join(rng.choice('ACGT') for _ in range(600)) + names, seqs_aligned, seqs_unaligned = [], [], [] + for clade, clade_rate in (('A', 0.16), ('B', 0.18)): + clade_consensus = evolve(ancestor, clade_rate, rng) + for i in range(1, 5): + s = evolve(clade_consensus, 0.03, rng) + name = f'{clade}{i}_testsp' + names.append(name) + seqs_aligned.append(s) + # unaligned variant: drop a random block from most sequences + if rng.random() < 0.8: + start = rng.randrange(50, 500) + ln = rng.randrange(8, 15) + s = s[:start] + s[start + ln:] + seqs_unaligned.append(s) + return names, seqs_aligned, seqs_unaligned + + +def write_fasta(path, names, seqs): + with open(path, 'w') as f: + for n, s in zip(names, seqs): + f.write(f'>{n}\n{s}\n') + + +@pytest.fixture +def synthetic(tmp_path): + """The dataset written to tmp_path: {'aligned': Path, 'unaligned': Path}.""" + names, aligned, unaligned = make_dataset() + folder = tmp_path / 'synthetic' + folder.mkdir() + paths = {'aligned': folder / 'test_prealigned.fasta', + 'unaligned': folder / 'test_unaligned.fasta'} + write_fasta(paths['aligned'], names, aligned) + write_fasta(paths['unaligned'], names, unaligned) + return paths + + +def phyml_config(binaries, **overrides): + """PipelineWorker config for a small PhyML run (MPI x2, HKY85, 20 bootstraps).""" + config = { + 'phyml_path': binaries['phyml'], 'muscle_path': binaries['muscle'], + 'mpirun_path': binaries['mpiexec'], 'mpi_np': '2', 'use_mpi': True, + 'datatype': 'nt', 'model': 'HKY85', + 'support_method': 'bootstrap', 'bootstrap_replicates': '20', + 'freq': 'default', 'tstv': 'e', 'pinv': '0.0', + 'classes': '4', 'gamma': 'e', 'search': 'NNI', + 'optimize': 'tlr', 'parsimony': False, + } + config.update(overrides) + return config diff --git a/tests/fetch_real_data.py b/tests/e2e/fetch_real_data.py similarity index 96% rename from tests/fetch_real_data.py rename to tests/e2e/fetch_real_data.py index 2272d83..b99db88 100644 --- a/tests/fetch_real_data.py +++ b/tests/e2e/fetch_real_data.py @@ -17,7 +17,8 @@ from Bio import SeqIO -RUN = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'testrun') +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +RUN = os.path.join(ROOT, 'testrun') os.makedirs(RUN, exist_ok=True) OUT_DIR = os.path.join(RUN, 'real') OUT_FA = os.path.join(OUT_DIR, 'cytb_mammals.fasta') diff --git a/tests/e2e/test_batch.py b/tests/e2e/test_batch.py new file mode 100644 index 0000000..23403cd --- /dev/null +++ b/tests/e2e/test_batch.py @@ -0,0 +1,107 @@ +"""Batch mode: a folder of FASTA files, CSV summary, queue and history.""" +import json +import shutil + +import pytest +from Bio import Phylo +from PySide6.QtWidgets import QMessageBox + +import phyml_ui.main_window as main_window +from datasets import phyml_config, synthetic # noqa: F401 (fixture) +from phyml_core.pipeline import PipelineWorker +from phyml_ui.main_window import PhyMLApp + +pytestmark = [pytest.mark.needs_phyml, pytest.mark.needs_mpi, pytest.mark.needs_muscle] + + +@pytest.fixture +def window(isolated_config, qtbot): + # isolated_config goes first so it is torn down last: qtbot closes the + # window at teardown and closeEvent still writes the (patched) config file + w = PhyMLApp() + qtbot.addWidget(w) + return w + + +@pytest.fixture +def batch_dir(tmp_path, synthetic): + """A folder with an unaligned (batchA) and a pre-aligned (batchB) FASTA.""" + run_dir = tmp_path / 't7_batch' + run_dir.mkdir() + shutil.copy(synthetic['unaligned'], run_dir / 'batchA.fasta') + shutil.copy(synthetic['aligned'], run_dir / 'batchB.fasta') + return run_dir + + +@pytest.fixture +def gui_batch(window, qtbot, monkeypatch, binaries, batch_dir, synthetic): + """The window after a finished batch run over batch_dir, started with Run.""" + # a MUSCLE output left by an earlier run: the queue must not pick it up + shutil.copy(synthetic['aligned'], batch_dir / 'batchA_aligned.fasta') + warned = [] + monkeypatch.setattr(QMessageBox, 'warning', + staticmethod(lambda *a, **k: warned.append(a) or QMessageBox.Ok)) + window.phyml_path_input.setText(binaries['phyml']) + window.mpirun_path_input.setText(binaries['mpiexec']) + window.muscle_path_input.setText(binaries['muscle']) + window.chk_use_mpi.setChecked(True) + window.set_combo_by_data(window.cb_support_method, 'bootstrap') + window.le_bootstrap_replicates.setText('20') + window.le_mpi_np.setText('2') + window.fasta_path_input.setText(str(batch_dir)) + window.btn_run.click() # real signal path: clicked(bool) must not break start_pipeline + assert not warned, f'run was blocked by a warning: {warned}' + qtbot.waitUntil(lambda: window.btn_run.isEnabled(), timeout=180_000) + window.worker.wait() + return window, batch_dir + + +def test_two_file_batch(qapp, qtbot, binaries, batch_dir): + """Batch of two files through the real worker (MPI + bootstrap).""" + files = [str(batch_dir / 'batchA.fasta'), str(batch_dir / 'batchB.fasta')] + done, logs = [], [] + worker = PipelineWorker(phyml_config(binaries), files) + worker.log_signal.connect(logs.append) + worker.finished_signal.connect(lambda ok, tree: done.append((ok, tree))) + worker.start() + try: + qtbot.waitUntil(lambda: bool(done), timeout=180_000) + finally: + worker.cancel() + worker.wait() + + assert done[0][0], 'batch failed:\n' + '\n'.join(logs[-10:]) + assert any('===' in line for line in logs), 'per-file headers missing' + assert any('2/2' in line for line in logs), 'batch summary missing' + assert (batch_dir / 'batchA_aligned.fasta').exists(), 'batchA not aligned' + assert (batch_dir / 'batchA_aligned.phy_phyml_tree.txt').exists() + tree_b = batch_dir / 'batchB.phy_phyml_tree.txt' + assert tree_b.exists() + assert len(Phylo.read(str(tree_b), 'newick').get_terminals()) == 8 + + +def test_gui_batch_writes_summary_csv(gui_batch): + """Full GUI path: a batch started from the window writes phymlgui_summary.csv.""" + window, run_dir = gui_batch + assert window.btn_run.isEnabled(), 'pipeline did not finish through GUI path' + csv_path = run_dir / 'phymlgui_summary.csv' + assert csv_path.exists(), 'summary CSV missing' + content = csv_path.read_text(encoding='utf-8-sig') + assert ('batchA.fasta' in content and 'batchB.fasta' in content + and 'OK' in content and 'HKY85' in content) + first_col = [line.split(';')[0] for line in content.splitlines()[1:] if line] + assert first_col == ['batchA.fasta', 'batchB.fasta'], f'queue leaked artifacts: {first_col}' + + +def test_queue_history_open_folder(gui_batch): + """Queue statuses, history record and the open-folder button after a batch.""" + window, _ = gui_batch + assert window.queue_list.count() == 2 + statuses = [window.queue_list.item(i).text().split(' ')[-1] for i in range(2)] + assert statuses == ['✓', '✓'], statuses + with open(main_window.HISTORY_FILE, encoding='utf-8') as f: + records = json.load(f) + assert records and records[0]['n'] == 2 and records[0]['ok'] == 2 + assert window.history_list.count() >= 1 + assert window.open_dir_btn.isEnabled() + assert not window.retry_btn.isVisible() or not window._failed_files diff --git a/tests/e2e/test_cpu_monitor.py b/tests/e2e/test_cpu_monitor.py new file mode 100644 index 0000000..b880191 --- /dev/null +++ b/tests/e2e/test_cpu_monitor.py @@ -0,0 +1,49 @@ +"""Live CPU monitoring of a real MPI run.""" +import shutil +import time + +import pytest + +from datasets import phyml_config, synthetic # noqa: F401 (fixture) +from phyml_core.pipeline import PipelineWorker +from phyml_ui.main_window import PhyMLApp + +pytestmark = [pytest.mark.needs_phyml, pytest.mark.needs_mpi, pytest.mark.needs_muscle] + + +@pytest.fixture +def window(isolated_config, qtbot): + # isolated_config goes first so it is torn down last: qtbot closes the + # window at teardown and closeEvent still writes the (patched) config file + w = PhyMLApp() + qtbot.addWidget(w) + return w + + +def test_cpu_samples_during_mpi_run(window, qapp, tmp_path, monkeypatch, binaries, synthetic): + run_dir = tmp_path / 't6_monitor' + run_dir.mkdir() + fasta = run_dir / 'test_unaligned.fasta' + shutil.copy(synthetic['unaligned'], fasta) + monkeypatch.chdir(run_dir) + + window._cpu_samples = [] + worker = PipelineWorker(phyml_config(binaries), str(fasta)) + window.worker = worker + done, wlogs = [], [] + worker.log_signal.connect(wlogs.append) + worker.finished_signal.connect(lambda ok, tree: done.append((ok, tree))) + worker.start() + ticks = 0 + while not done and ticks < 300: + qapp.processEvents() + time.sleep(0.3) + window.update_cpu_status() + ticks += 1 + if done: + worker.wait() # the signal is emitted just before the thread exits + assert done, 'worker did not finish' + assert done[0][0], 'pipeline failed:\n' + '\n'.join(wlogs[-10:]) + assert window._cpu_samples, 'no CPU samples collected' + peak = max(window._cpu_samples) + assert peak > 50, f'analysis CPU unexpectedly low: peak {peak}' diff --git a/tests/e2e/test_iqtree.py b/tests/e2e/test_iqtree.py new file mode 100644 index 0000000..bb79385 --- /dev/null +++ b/tests/e2e/test_iqtree.py @@ -0,0 +1,143 @@ +"""IQ-TREE engine, ModelFinder and the XLSX summary.""" +import random +import re +import shutil +import subprocess +from pathlib import Path + +import pytest +from Bio import AlignIO, Phylo +from PySide6.QtWidgets import QMessageBox + +from datasets import synthetic # noqa: F401 (fixture) +from phyml_core.pipeline import PipelineWorker +from phyml_ui.main_window import PhyMLApp + +pytestmark = pytest.mark.needs_iqtree + + +@pytest.fixture +def window(isolated_config, qtbot): + # isolated_config goes first so it is torn down last: qtbot closes the + # window at teardown and closeEvent still writes the (patched) config file + w = PhyMLApp() + qtbot.addWidget(w) + return w + + +@pytest.fixture +def run_dir(tmp_path, synthetic): + """A working folder holding the pre-aligned synthetic dataset.""" + folder = tmp_path / 't10_iqtree' + folder.mkdir() + shutil.copy(synthetic['aligned'], folder / 'test_prealigned.fasta') + return folder + + +def iqtree_config(binaries, **overrides): + config = { + 'engine': 'iqtree', 'iqtree_path': binaries['iqtree'], + 'use_mpi': False, 'mpi_np': '2', 'datatype': 'nt', 'model': 'MFP', + 'support_method': 'bootstrap', 'bootstrap_replicates': '1000', + 'freq': 'default', 'tstv': 'e', 'pinv': '0.0', + 'classes': '4', 'gamma': 'e', 'search': 'NNI', 'optimize': 'tlr', 'parsimony': False, + } + config.update(overrides) + return config + + +def run_worker(config, fasta): + """Run the worker synchronously; returns (worker, logs, finished results).""" + logs, done = [], [] + worker = PipelineWorker(config, str(fasta)) + worker.log_signal.connect(logs.append) + worker.finished_signal.connect(lambda ok, tree: done.append((ok, tree))) + worker.run() + return worker, logs, done + + +def test_mfp_ufboot_run(qapp, monkeypatch, binaries, run_dir): + """Worker-level: ModelFinder + ultrafast bootstrap on pre-aligned synthetic data.""" + monkeypatch.chdir(run_dir) + worker, logs, done = run_worker(iqtree_config(binaries), run_dir / 'test_prealigned.fasta') + + assert done[0][0], 'iqtree pipeline failed:\n' + '\n'.join(logs[-8:]) + tree_path = done[0][1] + assert Path(tree_path).exists(), tree_path + tree = Phylo.read(tree_path, 'newick') + assert len(tree.get_terminals()) == 8 + row = worker.results[0] + assert row['model'] and row['model'] != 'n/a', row + iq_report = (run_dir / 'test_prealigned_iqtree.iqtree').read_text(encoding='utf-8') + lnl_file = re.search(r'Log-likelihood of the tree:\s*(-?[\d.]+)', iq_report).group(1) + assert row['lnL'] == lnl_file, (row['lnL'], lnl_file) + + +def test_100_replicates_raised_to_1000(qapp, monkeypatch, binaries, run_dir): + """A PhyML-sized replicate count (100) must not break UFBoot (needs >= 1000).""" + monkeypatch.chdir(run_dir) + config = iqtree_config(binaries, bootstrap_replicates='100') + _, logs, done = run_worker(config, run_dir / 'test_prealigned.fasta') + assert done[0][0], 'iqtree with 100 replicates failed:\n' + '\n'.join(logs[-8:]) + assert any(' -B 1000' in line for line in logs), 'UFBoot replicates were not raised to 1000' + + +def test_gui_validation_and_run_with_xlsx(window, qtbot, monkeypatch, binaries, run_dir): + """GUI: engine validation, model list, full run + xlsx summary.""" + warned = [] + monkeypatch.setattr(QMessageBox, 'warning', + staticmethod(lambda *a, **k: warned.append(a) or QMessageBox.Ok)) + window.engine_setting = 'iqtree' + window.update_models() + assert window.cb_model.currentData() == 'MFP' + window.iqtree_path_input.setText('') # no binary -> blocked with warning + window.fasta_path_input.setText(str(run_dir / 'test_prealigned.fasta')) + window.start_pipeline() + assert warned and (window.worker is None or not window.worker.isRunning()) + + window.iqtree_path_input.setText(binaries['iqtree']) + window.set_combo_by_data(window.cb_support_method, 'bootstrap') + window.le_bootstrap_replicates.setText('100') + window.start_pipeline() + qtbot.waitUntil(lambda: window.btn_run.isEnabled(), timeout=180_000) + window.worker.wait() + assert window.worker.results[0]['status'] == 'OK', \ + 'GUI iqtree run failed:\n' + window.log_console.toPlainText()[-800:] + xlsx = run_dir / 'phymlgui_summary.xlsx' + assert xlsx.exists(), 'xlsx summary missing' + from openpyxl import load_workbook # optional dependency of the app + wb = load_workbook(str(xlsx)) + assert wb.active.max_row == 2 + + +def test_every_listed_model_accepted(window, binaries, run_dir): + """Every model offered in the IQ-TREE model list is accepted by IQ-TREE.""" + window.engine_setting = 'iqtree' + window.update_models() + + nt_phy = run_dir / 'test_prealigned.phy' + AlignIO.convert(str(run_dir / 'test_prealigned.fasta'), 'fasta', + str(nt_phy), 'phylip-relaxed') + rnd = random.Random(7) + aa_phy = run_dir / 'models_aa.phy' + with open(aa_phy, 'w') as f: + base = ''.join(rnd.choice('ACDEFGHIKLMNPQRSTVWY') for _ in range(120)) + f.write('6 120\n') + for i in range(6): + seq = ''.join(c if rnd.random() > 0.15 else rnd.choice('ACDEFGHIKLMNPQRSTVWY') + for c in base) + f.write(f'P{i} {seq}\n') + + rejected = [] + for dt, phy in {'nt': nt_phy, 'aa': aa_phy}.items(): + window.set_combo_by_data(window.cb_datatype, dt) + models = [window.cb_model.itemData(i) for i in range(window.cb_model.count())] + for m in models: + if m == 'MFP': + continue # ModelFinder is exercised by the runs above + r = subprocess.run([binaries['iqtree'], '-s', str(phy), '-m', m, '-n', '0', + '-T', '1', '-redo', '--prefix', str(run_dir / 'model_probe')], + capture_output=True, text=True) + if r.returncode != 0: + rejected.append(m) + assert not rejected, f'IQ-TREE rejects listed models: {rejected}' diff --git a/tests/e2e/test_phyml_pipeline.py b/tests/e2e/test_phyml_pipeline.py new file mode 100644 index 0000000..4f11bfa --- /dev/null +++ b/tests/e2e/test_phyml_pipeline.py @@ -0,0 +1,142 @@ +"""End-to-end tests for PipelineWorker with real MUSCLE + phyml-mpi. + +Runs the production pipeline on synthetic data with a known topology: two +clades A=(A1..A4), B=(B1..B4). After each run we check the output files and +that the A|B split is present in the tree. + +Scenarios: + t1: unaligned input, no MPI (MUSCLE + phyml-mpi standalone) + t2: unaligned input, MPI -np2 (MUSCLE + mpiexec) + t3: aligned input, no MPI (no MUSCLE) + t4: aligned input, MPI -np2 +""" +import re +import shutil +from pathlib import Path + +import pytest +from Bio import Phylo + +from datasets import phyml_config, synthetic # noqa: F401 (fixture) +from phyml_core.pipeline import PipelineWorker + +pytestmark = [pytest.mark.needs_phyml, pytest.mark.needs_muscle] + + +def run_pipeline(tmp_path, monkeypatch, source, folder, config): + """Copy `source` into tmp_path/folder and run the worker there synchronously.""" + wd = tmp_path / folder + wd.mkdir() + fasta = wd / Path(source).name + shutil.copy(source, fasta) + monkeypatch.chdir(wd) + logs = [] + worker = PipelineWorker(config, str(fasta)) + worker.log_signal.connect(logs.append) + worker.finished_signal.connect(lambda ok, tree: logs.append(f'__FINISHED__{ok}|{tree}')) + worker.run() # run synchronously in this thread + return wd, logs + + +def split_present(tree, names): + return any(not cl.is_terminal() and {t.name for t in cl.get_terminals()} == names + for cl in tree.find_clades()) + + +def check_run(wd, logs, expect_alignment): + ok = any(line.startswith('__FINISHED__True') for line in logs) + assert ok, 'pipeline reported failure:\n' + '\n'.join(logs) + + files = [f.name for f in wd.iterdir()] + aligned = [f for f in files if f.endswith('_aligned.fasta')] + assert bool(aligned) == expect_alignment, f'aligned file presence mismatch: {aligned}' + + tree_path = [f for f in files if f.endswith('_phyml_tree.txt')][0] + stats_path = [f for f in files if f.endswith('_phyml_stats.txt')][0] + tree = Phylo.read(str(wd / tree_path), 'newick') + terms = {t.name for t in tree.get_terminals()} + assert len(terms) == 8, f'expected 8 taxa, got {terms}' + + clade_a = {t for t in terms if t.startswith('A')} + clade_b = {t for t in terms if t.startswith('B')} + # Trees are UNROOTED with an arbitrary trifurcating root: the A|B split + # shows up as a node whose terminal set equals EITHER clade (whichever + # side faces away from the arbitrary root), never necessarily both. + assert split_present(tree, clade_a) or split_present(tree, clade_b), \ + 'A|B split missing - tree topology wrong' + + stats = (wd / stats_path).read_text() + assert 'Log-likelihood' in stats, 'stats file lacks log-likelihood' + stats_lines = [line for line in logs if 'HKY85' in line and '|' in line] + assert stats_lines, 'stats summary line missing from worker log' + m = re.search(r'lnL[^\d-]*(-?[\d.]+)', stats_lines[0]) + lnl_file = re.search(r'Log-likelihood:\s*(-?[\d.]+)', stats).group(1) + assert m and abs(float(m.group(1)) - float(lnl_file)) < 0.01, 'lnL mismatch in stats summary' + + +# phyml-mpi cannot distribute a serial bootstrap (asserts n_proc > 1), so +# standalone runs use aLRT branch support; MPI runs use bootstrap. +SCENARIOS = [ + pytest.param('t1_unaligned_serial_alrt', 'unaligned', False, True, '-4', id='t1'), + pytest.param('t2_unaligned_mpi_boot', 'unaligned', True, True, 'bootstrap', id='t2', + marks=pytest.mark.needs_mpi), + pytest.param('t3_aligned_serial_alrt', 'aligned', False, False, '-4', id='t3'), + pytest.param('t4_aligned_mpi_boot', 'aligned', True, False, 'bootstrap', id='t4', + marks=pytest.mark.needs_mpi), +] + + +@pytest.mark.parametrize('folder, dataset, use_mpi, expect_alignment, support', SCENARIOS) +def test_scenarios(qapp, tmp_path, monkeypatch, binaries, synthetic, + folder, dataset, use_mpi, expect_alignment, support): + config = phyml_config(binaries, use_mpi=use_mpi, support_method=support) + wd, logs = run_pipeline(tmp_path, monkeypatch, synthetic[dataset], folder, config) + check_run(wd, logs, expect_alignment) + + +def test_serial_bootstrap_on_mpi_build_fails_gracefully(qapp, tmp_path, monkeypatch, + binaries, synthetic): + """Serial bootstrap on the MPI build must fail gracefully (PhyML assert).""" + config = phyml_config(binaries, use_mpi=False, support_method='bootstrap') + _, logs = run_pipeline(tmp_path, monkeypatch, synthetic['unaligned'], + 't5_serial_boot_expected_fail', config) + failed_ok = any(line.startswith('__FINISHED__False') for line in logs) + assert failed_ok, 'expected graceful failure for serial bootstrap on MPI build' + + +@pytest.mark.needs_mpi +@pytest.mark.slow +def test_real_cytb_tree(qapp, tmp_path, monkeypatch, binaries): + """Real NCBI data (mammalian mitochondrial CytB) with a known phylogeny.""" + import fetch_real_data # imported lazily: it creates testrun/ on import + if not Path(fetch_real_data.OUT_FA).exists(): + try: + assert fetch_real_data.main() == 0 + except Exception as e: + pytest.skip(f'real data unavailable ({e})') + + # cross-order mammalian mtDNA needs GTR+gamma and SPR search; + # HKY85+NNI scrambles the short hominoid internal branches + config = phyml_config(binaries, use_mpi=True, support_method='bootstrap', + model='GTR', search='SPR') + wd, logs = run_pipeline(tmp_path, monkeypatch, fetch_real_data.OUT_FA, + 't9_real_ncbi', config) + ok = any(line.startswith('__FINISHED__True') for line in logs) + assert ok, 'real-data pipeline failed:\n' + '\n'.join(logs[-8:]) + tree_path = [f.name for f in wd.iterdir() if f.name.endswith('_phyml_tree.txt')][0] + tree = Phylo.read(str(wd / tree_path), 'newick') + terms = {t.name for t in tree.get_terminals()} + rodents = {'Mus_musculus', 'Rattus_norvegicus'} + primates = {'Homo_sapiens', 'Pan_troglodytes', 'Pan_paniscus', + 'Gorilla_gorilla', 'Pongo_pygmaeus', 'Macaca_mulatta'} + assert terms == primates | rodents, f'unexpected taxa: {terms}' + # rodents|primates is one split: either side may face the arbitrary root + assert split_present(tree, rodents) or split_present(tree, primates), \ + 'rodents/primates split missing' + # Pan paniscus + Pan troglodytes is the strongly supported sister pair + # (~19/20). NB: Homo+Pan and the exact hominoid backbone are NOT + # recoverable from a single 1141 bp mitochondrial gene (saturated 3rd + # codon positions, long macaque branch) - a known single-gene CytB + # limitation, not a pipeline defect, so we do not assert them. + assert split_present(tree, {'Pan_troglodytes', 'Pan_paniscus'}), \ + 'Pan sister pair missing in CytB tree' diff --git a/tests/gui/test_main_window.py b/tests/gui/test_main_window.py new file mode 100644 index 0000000..0786ba4 --- /dev/null +++ b/tests/gui/test_main_window.py @@ -0,0 +1,143 @@ +"""GUI behaviour of PhyMLApp that needs no analysis binaries.""" +import pytest +from PySide6.QtWidgets import QMessageBox, QSizePolicy + +from phyml_core.i18n import _t +from phyml_core.utils import detect_cores +from phyml_ui.main_window import PhyMLApp, SettingsDialog +from phyml_ui.themes import build_qss, resolve + + +@pytest.fixture +def window(isolated_config, qtbot): + # isolated_config goes first so it is torn down last: qtbot closes the + # window at teardown and closeEvent still writes the (patched) config file + w = PhyMLApp() + qtbot.addWidget(w) + return w + + +@pytest.fixture +def warned(monkeypatch): + """Replace QMessageBox.warning; the list collects the (parent, title, text) calls.""" + shown = [] + monkeypatch.setattr(QMessageBox, 'warning', + staticmethod(lambda *a, **k: shown.append(a) or QMessageBox.Ok)) + return shown + + +@pytest.fixture +def restore_stylesheet(qapp): + original = qapp.styleSheet() + yield + qapp.setStyleSheet(original) + + +def test_defaults_match_detected_cores(window): + """np defaults to the physical core count; the status bar shows the core info.""" + phys, logi = detect_cores() + assert window.le_mpi_np.text() == str(window.phys_cores), window.le_mpi_np.text() + assert (window.phys_cores, window.logi_cores) == (phys, logi) + assert str(phys) in window.cores_label.text() and str(logi) in window.cores_label.text() + + +def test_mpi_binary_autodetect(window, tmp_path): + window.chk_use_mpi.setChecked(False) + window.phyml_path_input.setText(str(tmp_path / 'phyml.exe')) + assert not window.chk_use_mpi.isChecked(), 'plain phyml.exe must not enable MPI' + window.phyml_path_input.setText(str(tmp_path / 'phyml-mpi.exe')) + assert window.chk_use_mpi.isChecked(), 'phyml-mpi.exe must auto-enable MPI' + log_text = window.log_console.toPlainText() + assert 'MPI' in log_text, 'detection message missing from log' + + +def test_serial_bootstrap_on_mpi_build_blocked(window, warned, tmp_path): + """MPI binary + MPI off + bootstrap -> blocked with a warning.""" + window.phyml_path_input.setText(str(tmp_path / 'phyml-mpi.exe')) + window.chk_use_mpi.setChecked(False) # after the path: the path edit auto-enables MPI + window.set_combo_by_data(window.cb_support_method, 'bootstrap') + fasta = tmp_path / 'test_unaligned.fasta' + fasta.write_text('>a\nACGT\n>b\nACGA\n') + window.fasta_file = str(fasta) + window.start_pipeline() + assert warned, 'warning box must be shown' + assert warned[0][1] == _t('mpi_boot_title'), 'blocked for the wrong reason' + assert window.btn_run.isEnabled(), 'Run must stay enabled after blocked start' + assert window.worker is None or not window.worker.isRunning() + + +def test_theme_tabs_presets_log_filter_recent(window, qapp, tmp_path, restore_stylesheet): + """New UI: themes, results tabs, structured log, recent list, cpu timer.""" + window.apply_theme('dark', 'blue') + theme, accent = resolve('dark', 'blue') + assert theme == 'dark' and accent == '#2D7DD2' + assert qapp.styleSheet() == build_qss(theme, accent) + assert window.log_console._role_colors['out'] == '#E6E8EB', 'phyml output must be themed' + assert window.log_console._role_colors['system'] == '#2D7DD2', 'system lines use accent' + assert window.cpu_timer.isActive(), 'CPU monitor timer must be running' + window.apply_theme('system', 'system') + assert window.result_tabs.count() == 4, 'expected 4 result tabs (log/stats/summary/queue)' + + # presets: built-ins listed, apply changes support method, user preset saves + assert window.presets_list.count() >= 3 + texts = [window.presets_list.item(i).text() for i in range(window.presets_list.count())] + screen_row = next(i for i, t in enumerate(texts) + if 'скрининг' in t.lower() or 'screening' in t.lower()) + window._preset_activated(window.presets_list.item(screen_row)) + assert window.cb_support_method.currentData() == '-4', 'screening preset must select aLRT' + window._user_presets['my-test'] = {k: window.get_current_config()[k] for k in ('model',)} + window._refresh_presets() + assert any(window.presets_list.item(i).text() == 'my-test' + for i in range(window.presets_list.count())) + + # tree viewer zoom / pan mode buttons + window.tree_view.btn_zoom.click() + assert (window.tree_view.btn_zoom.isChecked() + and window.tree_view.mode_label.text() == _t('tree_zoom')) + window.tree_view.btn_pan.click() + assert window.tree_view.btn_pan.isChecked() and not window.tree_view.btn_zoom.isChecked() + window.tree_view.btn_pan.click() + assert not window.tree_view.btn_pan.isChecked() + + # settings dialog: a path edit is applied and re-triggers the MPI autodetect + d = SettingsDialog(window, window) + assert d.edits[0].text() == window.phyml_path_input.text() + mpi_path = str(tmp_path / 'some' / 'phyml-mpi.exe') + d.edits[0].setText(mpi_path) + d.accept() + assert window.phyml_path_input.text() == mpi_path + assert window.chk_use_mpi.isChecked(), 'settings edit must re-trigger MPI autodetect' + + # structured log: filter by role + window.log_console.filter_box.setCurrentIndex(0) + window.log_console.clear() + window.log_console.append('[ERROR] test failure line') + window.log_console.append('plain phyml output line') + assert '[ERROR] test failure line' in window.log_console.toPlainText() + assert 'plain phyml output line' in window.log_console.toPlainText() + window.set_combo_by_data(window.log_console.filter_box, 'error') + assert '[ERROR]' in window.log_console.view.toPlainText() + assert 'plain phyml output line' not in window.log_console.view.toPlainText() + window.set_combo_by_data(window.log_console.filter_box, 'all') + assert 'plain phyml output line' in window.log_console.view.toPlainText() + + # recent list: most recent first, no duplicates + window._remember_recent('X:/a.fasta') + window._remember_recent('X:/b.fasta') + window._remember_recent('X:/a.fasta') + assert window.recent_list.count() == 2 and window._recent[0] == 'X:/a.fasta' + + +def test_cpu_bar_expands(window): + """The CPU bar fills the free status bar space.""" + assert window.cpu_sys_bar.sizePolicy().horizontalPolicy() == QSizePolicy.Policy.Expanding + + +def test_empty_folder_blocked(window, warned, tmp_path): + empty_dir = tmp_path / 't8_empty_dir' + empty_dir.mkdir() + window.fasta_path_input.setText(str(empty_dir)) + window.start_pipeline() + assert warned, 'empty folder must trigger a warning' + assert warned[0][2] == _t('batch_empty'), 'blocked for the wrong reason' + assert window.worker is None or not window.worker.isRunning() diff --git a/tests/run_tests.py b/tests/run_tests.py deleted file mode 100644 index 73ac2d9..0000000 --- a/tests/run_tests.py +++ /dev/null @@ -1,187 +0,0 @@ -"""End-to-end tests for phymlGUI PipelineWorker using real MUSCLE + phyml-mpi. - -Runs the production pipeline code (phymlgui.PipelineWorker) on synthetic data -with a known topology: two clades A=(A1..A4), B=(B1..B4). After each run we -check the output files and that both clades come out monophyletic. - -Scenarios: - t1: unaligned input, no MPI (MUSCLE + phyml-mpi standalone) - t2: unaligned input, MPI -np2 (MUSCLE + mpiexec) - t3: aligned input, no MPI (no MUSCLE) - t4: aligned input, MPI -np2 -""" -import os, sys, random, shutil, re - -os.environ['QT_QPA_PLATFORM'] = 'offscreen' -ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, ROOT) - -import phymlgui -from Bio import Phylo, SeqIO -from Bio.Seq import Seq -from Bio.SeqRecord import SeqRecord - -RUN = os.path.join(ROOT, 'testrun') -os.makedirs(RUN, exist_ok=True) -PHYML_EXE = os.path.join(ROOT, 'phymlgui v1.1', 'phyml-mpi.exe') -MUSCLE_EXE = os.path.join(ROOT, 'tools', 'muscle.exe') -MPIEXEC_EXE = r'C:\Program Files\Microsoft MPI\Bin\mpiexec.exe' - -if not all(map(os.path.exists, (PHYML_EXE, MUSCLE_EXE, MPIEXEC_EXE))): - print('SKIP: local analysis binaries not found (phymlgui v1.1/phyml-mpi.exe, tools/muscle.exe, MS-MPI)') - sys.exit(0) - -# ---------- test data generation ---------- -def evolve(seq, rate, rng): - bases = 'ACGT' - return ''.join(b if rng.random() > rate else rng.choice(bases) for b in seq) - -def make_dataset(): - rng = random.Random(42) - ancestor = ''.join(rng.choice('ACGT') for _ in range(600)) - names, seqs_aligned, seqs_unaligned = [], [], [] - for clade, clade_rate in (('A', 0.16), ('B', 0.18)): - clade_consensus = evolve(ancestor, clade_rate, rng) - for i in range(1, 5): - s = evolve(clade_consensus, 0.03, rng) - name = f'{clade}{i}_testsp' - names.append(name) - seqs_aligned.append(s) - # unaligned variant: drop a random block from most sequences - if rng.random() < 0.8: - start = rng.randrange(50, 500) - ln = rng.randrange(8, 15) - s = s[:start] + s[start + ln:] - seqs_unaligned.append(s) - return names, seqs_aligned, seqs_unaligned - -def write_fasta(path, names, seqs): - with open(path, 'w') as f: - for n, s in zip(names, seqs): - f.write(f'>{n}\n{s}\n') - -# ---------- pipeline runner ---------- -def run_pipeline(folder, fasta_name, use_mpi, support='bootstrap', model='HKY85', search='NNI'): - wd = os.path.join(RUN, folder) - shutil.rmtree(wd, ignore_errors=True) - os.makedirs(wd, exist_ok=True) - shutil.copy(os.path.join(RUN, fasta_name), wd) - os.chdir(wd) - app = phymlgui.QApplication.instance() or phymlgui.QApplication([]) - - config = { - 'phyml_path': PHYML_EXE, 'muscle_path': MUSCLE_EXE, - 'mpirun_path': MPIEXEC_EXE, 'mpi_np': '2', 'use_mpi': use_mpi, - 'datatype': 'nt', 'model': model, - 'support_method': support, 'bootstrap_replicates': '20', - 'freq': 'default', 'tstv': 'e', 'pinv': '0.0', - 'classes': '4', 'gamma': 'e', 'search': search, - 'optimize': 'tlr', 'parsimony': False, - } - logs = [] - worker = phymlgui.PipelineWorker(config, os.path.join(wd, fasta_name)) - worker.log_signal.connect(logs.append) - worker.finished_signal.connect(lambda ok, tree: logs.append(f'__FINISHED__{ok}|{tree}')) - worker.run() # run synchronously in this thread - - os.chdir(ROOT) - return wd, logs - -def check(wd, logs, expect_alignment): - ok = any(l.startswith('__FINISHED__True') for l in logs) - assert ok, 'pipeline reported failure:\n' + '\n'.join(logs) - - aligned = [f for f in os.listdir(wd) if f.endswith('_aligned.fasta')] - assert bool(aligned) == expect_alignment, f'aligned file presence mismatch: {aligned}' - - tree_path = [f for f in os.listdir(wd) if f.endswith('_phyml_tree.txt')][0] - stats_path = [f for f in os.listdir(wd) if f.endswith('_phyml_stats.txt')][0] - tree = Phylo.read(os.path.join(wd, tree_path), 'newick') - terms = {t.name for t in tree.get_terminals()} - assert len(terms) == 8, f'expected 8 taxa, got {terms}' - - cladeA = {t for t in terms if t.startswith('A')} - cladeB = {t for t in terms if t.startswith('B')} - # Trees are UNROOTED with an arbitrary trifurcating root: the A|B split - # shows up as a node whose terminal set equals EITHER clade (whichever - # side faces away from the arbitrary root), never necessarily both. - def is_split_present(names): - return any( - not cl.is_terminal() and {t.name for t in cl.get_terminals()} == names - for cl in tree.find_clades()) - split_ok = is_split_present(cladeA) or is_split_present(cladeB) - assert split_ok, 'A|B split missing — tree topology wrong' - - stats = open(os.path.join(wd, stats_path)).read() - assert 'Log-likelihood' in stats, 'stats file lacks log-likelihood' - stats_lines = [l for l in logs if 'HKY85' in l and '|' in l] - assert stats_lines, 'stats summary line missing from worker log' - m = re.search(r'lnL[^\d-]*(-?[\d.]+)', stats_lines[0]) - lnl_file = re.search(r'Log-likelihood:\s*(-?[\d.]+)', stats).group(1) - assert m and abs(float(m.group(1)) - float(lnl_file)) < 0.01, 'lnL mismatch in stats summary' - return tree_path, split_ok - -if __name__ == '__main__': - names, sa, su = make_dataset() - write_fasta(os.path.join(RUN, 'test_prealigned.fasta'), names, sa) - write_fasta(os.path.join(RUN, 'test_unaligned.fasta'), names, su) - - # phyml-mpi cannot distribute a serial bootstrap (asserts n_proc > 1), - # so standalone runs use aLRT branch support; MPI runs use bootstrap. - scenarios = [ - ('t1_unaligned_serial_alrt', 'test_unaligned.fasta', False, True, '-4'), - ('t2_unaligned_mpi_boot', 'test_unaligned.fasta', True, True, 'bootstrap'), - ('t3_aligned_serial_alrt', 'test_prealigned.fasta', False, False, '-4'), - ('t4_aligned_mpi_boot', 'test_prealigned.fasta', True, False, 'bootstrap'), - ] - for folder, fasta, mpi, expect_aln, support in scenarios: - wd, logs = run_pipeline(folder, fasta, mpi, support) - tree, _ = check(wd, logs, expect_aln) - mpi_tag = 'MPI x2 + bootstrap' if mpi else 'serial + aLRT' - aln_tag = 'MUSCLE ran' if expect_aln else 'pre-aligned' - print(f'PASS {folder}: {mpi_tag}, {aln_tag}, tree OK ({tree})') - - # t5: serial bootstrap on the MPI build must fail gracefully (PhyML assert) - wd, logs = run_pipeline('t5_serial_boot_expected_fail', 'test_unaligned.fasta', False, 'bootstrap') - failed_ok = any(l.startswith('__FINISHED__False') for l in logs) - assert failed_ok, 'expected graceful failure for serial bootstrap on MPI build' - print('PASS t5_serial_boot_expected_fail: PhyML error surfaced in log, no crash') - - # t9: real NCBI data (mammalian mitochondrial CytB) with known phylogeny - sys.path.insert(0, RUN) - import fetch_real_data - if not os.path.exists(fetch_real_data.OUT_FA): - try: - assert fetch_real_data.main() == 0 - except Exception as e: - print(f'SKIP t9_real_ncbi: real data unavailable ({e})') - if os.path.exists(fetch_real_data.OUT_FA): - shutil.copy(fetch_real_data.OUT_FA, os.path.join(RUN, 'cytb_mammals.fasta')) - # cross-order mammalian mtDNA needs GTR+gamma and SPR search; - # HKY85+NNI scrambles the short hominoid internal branches - wd, logs = run_pipeline('t9_real_ncbi', 'cytb_mammals.fasta', True, - 'bootstrap', model='GTR', search='SPR') - ok = any(l.startswith('__FINISHED__True') for l in logs) - assert ok, 'real-data pipeline failed:\n' + '\n'.join(logs[-8:]) - tree_path = [f for f in os.listdir(wd) if f.endswith('_phyml_tree.txt')][0] - tree = Phylo.read(os.path.join(wd, tree_path), 'newick') - terms = {t.name for t in tree.get_terminals()} - rodents = {'Mus_musculus', 'Rattus_norvegicus'} - primates = {'Homo_sapiens', 'Pan_troglodytes', 'Pan_paniscus', - 'Gorilla_gorilla', 'Pongo_pygmaeus', 'Macaca_mulatta'} - assert terms == primates | rodents, f'unexpected taxa: {terms}' - def split(names): - return any(not c.is_terminal() and {t.name for t in c.get_terminals()} == names - for c in tree.find_clades()) - # rodents|primates is one split: either side may face the arbitrary root - assert split(rodents) or split(primates), 'rodents/primates split missing' - # Pan paniscus + Pan troglodytes is the strongly supported sister pair - # (~19/20). NB: Homo+Pan and the exact hominoid backbone are NOT - # recoverable from a single 1141 bp mitochondrial gene (saturated 3rd - # codon positions, long macaque branch) — a known single-gene CytB - # limitation, not a pipeline defect, so we do not assert them. - pan_pair = {'Pan_troglodytes', 'Pan_paniscus'} - assert split(pan_pair), 'Pan sister pair missing in CytB tree' - print('PASS t9_real_ncbi: real CytB tree recovers rodents/primates split and Pan pair') - - print('ALL PIPELINE TESTS PASSED') diff --git a/tests/test_batch.py b/tests/test_batch.py deleted file mode 100644 index df7157e..0000000 --- a/tests/test_batch.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Tests for the stretching CPU bar and batch mode.""" -import os, sys, time, shutil - -os.environ['QT_QPA_PLATFORM'] = 'offscreen' -ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, ROOT) -os.chdir(ROOT) - -import phymlgui -from PySide6.QtWidgets import QSizePolicy, QMessageBox - -app = phymlgui.QApplication([]) -if not all(os.path.exists(p) for p in ( - os.path.join(ROOT, 'phymlgui v1.1', 'phyml-mpi.exe'), - os.path.join(ROOT, 'tools', 'muscle.exe'))): - print('SKIP: local analysis binaries not found') - sys.exit(0) - -w = phymlgui.PhyMLApp() - -# 1. CPU bar expands to fill free status bar space -assert w.cpu_sys_bar.sizePolicy().horizontalPolicy() == QSizePolicy.Expanding -print('PASS status bar: cpu bar has Expanding policy') - -# 2. empty-folder guard -empty_dir = os.path.abspath(os.path.join('testrun', 't8_empty_dir')) -os.makedirs(empty_dir, exist_ok=True) -warned = [] -phymlgui.QMessageBox.warning = staticmethod(lambda *a, **k: warned.append(a) or QMessageBox.Ok) -w.fasta_path_input.setText(empty_dir) -w.start_pipeline() -assert warned, 'empty folder must trigger a warning' -assert w.worker is None or not w.worker.isRunning() -print('PASS batch guard: empty folder blocked with warning') - -# 3. batch of two files through the real worker (MPI + bootstrap) -run_dir = os.path.join('testrun', 't7_batch') -shutil.rmtree(run_dir, ignore_errors=True) -os.makedirs(run_dir, exist_ok=True) -shutil.copy('testrun/test_unaligned.fasta', os.path.join(run_dir, 'batchA.fasta')) -shutil.copy('testrun/test_prealigned.fasta', os.path.join(run_dir, 'batchB.fasta')) - -config = { - 'phyml_path': os.path.join(ROOT, 'phymlgui v1.1', 'phyml-mpi.exe'), - 'muscle_path': os.path.join(ROOT, 'tools', 'muscle.exe'), - 'mpirun_path': r'C:\Program Files\Microsoft MPI\Bin\mpiexec.exe', - 'mpi_np': '2', 'use_mpi': True, 'datatype': 'nt', 'model': 'HKY85', - 'support_method': 'bootstrap', 'bootstrap_replicates': '20', - 'freq': 'default', 'tstv': 'e', 'pinv': '0.0', 'classes': '4', - 'gamma': 'e', 'search': 'NNI', 'optimize': 'tlr', 'parsimony': False, -} -files = [os.path.join(run_dir, 'batchA.fasta'), os.path.join(run_dir, 'batchB.fasta')] -done, logs = [], [] -worker = phymlgui.PipelineWorker(config, files) -worker.log_signal.connect(logs.append) -worker.finished_signal.connect(lambda ok, tree: done.append((ok, tree))) -worker.start() -while not done: - app.processEvents() - time.sleep(0.3) - -assert done[0][0], 'batch failed:\n' + '\n'.join(logs[-10:]) -assert any('===' in l for l in logs), 'per-file headers missing' -assert any('2/2' in l for l in logs), 'batch summary missing' -assert os.path.exists(os.path.join(run_dir, 'batchA_aligned.fasta')), 'batchA not aligned' -assert os.path.exists(os.path.join(run_dir, 'batchA_aligned.phy_phyml_tree.txt')) -assert os.path.exists(os.path.join(run_dir, 'batchB.phy_phyml_tree.txt')) -from Bio import Phylo -assert len(Phylo.read(os.path.join(run_dir, 'batchB.phy_phyml_tree.txt'), 'newick').get_terminals()) == 8 -print('PASS batch: 2/2 files processed, trees for both, headers and summary logged') - -# 4. full GUI path: batch via start_pipeline writes phymlgui_summary.csv -cfg_bak = None -if os.path.exists(phymlgui.CONFIG_FILE): - import shutil as _sh - cfg_bak = phymlgui.CONFIG_FILE + '.bak' - _sh.copy(phymlgui.CONFIG_FILE, cfg_bak) -try: - w.le_bootstrap_replicates.setText('20') # user config may hold 10000+ - w.le_mpi_np.setText('2') - run_dir_abs = os.path.abspath(run_dir) - w.fasta_path_input.setText(run_dir_abs) - w.btn_run.click() # real signal path: clicked(bool) must not break start_pipeline - guard = 0 - while (not w.btn_run.isEnabled()) and guard < 400: - app.processEvents(); time.sleep(0.3); guard += 1 - assert w.btn_run.isEnabled(), 'pipeline did not finish through GUI path' - csv_path = os.path.join(run_dir, 'phymlgui_summary.csv') - assert os.path.exists(csv_path), 'summary CSV missing' - content = open(csv_path, encoding='utf-8-sig').read() - assert 'batchA.fasta' in content and 'batchB.fasta' in content and 'OK' in content and 'HKY85' in content - first_col = [l.split(';')[0] for l in content.splitlines()[1:] if l] - assert first_col == ['batchA.fasta', 'batchB.fasta'], f'queue leaked artifacts: {first_col}' - print('PASS csv summary: phymlgui_summary.csv written with both files and stats') - - # 5. stage-3: queue statuses, history record, open-folder enabled - assert w.queue_list.count() == 2 - statuses = [w.queue_list.item(i).text().split(' ')[-1] for i in range(2)] - assert statuses == ['✓', '✓'], statuses - import json as _json - from phyml_ui.main_window import HISTORY_FILE - records = _json.load(open(HISTORY_FILE, encoding='utf-8')) - assert records and records[0]['n'] == 2 and records[0]['ok'] == 2 - assert w.history_list.count() >= 1 - assert w.open_dir_btn.isEnabled() - assert not w.retry_btn.isVisible() or not w._failed_files - print('PASS stage3: queue statuses, history record, open-folder enabled') -finally: - if cfg_bak: - import shutil as _sh - _sh.move(cfg_bak, phymlgui.CONFIG_FILE) -print('ALL BATCH TESTS PASSED') diff --git a/tests/test_gui_logic.py b/tests/test_gui_logic.py deleted file mode 100644 index 5352241..0000000 --- a/tests/test_gui_logic.py +++ /dev/null @@ -1,156 +0,0 @@ -"""GUI logic tests for the new MPI/core/CPU-monitoring features.""" -import os, sys, json, time, shutil - -os.environ['QT_QPA_PLATFORM'] = 'offscreen' -ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, ROOT) -os.chdir(ROOT) - -import phymlgui -from PySide6.QtWidgets import QMessageBox - -# 1. locales.json: valid JSON, all new keys present in both languages -BIN_ROOT = ROOT -PHYML_OK = all(os.path.exists(p) for p in ( - os.path.join(BIN_ROOT, 'phymlgui v1.1', 'phyml-mpi.exe'), - os.path.join(BIN_ROOT, 'tools', 'muscle.exe'), - r'C:\Program Files\Microsoft MPI\Bin\mpiexec.exe')) - -loc = json.load(open('locales.json', encoding='utf-8')) -new_keys = ['cores_info', 'cpu_sys', 'cpu_analysis', 'cpu_analysis_idle', - 'cpu_analysis_done', 'mpi_detected', 'mpi_boot_title', 'mpi_boot_msg'] -for lang in ('en', 'ru'): - for k in new_keys: - assert k in loc[lang], f'{lang} missing {k}' -print('PASS locales: new keys present in en+ru') - -app = phymlgui.QApplication([]) - -# keep existing config out of the way to test defaults -had_config = os.path.exists(phymlgui.CONFIG_FILE) -if had_config: - shutil.move(phymlgui.CONFIG_FILE, phymlgui.CONFIG_FILE + '.bak') - -try: - w = phymlgui.PhyMLApp() - - # 2. np default == physical cores; status bar shows core info - assert w.le_mpi_np.text() == str(w.phys_cores), w.le_mpi_np.text() - assert w.phys_cores == 8 and w.logi_cores == 16, (w.phys_cores, w.logi_cores) - assert '8' in w.cores_label.text() and '16' in w.cores_label.text() - print(f'PASS defaults: np={w.le_mpi_np.text()}, cores label: {w.cores_label.text()!r}') - - # 3. MPI binary autodetection - w.chk_use_mpi.setChecked(False) - w.phyml_path_input.setText(r'D:\tools\phyml.exe') - assert not w.chk_use_mpi.isChecked(), 'plain phyml.exe must not enable MPI' - w.phyml_path_input.setText(r'D:\phyml\phymlgui v1.1\phyml-mpi.exe') - assert w.chk_use_mpi.isChecked(), 'phyml-mpi.exe must auto-enable MPI' - log_text = w.log_console.toPlainText() - assert 'MPI' in log_text, 'detection message missing from log' - print('PASS mpi autodetect: checkbox auto-enabled + logged') - - # 4. validation: MPI binary + MPI off + bootstrap -> blocked with warning - w.chk_use_mpi.setChecked(False) - w.set_combo_by_data(w.cb_support_method, 'bootstrap') - w.fasta_file = os.path.join('testrun', 'test_unaligned.fasta') - warned = [] - phymlgui.QMessageBox.warning = staticmethod(lambda *a, **k: warned.append(a) or QMessageBox.Ok) - w.start_pipeline() - assert warned, 'warning box must be shown' - assert w.btn_run.isEnabled(), 'Run must stay enabled after blocked start' - assert w.worker is None or not w.worker.isRunning() - print('PASS validation: serial bootstrap on MPI build blocked with warning') - - # 5. CPU monitor during a real MPI run - run_dir = os.path.join('testrun', 't6_monitor') - shutil.rmtree(run_dir, ignore_errors=True) - os.makedirs(run_dir, exist_ok=True) - shutil.copy(w.fasta_file, run_dir) - fasta = os.path.join(run_dir, 'test_unaligned.fasta') - - config = { - 'phyml_path': os.path.join(ROOT, 'phymlgui v1.1', 'phyml-mpi.exe'), - 'muscle_path': os.path.join(ROOT, 'tools', 'muscle.exe'), - 'mpirun_path': r'C:\Program Files\Microsoft MPI\Bin\mpiexec.exe', - 'mpi_np': '2', 'use_mpi': True, 'datatype': 'nt', 'model': 'HKY85', - 'support_method': 'bootstrap', 'bootstrap_replicates': '20', - 'freq': 'default', 'tstv': 'e', 'pinv': '0.0', 'classes': '4', - 'gamma': 'e', 'search': 'NNI', 'optimize': 'tlr', 'parsimony': False, - } - os.chdir(run_dir) - fasta = os.path.abspath('test_unaligned.fasta') - w._cpu_samples = [] - worker = phymlgui.PipelineWorker(config, fasta) - w.worker = worker - done, wlogs = [], [] - worker.log_signal.connect(wlogs.append) - worker.finished_signal.connect(lambda ok, tree: done.append((ok, tree))) - worker.start() - ticks = 0 - while not done and ticks < 300: - app.processEvents() - time.sleep(0.3) - w.update_cpu_status() - ticks += 1 - os.chdir(ROOT) - assert done, 'worker did not finish' - assert done[0][0], 'pipeline failed:\n' + '\n'.join(wlogs[-10:]) - assert w._cpu_samples, 'no CPU samples collected' - peak = max(w._cpu_samples) - assert peak > 50, f'analysis CPU unexpectedly low: peak {peak}' - print(f'PASS cpu monitor: {len(w._cpu_samples)} samples, peak {peak:.0f}% ' - f'(~{peak/100:.1f} cores), final label: {w.cpu_run_label.text()!r}') - - # 6. new UI: themes, results tabs, structured log, recent list, cpu timer - from phyml_ui.themes import resolve, build_qss - w.apply_theme('dark', 'blue') - theme, accent = resolve('dark', 'blue') - assert theme == 'dark' and accent == '#2D7DD2' - assert app.styleSheet() == build_qss(theme, accent) - assert w.log_console._role_colors['out'] == '#E6E8EB', 'phyml output must be themed' - assert w.log_console._role_colors['system'] == '#2D7DD2', 'system lines use accent' - assert w.cpu_timer.isActive(), 'CPU monitor timer must be running' - w.apply_theme('system', 'system') - assert w.result_tabs.count() == 4, 'expected 4 result tabs (log/stats/summary/queue)' - # presets: built-ins listed, apply changes support method, user preset saves - assert w.presets_list.count() >= 3 - texts = [w.presets_list.item(i).text() for i in range(w.presets_list.count())] - screen_row = next(i for i, t in enumerate(texts) if 'скрининг' in t.lower() or 'screening' in t.lower()) - w._preset_activated(w.presets_list.item(screen_row)) - assert w.cb_support_method.currentData() == '-4', 'screening preset must select aLRT' - w._user_presets['my-test'] = {k: w.get_current_config()[k] for k in ('model',)} - w._refresh_presets() - assert any(w.presets_list.item(i).text() == 'my-test' for i in range(w.presets_list.count())) - w.tree_view.btn_zoom.click() - assert w.tree_view.btn_zoom.isChecked() and w.tree_view.mode_label.text() == phymlgui._t('tree_zoom') - w.tree_view.btn_pan.click() - assert w.tree_view.btn_pan.isChecked() and not w.tree_view.btn_zoom.isChecked() - w.tree_view.btn_pan.click() - assert not w.tree_view.btn_pan.isChecked() - from phyml_ui.main_window import SettingsDialog - d = SettingsDialog(w, w) - assert d.edits[0].text() == w.phyml_path_input.text() - d.edits[0].setText('D:/some/phyml-mpi.exe') - d.accept() - assert w.phyml_path_input.text() == 'D:/some/phyml-mpi.exe' - assert w.chk_use_mpi.isChecked(), 'settings edit must re-trigger MPI autodetect' - w.log_console.filter_box.setCurrentIndex(0) - w.log_console.clear() - w.log_console.append('[ERROR] test failure line') - w.log_console.append('plain phyml output line') - assert '[ERROR] test failure line' in w.log_console.toPlainText() - assert 'plain phyml output line' in w.log_console.toPlainText() - w.set_combo_by_data(w.log_console.filter_box, 'error') - assert '[ERROR]' in w.log_console.view.toPlainText() - assert 'plain phyml output line' not in w.log_console.view.toPlainText() - w.set_combo_by_data(w.log_console.filter_box, 'all') - assert 'plain phyml output line' in w.log_console.view.toPlainText() - w._remember_recent('X:/a.fasta'); w._remember_recent('X:/b.fasta') - w._remember_recent('X:/a.fasta') - assert w.recent_list.count() == 2 and w._recent[0] == 'X:/a.fasta' - print('PASS new UI: theme toggle, 4 result tabs, log filter, recent list') - print('ALL GUI LOGIC TESTS PASSED') -finally: - if had_config: - shutil.move(phymlgui.CONFIG_FILE + '.bak', phymlgui.CONFIG_FILE) diff --git a/tests/test_iqtree.py b/tests/test_iqtree.py deleted file mode 100644 index 3ecc483..0000000 --- a/tests/test_iqtree.py +++ /dev/null @@ -1,132 +0,0 @@ -"""Tests for the IQ-TREE engine, ModelFinder and the XLSX summary.""" -import os, sys, time, shutil, re, json - -os.environ['QT_QPA_PLATFORM'] = 'offscreen' -ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, ROOT) -os.chdir(ROOT) - -import phymlgui -from PySide6.QtWidgets import QMessageBox - -IQTREE_EXE = os.path.join(ROOT, 'tools', 'iqtree2', 'iqtree2.exe') -if not os.path.exists(IQTREE_EXE): - print('SKIP: IQ-TREE binary not available') - sys.exit(0) - -app = phymlgui.QApplication([]) - -# 1. worker-level: MFP + ultrafast bootstrap on pre-aligned synthetic data -run_dir = os.path.join('testrun', 't10_iqtree') -shutil.rmtree(run_dir, ignore_errors=True) -os.makedirs(run_dir, exist_ok=True) -shutil.copy('testrun/test_prealigned.fasta', run_dir) -os.chdir(run_dir) - -config = { - 'engine': 'iqtree', 'iqtree_path': IQTREE_EXE, - 'phyml_path': os.path.join(ROOT, 'phymlgui v1.1', 'phyml-mpi.exe'), - 'muscle_path': os.path.join(ROOT, 'tools', 'muscle.exe'), - 'use_mpi': False, 'mpi_np': '2', 'datatype': 'nt', 'model': 'MFP', - 'support_method': 'bootstrap', 'bootstrap_replicates': '1000', - 'freq': 'default', 'tstv': 'e', 'pinv': '0.0', - 'classes': '4', 'gamma': 'e', 'search': 'NNI', 'optimize': 'tlr', 'parsimony': False, -} -logs, done = [], [] -worker = phymlgui.PipelineWorker(config, os.path.abspath('test_prealigned.fasta')) -worker.log_signal.connect(logs.append) -worker.finished_signal.connect(lambda ok, tree: done.append((ok, tree))) -worker.run() -os.chdir(ROOT) - -assert done[0][0], 'iqtree pipeline failed:\n' + '\n'.join(logs[-8:]) -tree_rel = os.path.relpath(done[0][1], ROOT) -assert os.path.exists(done[0][1]), done[0][1] -from Bio import Phylo -tree = Phylo.read(done[0][1], 'newick') -assert len(tree.get_terminals()) == 8 -row = worker.results[0] -assert row['model'] and row['model'] != 'n/a', row -iq_report = open(os.path.join(run_dir, 'test_prealigned_iqtree.iqtree'), encoding='utf-8').read() -lnl_file = re.search(r'Log-likelihood of the tree:\s*(-?[\d.]+)', iq_report).group(1) -assert row['lnL'] == lnl_file, (row['lnL'], lnl_file) -print(f'PASS iqtree engine: MFP picked {row["model"]}, lnL {row["lnL"]}, 8 taxa') - -# 1b. PhyML-sized replicate count (100) must not break UFBoot (needs >= 1000) -os.chdir(run_dir) -logs, done = [], [] -worker = phymlgui.PipelineWorker(dict(config, bootstrap_replicates='100'), - os.path.abspath('test_prealigned.fasta')) -worker.log_signal.connect(logs.append) -worker.finished_signal.connect(lambda ok, tree: done.append((ok, tree))) -worker.run() -os.chdir(ROOT) -assert done[0][0], 'iqtree with 100 replicates failed:\n' + '\n'.join(logs[-8:]) -assert any(' -B 1000' in l for l in logs), 'UFBoot replicates were not raised to 1000' -print('PASS iqtree ufboot: 100 replicates raised to 1000, run succeeds') - -# 2. GUI: engine validation, model list, full run + xlsx summary -cfg_bak = phymlgui.CONFIG_FILE + '.bak' -had_cfg = os.path.exists(phymlgui.CONFIG_FILE) -if had_cfg: - shutil.copy(phymlgui.CONFIG_FILE, cfg_bak) -try: - w = phymlgui.PhyMLApp() - warned = [] - phymlgui.QMessageBox.warning = staticmethod(lambda *a, **k: warned.append(a) or QMessageBox.Ok) - w.engine_setting = 'iqtree' - w.update_models() - assert w.cb_model.currentData() == 'MFP' - w.iqtree_path_input.setText('') # no binary -> blocked with warning - w.fasta_path_input.setText(os.path.abspath(os.path.join(run_dir, 'test_prealigned.fasta'))) - w.start_pipeline() - assert warned and (w.worker is None or not w.worker.isRunning()) - - w.iqtree_path_input.setText(IQTREE_EXE) - w.set_combo_by_data(w.cb_support_method, 'bootstrap') - w.le_bootstrap_replicates.setText('100') - w.start_pipeline() - guard = 0 - while not w.btn_run.isEnabled() and guard < 400: - app.processEvents(); time.sleep(0.3); guard += 1 - assert w.btn_run.isEnabled(), 'GUI iqtree run did not finish' - assert w.worker.results[0]['status'] == 'OK', \ - 'GUI iqtree run failed:\n' + w.log_console.toPlainText()[-800:] - xlsx = os.path.join(run_dir, 'phymlgui_summary.xlsx') - assert os.path.exists(xlsx), 'xlsx summary missing' - from openpyxl import load_workbook - wb = load_workbook(xlsx) - assert wb.active.max_row == 2 - print('PASS iqtree GUI: validation, model list, run + xlsx summary') - - # 3. every model offered in the IQ-TREE model list is accepted by IQ-TREE - import random - import subprocess - rnd = random.Random(7) - aa_phy = os.path.join(run_dir, 'models_aa.phy') - with open(aa_phy, 'w') as f: - base = ''.join(rnd.choice('ACDEFGHIKLMNPQRSTVWY') for _ in range(120)) - f.write('6 120\n') - for i in range(6): - seq = ''.join(c if rnd.random() > 0.15 else rnd.choice('ACDEFGHIKLMNPQRSTVWY') - for c in base) - f.write(f'P{i} {seq}\n') - alignments = {'nt': os.path.join(run_dir, 'test_prealigned.phy'), 'aa': aa_phy} - rejected = [] - for dt, phy in alignments.items(): - w.set_combo_by_data(w.cb_datatype, dt) - models = [w.cb_model.itemData(i) for i in range(w.cb_model.count())] - for m in models: - if m == 'MFP': - continue # ModelFinder is exercised by the runs above - r = subprocess.run([IQTREE_EXE, '-s', phy, '-m', m, '-n', '0', '-T', '1', '-redo', - '--prefix', os.path.join(run_dir, 'model_probe')], - capture_output=True, text=True) - if r.returncode != 0: - rejected.append(m) - assert not rejected, f'IQ-TREE rejects listed models: {rejected}' - print('PASS iqtree models: every listed nt/aa model is accepted by IQ-TREE') -finally: - if had_cfg: - shutil.move(cfg_bak, phymlgui.CONFIG_FILE) -print('ALL IQ-TREE TESTS PASSED') From 6f15bf0f075e03de79743f0efb30f82d8fe66ea1 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:04:39 +0300 Subject: [PATCH 15/36] Share window and warned fixtures, bound the IQ-TREE probe, restore locale key check Co-Authored-By: Claude Opus 5.5 --- tests/conftest.py | 24 ++++++++++++++++++++++++ tests/e2e/test_batch.py | 16 +--------------- tests/e2e/test_cpu_monitor.py | 10 ---------- tests/e2e/test_iqtree.py | 18 ++---------------- tests/gui/test_main_window.py | 22 ++-------------------- tests/gui/test_main_window_smoke.py | 15 ++++----------- tests/unit/test_i18n.py | 10 ++++++++++ 7 files changed, 43 insertions(+), 72 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a305bff..427059c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -67,6 +67,30 @@ def isolated_config(tmp_path, monkeypatch): return tmp_path +@pytest.fixture +def window(isolated_config, qtbot): + """A PhyMLApp on isolated config/history files, closed by qtbot at teardown. + + isolated_config must be listed BEFORE qtbot: fixtures are torn down in + reverse order, and closing the window runs closeEvent -> auto_save(), which + must still write the patched (tmp_path) config, never the real one. + """ + from phyml_ui.main_window import PhyMLApp + w = PhyMLApp() + qtbot.addWidget(w) + return w + + +@pytest.fixture +def warned(monkeypatch): + """Replace QMessageBox.warning; the returned list collects the call args.""" + from PySide6.QtWidgets import QMessageBox + shown = [] + monkeypatch.setattr(QMessageBox, 'warning', + staticmethod(lambda *a, **k: shown.append(a) or QMessageBox.Ok)) + return shown + + def _digest(path): return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None diff --git a/tests/e2e/test_batch.py b/tests/e2e/test_batch.py index 23403cd..41bd83a 100644 --- a/tests/e2e/test_batch.py +++ b/tests/e2e/test_batch.py @@ -4,25 +4,14 @@ import pytest from Bio import Phylo -from PySide6.QtWidgets import QMessageBox import phyml_ui.main_window as main_window from datasets import phyml_config, synthetic # noqa: F401 (fixture) from phyml_core.pipeline import PipelineWorker -from phyml_ui.main_window import PhyMLApp pytestmark = [pytest.mark.needs_phyml, pytest.mark.needs_mpi, pytest.mark.needs_muscle] -@pytest.fixture -def window(isolated_config, qtbot): - # isolated_config goes first so it is torn down last: qtbot closes the - # window at teardown and closeEvent still writes the (patched) config file - w = PhyMLApp() - qtbot.addWidget(w) - return w - - @pytest.fixture def batch_dir(tmp_path, synthetic): """A folder with an unaligned (batchA) and a pre-aligned (batchB) FASTA.""" @@ -34,13 +23,10 @@ def batch_dir(tmp_path, synthetic): @pytest.fixture -def gui_batch(window, qtbot, monkeypatch, binaries, batch_dir, synthetic): +def gui_batch(window, warned, qtbot, binaries, batch_dir, synthetic): """The window after a finished batch run over batch_dir, started with Run.""" # a MUSCLE output left by an earlier run: the queue must not pick it up shutil.copy(synthetic['aligned'], batch_dir / 'batchA_aligned.fasta') - warned = [] - monkeypatch.setattr(QMessageBox, 'warning', - staticmethod(lambda *a, **k: warned.append(a) or QMessageBox.Ok)) window.phyml_path_input.setText(binaries['phyml']) window.mpirun_path_input.setText(binaries['mpiexec']) window.muscle_path_input.setText(binaries['muscle']) diff --git a/tests/e2e/test_cpu_monitor.py b/tests/e2e/test_cpu_monitor.py index b880191..c343e06 100644 --- a/tests/e2e/test_cpu_monitor.py +++ b/tests/e2e/test_cpu_monitor.py @@ -6,20 +6,10 @@ from datasets import phyml_config, synthetic # noqa: F401 (fixture) from phyml_core.pipeline import PipelineWorker -from phyml_ui.main_window import PhyMLApp pytestmark = [pytest.mark.needs_phyml, pytest.mark.needs_mpi, pytest.mark.needs_muscle] -@pytest.fixture -def window(isolated_config, qtbot): - # isolated_config goes first so it is torn down last: qtbot closes the - # window at teardown and closeEvent still writes the (patched) config file - w = PhyMLApp() - qtbot.addWidget(w) - return w - - def test_cpu_samples_during_mpi_run(window, qapp, tmp_path, monkeypatch, binaries, synthetic): run_dir = tmp_path / 't6_monitor' run_dir.mkdir() diff --git a/tests/e2e/test_iqtree.py b/tests/e2e/test_iqtree.py index bb79385..1175d53 100644 --- a/tests/e2e/test_iqtree.py +++ b/tests/e2e/test_iqtree.py @@ -7,24 +7,13 @@ import pytest from Bio import AlignIO, Phylo -from PySide6.QtWidgets import QMessageBox from datasets import synthetic # noqa: F401 (fixture) from phyml_core.pipeline import PipelineWorker -from phyml_ui.main_window import PhyMLApp pytestmark = pytest.mark.needs_iqtree -@pytest.fixture -def window(isolated_config, qtbot): - # isolated_config goes first so it is torn down last: qtbot closes the - # window at teardown and closeEvent still writes the (patched) config file - w = PhyMLApp() - qtbot.addWidget(w) - return w - - @pytest.fixture def run_dir(tmp_path, synthetic): """A working folder holding the pre-aligned synthetic dataset.""" @@ -82,11 +71,8 @@ def test_100_replicates_raised_to_1000(qapp, monkeypatch, binaries, run_dir): assert any(' -B 1000' in line for line in logs), 'UFBoot replicates were not raised to 1000' -def test_gui_validation_and_run_with_xlsx(window, qtbot, monkeypatch, binaries, run_dir): +def test_gui_validation_and_run_with_xlsx(window, warned, qtbot, binaries, run_dir): """GUI: engine validation, model list, full run + xlsx summary.""" - warned = [] - monkeypatch.setattr(QMessageBox, 'warning', - staticmethod(lambda *a, **k: warned.append(a) or QMessageBox.Ok)) window.engine_setting = 'iqtree' window.update_models() assert window.cb_model.currentData() == 'MFP' @@ -137,7 +123,7 @@ def test_every_listed_model_accepted(window, binaries, run_dir): continue # ModelFinder is exercised by the runs above r = subprocess.run([binaries['iqtree'], '-s', str(phy), '-m', m, '-n', '0', '-T', '1', '-redo', '--prefix', str(run_dir / 'model_probe')], - capture_output=True, text=True) + capture_output=True, text=True, timeout=120) if r.returncode != 0: rejected.append(m) assert not rejected, f'IQ-TREE rejects listed models: {rejected}' diff --git a/tests/gui/test_main_window.py b/tests/gui/test_main_window.py index 0786ba4..524e3c6 100644 --- a/tests/gui/test_main_window.py +++ b/tests/gui/test_main_window.py @@ -1,31 +1,13 @@ """GUI behaviour of PhyMLApp that needs no analysis binaries.""" import pytest -from PySide6.QtWidgets import QMessageBox, QSizePolicy +from PySide6.QtWidgets import QSizePolicy from phyml_core.i18n import _t from phyml_core.utils import detect_cores -from phyml_ui.main_window import PhyMLApp, SettingsDialog +from phyml_ui.main_window import SettingsDialog from phyml_ui.themes import build_qss, resolve -@pytest.fixture -def window(isolated_config, qtbot): - # isolated_config goes first so it is torn down last: qtbot closes the - # window at teardown and closeEvent still writes the (patched) config file - w = PhyMLApp() - qtbot.addWidget(w) - return w - - -@pytest.fixture -def warned(monkeypatch): - """Replace QMessageBox.warning; the list collects the (parent, title, text) calls.""" - shown = [] - monkeypatch.setattr(QMessageBox, 'warning', - staticmethod(lambda *a, **k: shown.append(a) or QMessageBox.Ok)) - return shown - - @pytest.fixture def restore_stylesheet(qapp): original = qapp.styleSheet() diff --git a/tests/gui/test_main_window_smoke.py b/tests/gui/test_main_window_smoke.py index 1112549..76d19dd 100644 --- a/tests/gui/test_main_window_smoke.py +++ b/tests/gui/test_main_window_smoke.py @@ -1,11 +1,4 @@ -from phyml_ui.main_window import PhyMLApp - - -def test_window_builds(isolated_config, qtbot): - # isolated_config goes first so it is torn down last: qtbot closes the - # window at teardown and closeEvent still writes the (patched) config file - w = PhyMLApp() - qtbot.addWidget(w) - assert w.result_tabs.count() == 4 - assert w.presets_list.count() >= 3 - assert w.cpu_timer.isActive() +def test_window_builds(window): + assert window.result_tabs.count() == 4 + assert window.presets_list.count() >= 3 + assert window.cpu_timer.isActive() diff --git a/tests/unit/test_i18n.py b/tests/unit/test_i18n.py index 3aabbe6..30edd1e 100644 --- a/tests/unit/test_i18n.py +++ b/tests/unit/test_i18n.py @@ -8,3 +8,13 @@ def test_locales_have_same_keys(): assert set(loc) >= {'en', 'ru'} assert len(loc['en']) == len(loc['ru']), 'locale key sets diverge' assert set(loc['en']) == set(loc['ru']), 'locale key sets diverge' + + +def test_mpi_and_cpu_keys_present(): + """The MPI autodetect and CPU monitor strings exist in both languages.""" + loc = json.loads((ROOT / 'locales.json').read_text(encoding='utf-8')) + keys = ['cores_info', 'cpu_sys', 'cpu_analysis', 'cpu_analysis_idle', + 'cpu_analysis_done', 'mpi_detected', 'mpi_boot_title', 'mpi_boot_msg'] + for lang in ('en', 'ru'): + for k in keys: + assert k in loc[lang], f'{lang} missing {k}' From 55bc295571a9736c0e1e81ebcdb0e2f7fc33d355 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:10:29 +0300 Subject: [PATCH 16/36] Add AppPaths and settings with atomic save, type coercion and legacy migration Co-Authored-By: Claude Opus 5.5 --- locales.json | 2 + phyml_core/paths.py | 113 ++++++++++++++++ phyml_core/settings.py | 166 +++++++++++++++++++++++ tests/unit/test_paths.py | 150 +++++++++++++++++++++ tests/unit/test_settings.py | 254 ++++++++++++++++++++++++++++++++++++ 5 files changed, 685 insertions(+) create mode 100644 phyml_core/paths.py create mode 100644 phyml_core/settings.py create mode 100644 tests/unit/test_paths.py create mode 100644 tests/unit/test_settings.py diff --git a/locales.json b/locales.json index 759f753..0ed7154 100644 --- a/locales.json +++ b/locales.json @@ -76,6 +76,7 @@ "file_fasta": "Select Input File", "sys_load": "[System] Parameters loaded.\n", "err_load": "[ERROR] Failed to read config: {}\n", + "err_settings_broken": "[ERROR] Settings file is damaged; defaults loaded, a copy is saved as {}.", "err_box_title": "Error", "err_box_msg": "Please specify a valid FASTA file.", "sys_interrupt": "\n[!] Sending interrupt signal...", @@ -263,6 +264,7 @@ "file_fasta": "Выбор входного файла", "sys_load": "[Система] Параметры загружены.\n", "err_load": "[ОШИБКА] Сбой чтения конфига: {}\n", + "err_settings_broken": "[ОШИБКА] Файл настроек повреждён: загружены настройки по умолчанию, копия сохранена как {}.", "err_box_title": "Ошибка", "err_box_msg": "Укажите существующий FASTA файл.", "sys_interrupt": "\n[!] Прерывание процесса...", diff --git a/phyml_core/paths.py b/phyml_core/paths.py new file mode 100644 index 0000000..5ffd95a --- /dev/null +++ b/phyml_core/paths.py @@ -0,0 +1,113 @@ +"""Where phymlGUI keeps its program files and its user data (no Qt, no cwd dependence).""" +import os +import sys +import tempfile +from dataclasses import dataclass +from typing import Callable, Mapping, Optional + +CONFIG_NAME = 'phyml_config.json' +HISTORY_NAME = 'phyml_history.json' +DATA_DIR_ENV = 'PHYMLGUI_DATA_DIR' +APPDATA_SUBDIR = 'phymlGUI' + + +def default_app_dir() -> str: + """Folder of the frozen exe, or the repository root when run from sources.""" + if getattr(sys, 'frozen', False): + return os.path.dirname(sys.executable) + return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def _dir_is_writable(directory: str) -> bool: + """Probe by creating and deleting a temporary file inside the folder.""" + try: + fd, probe = tempfile.mkstemp(dir=directory, prefix='.phymlgui-') + os.close(fd) + os.remove(probe) + return True + except OSError: + return False + + +def _norm(path: str) -> str: + return os.path.normcase(os.path.abspath(path)) + + +@dataclass(frozen=True) +class AppPaths: + app_dir: str # program files: bin/, locales.json, the exe + data_dir: str # user data: config and history + + @property + def bin_dir(self) -> str: + return os.path.join(self.app_dir, 'bin') + + @property + def config_file(self) -> str: + return os.path.join(self.data_dir, CONFIG_NAME) + + @property + def history_file(self) -> str: + return os.path.join(self.data_dir, HISTORY_NAME) + + def resource(self, name: str) -> str: + """A bundled resource: inside sys._MEIPASS when frozen, else next to the program.""" + bundle = getattr(sys, '_MEIPASS', None) + if bundle: + candidate = os.path.join(bundle, name) + if os.path.exists(candidate): + return candidate + return os.path.join(self.app_dir, name) + + def to_stored(self, path: str) -> str: + """Relative to app_dir when the path lies inside it, otherwise unchanged. + + A relative input (a bare command name such as 'mpiexec', or an already stored + value) has no defined location, so it is returned as is. + """ + if not path or not os.path.isabs(path): + return path + target, root = _norm(path), _norm(self.app_dir) + try: + inside = target != root and os.path.commonpath([target, root]) == root + except ValueError: # different drives + inside = False + return os.path.relpath(os.path.abspath(path), os.path.abspath(self.app_dir)) if inside else path + + def from_stored(self, stored: str) -> str: + """Inverse of to_stored: a relative value is resolved against app_dir.""" + if not stored or os.path.isabs(stored): + return stored + return os.path.join(self.app_dir, stored) + + def find_tool(self, stored: str, names: tuple) -> str: + """The configured tool if it exists, else the first of names in bin/, else the + configured path again (so the error message names what the user set).""" + configured = self.from_stored(stored) + if configured and os.path.isfile(configured): + return configured + for name in names: + candidate = os.path.join(self.bin_dir, name) + if os.path.isfile(candidate): + return candidate + return configured + + @classmethod + def detect(cls, env: Optional[Mapping[str, str]] = None, + is_writable: Optional[Callable[[str], bool]] = None, + app_dir: Optional[str] = None) -> 'AppPaths': + """Pick the data folder: $PHYMLGUI_DATA_DIR, then app_dir if writable, then + %APPDATA%\\phymlGUI. The chosen folder is created.""" + env = os.environ if env is None else env + is_writable = is_writable or _dir_is_writable + app = default_app_dir() if app_dir is None else app_dir + override = env.get(DATA_DIR_ENV) + if override: + data = override + elif is_writable(app): + data = app + else: + base = env.get('APPDATA') or os.path.join(os.path.expanduser('~'), 'AppData', 'Roaming') + data = os.path.join(base, APPDATA_SUBDIR) + os.makedirs(data, exist_ok=True) + return cls(app_dir=app, data_dir=data) diff --git a/phyml_core/settings.py b/phyml_core/settings.py new file mode 100644 index 0000000..4cee4b4 --- /dev/null +++ b/phyml_core/settings.py @@ -0,0 +1,166 @@ +"""Load, validate, atomically save and migrate the user settings (no Qt).""" +import copy +import json +import os +import shutil +import tempfile +from typing import Callable, List, Optional + +from .i18n import _t +from .paths import CONFIG_NAME, HISTORY_NAME, AppPaths + +SCHEMA_VERSION = 2 + +DEFAULTS = { + 'engine': 'phyml', + 'iqtree_path': '', + 'phyml_path': '', + 'mpirun_path': 'mpiexec', + 'muscle_path': '', + 'datatype': 'nt', + 'model': 'HKY85', + 'support_method': 'bootstrap', + 'bootstrap_replicates': '100', + 'use_mpi': False, + 'mpi_np': '', + 'freq': 'default', + 'tstv': 'e', + 'pinv': '0.0', + 'classes': '4', + 'gamma': 'e', + 'search': 'NNI', + 'optimize': 'tlr', + 'parsimony': False, + 'show_tree': True, + 'theme': 'system', + 'accent': 'green', + 'recent': [], + 'presets': {}, + 'window': {}, + 'schema_version': SCHEMA_VERSION, +} + +# settings holding a path to a tool; stored relative to app_dir when they lie inside it +PATH_KEYS = ('phyml_path', 'iqtree_path', 'muscle_path', 'mpirun_path') + + +def _coerce(value, default): + """Bring a value read from disk to the type of its default; fall back to the default.""" + if isinstance(default, bool): + return value if isinstance(value, bool) else default + if isinstance(default, str): + if isinstance(value, str): + return value + if isinstance(value, (int, float)) and not isinstance(value, bool): + return str(value) + return copy.deepcopy(default) + if isinstance(value, type(default)) and not isinstance(value, bool): + return value + return copy.deepcopy(default) + + +def load(path: str, on_error: Optional[Callable[[str], None]] = None) -> dict: + """Settings from path over DEFAULTS. Never raises: a missing, damaged or unreadable + file yields the defaults (a damaged one is first copied to path + '.broken').""" + if not os.path.exists(path): + return copy.deepcopy(DEFAULTS) + try: + with open(path, 'rb') as f: + raw = f.read() + except OSError as e: + if on_error: + on_error(_t('err_load').format(e)) + return copy.deepcopy(DEFAULTS) + try: + data = json.loads(raw.decode('utf-8-sig')) + if not isinstance(data, dict): + raise ValueError('settings must be a JSON object') + except ValueError: # includes JSONDecodeError and UnicodeDecodeError + broken = path + '.broken' + try: + with open(broken, 'wb') as f: + f.write(raw) + except OSError: + pass + if on_error: + on_error(_t('err_settings_broken').format(broken)) + return copy.deepcopy(DEFAULTS) + + cfg = {} + for key, default in DEFAULTS.items(): + cfg[key] = _coerce(data[key], default) if key in data else copy.deepcopy(default) + for key, value in data.items(): + if key not in DEFAULTS: + cfg[key] = value + cfg['schema_version'] = SCHEMA_VERSION + return cfg + + +def save(path: str, cfg: dict) -> None: + """Write cfg as JSON through a temporary file in the same folder and os.replace. + OSError is passed on to the caller; no temporary file is left behind.""" + directory = os.path.dirname(os.path.abspath(path)) + fd, tmp = tempfile.mkstemp(dir=directory, prefix=os.path.basename(path) + '.', suffix='.tmp') + try: + with os.fdopen(fd, 'w', encoding='utf-8') as f: + json.dump(cfg, f, indent=4, ensure_ascii=False) + os.replace(tmp, path) + except BaseException: + try: + os.remove(tmp) + except OSError: + pass + raise + + +def _is_command_name(value: str) -> bool: + """A bare name without a folder part, e.g. 'mpiexec' (looked up through PATH).""" + return bool(value) and not os.path.isabs(value) and os.path.basename(value) == value + + +def to_disk(cfg: dict, paths: AppPaths) -> dict: + """A copy of cfg for saving: tool paths and recent files inside app_dir become relative.""" + out = dict(cfg) + for key in PATH_KEYS: + if isinstance(out.get(key), str): + out[key] = paths.to_stored(out[key]) + if isinstance(out.get('recent'), list): + out['recent'] = [paths.to_stored(p) if isinstance(p, str) else p for p in out['recent']] + return out + + +def from_disk(cfg: dict, paths: AppPaths) -> dict: + """A copy of cfg with stored relative paths resolved against app_dir. + + A bare command name that is not a file in app_dir (the default 'mpiexec') is left + alone, so it keeps being resolved through PATH instead of becoming app_dir/mpiexec. + """ + out = dict(cfg) + for key in PATH_KEYS: + value = out.get(key) + if not isinstance(value, str): + continue + if _is_command_name(value) and not os.path.exists(os.path.join(paths.app_dir, value)): + continue + out[key] = paths.from_stored(value) + if isinstance(out.get('recent'), list): + out['recent'] = [paths.from_stored(p) if isinstance(p, str) else p for p in out['recent']] + return out + + +def migrate_legacy(paths: AppPaths, cwd: str) -> List[str]: + """Copy config/history from the old location (the working folder) into data_dir. + + Done only while data_dir has no config yet and cwd is a different folder, so it runs + once. The originals stay in place. Returns the names of the copied files. + """ + if os.path.realpath(cwd) == os.path.realpath(paths.data_dir): + return [] + if os.path.exists(paths.config_file): + return [] + found = [n for n in (CONFIG_NAME, HISTORY_NAME) if os.path.isfile(os.path.join(cwd, n))] + if found: + os.makedirs(paths.data_dir, exist_ok=True) + for name in found: + shutil.copy2(os.path.join(cwd, name), os.path.join(paths.data_dir, name)) + return found diff --git a/tests/unit/test_paths.py b/tests/unit/test_paths.py new file mode 100644 index 0000000..aaef52a --- /dev/null +++ b/tests/unit/test_paths.py @@ -0,0 +1,150 @@ +import os +import subprocess +import sys + +from conftest import ROOT + +from phyml_core.paths import CONFIG_NAME, HISTORY_NAME, AppPaths, default_app_dir + + +def test_file_names(): + assert CONFIG_NAME == 'phyml_config.json' + assert HISTORY_NAME == 'phyml_history.json' + + +def test_default_app_dir_is_repo_root_from_sources(monkeypatch): + monkeypatch.delattr(sys, 'frozen', raising=False) + assert os.path.samefile(default_app_dir(), ROOT) + + +def test_default_app_dir_is_exe_folder_when_frozen(tmp_path, monkeypatch): + monkeypatch.setattr(sys, 'frozen', True, raising=False) + monkeypatch.setattr(sys, 'executable', str(tmp_path / 'phymlgui.exe')) + assert default_app_dir() == str(tmp_path) + + +def test_detect_uses_env_override(tmp_path): + p = AppPaths.detect(env={'PHYMLGUI_DATA_DIR': str(tmp_path / 'd')}, app_dir=str(tmp_path)) + assert p.data_dir == str(tmp_path / 'd') and os.path.isdir(p.data_dir) + + +def test_detect_prefers_writable_app_dir(tmp_path): + p = AppPaths.detect(env={}, is_writable=lambda d: True, app_dir=str(tmp_path)) + assert p.data_dir == str(tmp_path) + + +def test_detect_falls_back_to_appdata(tmp_path): + p = AppPaths.detect(env={'APPDATA': str(tmp_path / 'ad')}, is_writable=lambda d: False, + app_dir=str(tmp_path / 'app')) + assert p.data_dir == str(tmp_path / 'ad' / 'phymlGUI') and os.path.isdir(p.data_dir) + + +def test_detect_default_probe_creates_and_removes_temp_file(tmp_path): + # the default is_writable probe must pick a writable app_dir and leave no trace in it + p = AppPaths.detect(env={}, app_dir=str(tmp_path)) + assert p.data_dir == str(tmp_path) + assert os.listdir(tmp_path) == [] + + +def test_detect_default_probe_rejects_missing_dir(tmp_path): + missing = tmp_path / 'no-such-app' + p = AppPaths.detect(env={'APPDATA': str(tmp_path / 'ad')}, app_dir=str(missing)) + assert p.data_dir == str(tmp_path / 'ad' / 'phymlGUI') + assert not missing.exists() + + +def test_detect_uses_cyrillic_and_spaces(tmp_path): + base = tmp_path / 'Мои данные' / 'проект 1' + p = AppPaths.detect(env={'PHYMLGUI_DATA_DIR': str(base)}, app_dir=str(tmp_path)) + assert os.path.isdir(base) and p.data_dir == str(base) + assert p.config_file == os.path.join(str(base), 'phyml_config.json') + assert p.history_file == os.path.join(str(base), 'phyml_history.json') + + +def test_derived_locations(tmp_path): + p = AppPaths(app_dir=str(tmp_path / 'app'), data_dir=str(tmp_path / 'data')) + assert p.bin_dir == os.path.join(str(tmp_path / 'app'), 'bin') + assert p.config_file == os.path.join(str(tmp_path / 'data'), CONFIG_NAME) + assert p.history_file == os.path.join(str(tmp_path / 'data'), HISTORY_NAME) + + +def test_to_stored_roundtrip_cyrillic(tmp_path): + app = tmp_path / 'Мои программы' / 'phymlGUI' + p = AppPaths(app_dir=str(app), data_dir=str(app)) + inside = str(app / 'bin' / 'muscle.exe') + assert p.to_stored(inside) == os.path.join('bin', 'muscle.exe') + assert p.from_stored(p.to_stored(inside)) == inside + outside = str(tmp_path / 'данные' / 'a.fasta') + assert p.to_stored(outside) == outside and p.to_stored('') == '' + + +def test_to_stored_ignores_prefix_lookalike_and_case(tmp_path): + app = tmp_path / 'phymlGUI' + p = AppPaths(app_dir=str(app), data_dir=str(app)) + lookalike = str(tmp_path / 'phymlGUI-old' / 'muscle.exe') + assert p.to_stored(lookalike) == lookalike # shares a string prefix, not a folder + if os.name == 'nt': # Windows paths compare case-insensitively + assert p.to_stored(str(app).upper() + '\\bin\\x.exe') == os.path.join('bin', 'x.exe') + + +def test_to_stored_keeps_relative_input_untouched(tmp_path): + # a bare command name (the default mpirun_path) must not be turned into a cwd-based path + p = AppPaths(app_dir=str(tmp_path), data_dir=str(tmp_path)) + assert p.to_stored('mpiexec') == 'mpiexec' + + +def test_from_stored_absolute_and_empty_unchanged(tmp_path): + p = AppPaths(app_dir=str(tmp_path / 'app'), data_dir=str(tmp_path)) + absolute = str(tmp_path / 'elsewhere' / 'x.exe') + assert p.from_stored(absolute) == absolute and p.from_stored('') == '' + assert p.from_stored('bin/x.exe') == os.path.join(str(tmp_path / 'app'), 'bin/x.exe') + + +def test_find_tool_prefers_stored_then_bin(tmp_path): + app = tmp_path / 'app' + (app / 'bin').mkdir(parents=True) + p = AppPaths(app_dir=str(app), data_dir=str(app)) + configured = tmp_path / 'custom' / 'muscle.exe' + configured.parent.mkdir() + configured.write_bytes(b'') + bundled = app / 'bin' / 'muscle.exe' + bundled.write_bytes(b'') + # the configured file exists -> it wins over bin/ + assert p.find_tool(str(configured), ('muscle.exe',)) == str(configured) + # configured file is gone, bin/muscle.exe exists -> bin/ + assert p.find_tool(str(tmp_path / 'gone.exe'), ('muscle.exe',)) == str(bundled) + # empty setting -> bin/, honouring the order of names + (app / 'bin' / 'muscle5.exe').write_bytes(b'') + assert p.find_tool('', ('muscle5.exe', 'muscle.exe')) == str(app / 'bin' / 'muscle5.exe') + assert p.find_tool('', ('absent.exe', 'muscle.exe')) == str(bundled) + # nothing exists -> the configured path, so the error message names it + bundled.unlink() + (app / 'bin' / 'muscle5.exe').unlink() + missing = str(tmp_path / 'gone.exe') + assert p.find_tool(missing, ('muscle.exe',)) == missing + assert p.find_tool(os.path.join('bin', 'gone.exe'), ('muscle.exe',)) == os.path.join( + str(app), 'bin', 'gone.exe') + assert p.find_tool('', ('muscle.exe',)) == '' + + +def test_resource_prefers_meipass(tmp_path, monkeypatch): + app = tmp_path / 'app' + bundle = tmp_path / 'bundle' + app.mkdir() + bundle.mkdir() + (bundle / 'locales.json').write_text('{}', 'utf-8') + p = AppPaths(app_dir=str(app), data_dir=str(app)) + monkeypatch.setattr(sys, '_MEIPASS', str(bundle), raising=False) + assert p.resource('locales.json') == str(bundle / 'locales.json') + # not in the bundle -> falls back to app_dir + assert p.resource('icon.ico') == str(app / 'icon.ico') + monkeypatch.delattr(sys, '_MEIPASS') + assert p.resource('locales.json') == str(app / 'locales.json') + + +def test_paths_module_is_qt_free(): + code = ("import sys, phyml_core.paths, phyml_core.settings; " + "assert not [m for m in sys.modules if m.split('.')[0] in ('PySide6', 'PyQt5', 'PyQt6')]") + result = subprocess.run([sys.executable, '-c', code], cwd=str(ROOT), + capture_output=True, text=True) + assert result.returncode == 0, result.stderr diff --git a/tests/unit/test_settings.py b/tests/unit/test_settings.py new file mode 100644 index 0000000..ab4b4af --- /dev/null +++ b/tests/unit/test_settings.py @@ -0,0 +1,254 @@ +import json +import os + +import pytest + +from phyml_core import settings +from phyml_core.i18n import _t +from phyml_core.paths import CONFIG_NAME, HISTORY_NAME, AppPaths + + +def test_defaults_match_the_window_config_keys(): + expected = { + 'engine': 'phyml', 'iqtree_path': '', 'phyml_path': '', 'mpirun_path': 'mpiexec', + 'muscle_path': '', 'datatype': 'nt', 'model': 'HKY85', 'support_method': 'bootstrap', + 'bootstrap_replicates': '100', 'use_mpi': False, 'mpi_np': '', 'freq': 'default', + 'tstv': 'e', 'pinv': '0.0', 'classes': '4', 'gamma': 'e', 'search': 'NNI', + 'optimize': 'tlr', 'parsimony': False, 'show_tree': True, 'theme': 'system', + 'accent': 'green', 'recent': [], 'presets': {}, 'window': {}, 'schema_version': 2, + } + assert settings.DEFAULTS == expected + assert settings.SCHEMA_VERSION == 2 + assert settings.PATH_KEYS == ('phyml_path', 'iqtree_path', 'muscle_path', 'mpirun_path') + + +def test_load_missing_file_returns_defaults(tmp_path): + cfg = settings.load(str(tmp_path / 'absent.json')) + assert cfg == settings.DEFAULTS and cfg is not settings.DEFAULTS + cfg['recent'].append('x') # mutating the result must not leak + cfg['window']['w'] = 1 + assert settings.DEFAULTS['recent'] == [] and settings.DEFAULTS['window'] == {} + + +def test_load_fills_missing_keys_and_coerces_types(tmp_path): + path = tmp_path / 'c.json' + path.write_text(json.dumps({'mpi_np': 8, 'recent': None, 'use_mpi': 'yes', 'custom_key': 1}), 'utf-8') + cfg = settings.load(str(path)) + assert cfg['mpi_np'] == '8' and cfg['recent'] == [] and cfg['use_mpi'] is False + assert cfg['custom_key'] == 1 and cfg['model'] == 'HKY85' and cfg['schema_version'] == 2 + + +def test_load_coercion_rules(tmp_path): + path = tmp_path / 'c.json' + path.write_text(json.dumps({ + 'bootstrap_replicates': 200, # number -> str + 'pinv': 0.5, # float -> str + 'model': None, # not str/number -> default + 'theme': ['dark'], # list for a str -> default + 'use_mpi': True, # real bool kept + 'parsimony': 1, # int is not a bool -> default + 'show_tree': 'false', # str is not a bool -> default + 'recent': ['a.fasta', 'b.fasta'], + 'presets': [], # list for a dict -> default + 'window': {'w': 800}, + 'schema_version': 1, # always rewritten to the current version + }), 'utf-8') + cfg = settings.load(str(path)) + assert cfg['bootstrap_replicates'] == '200' and cfg['pinv'] == '0.5' + assert cfg['model'] == 'HKY85' and cfg['theme'] == 'system' + assert cfg['use_mpi'] is True and cfg['parsimony'] is False and cfg['show_tree'] is True + assert cfg['recent'] == ['a.fasta', 'b.fasta'] and cfg['presets'] == {} + assert cfg['window'] == {'w': 800} and cfg['schema_version'] == 2 + + +def test_load_bool_is_not_a_string_number(tmp_path): + path = tmp_path / 'c.json' + path.write_text(json.dumps({'mpi_np': True}), 'utf-8') + assert settings.load(str(path))['mpi_np'] == '' + + +def test_load_broken_json_keeps_copy_and_reports(tmp_path): + path = tmp_path / 'c.json' + path.write_text('{oops', 'utf-8') + errors = [] + cfg = settings.load(str(path), on_error=errors.append) + assert cfg == settings.DEFAULTS + assert len(errors) == 1 and '.broken' in errors[0] + assert errors[0] == _t('err_settings_broken').format(str(path) + '.broken') + broken = tmp_path / 'c.json.broken' + assert broken.read_text('utf-8') == '{oops' + assert path.read_text('utf-8') == '{oops' # the damaged file itself is not rewritten + + +def test_load_broken_json_without_callback(tmp_path): + path = tmp_path / 'c.json' + path.write_text('', 'utf-8') # empty file is not valid JSON either + assert settings.load(str(path)) == settings.DEFAULTS + assert (tmp_path / 'c.json.broken').exists() + + +def test_load_non_object_json_is_broken(tmp_path): + path = tmp_path / 'c.json' + path.write_text('[1, 2]', 'utf-8') + errors = [] + assert settings.load(str(path), errors.append) == settings.DEFAULTS + assert len(errors) == 1 and (tmp_path / 'c.json.broken').read_text('utf-8') == '[1, 2]' + + +def test_load_non_utf8_file_is_broken(tmp_path): + path = tmp_path / 'c.json' + path.write_bytes('{"model": "Ж"}'.encode('cp1251')) + errors = [] + assert settings.load(str(path), errors.append) == settings.DEFAULTS + assert len(errors) == 1 and (tmp_path / 'c.json.broken').read_bytes() == path.read_bytes() + + +def test_load_unreadable_path_returns_defaults_and_reports(tmp_path): + errors = [] + cfg = settings.load(str(tmp_path), on_error=errors.append) # a directory, not a file + assert cfg == settings.DEFAULTS and len(errors) == 1 + + +def test_save_is_atomic_and_roundtrips(tmp_path): + path = tmp_path / 'c.json' + cfg = settings.load(str(path)) + cfg['model'] = 'GTR' + cfg['recent'] = [r'D:\данные\проект 1\a.fasta'] + cfg['custom_key'] = {'k': [1, 2]} + settings.save(str(path), cfg) + assert settings.load(str(path)) == cfg + assert os.listdir(tmp_path) == ['c.json'] + settings.save(str(path), dict(cfg, model='JC69')) # overwrite an existing file + assert settings.load(str(path))['model'] == 'JC69' + assert os.listdir(tmp_path) == ['c.json'] + + +def test_save_writes_readable_utf8_json(tmp_path): + path = tmp_path / 'c.json' + settings.save(str(path), {'recent': ['D:/данные/a.fasta']}) + text = path.read_text('utf-8') + assert 'данные' in text and '\\u' not in text # ensure_ascii=False + assert text.startswith('{\n "recent"') # indent=4 + + +def test_save_raises_oserror_for_directory_target(tmp_path): + target = tmp_path / 'sub' + target.mkdir() + with pytest.raises(OSError): + settings.save(str(target), {'a': 1}) + assert os.listdir(tmp_path) == ['sub'] # no temp file left behind + + +def test_save_failure_keeps_previous_file(tmp_path): + path = tmp_path / 'c.json' + settings.save(str(path), {'a': 1}) + with pytest.raises(TypeError): + settings.save(str(path), {'a': object()}) # not JSON-serialisable + assert json.loads(path.read_text('utf-8')) == {'a': 1} + assert os.listdir(tmp_path) == ['c.json'] + + +def _paths(tmp_path): + app = tmp_path / 'Мои программы' / 'phymlGUI' + (app / 'bin').mkdir(parents=True) + return AppPaths(app_dir=str(app), data_dir=str(app)) + + +def test_to_disk_from_disk_roundtrip(tmp_path): + p = _paths(tmp_path) + inside = str(tmp_path / 'Мои программы' / 'phymlGUI' / 'bin' / 'phyml-mpi.exe') + outside = str(tmp_path / 'данные' / 'a.fasta') + cfg = dict(settings.DEFAULTS, phyml_path=inside, muscle_path=r'C:\tools\muscle.exe', + recent=[inside, outside], model='GTR') + stored = settings.to_disk(cfg, p) + assert stored['phyml_path'] == os.path.join('bin', 'phyml-mpi.exe') # relative on disk + assert stored['muscle_path'] == r'C:\tools\muscle.exe' + assert stored['recent'] == [os.path.join('bin', 'phyml-mpi.exe'), outside] + assert stored['model'] == 'GTR' + assert cfg['phyml_path'] == inside # input not mutated + restored = settings.from_disk(stored, p) + assert restored == cfg # absolute again + + +def test_to_disk_from_disk_leave_missing_and_foreign_values_alone(tmp_path): + p = _paths(tmp_path) + assert settings.to_disk({'model': 'JC69'}, p) == {'model': 'JC69'} + assert settings.from_disk({'model': 'JC69'}, p) == {'model': 'JC69'} + odd = {'phyml_path': None, 'recent': 'not-a-list'} + assert settings.from_disk(settings.to_disk(odd, p), p) == odd + + +def test_default_mpiexec_command_name_survives_a_save_load_cycle(tmp_path): + # 'mpiexec' is resolved through PATH; it must never become \mpiexec + p = _paths(tmp_path) + cfg = dict(settings.DEFAULTS) + assert settings.from_disk(settings.to_disk(cfg, p), p)['mpirun_path'] == 'mpiexec' + # but a bare file name that really sits in app_dir is still resolved against it + tool = os.path.join(p.app_dir, 'muscle.exe') + open(tool, 'wb').close() + stored = settings.to_disk(dict(cfg, muscle_path=tool), p) + assert stored['muscle_path'] == 'muscle.exe' + assert settings.from_disk(stored, p)['muscle_path'] == tool + + +def test_migrate_legacy_copies_once(tmp_path): + cwd = tmp_path / 'old cwd' + data = tmp_path / 'новая папка' + cwd.mkdir() + data.mkdir() + (cwd / CONFIG_NAME).write_text('{"model": "GTR"}', 'utf-8') + (cwd / HISTORY_NAME).write_text('[]', 'utf-8') + p = AppPaths(app_dir=str(tmp_path), data_dir=str(data)) + assert sorted(settings.migrate_legacy(p, str(cwd))) == sorted([CONFIG_NAME, HISTORY_NAME]) + assert (data / CONFIG_NAME).read_text('utf-8') == '{"model": "GTR"}' + assert (data / HISTORY_NAME).read_text('utf-8') == '[]' + assert (cwd / CONFIG_NAME).exists() and (cwd / HISTORY_NAME).exists() # originals stay + assert settings.migrate_legacy(p, str(cwd)) == [] # once only + + +def test_migrate_legacy_copies_what_exists(tmp_path): + cwd = tmp_path / 'cwd' + data = tmp_path / 'data' + cwd.mkdir() + data.mkdir() + (cwd / HISTORY_NAME).write_text('[]', 'utf-8') + p = AppPaths(app_dir=str(tmp_path), data_dir=str(data)) + assert settings.migrate_legacy(p, str(cwd)) == [HISTORY_NAME] + assert not (data / CONFIG_NAME).exists() + + +def test_migrate_legacy_nothing_to_copy(tmp_path): + cwd = tmp_path / 'cwd' + cwd.mkdir() + p = AppPaths(app_dir=str(tmp_path), data_dir=str(tmp_path / 'not created yet')) + assert settings.migrate_legacy(p, str(cwd)) == [] + assert not (tmp_path / 'not created yet').exists() + + +def test_migrate_legacy_creates_missing_data_dir(tmp_path): + cwd = tmp_path / 'cwd' + cwd.mkdir() + (cwd / CONFIG_NAME).write_text('{}', 'utf-8') + data = tmp_path / 'not created yet' + p = AppPaths(app_dir=str(tmp_path), data_dir=str(data)) + assert settings.migrate_legacy(p, str(cwd)) == [CONFIG_NAME] + assert (data / CONFIG_NAME).exists() + + +def test_migrate_legacy_noop_when_cwd_is_data_dir(tmp_path): + (tmp_path / CONFIG_NAME).write_text('{"model": "GTR"}', 'utf-8') + p = AppPaths(app_dir=str(tmp_path), data_dir=str(tmp_path)) + assert settings.migrate_legacy(p, str(tmp_path)) == [] + assert (tmp_path / CONFIG_NAME).read_text('utf-8') == '{"model": "GTR"}' + + +def test_migrate_legacy_noop_when_data_dir_has_config(tmp_path): + cwd = tmp_path / 'cwd' + data = tmp_path / 'data' + cwd.mkdir() + data.mkdir() + (cwd / CONFIG_NAME).write_text('{"model": "GTR"}', 'utf-8') + (data / CONFIG_NAME).write_text('{"model": "JC69"}', 'utf-8') + p = AppPaths(app_dir=str(tmp_path), data_dir=str(data)) + assert settings.migrate_legacy(p, str(cwd)) == [] + assert (data / CONFIG_NAME).read_text('utf-8') == '{"model": "JC69"}' # never overwritten From 171f02ff0ff4837772d8e4e4bd32b0732ed75f1e Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:16:35 +0300 Subject: [PATCH 17/36] Make legacy migration copy-once without overwrites and let load raise on unreadable files Co-Authored-By: Claude Opus 5.5 --- phyml_core/settings.py | 34 ++++++++++++-------- tests/unit/test_settings.py | 62 ++++++++++++++++++++++++++++++++++--- 2 files changed, 79 insertions(+), 17 deletions(-) diff --git a/phyml_core/settings.py b/phyml_core/settings.py index 4cee4b4..f669615 100644 --- a/phyml_core/settings.py +++ b/phyml_core/settings.py @@ -60,16 +60,18 @@ def _coerce(value, default): def load(path: str, on_error: Optional[Callable[[str], None]] = None) -> dict: - """Settings from path over DEFAULTS. Never raises: a missing, damaged or unreadable - file yields the defaults (a damaged one is first copied to path + '.broken').""" - if not os.path.exists(path): - return copy.deepcopy(DEFAULTS) + """Settings from path over DEFAULTS. + + A missing file yields the defaults. A file with damaged content (not JSON, not an + object, not UTF-8) yields the defaults too, after its bytes are copied to + path + '.broken' and on_error is told. A file that exists but cannot be read raises + its OSError: returning defaults there would let a later save overwrite the user's + real settings after a transient lock or permission error. + """ try: with open(path, 'rb') as f: raw = f.read() - except OSError as e: - if on_error: - on_error(_t('err_load').format(e)) + except FileNotFoundError: return copy.deepcopy(DEFAULTS) try: data = json.loads(raw.decode('utf-8-sig')) @@ -151,16 +153,22 @@ def from_disk(cfg: dict, paths: AppPaths) -> dict: def migrate_legacy(paths: AppPaths, cwd: str) -> List[str]: """Copy config/history from the old location (the working folder) into data_dir. - Done only while data_dir has no config yet and cwd is a different folder, so it runs - once. The originals stay in place. Returns the names of the copied files. + Runs only while data_dir has no config yet and cwd is a different folder. A file that + already exists in data_dir is never overwritten, so a repeated call (history-only + legacy folder, config deleted to reset settings, crash before the first save) copies + nothing new. The history goes first and the config last: the config is what closes + the gate, so a failed history copy leaves the migration to be retried. The originals + stay in place. Returns the names of the files copied. """ if os.path.realpath(cwd) == os.path.realpath(paths.data_dir): return [] if os.path.exists(paths.config_file): return [] - found = [n for n in (CONFIG_NAME, HISTORY_NAME) if os.path.isfile(os.path.join(cwd, n))] - if found: + pending = [n for n in (HISTORY_NAME, CONFIG_NAME) + if os.path.isfile(os.path.join(cwd, n)) + and not os.path.exists(os.path.join(paths.data_dir, n))] + if pending: os.makedirs(paths.data_dir, exist_ok=True) - for name in found: + for name in pending: shutil.copy2(os.path.join(cwd, name), os.path.join(paths.data_dir, name)) - return found + return pending diff --git a/tests/unit/test_settings.py b/tests/unit/test_settings.py index ab4b4af..8c50c34 100644 --- a/tests/unit/test_settings.py +++ b/tests/unit/test_settings.py @@ -103,10 +103,13 @@ def test_load_non_utf8_file_is_broken(tmp_path): assert len(errors) == 1 and (tmp_path / 'c.json.broken').read_bytes() == path.read_bytes() -def test_load_unreadable_path_returns_defaults_and_reports(tmp_path): +def test_load_unreadable_existing_file_raises(tmp_path): + # the path exists but cannot be read (a directory is deterministic on Windows and POSIX): + # load must not hand out defaults that a later save would write over the real settings errors = [] - cfg = settings.load(str(tmp_path), on_error=errors.append) # a directory, not a file - assert cfg == settings.DEFAULTS and len(errors) == 1 + with pytest.raises(OSError): + settings.load(str(tmp_path), on_error=errors.append) + assert errors == [] and not os.path.exists(str(tmp_path) + '.broken') def test_save_is_atomic_and_roundtrips(tmp_path): @@ -199,7 +202,7 @@ def test_migrate_legacy_copies_once(tmp_path): (cwd / CONFIG_NAME).write_text('{"model": "GTR"}', 'utf-8') (cwd / HISTORY_NAME).write_text('[]', 'utf-8') p = AppPaths(app_dir=str(tmp_path), data_dir=str(data)) - assert sorted(settings.migrate_legacy(p, str(cwd))) == sorted([CONFIG_NAME, HISTORY_NAME]) + assert settings.migrate_legacy(p, str(cwd)) == [HISTORY_NAME, CONFIG_NAME] # config last assert (data / CONFIG_NAME).read_text('utf-8') == '{"model": "GTR"}' assert (data / HISTORY_NAME).read_text('utf-8') == '[]' assert (cwd / CONFIG_NAME).exists() and (cwd / HISTORY_NAME).exists() # originals stay @@ -217,6 +220,57 @@ def test_migrate_legacy_copies_what_exists(tmp_path): assert not (data / CONFIG_NAME).exists() +def test_migrate_legacy_history_only_copies_once(tmp_path): + cwd = tmp_path / 'cwd' + data = tmp_path / 'data' + cwd.mkdir() + data.mkdir() + (cwd / HISTORY_NAME).write_text('["old"]', 'utf-8') + p = AppPaths(app_dir=str(tmp_path), data_dir=str(data)) + assert settings.migrate_legacy(p, str(cwd)) == [HISTORY_NAME] + (data / HISTORY_NAME).write_text('["newer, written by the app"]', 'utf-8') + assert settings.migrate_legacy(p, str(cwd)) == [] # no config in data_dir, still once + assert (data / HISTORY_NAME).read_text('utf-8') == '["newer, written by the app"]' + assert not (data / CONFIG_NAME).exists() + + +def test_migrate_legacy_never_overwrites_existing_files(tmp_path): + cwd = tmp_path / 'cwd' + data = tmp_path / 'data' + cwd.mkdir() + data.mkdir() + (cwd / CONFIG_NAME).write_text('{"model": "GTR"}', 'utf-8') + (cwd / HISTORY_NAME).write_text('["old"]', 'utf-8') + (data / HISTORY_NAME).write_text('["keep me"]', 'utf-8') # the app died before saving a config + p = AppPaths(app_dir=str(tmp_path), data_dir=str(data)) + assert settings.migrate_legacy(p, str(cwd)) == [CONFIG_NAME] + assert (data / HISTORY_NAME).read_text('utf-8') == '["keep me"]' + assert (data / CONFIG_NAME).read_text('utf-8') == '{"model": "GTR"}' + + +def test_migrate_legacy_failed_history_copy_keeps_gate_open(tmp_path, monkeypatch): + cwd = tmp_path / 'cwd' + data = tmp_path / 'data' + cwd.mkdir() + data.mkdir() + (cwd / CONFIG_NAME).write_text('{"model": "GTR"}', 'utf-8') + (cwd / HISTORY_NAME).write_text('[]', 'utf-8') + p = AppPaths(app_dir=str(tmp_path), data_dir=str(data)) + real_copy2 = settings.shutil.copy2 + + def failing_copy2(src, dst, **kwargs): + if os.path.basename(src) == HISTORY_NAME: + raise OSError('disk full') + return real_copy2(src, dst, **kwargs) + + monkeypatch.setattr(settings.shutil, 'copy2', failing_copy2) + with pytest.raises(OSError): + settings.migrate_legacy(p, str(cwd)) + assert not (data / CONFIG_NAME).exists() # config not copied: the gate stays open + monkeypatch.undo() + assert settings.migrate_legacy(p, str(cwd)) == [HISTORY_NAME, CONFIG_NAME] # retried in full + + def test_migrate_legacy_nothing_to_copy(tmp_path): cwd = tmp_path / 'cwd' cwd.mkdir() From 72d9eb8d55dbacaa6bb1dd0e42329895f828f5c0 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:21:14 +0300 Subject: [PATCH 18/36] Add HistoryStore with full run params and engine-aware presets Co-Authored-By: Claude Opus 5.5 --- locales.json | 2 + phyml_core/history.py | 125 +++++++++++++++++++ phyml_core/presets.py | 36 ++++++ phyml_core/settings.py | 11 +- tests/unit/test_history.py | 231 ++++++++++++++++++++++++++++++++++++ tests/unit/test_presets.py | 91 ++++++++++++++ tests/unit/test_settings.py | 16 +++ 7 files changed, 509 insertions(+), 3 deletions(-) create mode 100644 phyml_core/history.py create mode 100644 phyml_core/presets.py create mode 100644 tests/unit/test_history.py create mode 100644 tests/unit/test_presets.py diff --git a/locales.json b/locales.json index 0ed7154..032a480 100644 --- a/locales.json +++ b/locales.json @@ -77,6 +77,7 @@ "sys_load": "[System] Parameters loaded.\n", "err_load": "[ERROR] Failed to read config: {}\n", "err_settings_broken": "[ERROR] Settings file is damaged; defaults loaded, a copy is saved as {}.", + "err_history_broken": "[ERROR] Run history file is damaged and was ignored: {}", "err_box_title": "Error", "err_box_msg": "Please specify a valid FASTA file.", "sys_interrupt": "\n[!] Sending interrupt signal...", @@ -265,6 +266,7 @@ "sys_load": "[Система] Параметры загружены.\n", "err_load": "[ОШИБКА] Сбой чтения конфига: {}\n", "err_settings_broken": "[ОШИБКА] Файл настроек повреждён: загружены настройки по умолчанию, копия сохранена как {}.", + "err_history_broken": "[ОШИБКА] Файл истории запусков повреждён и пропущен: {}", "err_box_title": "Ошибка", "err_box_msg": "Укажите существующий FASTA файл.", "sys_interrupt": "\n[!] Прерывание процесса...", diff --git a/phyml_core/history.py b/phyml_core/history.py new file mode 100644 index 0000000..d91a6f0 --- /dev/null +++ b/phyml_core/history.py @@ -0,0 +1,125 @@ +"""Run history: a JSON list of run records, newest first (no Qt). + +Two record shapes live in the same file. Records written by older versions are flat: +{'ts', 'input', 'ok', 'n', 'time', 'model', 'engine', 'tree'}. Records made by new_record carry +the full run: inputs, parameters and a result entry per file. display_input and best_tree read +both shapes, so a history file never needs converting. +""" +import copy +import json +import os +from datetime import datetime +from typing import Callable, List, Optional + +from .i18n import _t +from .settings import write_json_atomic + +HISTORY_LIMIT = 200 +_TS_FORMAT = '%Y-%m-%d %H:%M' + + +class HistoryStore: + """The history file at path, newest record first, at most `limit` records.""" + + def __init__(self, path: str, limit: int = HISTORY_LIMIT): + self.path = path + self.limit = limit + + def load(self, on_error: Optional[Callable[[str], None]] = None) -> List[dict]: + """All records, newest first. + + A missing file yields []. A file with damaged content (not JSON, not a list, not + UTF-8) yields [] too, after its bytes are copied to path + '.broken' and on_error is + told. A file that exists but cannot be read raises its OSError: returning [] there + would let the next append replace the user's real history. + """ + try: + with open(self.path, 'rb') as f: + raw = f.read() + except FileNotFoundError: + return [] + try: + data = json.loads(raw.decode('utf-8-sig')) + if not isinstance(data, list): + raise ValueError('history must be a JSON list') + except ValueError: # includes JSONDecodeError and UnicodeDecodeError + try: + with open(self.path + '.broken', 'wb') as f: + f.write(raw) + except OSError: + pass + if on_error: + on_error(_t('err_history_broken').format(self.path)) + return [] + return data + + def append(self, record: dict) -> None: + """Put record first, cut the list to `limit` and write the file atomically. + A damaged file is kept as path + '.broken' before it is replaced. OSError is passed on.""" + records = self.load() + records.insert(0, record) + write_json_atomic(self.path, records[:self.limit], indent=2) + + def get(self, index: int) -> Optional[dict]: + """The record at index (0 is the newest), or None when there is none.""" + records = self.load() + if 0 <= index < len(records): + return records[index] + return None + + +def new_record(*, inputs: List[str], engine: str, params: dict, files: List[dict], + duration_s: float, ts: Optional[datetime] = None) -> dict: + """A history record for a finished run. + + files holds one entry per processed file: {'file', 'status', 'tree', 'stats', 'error'}. + 'ok' counts the entries whose status is 'ok', 'n' all of them. + """ + ts = ts or datetime.now() + return { + 'ts': ts.strftime(_TS_FORMAT), + 'inputs': list(inputs), + 'engine': engine, + 'params': copy.deepcopy(params), + 'files': copy.deepcopy(files), + 'ok': sum(1 for f in files if f.get('status') == 'ok'), + 'n': len(files), + 'duration_s': duration_s, + } + + +def _name(path: str) -> str: + """Last component of path, ignoring a trailing separator (a drive root stays whole).""" + path = os.path.normpath(path) + return os.path.basename(path) or path + + +def display_input(rec: dict) -> str: + """Short text for what the run processed. + + A file shows its name, a folder its name plus '/', several files the name of the folder + they share plus '/'. Old records just hold the name in 'input'. + """ + if 'inputs' not in rec: + return rec.get('input', '') + inputs = rec['inputs'] + if not inputs: + return '' + if len(inputs) == 1: + return _name(inputs[0]) + ('/' if os.path.isdir(inputs[0]) else '') + try: + common = os.path.commonpath([os.path.dirname(os.path.normpath(p)) for p in inputs]) + except ValueError: # different drives, or absolute mixed with relative + common = '' + return _name(common) + '/' if common else _name(inputs[0]) + + +def best_tree(rec: dict) -> str: + """The tree of the last file that finished with status 'ok', '' if there is none. + Old records hold a single path in 'tree'.""" + if 'files' not in rec: + return rec.get('tree') or '' + for entry in reversed(rec['files']): + if entry.get('status') == 'ok': + return entry.get('tree') or '' + return '' diff --git a/phyml_core/presets.py b/phyml_core/presets.py new file mode 100644 index 0000000..62b4033 --- /dev/null +++ b/phyml_core/presets.py @@ -0,0 +1,36 @@ +"""Analysis presets: the parameter subset that is saved and re-applied (no Qt). + +A preset holds the model and search parameters and the engine. Tool paths and display +options are deliberately outside it, so applying a preset never repoints a program. +""" +import copy + +PRESET_KEYS = ('datatype', 'model', 'support_method', 'bootstrap_replicates', + 'freq', 'tstv', 'pinv', 'classes', 'gamma', 'search', + 'optimize', 'parsimony', 'engine') + +BUILTIN = { + 'preset_builtin_screen': { + 'datatype': 'nt', 'model': 'HKY85', 'support_method': '-4', + 'bootstrap_replicates': '100', 'freq': 'default', 'tstv': 'e', + 'pinv': '0.0', 'classes': '4', 'gamma': 'e', 'search': 'NNI', + 'optimize': 'tlr', 'parsimony': False, 'engine': 'phyml'}, + 'preset_builtin_pub': { + 'datatype': 'nt', 'model': 'GTR', 'support_method': 'bootstrap', + 'bootstrap_replicates': '1000', 'freq': 'default', 'tstv': 'e', + 'pinv': 'e', 'classes': '4', 'gamma': 'e', 'search': 'BEST', + 'optimize': 'tlr', 'parsimony': False, 'engine': 'phyml'}, +} + + +def extract(params: dict) -> dict: + """The part of params that a preset stores: PRESET_KEYS that params has.""" + return {k: params[k] for k in PRESET_KEYS if k in params} + + +def apply(preset: dict, params: dict) -> dict: + """A copy of params with the PRESET_KEYS of preset laid over it (the engine included). + A preset saved without an engine leaves the current engine alone.""" + out = copy.deepcopy(params) + out.update({k: preset[k] for k in PRESET_KEYS if k in preset}) + return out diff --git a/phyml_core/settings.py b/phyml_core/settings.py index f669615..465f5dc 100644 --- a/phyml_core/settings.py +++ b/phyml_core/settings.py @@ -98,14 +98,14 @@ def load(path: str, on_error: Optional[Callable[[str], None]] = None) -> dict: return cfg -def save(path: str, cfg: dict) -> None: - """Write cfg as JSON through a temporary file in the same folder and os.replace. +def write_json_atomic(path: str, obj, indent: int = 4) -> None: + """Write obj as UTF-8 JSON through a temporary file in the same folder and os.replace. OSError is passed on to the caller; no temporary file is left behind.""" directory = os.path.dirname(os.path.abspath(path)) fd, tmp = tempfile.mkstemp(dir=directory, prefix=os.path.basename(path) + '.', suffix='.tmp') try: with os.fdopen(fd, 'w', encoding='utf-8') as f: - json.dump(cfg, f, indent=4, ensure_ascii=False) + json.dump(obj, f, indent=indent, ensure_ascii=False) os.replace(tmp, path) except BaseException: try: @@ -115,6 +115,11 @@ def save(path: str, cfg: dict) -> None: raise +def save(path: str, cfg: dict) -> None: + """Write cfg atomically (see write_json_atomic), indented by 4.""" + write_json_atomic(path, cfg, indent=4) + + def _is_command_name(value: str) -> bool: """A bare name without a folder part, e.g. 'mpiexec' (looked up through PATH).""" return bool(value) and not os.path.isabs(value) and os.path.basename(value) == value diff --git a/tests/unit/test_history.py b/tests/unit/test_history.py new file mode 100644 index 0000000..e32df75 --- /dev/null +++ b/tests/unit/test_history.py @@ -0,0 +1,231 @@ +import json +import os +from datetime import datetime + +import pytest + +from phyml_core import history +from phyml_core.history import HistoryStore +from phyml_core.i18n import STRINGS, _t + + +def _rec(i): + return history.new_record(inputs=[f'D:/данные/{i}.fasta'], engine='phyml', params={'model': 'GTR'}, + files=[], duration_s=1.0, ts=datetime(2026, 1, 1, 12, 0)) + + +def test_append_keeps_newest_first_and_limit(tmp_path): + store = HistoryStore(str(tmp_path / 'h.json')) + for i in range(205): + store.append({'n': i}) + records = store.load() + assert len(records) == 200 + assert records[0] == {'n': 204} # the last appended record is first + assert records[-1] == {'n': 5} # 0..4 fell off the end + + +def test_custom_limit(tmp_path): + store = HistoryStore(str(tmp_path / 'h.json'), limit=3) + for i in range(5): + store.append({'n': i}) + assert [r['n'] for r in store.load()] == [4, 3, 2] + + +def test_load_missing_file_returns_empty_without_creating_it(tmp_path): + path = tmp_path / 'h.json' + errors = [] + assert HistoryStore(str(path)).load(on_error=errors.append) == [] + assert errors == [] and not path.exists() + + +def test_load_corrupt_file_reports_and_returns_empty(tmp_path): + path = tmp_path / 'h.json' + path.write_text('[{oops', 'utf-8') + errors = [] + assert HistoryStore(str(path)).load(on_error=errors.append) == [] + assert errors == [_t('err_history_broken').format(str(path))] + + +def test_load_corrupt_file_without_callback(tmp_path): + path = tmp_path / 'h.json' + path.write_text('', 'utf-8') # an empty file is not valid JSON + assert HistoryStore(str(path)).load() == [] + assert (tmp_path / 'h.json.broken').exists() + + +def test_load_non_list_json_is_broken(tmp_path): + path = tmp_path / 'h.json' + path.write_text('{"ts": "x"}', 'utf-8') + errors = [] + assert HistoryStore(str(path)).load(errors.append) == [] + assert len(errors) == 1 + assert (tmp_path / 'h.json.broken').read_text('utf-8') == '{"ts": "x"}' + + +def test_load_non_utf8_file_is_broken(tmp_path): + path = tmp_path / 'h.json' + path.write_bytes('[{"input": "Ж"}]'.encode('cp1251')) + errors = [] + assert HistoryStore(str(path)).load(errors.append) == [] + assert len(errors) == 1 and (tmp_path / 'h.json.broken').read_bytes() == path.read_bytes() + + +def test_load_unreadable_existing_history_raises(tmp_path): + # the path exists but cannot be read (a directory is deterministic on Windows and POSIX): + # returning [] would let the next append replace the user's real history + errors = [] + with pytest.raises(OSError): + HistoryStore(str(tmp_path)).load(on_error=errors.append) + assert errors == [] and not os.path.exists(str(tmp_path) + '.broken') + + +def test_append_to_unreadable_existing_history_raises_and_changes_nothing(tmp_path): + target = tmp_path / 'h.json' + target.mkdir() + with pytest.raises(OSError): + HistoryStore(str(target)).append({'n': 1}) + assert os.listdir(tmp_path) == ['h.json'] and os.listdir(target) == [] + + +def test_damaged_history_kept_as_broken_copy(tmp_path): + path = tmp_path / 'h.json' + path.write_bytes(b'[{"ts": "2026-01-01 10:00", "input": "a.fasta"}, {oops') + errors = [] + store = HistoryStore(str(path)) + assert store.load(on_error=errors.append) == [] + assert path.read_bytes().endswith(b'{oops') # load alone does not rewrite the damaged file + store.append({'n': 1}) # append loads first: the raw bytes survive + broken = tmp_path / 'h.json.broken' + assert broken.read_bytes() == b'[{"ts": "2026-01-01 10:00", "input": "a.fasta"}, {oops' + assert store.load() == [{'n': 1}] # and the history file is healthy again + assert sorted(os.listdir(tmp_path)) == ['h.json', 'h.json.broken'] + + +def test_append_roundtrips_readable_utf8_indent_2(tmp_path): + path = tmp_path / 'h.json' + store = HistoryStore(str(path)) + rec = _rec(1) + store.append(rec) + assert store.load() == [rec] + text = path.read_text('utf-8') + assert 'данные' in text and '\\u' not in text # ensure_ascii=False + assert text.startswith('[\n {\n "ts"') # indent=2, like the files written so far + assert os.listdir(tmp_path) == ['h.json'] # no temp file left behind + + +def test_append_propagates_oserror(tmp_path): + store = HistoryStore(str(tmp_path / 'no such folder' / 'h.json')) + with pytest.raises(OSError): + store.append({'n': 1}) + assert os.listdir(tmp_path) == [] + + +def test_append_failure_keeps_previous_history(tmp_path): + path = tmp_path / 'h.json' + store = HistoryStore(str(path)) + store.append({'n': 1}) + with pytest.raises(TypeError): + store.append({'bad': object()}) # not JSON-serialisable + assert json.loads(path.read_text('utf-8')) == [{'n': 1}] + assert os.listdir(tmp_path) == ['h.json'] + + +def test_get_returns_record_or_none(tmp_path): + store = HistoryStore(str(tmp_path / 'h.json')) + assert store.get(0) is None # no file yet + store.append({'n': 1}) + store.append({'n': 2}) + assert store.get(0) == {'n': 2} and store.get(1) == {'n': 1} + assert store.get(2) is None and store.get(-1) is None + + +def test_new_record_counts_ok_and_n(): + files = [ + {'file': 'a.fasta', 'status': 'ok', 'tree': 'a.txt', 'stats': {'lnl': -1.5}, 'error': ''}, + {'file': 'b.fasta', 'status': 'error', 'tree': '', 'stats': {}, 'error': 'boom'}, + {'file': 'c.fasta', 'status': 'ok', 'tree': 'c.txt', 'stats': {}, 'error': ''}, + ] + params = {'model': 'GTR', 'engine': 'iqtree'} + rec = history.new_record(inputs=['D:/data'], engine='iqtree', params=params, files=files, + duration_s=12.5, ts=datetime(2026, 3, 7, 9, 4)) + assert set(rec) == {'ts', 'inputs', 'engine', 'params', 'files', 'ok', 'n', 'duration_s'} + assert rec['ts'] == '2026-03-07 09:04' + assert rec['inputs'] == ['D:/data'] and rec['engine'] == 'iqtree' + assert rec['params'] == params and rec['files'] == files + assert rec['ok'] == 2 and rec['n'] == 3 and rec['duration_s'] == 12.5 + json.dumps(rec) # JSON-serialisable as is + + +def test_new_record_does_not_alias_its_arguments(): + inputs, params, files = ['a.fasta'], {'model': 'GTR'}, [{'file': 'a.fasta', 'status': 'ok'}] + rec = history.new_record(inputs=inputs, engine='phyml', params=params, files=files, duration_s=0.0) + inputs.append('b.fasta') + params['model'] = 'JC69' + files[0]['status'] = 'error' + assert rec['inputs'] == ['a.fasta'] and rec['params'] == {'model': 'GTR'} + assert rec['files'][0]['status'] == 'ok' and rec['ok'] == 1 + + +def test_new_record_timestamp_defaults_to_now(): + before = datetime.now().replace(second=0, microsecond=0) + rec = history.new_record(inputs=[], engine='phyml', params={}, files=[], duration_s=0.0) + stamp = datetime.strptime(rec['ts'], '%Y-%m-%d %H:%M') + assert before <= stamp <= datetime.now() + assert rec['ok'] == 0 and rec['n'] == 0 + + +def test_old_record_display_and_tree(): + old = {'ts': '2025-05-01 10:00', 'input': 'a.fasta', 'ok': 1, 'n': 1, 'time': '30s', + 'model': 'GTR', 'engine': 'phyml', 'tree': 'x.txt'} + assert history.display_input(old) == 'a.fasta' + assert history.best_tree(old) == 'x.txt' + assert history.display_input({}) == '' and history.best_tree({}) == '' + assert history.best_tree({'input': 'a.fasta', 'tree': None}) == '' + + +def test_new_record_display_input_file_folder_and_batch(tmp_path): + folder = tmp_path / 'Мои данные' + folder.mkdir() + single = str(folder / 'a.fasta') + assert history.display_input({'inputs': [single]}) == 'a.fasta' + assert history.display_input({'inputs': [str(folder)]}) == 'Мои данные/' + assert history.display_input({'inputs': [str(folder) + os.sep]}) == 'Мои данные/' + batch = [str(folder / 'a.fasta'), str(folder / 'b.fasta'), str(folder / 'c.fasta')] + assert history.display_input({'inputs': batch}) == 'Мои данные/' + assert history.display_input({'inputs': []}) == '' + + +def test_display_input_of_files_without_a_common_folder_falls_back_to_the_first_name(): + # relative names have no common parent folder to show + assert history.display_input({'inputs': ['a.fasta', 'b.fasta']}) == 'a.fasta' + + +def test_new_record_best_tree_is_last_ok(): + rec = {'files': [ + {'file': 'a', 'status': 'ok', 'tree': 'a.txt'}, + {'file': 'b', 'status': 'ok', 'tree': 'b.txt'}, + {'file': 'c', 'status': 'error', 'tree': ''}, + ]} + assert history.best_tree(rec) == 'b.txt' + assert history.best_tree({'files': [{'file': 'c', 'status': 'error', 'tree': 't.txt'}]}) == '' + assert history.best_tree({'files': []}) == '' + assert history.best_tree({'files': [{'file': 'a', 'status': 'ok'}]}) == '' # no tree recorded + + +def test_mixed_old_and_new_records_survive_append(tmp_path): + path = tmp_path / 'h.json' + old = {'ts': '2025-05-01 10:00', 'input': 'a.fasta', 'ok': 1, 'n': 1, 'time': '30s', + 'model': 'GTR', 'engine': 'phyml', 'tree': 'x.txt'} + path.write_text(json.dumps([old], indent=2), 'utf-8') # a history written before this version + store = HistoryStore(str(path)) + new = _rec(2) + store.append(new) + assert store.load() == [new, old] + assert history.display_input(store.get(1)) == 'a.fasta' + + +def test_err_history_broken_is_localised_with_a_placeholder(): + for lang in ('en', 'ru'): + assert '{}' in STRINGS[lang]['err_history_broken'] + assert STRINGS['en']['err_history_broken'] == '[ERROR] Run history file is damaged and was ignored: {}' + assert STRINGS['ru']['err_history_broken'] == '[ОШИБКА] Файл истории запусков повреждён и пропущен: {}' diff --git a/tests/unit/test_presets.py b/tests/unit/test_presets.py new file mode 100644 index 0000000..0275639 --- /dev/null +++ b/tests/unit/test_presets.py @@ -0,0 +1,91 @@ +from phyml_core import presets, settings + +OLD_KEYS = ('datatype', 'model', 'support_method', 'bootstrap_replicates', + 'freq', 'tstv', 'pinv', 'classes', 'gamma', 'search', + 'optimize', 'parsimony') + + +def test_preset_keys_are_the_old_twelve_plus_engine(): + assert presets.PRESET_KEYS == OLD_KEYS + ('engine',) + + +def test_preset_keys_are_all_settings_keys(): + assert set(presets.PRESET_KEYS) <= set(settings.DEFAULTS) + + +def test_builtin_screening_uses_alrt(): + assert presets.BUILTIN['preset_builtin_screen']['support_method'] == '-4' + + +def test_builtin_values_are_the_current_ones_plus_phyml_engine(): + assert set(presets.BUILTIN) == {'preset_builtin_screen', 'preset_builtin_pub'} + assert presets.BUILTIN['preset_builtin_screen'] == { + 'datatype': 'nt', 'model': 'HKY85', 'support_method': '-4', + 'bootstrap_replicates': '100', 'freq': 'default', 'tstv': 'e', + 'pinv': '0.0', 'classes': '4', 'gamma': 'e', 'search': 'NNI', + 'optimize': 'tlr', 'parsimony': False, 'engine': 'phyml'} + assert presets.BUILTIN['preset_builtin_pub'] == { + 'datatype': 'nt', 'model': 'GTR', 'support_method': 'bootstrap', + 'bootstrap_replicates': '1000', 'freq': 'default', 'tstv': 'e', + 'pinv': 'e', 'classes': '4', 'gamma': 'e', 'search': 'BEST', + 'optimize': 'tlr', 'parsimony': False, 'engine': 'phyml'} + for preset in presets.BUILTIN.values(): + assert set(preset) == set(presets.PRESET_KEYS) + + +def test_extract_excludes_paths_and_display(): + cfg = dict(settings.DEFAULTS, engine='iqtree', model='GTR', phyml_path=r'D:\bin\phyml.exe', + iqtree_path=r'D:\bin\iqtree2.exe', theme='dark', accent='blue', + recent=['a.fasta'], show_tree=False, use_mpi=True, mpi_np='4') + got = presets.extract(cfg) + assert set(got) == set(presets.PRESET_KEYS) + assert got['engine'] == 'iqtree' and got['model'] == 'GTR' + for excluded in ('phyml_path', 'iqtree_path', 'theme', 'accent', 'recent', 'show_tree', + 'use_mpi', 'mpi_np', 'presets', 'window', 'schema_version'): + assert excluded not in got + + +def test_extract_takes_only_the_keys_that_are_present(): + assert presets.extract({'model': 'JC69', 'theme': 'dark'}) == {'model': 'JC69'} + assert presets.extract({}) == {} + + +def test_apply_switches_engine_and_keeps_paths(): + params = dict(settings.DEFAULTS, engine='phyml', phyml_path=r'D:\bin\phyml.exe', + iqtree_path=r'D:\bin\iqtree2.exe', theme='dark', model='HKY85') + preset = dict(presets.BUILTIN['preset_builtin_pub'], engine='iqtree') + out = presets.apply(preset, params) + assert out['engine'] == 'iqtree' and out['model'] == 'GTR' and out['search'] == 'BEST' + assert out['bootstrap_replicates'] == '1000' and out['pinv'] == 'e' + assert out['phyml_path'] == r'D:\bin\phyml.exe' and out['iqtree_path'] == r'D:\bin\iqtree2.exe' + assert out['theme'] == 'dark' + assert out is not params and params['engine'] == 'phyml' and params['model'] == 'HKY85' + + +def test_apply_ignores_keys_outside_the_preset_keys(): + preset = {'model': 'GTR', 'phyml_path': 'evil.exe', 'theme': 'dark', 'recent': ['x']} + params = {'model': 'HKY85', 'phyml_path': 'good.exe', 'theme': 'light', 'recent': []} + out = presets.apply(preset, params) + assert out == {'model': 'GTR', 'phyml_path': 'good.exe', 'theme': 'light', 'recent': []} + + +def test_apply_preset_without_engine_keeps_the_current_engine(): + # presets saved before the engine became part of them + old_user_preset = {k: presets.BUILTIN['preset_builtin_pub'][k] for k in OLD_KEYS} + assert presets.apply(old_user_preset, {'engine': 'iqtree', 'model': 'HKY85'}) == dict( + old_user_preset, engine='iqtree') + + +def test_apply_does_not_share_nested_values_with_params(): + params = {'recent': ['a.fasta']} + out = presets.apply({'model': 'GTR'}, params) + out['recent'].append('b.fasta') + assert params['recent'] == ['a.fasta'] + + +def test_extract_then_apply_roundtrip(): + cfg = dict(settings.DEFAULTS, engine='iqtree', model='GTR', search='BEST', parsimony=True) + saved = presets.extract(cfg) + fresh = dict(settings.DEFAULTS) + assert presets.apply(saved, fresh) == dict(fresh, engine='iqtree', model='GTR', search='BEST', + parsimony=True) diff --git a/tests/unit/test_settings.py b/tests/unit/test_settings.py index 8c50c34..c6526a5 100644 --- a/tests/unit/test_settings.py +++ b/tests/unit/test_settings.py @@ -151,6 +151,22 @@ def test_save_failure_keeps_previous_file(tmp_path): assert os.listdir(tmp_path) == ['c.json'] +def test_write_json_atomic_indent_roundtrip_and_cleanup(tmp_path): + path = tmp_path / 'sub' / 'o.json' + path.parent.mkdir() + settings.write_json_atomic(str(path), {'a': ['б']}) + assert path.read_text('utf-8') == '{\n "a": [\n "б"\n ]\n}' # indent 4 by default + settings.write_json_atomic(str(path), {'a': ['б']}, indent=2) + assert path.read_text('utf-8') == '{\n "a": [\n "б"\n ]\n}' + with pytest.raises(TypeError): + settings.write_json_atomic(str(path), {'a': object()}) + assert json.loads(path.read_text('utf-8')) == {'a': ['б']} # previous file kept + assert os.listdir(path.parent) == ['o.json'] # no temp file left + with pytest.raises(OSError): + settings.write_json_atomic(str(path.parent), []) # a directory target + assert os.listdir(tmp_path) == ['sub'] and os.listdir(path.parent) == ['o.json'] + + def _paths(tmp_path): app = tmp_path / 'Мои программы' / 'phymlGUI' (app / 'bin').mkdir(parents=True) From c7054818e766e94d3bc4373cf68d6f4e5561b49c Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:27:51 +0300 Subject: [PATCH 19/36] Drop non-dict history entries on load and derive the folder label from the record Co-Authored-By: Claude Opus 5.5 --- phyml_core/history.py | 82 +++++++++++++++------- tests/unit/test_history.py | 137 +++++++++++++++++++++++++++++++++++-- 2 files changed, 189 insertions(+), 30 deletions(-) diff --git a/phyml_core/history.py b/phyml_core/history.py index d91a6f0..0deeb41 100644 --- a/phyml_core/history.py +++ b/phyml_core/history.py @@ -3,7 +3,8 @@ Two record shapes live in the same file. Records written by older versions are flat: {'ts', 'input', 'ok', 'n', 'time', 'model', 'engine', 'tree'}. Records made by new_record carry the full run: inputs, parameters and a result entry per file. display_input and best_tree read -both shapes, so a history file never needs converting. +both shapes, so a history file never needs converting. Both helpers also survive records +whose fields have the wrong type (a hand-edited or damaged file) and always return a str. """ import copy import json @@ -30,8 +31,10 @@ def load(self, on_error: Optional[Callable[[str], None]] = None) -> List[dict]: A missing file yields []. A file with damaged content (not JSON, not a list, not UTF-8) yields [] too, after its bytes are copied to path + '.broken' and on_error is - told. A file that exists but cannot be read raises its OSError: returning [] there - would let the next append replace the user's real history. + told. A list that holds entries other than objects keeps its dict entries and is + reported the same way, so one stray value neither hides the real history nor crashes + whoever shows it. A file that exists but cannot be read raises its OSError: returning + [] there would let the next append replace the user's real history. """ try: with open(self.path, 'rb') as f: @@ -40,18 +43,25 @@ def load(self, on_error: Optional[Callable[[str], None]] = None) -> List[dict]: return [] try: data = json.loads(raw.decode('utf-8-sig')) - if not isinstance(data, list): - raise ValueError('history must be a JSON list') except ValueError: # includes JSONDecodeError and UnicodeDecodeError - try: - with open(self.path + '.broken', 'wb') as f: - f.write(raw) - except OSError: - pass - if on_error: - on_error(_t('err_history_broken').format(self.path)) + data = None + if not isinstance(data, list): + self._report_broken(raw, on_error) return [] - return data + records = [entry for entry in data if isinstance(entry, dict)] + if len(records) != len(data): + self._report_broken(raw, on_error) + return records + + def _report_broken(self, raw: bytes, on_error: Optional[Callable[[str], None]]) -> None: + """Keep the unusable bytes next to the history file and tell the caller.""" + try: + with open(self.path + '.broken', 'wb') as f: + f.write(raw) + except OSError: + pass + if on_error: + on_error(_t('err_history_broken').format(self.path)) def append(self, record: dict) -> None: """Put record first, cut the list to `limit` and write the file atomically. @@ -88,25 +98,47 @@ def new_record(*, inputs: List[str], engine: str, params: dict, files: List[dict } +def _text(value) -> str: + """value if it is a str, else ''; a malformed field must not leak a non-str to the caller.""" + return value if isinstance(value, str) else '' + + def _name(path: str) -> str: """Last component of path, ignoring a trailing separator (a drive root stays whole).""" path = os.path.normpath(path) return os.path.basename(path) or path +def _same_path(entry_file: str, input_path: str) -> bool: + """Whether a file entry names the input itself; a bare file name matches by its name.""" + a = os.path.normcase(os.path.normpath(entry_file)) + b = os.path.normcase(os.path.normpath(input_path)) + return a == b or (not os.path.dirname(a) and a == os.path.basename(b)) + + +def _file_entries(rec: dict) -> List[dict]: + """The dict entries of rec['files'] (none when it is missing or not a list).""" + files = rec.get('files') + return [e for e in files if isinstance(e, dict)] if isinstance(files, list) else [] + + def display_input(rec: dict) -> str: - """Short text for what the run processed. + """Short text for what the run processed, derived from the record alone (the files may + be gone, or sit on a share that no longer answers). - A file shows its name, a folder its name plus '/', several files the name of the folder - they share plus '/'. Old records just hold the name in 'input'. + A single input that is the path of its only file entry shows its name. A single input + with other paths in 'files' is a folder: its name plus '/'. With no file entries there + is nothing to tell it by, so the plain name is shown. Several inputs show the name of + the folder they share plus '/'. Old records just hold the name in 'input'. """ - if 'inputs' not in rec: - return rec.get('input', '') - inputs = rec['inputs'] + inputs = rec.get('inputs') + inputs = [p for p in inputs if isinstance(p, str) and p] if isinstance(inputs, list) else [] if not inputs: - return '' + return _text(rec.get('input')) if len(inputs) == 1: - return _name(inputs[0]) + ('/' if os.path.isdir(inputs[0]) else '') + entries = [e['file'] for e in _file_entries(rec) if isinstance(e.get('file'), str)] + is_folder = bool(entries) and not any(_same_path(f, inputs[0]) for f in entries) + return _name(inputs[0]) + ('/' if is_folder else '') try: common = os.path.commonpath([os.path.dirname(os.path.normpath(p)) for p in inputs]) except ValueError: # different drives, or absolute mixed with relative @@ -117,9 +149,9 @@ def display_input(rec: dict) -> str: def best_tree(rec: dict) -> str: """The tree of the last file that finished with status 'ok', '' if there is none. Old records hold a single path in 'tree'.""" - if 'files' not in rec: - return rec.get('tree') or '' - for entry in reversed(rec['files']): + if not isinstance(rec.get('files'), list): + return _text(rec.get('tree')) + for entry in reversed(_file_entries(rec)): if entry.get('status') == 'ok': - return entry.get('tree') or '' + return _text(entry.get('tree')) return '' diff --git a/tests/unit/test_history.py b/tests/unit/test_history.py index e32df75..2b39d25 100644 --- a/tests/unit/test_history.py +++ b/tests/unit/test_history.py @@ -154,6 +154,7 @@ def test_new_record_counts_ok_and_n(): assert rec['params'] == params and rec['files'] == files assert rec['ok'] == 2 and rec['n'] == 3 and rec['duration_s'] == 12.5 json.dumps(rec) # JSON-serialisable as is + assert history.display_input(rec) == 'data/' # one input, other paths in files: a folder def test_new_record_does_not_alias_its_arguments(): @@ -183,16 +184,74 @@ def test_old_record_display_and_tree(): assert history.best_tree({'input': 'a.fasta', 'tree': None}) == '' +def _entry(path, status='ok'): + return {'file': path, 'status': status, 'tree': '', 'stats': {}, 'error': ''} + + def test_new_record_display_input_file_folder_and_batch(tmp_path): + # nothing here exists on disk: the label comes from the record alone folder = tmp_path / 'Мои данные' - folder.mkdir() single = str(folder / 'a.fasta') - assert history.display_input({'inputs': [single]}) == 'a.fasta' - assert history.display_input({'inputs': [str(folder)]}) == 'Мои данные/' - assert history.display_input({'inputs': [str(folder) + os.sep]}) == 'Мои данные/' + assert history.display_input({'inputs': [single], 'files': [_entry(single)]}) == 'a.fasta' + inside = [_entry(str(folder / 'a.fasta')), _entry(str(folder / 'b.fasta'))] + assert history.display_input({'inputs': [str(folder)], 'files': inside}) == 'Мои данные/' + assert history.display_input({'inputs': [str(folder) + os.sep], 'files': inside}) == 'Мои данные/' batch = [str(folder / 'a.fasta'), str(folder / 'b.fasta'), str(folder / 'c.fasta')] - assert history.display_input({'inputs': batch}) == 'Мои данные/' + assert history.display_input({'inputs': batch, 'files': [_entry(p) for p in batch]}) == 'Мои данные/' assert history.display_input({'inputs': []}) == '' + assert history.display_input({'inputs': ['']}) == '' + + +def test_single_input_is_a_file_when_its_only_file_entry_is_that_path(tmp_path): + single = str(tmp_path / 'a.fasta') + assert history.display_input({'inputs': [single], 'files': [_entry(single)]}) == 'a.fasta' + # the same path spelled differently (case on Windows, bare file name), or a failed file + assert history.display_input({'inputs': [single], 'files': [_entry(os.path.normcase(single))]}) == 'a.fasta' + assert history.display_input({'inputs': [single], 'files': [_entry('a.fasta')]}) == 'a.fasta' + assert history.display_input({'inputs': [single], 'files': [_entry(single, 'error')]}) == 'a.fasta' + + +def test_single_input_is_a_folder_when_its_file_entries_are_other_paths(tmp_path): + folder = str(tmp_path / 'set') + files = [_entry(os.path.join(folder, 'a.fasta')), _entry(os.path.join(folder, 'b.fasta'))] + assert history.display_input({'inputs': [folder], 'files': files}) == 'set/' + assert history.display_input({'inputs': [folder], 'files': files[:1]}) == 'set/' # one file inside + assert history.display_input({'inputs': [folder], 'files': [_entry('a.fasta')]}) == 'set/' + + +def test_single_input_without_file_entries_shows_the_plain_name(tmp_path): + # no evidence of a folder: the name is shown as it is + path = str(tmp_path / 'set') + assert history.display_input({'inputs': [path]}) == 'set' + assert history.display_input({'inputs': [path], 'files': []}) == 'set' + + +class _NoFilesystem: + """Stands in for the os module inside phyml_core.history: pure path functions work, + everything that asks the filesystem fails the test.""" + _FORBIDDEN = {'isdir', 'isfile', 'exists', 'lexists', 'islink', 'ismount', 'realpath', 'abspath', + 'getsize', 'samefile', 'stat', 'lstat', 'listdir', 'scandir', 'getcwd'} + + def __init__(self, real): + self._real = real + + def __getattr__(self, name): + if name in self._FORBIDDEN: + raise AssertionError(f'display_input must not query the filesystem ({name})') + return _NoFilesystem(getattr(self._real, name)) if name == 'path' else getattr(self._real, name) + + +def test_display_input_does_not_touch_the_filesystem(tmp_path, monkeypatch): + folder = tmp_path / 'set' + folder.mkdir() # it exists, yet display_input must not ask + files = [_entry(str(folder / 'a.fasta')), _entry(str(folder / 'b.fasta'))] + monkeypatch.setattr(history, 'os', _NoFilesystem(os)) + with pytest.raises(AssertionError): # the guard itself works + history.os.path.isdir(str(folder)) + assert history.display_input({'inputs': [str(folder)], 'files': files}) == 'set/' + assert history.display_input({'inputs': [files[0]['file']], 'files': files[:1]}) == 'a.fasta' + assert history.display_input({'inputs': [e['file'] for e in files], 'files': files}) == 'set/' + assert history.display_input({'input': 'old.fasta'}) == 'old.fasta' def test_display_input_of_files_without_a_common_folder_falls_back_to_the_first_name(): @@ -229,3 +288,71 @@ def test_err_history_broken_is_localised_with_a_placeholder(): assert '{}' in STRINGS[lang]['err_history_broken'] assert STRINGS['en']['err_history_broken'] == '[ERROR] Run history file is damaged and was ignored: {}' assert STRINGS['ru']['err_history_broken'] == '[ОШИБКА] Файл истории запусков повреждён и пропущен: {}' + + +def test_load_drops_non_dict_entries_and_keeps_broken_copy(tmp_path): + path = tmp_path / 'h.json' + valid = {'ts': '2026-01-01 10:00', 'input': 'a.fasta', 'ok': 1, 'n': 1, 'tree': 'x.txt'} + raw = json.dumps([1, 'x', None, valid, [2], True]).encode('utf-8') + path.write_bytes(raw) + errors = [] + store = HistoryStore(str(path)) + assert store.load(on_error=errors.append) == [valid] # the good history is not thrown away + assert errors == [_t('err_history_broken').format(str(path))] + assert (tmp_path / 'h.json.broken').read_bytes() == raw + assert path.read_bytes() == raw # load alone does not rewrite the file + assert store.get(0) == valid and store.get(1) is None # indexes follow the cleaned list + + +def test_load_list_without_any_dict_is_empty_and_broken(tmp_path): + path = tmp_path / 'h.json' + path.write_text('[1, "x"]', 'utf-8') + errors = [] + assert HistoryStore(str(path)).load(errors.append) == [] + assert len(errors) == 1 and (tmp_path / 'h.json.broken').read_text('utf-8') == '[1, "x"]' + + +def test_load_clean_list_is_not_reported(tmp_path): + path = tmp_path / 'h.json' + path.write_text('[{"n": 1}, {"n": 2}]', 'utf-8') + errors = [] + assert HistoryStore(str(path)).load(errors.append) == [{'n': 1}, {'n': 2}] + assert errors == [] and not (tmp_path / 'h.json.broken').exists() + + +def test_append_after_non_dict_entries_keeps_the_valid_ones_and_the_original_bytes(tmp_path): + path = tmp_path / 'h.json' + raw = b'[1, {"n": 1}, "x"]' + path.write_bytes(raw) + store = HistoryStore(str(path)) + store.append({'n': 2}) + assert store.load() == [{'n': 2}, {'n': 1}] # junk is gone from the live file + assert (tmp_path / 'h.json.broken').read_bytes() == raw # but kept in the copy + + +@pytest.mark.parametrize('rec', [ + {'files': None}, {'files': [None]}, {'files': 'abc'}, {'files': 7}, {'files': [1, 'x', []]}, + {'files': {'file': 'a'}}, {'inputs': 'abc'}, {'inputs': None}, {'inputs': 5}, {'inputs': [None, 5]}, + {'inputs': {'a': 1}}, {'input': None}, {'input': 5}, {'input': ['a']}, {'tree': None}, {'tree': 7}, + {'files': [{'status': 'ok', 'tree': None}]}, {'files': [{'status': 'ok', 'tree': 3}]}, + {'files': [{'status': 'ok', 'tree': ['t']}]}, {'files': [{'status': ['ok']}]}, {}, + {'inputs': ['a.fasta'], 'files': [None, 5, {'file': None}]}, +]) +def test_helpers_tolerate_malformed_records(rec): + assert isinstance(history.display_input(rec), str) + assert isinstance(history.best_tree(rec), str) + + +def test_malformed_new_style_fields_fall_back_to_the_old_record_keys(): + rec = {'inputs': 'abc', 'files': None, 'input': 'old.fasta', 'tree': 'x.txt'} + assert history.display_input(rec) == 'old.fasta' + assert history.best_tree(rec) == 'x.txt' + assert history.display_input({'inputs': None, 'input': None}) == '' + assert history.best_tree({'files': 'abc', 'tree': None}) == '' + + +def test_malformed_items_inside_lists_are_skipped(): + rec = {'inputs': [None, 'D:/data/a.fasta', 5], + 'files': [None, {'status': 'ok', 'tree': 't.txt'}, 'x', 3, {'status': 'error', 'tree': 'bad.txt'}]} + assert history.display_input(rec) == 'a.fasta' + assert history.best_tree(rec) == 't.txt' From 14f943ff80a5d5821e175aa1e431cfd76fc1bf5f Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:55:16 +0300 Subject: [PATCH 20/36] Store settings and history in the data dir via AppPaths, migrate legacy files once The main window now takes AppPaths (detected by default) and reads and writes its config and history through settings and HistoryStore instead of cwd-relative files. Legacy files are copied once on start and logged. Tool paths and recent files inside app_dir are stored relative. An unreadable config disables auto-save for the session so defaults never replace it; save and history write errors go to the log. History records carry the run params and per-file results; presets come from phyml_core.presets and switch the engine. resource_path no longer depends on the cwd. Tests get a data_dir fixture and a make_window factory that destroys its windows. Co-Authored-By: Claude Sonnet 5.5 --- locales.json | 4 + phyml_core/utils.py | 10 +- phyml_ui/main_window.py | 267 ++++++++++--------- phymlgui.py | 2 +- tests/README.md | 5 +- tests/conftest.py | 61 ++++- tests/e2e/test_batch.py | 6 +- tests/gui/test_main_window_storage.py | 360 ++++++++++++++++++++++++++ tests/unit/test_utils.py | 18 ++ 9 files changed, 587 insertions(+), 146 deletions(-) create mode 100644 tests/gui/test_main_window_storage.py create mode 100644 tests/unit/test_utils.py diff --git a/locales.json b/locales.json index 032a480..613871f 100644 --- a/locales.json +++ b/locales.json @@ -76,6 +76,8 @@ "file_fasta": "Select Input File", "sys_load": "[System] Parameters loaded.\n", "err_load": "[ERROR] Failed to read config: {}\n", + "settings_migrated": "[System] Settings copied from {} to {}.", + "err_settings_save": "[ERROR] Cannot save settings: {}", "err_settings_broken": "[ERROR] Settings file is damaged; defaults loaded, a copy is saved as {}.", "err_history_broken": "[ERROR] Run history file is damaged and was ignored: {}", "err_box_title": "Error", @@ -265,6 +267,8 @@ "file_fasta": "Выбор входного файла", "sys_load": "[Система] Параметры загружены.\n", "err_load": "[ОШИБКА] Сбой чтения конфига: {}\n", + "settings_migrated": "[Система] Настройки скопированы из {} в {}.", + "err_settings_save": "[ОШИБКА] Не удалось сохранить настройки: {}", "err_settings_broken": "[ОШИБКА] Файл настроек повреждён: загружены настройки по умолчанию, копия сохранена как {}.", "err_history_broken": "[ОШИБКА] Файл истории запусков повреждён и пропущен: {}", "err_box_title": "Ошибка", diff --git a/phyml_core/utils.py b/phyml_core/utils.py index d8bb7e7..f219d00 100644 --- a/phyml_core/utils.py +++ b/phyml_core/utils.py @@ -1,6 +1,8 @@ import os import sys +from .paths import default_app_dir + try: import psutil # optional: CPU monitoring and physical core detection except ImportError: @@ -8,11 +10,9 @@ def resource_path(relative_path): - """ Get the absolute path to a resource (required for correct operation within .exe) """ - try: - base_path = sys._MEIPASS - except Exception: - base_path = os.path.abspath(".") + """Absolute path to a bundled resource: inside the PyInstaller bundle when frozen, + otherwise next to the program (never relative to the current working folder).""" + base_path = getattr(sys, '_MEIPASS', None) or default_app_dir() return os.path.join(base_path, relative_path) diff --git a/phyml_ui/main_window.py b/phyml_ui/main_window.py index a8becfb..a54e4b3 100644 --- a/phyml_ui/main_window.py +++ b/phyml_ui/main_window.py @@ -1,5 +1,5 @@ +import copy import csv -import json import os import shutil from datetime import datetime @@ -17,33 +17,19 @@ from PySide6.QtCore import QTimer, Qt from PySide6.QtGui import QIcon, QFont, QShortcut, QKeySequence -from phyml_core import __version__ +from phyml_core import __version__, presets, settings +from phyml_core.history import HistoryStore, best_tree, display_input, new_record from phyml_core.i18n import _t +from phyml_core.paths import AppPaths from phyml_core.utils import psutil, detect_cores, is_mpi_binary, resource_path from phyml_core.pipeline import PipelineWorker from phyml_ui.tree_viewer import TreeViewerWidget from phyml_ui.themes import resolve, build_qss, role_colors, ACCENTS, PALETTES -CONFIG_FILE = 'phyml_config.json' -HISTORY_FILE = 'phyml_history.json' - -# parameter subset persisted in presets (no paths, no display options) -_PRESET_KEYS = ('datatype', 'model', 'support_method', 'bootstrap_replicates', - 'freq', 'tstv', 'pinv', 'classes', 'gamma', 'search', - 'optimize', 'parsimony') - -_BUILTIN_PRESETS = { - 'preset_builtin_screen': { - 'datatype': 'nt', 'model': 'HKY85', 'support_method': '-4', - 'bootstrap_replicates': '100', 'freq': 'default', 'tstv': 'e', - 'pinv': '0.0', 'classes': '4', 'gamma': 'e', 'search': 'NNI', - 'optimize': 'tlr', 'parsimony': False}, - 'preset_builtin_pub': { - 'datatype': 'nt', 'model': 'GTR', 'support_method': 'bootstrap', - 'bootstrap_replicates': '1000', 'freq': 'default', 'tstv': 'e', - 'pinv': 'e', 'classes': '4', 'gamma': 'e', 'search': 'BEST', - 'optimize': 'tlr', 'parsimony': False}, -} +# per-file values of a worker row that go into the history +_STAT_KEYS = ('model', 'taxa', 'lnL', 'gamma', 'tstv', 'time') +# settings that say nothing about how a run was made: left out of a history record +_NOT_RUN_PARAMS = ('presets', 'recent', 'window') class LogConsole(QWidget): @@ -223,9 +209,15 @@ def accept(self): # --- Main Application Window --- class PhyMLApp(QWidget): - def __init__(self): + def __init__(self, paths=None): super().__init__() + self.paths = paths if paths is not None else AppPaths.detect() + self.history_store = HistoryStore(self.paths.history_file) self._is_loading = True + self._autosave_blocked = False # set when the settings file could not be read + self._save_failed = False # the last auto_save failed (the error is logged once) + self._extra_settings = {} # keys of the settings file this version does not know + self._history_records = [] # the records shown in history_list, same order self.worker = None self.fasta_file = "" self._mpi_auto_path = "" @@ -643,64 +635,104 @@ def get_current_config(self): } } + def _log(self, text): + """A system line for the log; messages from the core modules may end with a newline.""" + self.log_console.append(text.rstrip('\n')) + def auto_save(self): - config = self.get_current_config() + if self._autosave_blocked: + return + data = dict(self._extra_settings) + data.update(settings.to_disk(self.get_current_config(), self.paths)) + data['schema_version'] = settings.SCHEMA_VERSION + try: + settings.save(self.paths.config_file, data) + self._save_failed = False + except OSError as e: + if not self._save_failed: # every field change saves: report a failure once + self._log(_t('err_settings_save').format(e)) + self._save_failed = True + + def _migrate_legacy_files(self): + """Copy config/history from the working folder of older versions, once.""" try: - with open(CONFIG_FILE, 'w', encoding='utf-8') as f: - json.dump(config, f, indent=4, ensure_ascii=False) - except: pass + cwd = os.getcwd() + copied = settings.migrate_legacy(self.paths, cwd) + except OSError as e: + self._log(_t('err_load').format(e)) + return + for name in copied: + self._log(_t('settings_migrated').format( + os.path.join(cwd, name), os.path.join(self.paths.data_dir, name))) def load_config(self): self.update_models() - if os.path.exists(CONFIG_FILE): - try: - with open(CONFIG_FILE, 'r', encoding='utf-8') as f: - config = json.load(f) - self.engine_setting = config.get("engine", "phyml") - self.iqtree_path_input.setText(config.get("iqtree_path", "")) - self.phyml_path_input.setText(config.get("phyml_path", "")) - mpirun = config.get("mpirun_path", "mpiexec") - if mpirun in ("mpirun", "mpiexec"): - mpirun = shutil.which("mpiexec") or shutil.which("mpirun") or mpirun - self.mpirun_path_input.setText(mpirun) - self.muscle_path_input.setText(config.get("muscle_path", "")) - self.set_combo_by_data(self.cb_datatype, config.get("datatype", "nt")) - self.update_models() - self.set_combo_by_data(self.cb_model, config.get("model", "HKY85")) - self.set_combo_by_data(self.cb_freq, config.get("freq", "default")) - self.set_combo_by_data(self.cb_support_method, config.get("support_method", "bootstrap")) - self.set_combo_by_data(self.cb_search, config.get("search", "NNI")) - self.set_combo_by_data(self.cb_optimize, config.get("optimize", "tlr")) - self.le_bootstrap_replicates.setText(config.get("bootstrap_replicates", "100")) - self.chk_use_mpi.setChecked(config.get("use_mpi", False)) - self.le_mpi_np.setText(config.get("mpi_np", str(self.phys_cores))) - self.le_tstv.setText(config.get("tstv", "e")) - self.le_pinv.setText(config.get("pinv", "0.0")) - self.le_classes.setText(config.get("classes", "4")) - self.le_gamma.setText(config.get("gamma", "e")) - self.chk_parsimony.setChecked(config.get("parsimony", False)) - self.chk_show_tree.setChecked(config.get("show_tree", True)) - self._recent = [p for p in config.get("recent", []) if os.path.exists(p)] - self._refresh_recent() - self._user_presets = dict(config.get("presets", {})) - self._refresh_presets() - self._refresh_history() - self.apply_theme(config.get("theme", "system"), - config.get("accent", "green")) - win = config.get("window") or {} - if win.get("w") and win.get("h"): - self.resize(min(win["w"], 4096), min(win["h"], 4096)) - if isinstance(win.get("splitter"), list) and len(win["splitter"]) == 2: - self.splitter.setSizes(win["splitter"]) - if isinstance(win.get("vsplit"), list) and len(win["vsplit"]) == 2: - self.right_split.setSizes(win["vsplit"]) - self.toggle_bootstrap_input() - self.log_console.append(_t("sys_load")) - except Exception as e: self.log_console.append(_t("err_load").format(e)) + self._migrate_legacy_files() + problems = [] + try: + raw = settings.load(self.paths.config_file, on_error=problems.append) + except OSError as e: + # The file is there but unreadable: work with defaults, but never save them + # over the user's real settings. + self._autosave_blocked = True + problems.append(_t('err_load').format(e)) + raw = copy.deepcopy(settings.DEFAULTS) + for message in problems: + self._log(message) + self._extra_settings = {k: v for k, v in raw.items() if k not in settings.DEFAULTS} + try: + self._apply_config(settings.from_disk(raw, self.paths)) + if not problems and os.path.isfile(self.paths.config_file): + self._log(_t("sys_load")) + except Exception as e: + self._log(_t("err_load").format(e)) # lists must be populated even without a config file self._refresh_presets() self._refresh_history() + def _apply_config(self, config): + """Show a settings dict (as returned by settings.from_disk) in the window.""" + self.engine_setting = config["engine"] + self.iqtree_path_input.setText(config["iqtree_path"]) + self.phyml_path_input.setText(config["phyml_path"]) + mpirun = config["mpirun_path"] + if mpirun in ("mpirun", "mpiexec"): + mpirun = shutil.which("mpiexec") or shutil.which("mpirun") or mpirun + self.mpirun_path_input.setText(mpirun) + self.muscle_path_input.setText(config["muscle_path"]) + self._apply_model_params(config) + self.chk_use_mpi.setChecked(config["use_mpi"]) + self.le_mpi_np.setText(config["mpi_np"] or str(self.phys_cores)) # '' = physical cores + self.chk_show_tree.setChecked(config["show_tree"]) + self._recent = [p for p in config["recent"] if isinstance(p, str) and os.path.exists(p)] + self._refresh_recent() + self._user_presets = dict(config["presets"]) + self.apply_theme(config["theme"], config["accent"]) + win = config["window"] + if win.get("w") and win.get("h"): + self.resize(min(win["w"], 4096), min(win["h"], 4096)) + if isinstance(win.get("splitter"), list) and len(win["splitter"]) == 2: + self.splitter.setSizes(win["splitter"]) + if isinstance(win.get("vsplit"), list) and len(win["vsplit"]) == 2: + self.right_split.setSizes(win["vsplit"]) + self.toggle_bootstrap_input() + + def _apply_model_params(self, params): + """Show the model and search parameters (the part presets also carry).""" + self.set_combo_by_data(self.cb_datatype, params["datatype"]) + self.update_models() + self.set_combo_by_data(self.cb_model, params["model"]) + self.set_combo_by_data(self.cb_freq, params["freq"]) + self.set_combo_by_data(self.cb_support_method, params["support_method"]) + self.set_combo_by_data(self.cb_search, params["search"]) + self.set_combo_by_data(self.cb_optimize, params["optimize"]) + self.le_bootstrap_replicates.setText(params["bootstrap_replicates"]) + self.le_tstv.setText(params["tstv"]) + self.le_pinv.setText(params["pinv"]) + self.le_classes.setText(params["classes"]) + self.le_gamma.setText(params["gamma"]) + self.chk_parsimony.setChecked(params["parsimony"]) + def set_combo_by_data(self, combo, data_val): idx = combo.findData(data_val) if idx >= 0: combo.setCurrentIndex(idx) @@ -804,46 +836,52 @@ def _open_output_dir(self): # ---------- history ---------- - def _load_history(self): - try: - with open(HISTORY_FILE, 'r', encoding='utf-8') as f: - return json.load(f) - except (OSError, ValueError): - return [] - def _append_history(self, n_ok, n_all, tree_path): - duration = '' - if self._run_started: - duration = f'{(datetime.now() - self._run_started).total_seconds():.0f}s' - rec = { - 'ts': datetime.now().strftime('%Y-%m-%d %H:%M'), - 'input': os.path.basename(self.fasta_path_input.text() or (self.worker.fasta_files[0] if getattr(self.worker, 'fasta_files', None) else '')), - 'ok': n_ok, 'n': n_all, 'time': duration, - 'model': (self.cb_model.currentData() or ''), - 'engine': self.engine_setting, - 'tree': tree_path if tree_path else '', - } - hist = self._load_history() - hist.insert(0, rec) + """Add the finished run to the history, as the worker's rows describe it. + + The counts and the best tree follow from those rows, so n_ok, n_all and tree_path + only keep the signature of the call. + """ + worker = self.worker + fasta_files = list(getattr(worker, 'fasta_files', None) or []) + rows = list(getattr(worker, 'results', None) or []) + config = getattr(worker, 'config', None) # what the run used, not what the form shows now + if not isinstance(config, dict): + config = self.get_current_config() + params = {k: v for k, v in config.items() if k not in _NOT_RUN_PARAMS} + files = [{'file': path, + 'status': 'ok' if row.get('status') == 'OK' else 'fail', + 'tree': row.get('tree', ''), + 'stats': {k: row.get(k, '') for k in _STAT_KEYS}, + 'error': row.get('error', '')} + for path, row in zip(fasta_files, rows)] + target = self.fasta_path_input.text() or self.fasta_file or (fasta_files[0] if fasta_files else '') + duration = (datetime.now() - self._run_started).total_seconds() if self._run_started else 0.0 + rec = new_record(inputs=[target] if target else [], + engine=params.get('engine') or self.engine_setting, + params=params, files=files, duration_s=duration) try: - with open(HISTORY_FILE, 'w', encoding='utf-8') as f: - json.dump(hist[:200], f, indent=2, ensure_ascii=False) - except OSError: - pass + self.history_store.append(rec) + except OSError as e: + self._log(_t('err_settings_save').format(e)) self._refresh_history() def _refresh_history(self): self.history_list.clear() - for rec in self._load_history()[:50]: + try: + self._history_records = self.history_store.load(on_error=self._log)[:50] + except OSError as e: + self._history_records = [] + self._log(_t('err_load').format(e)) + for rec in self._history_records: self.history_list.addItem(_t('hist_entry').format( - rec.get('ts', ''), rec.get('input', ''), rec.get('ok', 0), rec.get('n', 0))) + rec.get('ts', ''), display_input(rec), rec.get('ok', 0), rec.get('n', 0))) def _open_history_entry(self, item): row = self.history_list.row(item) - hist = self._load_history()[:50] - if row >= len(hist): + if row >= len(self._history_records): return - tree = hist[row].get('tree', '') + tree = best_tree(self._history_records[row]) if tree and os.path.exists(tree): self.tree_view.load_tree(tree) else: @@ -856,7 +894,7 @@ def _refresh_presets(self): save_item = QListWidgetItem(_t('preset_save_current')) save_item.setData(Qt.UserRole, '__save__') self.presets_list.addItem(save_item) - for label_key in _BUILTIN_PRESETS: + for label_key in presets.BUILTIN: it = QListWidgetItem(_t(label_key)) it.setData(Qt.UserRole, ('builtin', label_key)) self.presets_list.addItem(it) @@ -871,26 +909,16 @@ def _preset_activated(self, item): self._save_preset() return kind, key = data - params = _BUILTIN_PRESETS[key] if kind == 'builtin' else self._user_presets.get(key, {}) - self._apply_preset(params) + preset = presets.BUILTIN[key] if kind == 'builtin' else self._user_presets.get(key, {}) + self._apply_preset(preset) self.log_console.append(_t('preset_applied').format(item.text())) - def _apply_preset(self, params): + def _apply_preset(self, preset): + params = presets.apply(preset, self.get_current_config()) self._is_loading = True try: - self.set_combo_by_data(self.cb_datatype, params.get('datatype', 'nt')) - self.update_models() - self.set_combo_by_data(self.cb_model, params.get('model', 'HKY85')) - self.set_combo_by_data(self.cb_support_method, params.get('support_method', 'bootstrap')) - self.set_combo_by_data(self.cb_freq, params.get('freq', 'default')) - self.set_combo_by_data(self.cb_search, params.get('search', 'NNI')) - self.set_combo_by_data(self.cb_optimize, params.get('optimize', 'tlr')) - self.le_bootstrap_replicates.setText(params.get('bootstrap_replicates', '100')) - self.le_tstv.setText(params.get('tstv', 'e')) - self.le_pinv.setText(params.get('pinv', '0.0')) - self.le_classes.setText(params.get('classes', '4')) - self.le_gamma.setText(params.get('gamma', 'e')) - self.chk_parsimony.setChecked(params.get('parsimony', False)) + self.engine_setting = params['engine'] + self._apply_model_params(params) finally: self._is_loading = False self.toggle_bootstrap_input() @@ -901,8 +929,7 @@ def _save_preset(self): if not ok or not name.strip(): return name = name.strip() - cfg = self.get_current_config() - self._user_presets[name] = {k: cfg.get(k) for k in _PRESET_KEYS} + self._user_presets[name] = presets.extract(self.get_current_config()) self.auto_save() self._refresh_presets() self.log_console.append(_t('preset_saved').format(name)) diff --git a/phymlgui.py b/phymlgui.py index 3e0b882..b821b91 100644 --- a/phymlgui.py +++ b/phymlgui.py @@ -7,7 +7,7 @@ from phyml_core.i18n import _t, LANG, STRINGS # noqa: F401 from phyml_core.pipeline import PipelineWorker # noqa: F401 from phyml_core.utils import psutil, detect_cores, is_mpi_binary, resource_path # noqa: F401 -from phyml_ui.main_window import PhyMLApp, CONFIG_FILE # noqa: F401 +from phyml_ui.main_window import PhyMLApp # noqa: F401 from phyml_ui.tree_viewer import TreeViewerDialog # noqa: F401 diff --git a/tests/README.md b/tests/README.md index 3133ca7..064b966 100644 --- a/tests/README.md +++ b/tests/README.md @@ -13,8 +13,9 @@ Layout: - `e2e/` - end-to-end runs of the pipeline, batch mode, CPU monitor and IQ-TREE engine. `e2e/datasets.py` holds the synthetic two-clade dataset (fixture `synthetic`). - `data/` - static fixtures (stats files, trees). -- `conftest.py` - binary lookup, markers, the `isolated_config` fixture and a session guard - that fails the run if a test rewrites the real `phyml_config.json` / `phyml_history.json`. +- `conftest.py` - binary lookup, markers, the `data_dir` / `make_window` / `window` fixtures + (config and history in a temporary folder) and a session guard that fails the run if a test + rewrites the real `phyml_config.json` / `phyml_history.json`. ## Binaries and markers diff --git a/tests/conftest.py b/tests/conftest.py index 427059c..9effe55 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -59,26 +59,59 @@ def binaries(): @pytest.fixture -def isolated_config(tmp_path, monkeypatch): - """Point the window's config/history files at tmp_path (interim: Task 5 uses data_dir).""" - import phyml_ui.main_window as main_window - monkeypatch.setattr(main_window, 'CONFIG_FILE', str(tmp_path / 'phyml_config.json')) - monkeypatch.setattr(main_window, 'HISTORY_FILE', str(tmp_path / 'phyml_history.json')) - return tmp_path +def data_dir(tmp_path, monkeypatch): + """An empty user-data folder for the window, and an empty working folder. + + PHYMLGUI_DATA_DIR sends config and history to tmp_path/data. The cwd is moved to the + empty tmp_path/cwd, so the legacy migration finds no phyml_config.json there (the real + one sits in the repository root) and nothing a test does can reach the real files. + """ + data = tmp_path / 'data' + cwd = tmp_path / 'cwd' + data.mkdir() + cwd.mkdir() + monkeypatch.setenv('PHYMLGUI_DATA_DIR', str(data)) + monkeypatch.chdir(cwd) + return data @pytest.fixture -def window(isolated_config, qtbot): - """A PhyMLApp on isolated config/history files, closed by qtbot at teardown. +def make_window(data_dir, qtbot, qapp): + """Factory of PhyMLApp windows on the isolated data folder: make_window(paths=None). - isolated_config must be listed BEFORE qtbot: fixtures are torn down in - reverse order, and closing the window runs closeEvent -> auto_save(), which - must still write the patched (tmp_path) config, never the real one. + The windows are destroyed at teardown, not just closed, and the application stylesheet + is put back. A started window styles the whole application, and a closed window that + stays alive makes every later one slower to build (a deleteLater() that only goes + through processEvents() never deletes anything). + + data_dir is a dependency, so it is torn down after the windows: closing a window runs + closeEvent -> auto_save(), which must still see the isolated folder (the env override + and the cwd), never the real one. """ + from PySide6.QtCore import QEvent + from PySide6.QtWidgets import QApplication from phyml_ui.main_window import PhyMLApp - w = PhyMLApp() - qtbot.addWidget(w) - return w + original_style = qapp.styleSheet() + created = [] + + def make(paths=None): + w = PhyMLApp(paths=paths) + qtbot.addWidget(w) + created.append(w) + return w + + yield make + for w in created: + w.close() + w.deleteLater() + QApplication.sendPostedEvents(None, QEvent.DeferredDelete) + qapp.setStyleSheet(original_style) + + +@pytest.fixture +def window(make_window): + """A PhyMLApp on an isolated data folder (see make_window).""" + return make_window() @pytest.fixture diff --git a/tests/e2e/test_batch.py b/tests/e2e/test_batch.py index 41bd83a..71eb962 100644 --- a/tests/e2e/test_batch.py +++ b/tests/e2e/test_batch.py @@ -1,12 +1,11 @@ """Batch mode: a folder of FASTA files, CSV summary, queue and history.""" -import json import shutil import pytest from Bio import Phylo -import phyml_ui.main_window as main_window from datasets import phyml_config, synthetic # noqa: F401 (fixture) +from phyml_core.history import HistoryStore from phyml_core.pipeline import PipelineWorker pytestmark = [pytest.mark.needs_phyml, pytest.mark.needs_mpi, pytest.mark.needs_muscle] @@ -85,8 +84,7 @@ def test_queue_history_open_folder(gui_batch): assert window.queue_list.count() == 2 statuses = [window.queue_list.item(i).text().split(' ')[-1] for i in range(2)] assert statuses == ['✓', '✓'], statuses - with open(main_window.HISTORY_FILE, encoding='utf-8') as f: - records = json.load(f) + records = HistoryStore(window.paths.history_file).load() assert records and records[0]['n'] == 2 and records[0]['ok'] == 2 assert window.history_list.count() >= 1 assert window.open_dir_btn.isEnabled() diff --git a/tests/gui/test_main_window_storage.py b/tests/gui/test_main_window_storage.py new file mode 100644 index 0000000..c55c6eb --- /dev/null +++ b/tests/gui/test_main_window_storage.py @@ -0,0 +1,360 @@ +"""PhyMLApp keeps its settings and history in the data folder (AppPaths), never in the cwd.""" +import json +import os +import shutil +import types +from pathlib import Path + +import pytest +from PySide6.QtWidgets import QInputDialog + +import phyml_ui.main_window as main_window +from phyml_core import presets +from phyml_core.history import HistoryStore, display_input, new_record +from phyml_core.i18n import _t +from phyml_core.paths import AppPaths +from phyml_core.utils import detect_cores + + +def read_json(path): + with open(path, encoding='utf-8') as f: + return json.load(f) + + +def log_text(window): + return window.log_console.toPlainText() + + +def stub_worker(files, rows, **extra): + """What _append_history reads from a finished PipelineWorker. isRunning and active_process + are there for closeEvent and the CPU timer, which keep asking the worker.""" + return types.SimpleNamespace(fasta_files=list(files), results=list(rows), + isRunning=lambda: False, active_process=None, **extra) + + +def worker_row(path, status='OK', tree='', **stats): + row = {'file': os.path.basename(path), 'status': status, 'tree': tree, + 'model': '', 'taxa': '', 'lnL': '', 'gamma': '', 'tstv': '', 'time': ''} + row.update(stats) + return row + + +def raising(exc): + def fail(*args, **kwargs): + raise exc + return fail + + +# ---------- where the files go ---------- + +def test_config_saved_in_data_dir(window, data_dir): + window.set_combo_by_data(window.cb_model, 'GTR') + saved = read_json(data_dir / 'phyml_config.json') + assert saved['model'] == 'GTR' + assert saved['schema_version'] == 2 + assert not (Path.cwd() / 'phyml_config.json').exists() + + +def test_window_uses_detected_paths_by_default(window, data_dir): + assert window.paths.data_dir == str(data_dir) + assert window.paths.config_file == str(data_dir / 'phyml_config.json') + assert os.path.samefile(window.paths.app_dir, Path(__file__).resolve().parents[2]) + + +def test_history_written_to_data_dir(window, data_dir): + window.worker = stub_worker(['x.fasta'], [worker_row('x.fasta', tree='x.txt')]) + window._append_history(1, 1, 'x.txt') + records = HistoryStore(window.paths.history_file).load() + assert records[0]['ok'] == 1 + assert (data_dir / 'phyml_history.json').is_file() + assert not (Path.cwd() / 'phyml_history.json').exists() + assert window.history_list.count() == 1 + + +def test_tool_path_inside_app_dir_saved_relative(data_dir, tmp_path, make_window): + paths = AppPaths(app_dir=str(tmp_path / 'app'), data_dir=str(tmp_path / 'd')) + os.makedirs(paths.data_dir) + os.makedirs(paths.bin_dir) + exe = os.path.join(paths.bin_dir, 'phyml-mpi.exe') + sample = os.path.join(paths.app_dir, 'проба 1.fasta') + Path(sample).write_text('>a\nACGT\n', encoding='utf-8') + w = make_window(paths) + w._remember_recent(sample) + w.phyml_path_input.setText(exe) + w.auto_save() # the path holders are not bound to auto-save; the Settings dialog saves + saved = read_json(paths.config_file) + assert saved['phyml_path'] == os.path.join('bin', 'phyml-mpi.exe') + assert saved['recent'] == ['проба 1.fasta'] + assert w.phyml_path_input.text() == exe, 'the field keeps the absolute path' + w2 = make_window(paths) + assert w2.phyml_path_input.text() == exe, 'a stored relative path is resolved on load' + assert w2._recent == [sample] + + +def test_tool_path_outside_app_dir_stays_absolute(data_dir, tmp_path, make_window): + paths = AppPaths(app_dir=str(tmp_path / 'app'), data_dir=str(tmp_path / 'd')) + os.makedirs(paths.data_dir) + exe = str(tmp_path / 'elsewhere' / 'muscle.exe') + w = make_window(paths) + w.muscle_path_input.setText(exe) + w.auto_save() + assert read_json(paths.config_file)['muscle_path'] == exe + + +# ---------- legacy files ---------- + +def test_legacy_config_migrated_on_first_start(data_dir, make_window): + legacy = Path.cwd() / 'phyml_config.json' + legacy.write_text(json.dumps({'model': 'GTR'}), encoding='utf-8') + w = make_window() + assert w.cb_model.currentData() == 'GTR' + expected = _t('settings_migrated').format(str(legacy), str(data_dir / 'phyml_config.json')) + assert expected in log_text(w) + assert legacy.is_file(), 'the original stays in place' + assert read_json(data_dir / 'phyml_config.json')['model'] == 'GTR' + + +def test_legacy_history_migrated_and_listed(data_dir, make_window): + record = new_record(inputs=['a.fasta'], engine='phyml', params={}, duration_s=1.0, + files=[{'file': 'a.fasta', 'status': 'ok', 'tree': '', 'stats': {}, 'error': ''}]) + (Path.cwd() / 'phyml_history.json').write_text(json.dumps([record]), encoding='utf-8') + w = make_window() + assert w.history_list.count() == 1 + assert log_text(w).count(_t('settings_migrated').split('{}')[0]) == 1 # no config to copy + + +def test_migration_runs_once(data_dir, make_window): + (Path.cwd() / 'phyml_config.json').write_text('{"model": "GTR"}', encoding='utf-8') + first = make_window() + assert _t('settings_migrated').split('{}')[0] in log_text(first) + second = make_window() + assert _t('settings_migrated').split('{}')[0] not in log_text(second) + + +def test_failed_migration_is_logged_and_start_continues(data_dir, make_window, monkeypatch): + monkeypatch.setattr(main_window.settings, 'migrate_legacy', + raising(PermissionError(13, 'copy denied'))) + w = make_window() + assert 'copy denied' in log_text(w) + assert _t('err_load').split('{}')[0].strip() in log_text(w) + assert w.cb_model.currentData() == 'HKY85' + + +# ---------- loading ---------- + +def test_missing_config_gives_defaults_without_load_message(window): + assert window.cb_model.currentData() == 'HKY85' + assert _t('sys_load').strip() not in log_text(window) + + +def test_existing_config_logs_load_message(data_dir, make_window): + (data_dir / 'phyml_config.json').write_text( + json.dumps({'model': 'GTR', 'bootstrap_replicates': '333'}), encoding='utf-8') + w = make_window() + assert w.cb_model.currentData() == 'GTR' and w.le_bootstrap_replicates.text() == '333' + assert _t('sys_load').strip() in log_text(w) + assert not any(text.endswith('\n') for _, text in w.log_console._entries) + + +@pytest.mark.parametrize('stored, shown', [('', None), ('3', '3')]) +def test_empty_mpi_np_means_physical_cores(data_dir, make_window, stored, shown): + (data_dir / 'phyml_config.json').write_text(json.dumps({'mpi_np': stored}), encoding='utf-8') + w = make_window() + assert w.le_mpi_np.text() == (shown or str(detect_cores()[0])) + + +def test_bare_mpiexec_is_resolved_through_path(data_dir, make_window, monkeypatch): + found = str(data_dir / 'bin' / 'mpiexec.exe') + monkeypatch.setattr(shutil, 'which', lambda name: found if name == 'mpiexec' else None) + w = make_window() + assert w.mpirun_path_input.text() == found + + +def test_bare_mpiexec_kept_when_not_found(data_dir, make_window, monkeypatch): + monkeypatch.setattr(shutil, 'which', lambda name: None) + w = make_window() + assert w.mpirun_path_input.text() == 'mpiexec' + + +def test_damaged_config_reported_and_replaced_by_defaults(data_dir, make_window): + config = data_dir / 'phyml_config.json' + config.write_bytes(b'{ not json') + w = make_window() + broken = str(config) + '.broken' + assert _t('err_settings_broken').format(broken) in log_text(w) + assert Path(broken).read_bytes() == b'{ not json' + assert w.cb_model.currentData() == 'HKY85' + w.set_combo_by_data(w.cb_model, 'GTR') + assert read_json(config)['model'] == 'GTR', 'a damaged file does not stop saving' + + +def test_unreadable_config_keeps_the_file_and_disables_autosave(data_dir, make_window, monkeypatch): + config = data_dir / 'phyml_config.json' + config.write_text('{"model": "GTR"}', encoding='utf-8') + before = config.read_bytes() + monkeypatch.setattr(main_window.settings, 'load', raising(PermissionError(13, 'locked'))) + w = make_window() + assert 'locked' in log_text(w) + assert _t('err_load').split('{}')[0].strip() in log_text(w) + assert not any(text.endswith('\n') for _, text in w.log_console._entries) + w.set_combo_by_data(w.cb_model, 'JC69') + w.auto_save() + w.close() + assert config.read_bytes() == before, 'defaults must never replace the unreadable file' + + +def test_unknown_settings_survive_a_save(data_dir, make_window): + config = data_dir / 'phyml_config.json' + config.write_text(json.dumps({'future_key': {'a': 1}}), encoding='utf-8') + w = make_window() + w.set_combo_by_data(w.cb_model, 'GTR') + saved = read_json(config) + assert saved['future_key'] == {'a': 1} and saved['model'] == 'GTR' + + +def test_save_error_is_logged_once_per_failure(window, monkeypatch): + monkeypatch.setattr(main_window.settings, 'save', raising(OSError('read-only disk'))) + for model in ('GTR', 'JC69', 'K80'): + window.set_combo_by_data(window.cb_model, model) + prefix = _t('err_settings_save').split('{}')[0] + assert log_text(window).count('read-only disk') == 1 + assert prefix in log_text(window) + + +# ---------- history ---------- + +def test_history_record_holds_params_and_full_file_paths(window, tmp_path): + folder = tmp_path / 'run folder' + files = [str(folder / 'a.fasta'), str(folder / 'б.fasta')] + trees = [str(folder / 'a.phy_phyml_tree.txt'), ''] + window.fasta_path_input.setText(str(folder)) + window.set_combo_by_data(window.cb_model, 'GTR') + window.worker = stub_worker(files, [ + worker_row(files[0], 'OK', trees[0], model='GTR', taxa='8', lnL='-123.4', + gamma='0.5', tstv='2.1', time='3s'), + worker_row(files[1], 'FAIL')]) + window._append_history(1, 2, trees[0]) + + rec = HistoryStore(window.paths.history_file).load()[0] + assert rec['inputs'] == [str(folder)] + assert rec['engine'] == 'phyml' + assert (rec['ok'], rec['n']) == (1, 2) + assert rec['params']['model'] == 'GTR' + for key in ('presets', 'recent', 'window'): + assert key not in rec['params'] + assert [e['file'] for e in rec['files']] == files + assert [e['status'] for e in rec['files']] == ['ok', 'fail'] + assert rec['files'][0]['tree'] == trees[0] + assert rec['files'][0]['stats'] == {'model': 'GTR', 'taxa': '8', 'lnL': '-123.4', + 'gamma': '0.5', 'tstv': '2.1', 'time': '3s'} + assert 'error' in rec['files'][0] + assert isinstance(rec['duration_s'], float) + assert window.history_list.item(0).text() == _t('hist_entry').format( + rec['ts'], 'run folder/', 1, 2) + + +def test_history_params_come_from_the_run_config(window): + """A change made to the form while the run was going must not rewrite what was run.""" + window.set_combo_by_data(window.cb_model, 'GTR') + config = window.get_current_config() + window.set_combo_by_data(window.cb_model, 'JC69') + window.worker = stub_worker(['x.fasta'], [worker_row('x.fasta')], config=config) + window._append_history(1, 1, '') + assert HistoryStore(window.paths.history_file).load()[0]['params']['model'] == 'GTR' + + +def test_single_file_run_is_listed_by_its_name(window, tmp_path): + fasta = str(tmp_path / 'one.fasta') + window.fasta_path_input.setText(fasta) + window.worker = stub_worker([fasta], [worker_row(fasta, tree='t.txt')]) + window._append_history(1, 1, 't.txt') + rec = HistoryStore(window.paths.history_file).load()[0] + assert display_input(rec) == 'one.fasta' + assert window.history_list.item(0).text() == _t('hist_entry').format(rec['ts'], 'one.fasta', 1, 1) + + +def test_history_list_shows_old_and_new_records_and_opens_the_best_tree(window, tmp_path, monkeypatch): + tree = tmp_path / 'best.txt' + tree.write_text('(a,b);', encoding='utf-8') + store = HistoryStore(window.paths.history_file) + store.append({'ts': '2020-01-01 10:00', 'input': 'old.fasta', 'ok': 2, 'n': 3, 'time': '5s', + 'model': 'HKY85', 'engine': 'phyml', 'tree': str(tmp_path / 'gone.txt')}) + store.append(new_record( + inputs=[str(tmp_path / 'batch')], engine='phyml', params={}, duration_s=2.0, + files=[{'file': str(tmp_path / 'batch' / 'a.fasta'), 'status': 'fail', 'tree': '', + 'stats': {}, 'error': ''}, + {'file': str(tmp_path / 'batch' / 'b.fasta'), 'status': 'ok', 'tree': str(tree), + 'stats': {}, 'error': ''}])) + window._refresh_history() + assert window.history_list.count() == 2 + new_item, old_item = window.history_list.item(0), window.history_list.item(1) + assert new_item.text().split(' | ')[1] == 'batch/' + assert old_item.text() == _t('hist_entry').format('2020-01-01 10:00', 'old.fasta', 2, 3) + + shown = [] + monkeypatch.setattr(window.tree_view, 'load_tree', shown.append) + window._open_history_entry(new_item) + assert shown == [str(tree)] + window._open_history_entry(old_item) + assert shown == [str(tree)], 'a vanished tree is not loaded' + assert _t('hist_no_tree') in log_text(window) + + +def test_history_write_error_is_logged_without_crash(window, monkeypatch): + monkeypatch.setattr(HistoryStore, 'append', raising(OSError('disk full'))) + window.worker = stub_worker(['x.fasta'], [worker_row('x.fasta')]) + window._append_history(1, 1, '') + assert 'disk full' in log_text(window) + assert _t('err_settings_save').split('{}')[0] in log_text(window) + + +def test_unreadable_history_is_logged_without_crash(data_dir, make_window, monkeypatch): + monkeypatch.setattr(HistoryStore, 'load', raising(PermissionError(13, 'history locked'))) + w = make_window() + assert 'history locked' in log_text(w) + assert w.history_list.count() == 0 + w.worker = stub_worker(['x.fasta'], [worker_row('x.fasta')]) + w._append_history(1, 1, '') # must not raise either + assert not (data_dir / 'phyml_history.json').exists() + + +# ---------- presets ---------- + +def preset_item(window, label_key): + for i in range(window.presets_list.count()): + item = window.presets_list.item(i) + if item.data(main_window.Qt.UserRole) == ('builtin', label_key): + return item + raise AssertionError(label_key) + + +def test_builtin_preset_switches_the_engine(window): + window.engine_setting = 'iqtree' + window.update_models() + window._preset_activated(preset_item(window, 'preset_builtin_pub')) + assert window.engine_setting == 'phyml' + assert window.cb_model.currentData() == 'GTR' + assert window.cb_search.currentData() == 'BEST' + assert read_json(window.paths.config_file)['engine'] == 'phyml' + + +def test_user_preset_without_engine_keeps_the_current_one(window): + window.engine_setting = 'iqtree' + window.update_models() + window._user_presets['old'] = {'datatype': 'nt', 'model': 'HKY+F+G4', 'search': 'SPR'} + window._refresh_presets() + item = next(window.presets_list.item(i) for i in range(window.presets_list.count()) + if window.presets_list.item(i).text() == 'old') + window._preset_activated(item) + assert window.engine_setting == 'iqtree' + assert window.cb_model.currentData() == 'HKY+F+G4' and window.cb_search.currentData() == 'SPR' + + +def test_saved_user_preset_holds_the_engine_and_is_persisted(window, monkeypatch): + monkeypatch.setattr(QInputDialog, 'getText', staticmethod(lambda *a, **k: ('mine', True))) + window.set_combo_by_data(window.cb_model, 'GTR') + window._save_preset() + expected = presets.extract(window.get_current_config()) + assert window._user_presets['mine'] == expected + assert expected['engine'] == 'phyml' + assert read_json(window.paths.config_file)['presets']['mine'] == expected diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py new file mode 100644 index 0000000..124340c --- /dev/null +++ b/tests/unit/test_utils.py @@ -0,0 +1,18 @@ +import os +import sys + +from conftest import ROOT + +from phyml_core.utils import resource_path + + +def test_resource_path_does_not_depend_on_the_cwd(tmp_path, monkeypatch): + monkeypatch.delattr(sys, '_MEIPASS', raising=False) + monkeypatch.chdir(tmp_path) + for name in ('icon.png', 'locales.json'): + assert os.path.samefile(resource_path(name), ROOT / name) + + +def test_resource_path_uses_the_bundle_folder_when_frozen(tmp_path, monkeypatch): + monkeypatch.setattr(sys, '_MEIPASS', str(tmp_path), raising=False) + assert resource_path('icon.png') == str(tmp_path / 'icon.png') From 38ed4582953df7aadd17ccc647bb3fbe743b1aa7 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:05:27 +0300 Subject: [PATCH 21/36] Block auto-save after a failed migration while the data dir has no config Co-Authored-By: Claude Opus 5.5 --- phyml_ui/main_window.py | 4 ++++ tests/gui/test_main_window_storage.py | 32 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/phyml_ui/main_window.py b/phyml_ui/main_window.py index a54e4b3..0936a43 100644 --- a/phyml_ui/main_window.py +++ b/phyml_ui/main_window.py @@ -660,6 +660,10 @@ def _migrate_legacy_files(self): copied = settings.migrate_legacy(self.paths, cwd) except OSError as e: self._log(_t('err_load').format(e)) + if not os.path.exists(self.paths.config_file): + # Saving the defaults now would create the config that closes the migration + # gate, and the legacy settings would never be copied. + self._autosave_blocked = True return for name in copied: self._log(_t('settings_migrated').format( diff --git a/tests/gui/test_main_window_storage.py b/tests/gui/test_main_window_storage.py index c55c6eb..ca082b5 100644 --- a/tests/gui/test_main_window_storage.py +++ b/tests/gui/test_main_window_storage.py @@ -140,6 +140,38 @@ def test_failed_migration_is_logged_and_start_continues(data_dir, make_window, m assert w.cb_model.currentData() == 'HKY85' +def test_failed_migration_blocks_autosave_and_retries_next_start(data_dir, make_window, monkeypatch): + """Defaults must not close the migration gate: with no config in data_dir yet, the next + start has to find the legacy files still to be copied.""" + legacy = Path.cwd() / 'phyml_config.json' + legacy.write_text(json.dumps({'model': 'GTR'}), encoding='utf-8') + target = data_dir / 'phyml_config.json' + with monkeypatch.context() as broken: + broken.setattr(main_window.settings, 'migrate_legacy', + raising(PermissionError(13, 'copy denied'))) + w = make_window() + assert w.cb_model.currentData() == 'HKY85' + w.set_combo_by_data(w.cb_model, 'JC69') + w.auto_save() + w.close() + assert not target.exists(), 'defaults were written over the config that was to be migrated' + + second = make_window() + assert second.cb_model.currentData() == 'GTR' + assert _t('settings_migrated').format(str(legacy), str(target)) in log_text(second) + assert read_json(target)['model'] == 'GTR' + + +def test_failed_migration_keeps_autosave_when_the_config_exists(data_dir, make_window, monkeypatch): + """The gate is already closed by a real config: nothing to protect, saving goes on.""" + (data_dir / 'phyml_config.json').write_text('{"model": "GTR"}', encoding='utf-8') + monkeypatch.setattr(main_window.settings, 'migrate_legacy', + raising(PermissionError(13, 'copy denied'))) + w = make_window() + w.set_combo_by_data(w.cb_model, 'JC69') + assert read_json(data_dir / 'phyml_config.json')['model'] == 'JC69' + + # ---------- loading ---------- def test_missing_config_gives_defaults_without_load_message(window): From 0945772dd6210b0dfee0eb16def705d44a7f67ad Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:12:21 +0300 Subject: [PATCH 22/36] Add launchers, the Engine interface and the PhyML engine with argv tests Native and Mpi launchers, the Engine ABC with Issue/EngineOutputs, and PhyML(command, validate, executable, outputs, launcher, models). The parse_stats function moves to engines/phyml.py as parse_phyml_stats and pipeline.py imports it. Adds val_not_number, val_no_exe and val_no_mpiexec to both locales. Co-Authored-By: Claude Opus 5.5 --- locales.json | 10 +- phyml_core/engines/__init__.py | 7 + phyml_core/engines/base.py | 94 ++++++++ phyml_core/engines/phyml.py | 144 ++++++++++++ phyml_core/launchers.py | 21 ++ phyml_core/pipeline.py | 23 +- tests/unit/test_engine_phyml.py | 380 +++++++++++++++++++++++++++++++ tests/unit/test_launchers.py | 33 +++ tests/unit/test_stats_parsers.py | 5 +- 9 files changed, 692 insertions(+), 25 deletions(-) create mode 100644 phyml_core/engines/__init__.py create mode 100644 phyml_core/engines/base.py create mode 100644 phyml_core/engines/phyml.py create mode 100644 phyml_core/launchers.py create mode 100644 tests/unit/test_engine_phyml.py create mode 100644 tests/unit/test_launchers.py diff --git a/locales.json b/locales.json index 613871f..b2e2bb8 100644 --- a/locales.json +++ b/locales.json @@ -188,7 +188,10 @@ "sum_col_lnl": "lnL", "sum_col_gamma": "Gamma", "sum_col_tstv": "Ts/Tv", - "sum_col_time": "Time" + "sum_col_time": "Time", + "val_not_number": "Parameter \"{}\" must be a positive number, got \"{}\".", + "val_no_exe": "{} executable not found: {}", + "val_no_mpiexec": "MPI is enabled but mpiexec was not found: {}" }, "ru": { "tree_result": "Результат PhyML: {}", @@ -379,6 +382,9 @@ "sum_col_lnl": "lnL", "sum_col_gamma": "Гамма", "sum_col_tstv": "Ts/Tv", - "sum_col_time": "Время" + "sum_col_time": "Время", + "val_not_number": "Параметр «{}» должен быть положительным числом, получено «{}».", + "val_no_exe": "Не найден исполняемый файл {}: {}", + "val_no_mpiexec": "MPI включён, но mpiexec не найден: {}" } } \ No newline at end of file diff --git a/phyml_core/engines/__init__.py b/phyml_core/engines/__init__.py new file mode 100644 index 0000000..be1d7e4 --- /dev/null +++ b/phyml_core/engines/__init__.py @@ -0,0 +1,7 @@ +"""Registered engines, by key.""" +from typing import Dict + +from .base import Engine +from .phyml import PhyML + +ENGINES: Dict[str, Engine] = {'phyml': PhyML()} diff --git a/phyml_core/engines/base.py b/phyml_core/engines/base.py new file mode 100644 index 0000000..7d575bc --- /dev/null +++ b/phyml_core/engines/base.py @@ -0,0 +1,94 @@ +"""The Engine interface every phylogeny program implements (no Qt).""" +import re +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Dict, List, Literal, Optional, Tuple, Union + +from ..i18n import _t +from ..launchers import Mpi, Native +from ..paths import AppPaths + + +@dataclass(frozen=True) +class Issue: + """A problem found before a run; `key` is a localization key, `args` fill its {} slots.""" + level: Literal['error', 'warning'] + key: str + args: tuple = () + + +@dataclass(frozen=True) +class EngineOutputs: + """Files an engine writes next to the PHYLIP input; None where it writes no such file.""" + tree: str + stats: str + boot_trees: Optional[str] + log: Optional[str] + + +def format_issue(issue: Issue) -> str: + return _t(issue.key).format(*issue.args) + + +_DECIMAL = re.compile(r'[0-9]+(?:\.[0-9]*)?|\.[0-9]+') +_INTEGER = re.compile(r'[0-9]+') + + +def parse_decimal(value) -> Optional[float]: + """A plain non-negative decimal ('3', '0.5', '.5', '2.') as a float, else None. + + Unlike float() this rejects signs, exponents, 'nan'/'inf', digit separators and + non-ASCII digits, i.e. everything an engine's own argument parser may not accept. + """ + text = str(value).strip() + return float(text) if _DECIMAL.fullmatch(text) else None + + +def positive_number(value) -> bool: + """An integer or a decimal fraction greater than zero.""" + number = parse_decimal(value) + return number is not None and number > 0 + + +def positive_int(value) -> bool: + """A whole number greater than zero.""" + text = str(value).strip() + return bool(_INTEGER.fullmatch(text)) and int(text) > 0 + + +class Engine(ABC): + key: str # 'phyml' | 'iqtree' | later 'raxml' + name: str # shown in messages: 'PhyML', 'IQ-TREE' + label_key: str # localization key of the name in the engine selector + + @abstractmethod + def models(self, datatype: str) -> List[str]: + """Substitution models offered for 'nt' or 'aa' data.""" + + @abstractmethod + def validate(self, params: dict, paths: AppPaths) -> List[Issue]: + """Everything that would make the run fail or mislead, in one pass.""" + + @abstractmethod + def normalize(self, params: dict) -> Tuple[dict, List[Issue]]: + """A copy of params adjusted to what the engine accepts, with a note per change.""" + + @abstractmethod + def executable(self, params: dict, paths: AppPaths) -> str: + """The configured program, else the one shipped in bin/ (may not exist).""" + + @abstractmethod + def command(self, exe: str, phylip: str, params: dict) -> List[str]: + """The engine's own argv, without any launcher prefix. A pure function.""" + + @abstractmethod + def outputs(self, phylip: str) -> EngineOutputs: + """Where the engine writes its results for this input.""" + + @abstractmethod + def parse_stats(self, outputs: EngineOutputs) -> Dict[str, str]: + """model, taxa, lnL, gamma, tstv, time (empty strings where unknown).""" + + @abstractmethod + def launcher(self, params: dict) -> Union[Native, Mpi]: + """How to start the command: directly or under mpiexec.""" diff --git a/phyml_core/engines/phyml.py b/phyml_core/engines/phyml.py new file mode 100644 index 0000000..a419395 --- /dev/null +++ b/phyml_core/engines/phyml.py @@ -0,0 +1,144 @@ +"""The PhyML engine: argv, validation, outputs and the stats parser (no Qt).""" +import os +import re +import shutil +from typing import Dict, List, Tuple, Union + +from ..launchers import Mpi, Native +from ..paths import AppPaths +from ..utils import is_mpi_binary +from .base import Engine, EngineOutputs, Issue, parse_decimal, positive_int, positive_number + +NT_MODELS = ("HKY85", "JC69", "K80", "F81", "F84", "TN93", "GTR", "custom") +AA_MODELS = ("LG", "WAG", "JTT", "MtREV", "Dayhoff", "DCMut", "RtREV", "CPREV", "VT", "Blosum62", + "MtMam", "MtArt", "HIVw", "HIVb", "custom") + +EXECUTABLE_NAMES = ('phyml-mpi.exe', 'phyml.exe') + + +def parse_phyml_stats(stats_path: str) -> dict: + """Parse key values from a PhyML 3.3 stats file into a dict.""" + try: + with open(stats_path, encoding='utf-8', errors='replace') as f: + content = f.read() + except OSError: + return {} + def grab(pattern): + m = re.search(pattern, content) + return m.group(1) if m else "" + return { + 'model': grab(r'Model of \w+ substitution:\s*(\S+)'), + 'taxa': grab(r'Number of taxa:\s*(\d+)'), + 'lnL': grab(r'Log-likelihood:\s*(-?[\d.]+)'), + 'gamma': grab(r'Gamma shape parameter:\s*(-?[\d.]+)'), + 'tstv': grab(r'transition/transversion ratio:\s*(-?[\d.]+)'), + 'time': grab(r'Time used:\s*(\S+)'), + } + + +def _text(params: dict, key: str, default: str = '') -> str: + """A parameter as trimmed text; a missing or None value gives the default.""" + value = params.get(key) + return default if value is None else str(value).strip() + + +def _estimate_or_positive(value: str) -> bool: + return value == 'e' or positive_number(value) + + +def _estimate_or_fraction(value: str) -> bool: + """PhyML -v: 'e' (estimate) or a proportion of invariable sites in [0, 1).""" + if value == 'e': + return True + number = parse_decimal(value) + return number is not None and number < 1 + + +def _mpiexec_found(mpiexec: str) -> bool: + return bool(mpiexec) and (os.path.isfile(mpiexec) or shutil.which(mpiexec) is not None) + + +class PhyML(Engine): + key = 'phyml' + name = 'PhyML' + label_key = 'engine_phyml' + + def models(self, datatype: str) -> List[str]: + return list(NT_MODELS if datatype == 'nt' else AA_MODELS) + + def validate(self, params: dict, paths: AppPaths) -> List[Issue]: + issues: List[Issue] = [] + use_mpi = bool(params.get('use_mpi', False)) + bootstrap = _text(params, 'support_method', 'bootstrap') == 'bootstrap' + + exe = self.executable(params, paths) + if not exe or not os.path.isfile(exe): + issues.append(Issue('error', 'val_no_exe', (self.name, exe))) + elif is_mpi_binary(exe) and not use_mpi and bootstrap: + issues.append(Issue('error', 'mpi_boot_msg')) + + if use_mpi: + mpiexec = _text(params, 'mpirun_path', 'mpiexec') + if not _mpiexec_found(mpiexec): + issues.append(Issue('error', 'val_no_mpiexec', (mpiexec,))) + + def check(name: str, value: str, ok: bool) -> None: + if not ok: + issues.append(Issue('error', 'val_not_number', (name, value))) + + if bootstrap: + value = _text(params, 'bootstrap_replicates', '100') + check('bootstrap_replicates', value, positive_int(value)) + # empty tstv / pinv / classes / gamma are legal: they are left out of the argv + value = _text(params, 'tstv') + check('tstv', value, not value or _estimate_or_positive(value)) + value = _text(params, 'pinv') + check('pinv', value, not value or _estimate_or_fraction(value)) + value = _text(params, 'classes') + check('classes', value, not value or positive_int(value)) + value = _text(params, 'gamma') + check('gamma', value, not value or _estimate_or_positive(value)) + if use_mpi: + value = _text(params, 'mpi_np', '2') + check('mpi_np', value, positive_int(value)) + return issues + + def normalize(self, params: dict) -> Tuple[dict, List[Issue]]: + return dict(params), [] + + def executable(self, params: dict, paths: AppPaths) -> str: + return paths.find_tool(params.get('phyml_path', ''), EXECUTABLE_NAMES) + + def command(self, exe: str, phylip: str, params: dict) -> List[str]: + argv = [exe, '-i', phylip, + '-d', _text(params, 'datatype', 'nt'), + '-m', _text(params, 'model', 'HKY85')] + support = _text(params, 'support_method', 'bootstrap') + argv += ['-b', _text(params, 'bootstrap_replicates', '100') if support == 'bootstrap' else support] + freq = _text(params, 'freq') + if freq and freq != 'default': + argv += ['-f', freq] + for flag, key in (('-t', 'tstv'), ('-v', 'pinv'), ('-c', 'classes'), ('-a', 'gamma')): + value = _text(params, key) + if value: + argv += [flag, value] + argv += ['-s', _text(params, 'search', 'NNI'), '-o', _text(params, 'optimize', 'tlr')] + if params.get('parsimony'): + argv.append('-p') + return argv + + def outputs(self, phylip: str) -> EngineOutputs: + return EngineOutputs(tree=phylip + '_phyml_tree.txt', + stats=phylip + '_phyml_stats.txt', + boot_trees=phylip + '_phyml_boot_trees.txt', + log=None) + + def parse_stats(self, outputs: EngineOutputs) -> Dict[str, str]: + return parse_phyml_stats(outputs.stats) + + def launcher(self, params: dict) -> Union[Native, Mpi]: + """Call validate() first: a bad mpi_np raises ValueError here.""" + if not params.get('use_mpi', False): + return Native() + mpiexec = _text(params, 'mpirun_path', 'mpiexec') + return Mpi((mpiexec and shutil.which(mpiexec)) or mpiexec, int(_text(params, 'mpi_np', '2'))) diff --git a/phyml_core/launchers.py b/phyml_core/launchers.py new file mode 100644 index 0000000..e48ad3e --- /dev/null +++ b/phyml_core/launchers.py @@ -0,0 +1,21 @@ +"""How an engine command is started: directly, or under mpiexec (no Qt).""" +from dataclasses import dataclass +from typing import List + + +@dataclass(frozen=True) +class Native: + """Run the command as it is.""" + + def wrap(self, argv: List[str]) -> List[str]: + return list(argv) + + +@dataclass(frozen=True) +class Mpi: + """Run the command under `mpiexec -np `.""" + mpiexec: str + np: int + + def wrap(self, argv: List[str]) -> List[str]: + return [self.mpiexec, '-np', str(self.np), *argv] diff --git a/phyml_core/pipeline.py b/phyml_core/pipeline.py index 1f94066..7e75852 100644 --- a/phyml_core/pipeline.py +++ b/phyml_core/pipeline.py @@ -6,29 +6,10 @@ from Bio import AlignIO, SeqIO from Bio.Seq import Seq +from .engines.phyml import parse_phyml_stats from .i18n import _t -def parse_stats(stats_path): - """Parse key values from a PhyML 3.3 stats file into a dict.""" - try: - with open(stats_path, encoding='utf-8', errors='replace') as f: - content = f.read() - except OSError: - return {} - def grab(pattern): - m = re.search(pattern, content) - return m.group(1) if m else "" - return { - 'model': grab(r'Model of \w+ substitution:\s*(\S+)'), - 'taxa': grab(r'Number of taxa:\s*(\d+)'), - 'lnL': grab(r'Log-likelihood:\s*(-?[\d.]+)'), - 'gamma': grab(r'Gamma shape parameter:\s*(-?[\d.]+)'), - 'tstv': grab(r'transition/transversion ratio:\s*(-?[\d.]+)'), - 'time': grab(r'Time used:\s*(\S+)'), - } - - def parse_iqtree_stats(stats_path): """Parse key values from an IQ-TREE .iqtree report into a dict.""" try: @@ -241,7 +222,7 @@ def _run_phyml(self, phylip_path, creation_flags): else: self.log_signal.emit(_t("success_pipe")) tree_path = f"{phylip_path}_phyml_tree.txt" - stats = parse_stats(tree_path.replace('_phyml_tree.txt', '_phyml_stats.txt')) + stats = parse_phyml_stats(tree_path.replace('_phyml_tree.txt', '_phyml_stats.txt')) self.log_signal.emit(_t("stats_summary").format( *(stats.get(k) or "n/a" for k in ('model', 'taxa', 'lnL', 'gamma', 'tstv', 'time')))) return True, tree_path, stats diff --git a/tests/unit/test_engine_phyml.py b/tests/unit/test_engine_phyml.py new file mode 100644 index 0000000..523348b --- /dev/null +++ b/tests/unit/test_engine_phyml.py @@ -0,0 +1,380 @@ +import json +import os +import shutil + +import pytest +from conftest import DATA, ROOT + +from phyml_core import settings +from phyml_core.engines import ENGINES +from phyml_core.engines.base import (Engine, EngineOutputs, Issue, format_issue, + positive_int, positive_number) +from phyml_core.engines.phyml import PhyML, parse_phyml_stats +from phyml_core.launchers import Mpi, Native +from phyml_core.paths import AppPaths + +P = {'datatype': 'nt', 'model': 'HKY85', 'support_method': 'bootstrap', 'bootstrap_replicates': '100', + 'freq': 'default', 'tstv': 'e', 'pinv': '0.0', 'classes': '4', 'gamma': 'e', + 'search': 'NNI', 'optimize': 'tlr', 'parsimony': False} + +NT_MODELS = ["HKY85", "JC69", "K80", "F81", "F84", "TN93", "GTR", "custom"] +AA_MODELS = ["LG", "WAG", "JTT", "MtREV", "Dayhoff", "DCMut", "RtREV", "CPREV", "VT", "Blosum62", + "MtMam", "MtArt", "HIVw", "HIVb", "custom"] + + +@pytest.fixture +def paths(tmp_path): + return AppPaths(app_dir=str(tmp_path / 'app'), data_dir=str(tmp_path / 'data')) + + +@pytest.fixture +def exe(tmp_path): + """An existing PhyML-looking file outside bin/.""" + path = tmp_path / 'phyml.exe' + path.write_bytes(b'') + return str(path) + + +def keys(issues): + return [i.key for i in issues] + + +# --- base helpers ------------------------------------------------------------------------- + +def test_engine_is_abstract(): + with pytest.raises(TypeError): + Engine() + + +def test_phyml_is_registered_with_its_identity(): + assert isinstance(ENGINES['phyml'], PhyML) + engine = PhyML() + assert (engine.key, engine.name, engine.label_key) == ('phyml', 'PhyML', 'engine_phyml') + + +def test_issue_and_outputs_are_frozen_values(): + assert Issue('error', 'k') == Issue('error', 'k', ()) + assert Issue('warning', 'k', ('a',)).args == ('a',) + assert EngineOutputs('t', 's', None, None).boot_trees is None + with pytest.raises(Exception): + Issue('error', 'k').key = 'other' + + +def test_format_issue_fills_the_localized_template(): + from phyml_core.i18n import _t + issue = Issue('error', 'val_not_number', ('classes', '0')) + assert format_issue(issue) == _t('val_not_number').format('classes', '0') + assert 'classes' in format_issue(issue) and '0' in format_issue(issue) + + +@pytest.mark.parametrize('key', ['val_not_number', 'val_no_exe', 'val_no_mpiexec', 'mpi_boot_msg']) +def test_validation_messages_exist_in_both_languages(key): + loc = json.loads((ROOT / 'locales.json').read_text(encoding='utf-8')) + assert loc['en'][key] and loc['ru'][key] + assert loc['en'][key].count('{}') == loc['ru'][key].count('{}') + + +@pytest.mark.parametrize('value', ['1', '100', '0.5', '2.', '.5', '0.001', ' 3 ']) +def test_positive_number_accepts_plain_positive_decimals(value): + assert positive_number(value) + + +@pytest.mark.parametrize('value', ['', '0', '0.0', '-1', 'abc', 'e', '1e3', 'nan', 'inf', '1_0', '1,5', '\u0661\u0662']) +def test_positive_number_rejects_everything_else(value): + assert not positive_number(value) + + +@pytest.mark.parametrize('value, ok', [('1', True), ('100', True), (' 4 ', True), ('0', False), ('-2', False), + ('2.5', False), ('', False), ('x', False)]) +def test_positive_int(value, ok): + assert positive_int(value) is ok + + +# --- command ------------------------------------------------------------------------------ + +def test_command_matches_current_argv(): + assert PhyML().command('C:/b/phyml.exe', 'D:/d/a.phy', P) == [ + 'C:/b/phyml.exe', '-i', 'D:/d/a.phy', '-d', 'nt', '-m', 'HKY85', '-b', '100', + '-t', 'e', '-v', '0.0', '-c', '4', '-a', 'e', '-s', 'NNI', '-o', 'tlr'] + + +def test_command_alrt_freq_parsimony_and_empty_values(): + argv = PhyML().command('x', 'a.phy', dict(P, support_method='-4', freq='m', parsimony=True, tstv='')) + assert argv[argv.index('-b') + 1] == '-4' and argv[argv.index('-f') + 1] == 'm' + assert '-t' not in argv and argv[-1] == '-p' + + +def test_command_omits_freq_when_missing(): + params = {k: v for k, v in P.items() if k != 'freq'} + argv = PhyML().command('x', 'a.phy', params) + assert '-f' not in argv and None not in argv + + +def test_command_omits_every_empty_optional_value(): + argv = PhyML().command('x', 'a.phy', dict(P, tstv='', pinv='', classes='', gamma='', freq='')) + assert argv == ['x', '-i', 'a.phy', '-d', 'nt', '-m', 'HKY85', '-b', '100', '-s', 'NNI', '-o', 'tlr'] + + +def test_command_freq_default_is_not_passed(): + assert '-f' not in PhyML().command('x', 'a.phy', dict(P, freq='default')) + + +def test_command_uses_the_documented_defaults_for_a_bare_dict(): + assert PhyML().command('x', 'a.phy', {}) == [ + 'x', '-i', 'a.phy', '-d', 'nt', '-m', 'HKY85', '-b', '100', '-s', 'NNI', '-o', 'tlr'] + + +def test_command_has_no_launcher_prefix_and_ignores_mpi_settings(): + argv = PhyML().command('x', 'a.phy', dict(P, use_mpi=True, mpirun_path='mpiexec', mpi_np='4')) + assert argv[0] == 'x' and 'mpiexec' not in argv and '-np' not in argv + + +# --- launcher / outputs / stats / models -------------------------------------------------- + +def test_launcher_mpi_and_native(): + launcher = PhyML().launcher(dict(P, use_mpi=True, mpirun_path='C:/MPI/mpiexec.exe', mpi_np='2')) + assert launcher == Mpi('C:/MPI/mpiexec.exe', 2) + assert launcher.wrap(['x']) == ['C:/MPI/mpiexec.exe', '-np', '2', 'x'] + assert PhyML().launcher(dict(P, use_mpi=False)).wrap(['x']) == ['x'] + assert PhyML().launcher(P) == Native() + + +def test_launcher_resolves_a_bare_mpiexec_through_path(monkeypatch): + monkeypatch.setattr(shutil, 'which', lambda name, *a, **k: 'C:/MS/mpiexec.exe' if name == 'mpiexec' else None) + launcher = PhyML().launcher(dict(P, use_mpi=True, mpirun_path='mpiexec', mpi_np='3')) + assert launcher == Mpi('C:/MS/mpiexec.exe', 3) + + +def test_launcher_keeps_the_stored_name_when_which_finds_nothing(monkeypatch): + monkeypatch.setattr(shutil, 'which', lambda *a, **k: None) + assert PhyML().launcher(dict(P, use_mpi=True, mpirun_path='mpiexec', mpi_np='2')) == Mpi('mpiexec', 2) + + +def test_outputs(): + out = PhyML().outputs('D:/d/a.phy') + assert out == EngineOutputs(tree='D:/d/a.phy_phyml_tree.txt', stats='D:/d/a.phy_phyml_stats.txt', + boot_trees='D:/d/a.phy_phyml_boot_trees.txt', log=None) + + +def test_parse_phyml_stats_reads_the_real_report(): + s = parse_phyml_stats(str(DATA / 'x_phyml_stats.txt')) + assert s['model'] == 'HKY85' and s['taxa'] == '8' and s['lnL'] == '-1811.15467' + + +def test_engine_parse_stats_reads_the_stats_output(tmp_path): + phylip = tmp_path / 'a.phy' + shutil.copy(DATA / 'x_phyml_stats.txt', str(phylip) + '_phyml_stats.txt') + stats = PhyML().parse_stats(PhyML().outputs(str(phylip))) + assert stats['model'] == 'HKY85' and stats['time'] + + +def test_parse_stats_of_a_missing_file_is_empty(tmp_path): + assert PhyML().parse_stats(PhyML().outputs(str(tmp_path / 'nope.phy'))) == {} + + +def test_models_unchanged(): + assert PhyML().models('nt') == NT_MODELS + assert PhyML().models('aa') == AA_MODELS + + +def test_models_returns_a_fresh_list(): + PhyML().models('nt').append('junk') + assert PhyML().models('nt') == NT_MODELS + + +def test_normalize_changes_nothing(): + params = dict(P) + normalized, issues = PhyML().normalize(params) + assert normalized == P and issues == [] + assert normalized is not params + + +# --- executable --------------------------------------------------------------------------- + +def test_executable_prefers_the_configured_file(paths, exe): + assert PhyML().executable({'phyml_path': exe}, paths) == exe + + +def test_executable_falls_back_to_bin(paths): + bin_exe = os.path.join(paths.bin_dir, 'phyml-mpi.exe') + os.makedirs(paths.bin_dir) + open(bin_exe, 'wb').close() + assert PhyML().executable({'phyml_path': ''}, paths) == bin_exe + assert PhyML().executable({}, paths) == bin_exe + + +def test_executable_prefers_the_mpi_build_in_bin(paths): + os.makedirs(paths.bin_dir) + for name in ('phyml.exe', 'phyml-mpi.exe'): + open(os.path.join(paths.bin_dir, name), 'wb').close() + assert PhyML().executable({}, paths) == os.path.join(paths.bin_dir, 'phyml-mpi.exe') + + +def test_executable_resolves_a_stored_relative_path(paths): + os.makedirs(os.path.join(paths.app_dir, 'tools')) + target = os.path.join(paths.app_dir, 'tools', 'phyml.exe') + open(target, 'wb').close() + assert PhyML().executable({'phyml_path': os.path.join('tools', 'phyml.exe')}, paths) == target + + +def test_executable_returns_the_configured_path_when_nothing_exists(paths, tmp_path): + missing = str(tmp_path / 'gone.exe') + assert PhyML().executable({'phyml_path': missing}, paths) == missing + + +# --- validate ----------------------------------------------------------------------------- + +def test_validate_ok(paths, exe): + assert PhyML().validate(dict(P, phyml_path=exe, use_mpi=False, support_method='-4'), paths) == [] + + +def test_validate_accepts_the_shipped_defaults(paths, exe): + """pinv '0.0' and the other defaults must validate.""" + assert PhyML().validate(dict(settings.DEFAULTS, phyml_path=exe), paths) == [] + + +def test_validate_missing_exe(paths, tmp_path): + missing = str(tmp_path / 'missing.exe') + assert PhyML().validate(dict(P, phyml_path=missing), paths) == [ + Issue('error', 'val_no_exe', ('PhyML', missing))] + + +def test_validate_a_directory_is_not_an_exe(paths, tmp_path): + assert keys(PhyML().validate(dict(P, phyml_path=str(tmp_path)), paths)) == ['val_no_exe'] + + +def test_validate_empty_path_without_bin_reports_the_missing_exe(paths): + issues = PhyML().validate(dict(P, phyml_path=''), paths) + assert keys(issues) == ['val_no_exe'] and issues[0].args[0] == 'PhyML' + + +def test_validate_uses_the_bin_fallback(paths): + os.makedirs(paths.bin_dir) + open(os.path.join(paths.bin_dir, 'phyml.exe'), 'wb').close() + assert PhyML().validate(dict(P, phyml_path=''), paths) == [] + + +def test_validate_serial_bootstrap_on_mpi_build(paths, tmp_path): + mpi_exe = tmp_path / 'phyml-mpi.exe' + mpi_exe.write_bytes(b'') + issues = PhyML().validate(dict(P, phyml_path=str(mpi_exe), use_mpi=False), paths) + assert issues == [Issue('error', 'mpi_boot_msg')] + + +def test_validate_mpi_build_is_fine_with_mpi_alrt_or_a_serial_build(paths, tmp_path, monkeypatch): + monkeypatch.setattr(shutil, 'which', lambda *a, **k: 'C:/MS/mpiexec.exe') + mpi_exe = tmp_path / 'phyml-mpi.exe' + mpi_exe.write_bytes(b'') + base = dict(P, phyml_path=str(mpi_exe), mpirun_path='mpiexec', mpi_np='2') + assert PhyML().validate(dict(base, use_mpi=True), paths) == [] + assert PhyML().validate(dict(base, use_mpi=False, support_method='-4'), paths) == [] + serial = tmp_path / 'phyml.exe' + serial.write_bytes(b'') + assert PhyML().validate(dict(base, phyml_path=str(serial), use_mpi=False), paths) == [] + + +def test_validate_serial_bootstrap_check_sees_the_bin_fallback(paths): + os.makedirs(paths.bin_dir) + open(os.path.join(paths.bin_dir, 'phyml-mpi.exe'), 'wb').close() + assert keys(PhyML().validate(dict(P, phyml_path='', use_mpi=False), paths)) == ['mpi_boot_msg'] + + +def test_validate_reports_only_the_missing_exe_not_a_guess_about_its_build(paths, tmp_path): + missing = str(tmp_path / 'phyml-mpi.exe') + assert keys(PhyML().validate(dict(P, phyml_path=missing, use_mpi=False), paths)) == ['val_no_exe'] + + +def test_validate_numbers(paths, exe): + issues = PhyML().validate(dict(P, phyml_path=exe, bootstrap_replicates='abc', classes='0', pinv='x'), paths) + assert keys(issues) == ['val_not_number'] * 3 + assert {i.args for i in issues} == {('bootstrap_replicates', 'abc'), ('classes', '0'), ('pinv', 'x')} + assert all(i.level == 'error' for i in issues) + # 'e' means "estimate" and is a legal value for pinv and gamma (and tstv) + assert PhyML().validate(dict(P, phyml_path=exe, pinv='e', gamma='e', tstv='e'), paths) == [] + + +@pytest.mark.parametrize('pinv, ok', [('0', True), ('0.0', True), ('0.35', True), ('.5', True), ('0.999', True), + ('e', True), ('', True), ('1', False), ('1.0', False), ('-0.1', False), + ('2', False), ('abc', False)]) +def test_validate_pinv_is_e_or_a_fraction_below_one(paths, exe, pinv, ok): + issues = PhyML().validate(dict(P, phyml_path=exe, pinv=pinv), paths) + assert (issues == []) is ok + if not ok: + assert issues == [Issue('error', 'val_not_number', ('pinv', pinv))] + + +@pytest.mark.parametrize('field', ['tstv', 'gamma']) +@pytest.mark.parametrize('value, ok', [('e', True), ('', True), ('4.0', True), ('2', True), ('0.5', True), + ('0', False), ('-1', False), ('x', False), ('1e3', False)]) +def test_validate_tstv_and_gamma_are_e_or_positive(paths, exe, field, value, ok): + issues = PhyML().validate(dict(P, phyml_path=exe, **{field: value}), paths) + assert (issues == []) is ok + if not ok: + assert issues == [Issue('error', 'val_not_number', (field, value))] + + +@pytest.mark.parametrize('value, ok', [('4', True), ('1', True), ('', True), ('0', False), ('2.5', False), + ('-3', False), ('e', False), ('x', False)]) +def test_validate_classes_are_a_positive_integer_or_empty(paths, exe, value, ok): + issues = PhyML().validate(dict(P, phyml_path=exe, classes=value), paths) + assert (issues == []) is ok + + +@pytest.mark.parametrize('value, ok', [('100', True), ('1', True), ('', False), ('0', False), ('10.5', False), + ('abc', False)]) +def test_validate_bootstrap_replicates_when_bootstrap(paths, exe, value, ok): + issues = PhyML().validate(dict(P, phyml_path=exe, bootstrap_replicates=value), paths) + assert (issues == []) is ok + + +def test_validate_ignores_bootstrap_replicates_for_alrt(paths, exe): + assert PhyML().validate(dict(P, phyml_path=exe, support_method='-4', bootstrap_replicates='abc'), paths) == [] + + +def test_validate_ignores_mpi_np_when_mpi_is_off(paths, exe): + assert PhyML().validate(dict(P, phyml_path=exe, use_mpi=False, mpi_np=''), paths) == [] + + +@pytest.mark.parametrize('value, ok', [('2', True), ('16', True), ('', False), ('0', False), ('x', False), + ('1.5', False)]) +def test_validate_mpi_np_when_mpi_is_on(paths, exe, monkeypatch, value, ok): + monkeypatch.setattr(shutil, 'which', lambda *a, **k: 'C:/MS/mpiexec.exe') + issues = PhyML().validate(dict(P, phyml_path=exe, use_mpi=True, mpi_np=value), paths) + assert (issues == []) is ok + if not ok: + assert issues == [Issue('error', 'val_not_number', ('mpi_np', value))] + + +def test_validate_mpiexec_missing(paths, exe, tmp_path, monkeypatch): + monkeypatch.setattr(shutil, 'which', lambda *a, **k: None) + gone = str(tmp_path / 'MPI' / 'mpiexec.exe') + issues = PhyML().validate(dict(P, phyml_path=exe, use_mpi=True, mpirun_path=gone, mpi_np='2'), paths) + assert issues == [Issue('error', 'val_no_mpiexec', (gone,))] + bare = PhyML().validate(dict(P, phyml_path=exe, use_mpi=True, mpirun_path='mpiexec', mpi_np='2'), paths) + assert bare == [Issue('error', 'val_no_mpiexec', ('mpiexec',))] + + +def test_validate_mpiexec_present_as_a_file_or_a_command_name(paths, exe, tmp_path, monkeypatch): + monkeypatch.setattr(shutil, 'which', lambda *a, **k: None) + real = tmp_path / 'mpiexec.exe' + real.write_bytes(b'') + params = dict(P, phyml_path=exe, use_mpi=True, mpi_np='2') + assert PhyML().validate(dict(params, mpirun_path=str(real)), paths) == [] + monkeypatch.setattr(shutil, 'which', lambda name, *a, **k: 'C:/MS/mpiexec.exe' if name == 'mpiexec' else None) + assert PhyML().validate(dict(params, mpirun_path='mpiexec'), paths) == [] + + +def test_validate_mpiexec_is_not_checked_when_mpi_is_off(paths, exe, monkeypatch): + monkeypatch.setattr(shutil, 'which', lambda *a, **k: None) + assert PhyML().validate(dict(P, phyml_path=exe, use_mpi=False, mpirun_path='nowhere'), paths) == [] + + +def test_validate_collects_all_problems_at_once(paths, tmp_path, monkeypatch): + monkeypatch.setattr(shutil, 'which', lambda *a, **k: None) + issues = PhyML().validate(dict(P, phyml_path=str(tmp_path / 'no.exe'), use_mpi=True, mpirun_path='nowhere', + mpi_np='x', classes='0'), paths) + assert sorted(keys(issues)) == ['val_no_exe', 'val_no_mpiexec', 'val_not_number', 'val_not_number'] + + +def test_validate_survives_none_values(paths, exe): + assert PhyML().validate(dict(P, phyml_path=exe, tstv=None, pinv=None, classes=None, gamma=None), paths) == [] diff --git a/tests/unit/test_launchers.py b/tests/unit/test_launchers.py new file mode 100644 index 0000000..11c8f0e --- /dev/null +++ b/tests/unit/test_launchers.py @@ -0,0 +1,33 @@ +import dataclasses + +import pytest + +from phyml_core.launchers import Mpi, Native + + +def test_native_returns_the_argv_unchanged(): + argv = ['phyml.exe', '-i', 'a.phy'] + assert Native().wrap(argv) == argv + + +def test_native_does_not_alias_the_input_list(): + argv = ['x'] + wrapped = Native().wrap(argv) + wrapped.append('y') + assert argv == ['x'] + + +def test_mpi_prefixes_mpiexec_np_and_the_argv(): + assert Mpi('C:/MPI/mpiexec.exe', 4).wrap(['phyml.exe', '-i', 'a.phy']) == [ + 'C:/MPI/mpiexec.exe', '-np', '4', 'phyml.exe', '-i', 'a.phy'] + + +def test_mpi_formats_np_as_a_string(): + assert Mpi('mpiexec', 2).wrap([])[2] == '2' + + +def test_launchers_compare_by_value_and_are_immutable(): + assert Native() == Native() + assert Mpi('m', 2) == Mpi('m', 2) and Mpi('m', 2) != Mpi('m', 3) + with pytest.raises(dataclasses.FrozenInstanceError): + Mpi('m', 2).np = 3 diff --git a/tests/unit/test_stats_parsers.py b/tests/unit/test_stats_parsers.py index 4a7702c..33a61c3 100644 --- a/tests/unit/test_stats_parsers.py +++ b/tests/unit/test_stats_parsers.py @@ -1,9 +1,10 @@ from conftest import DATA -from phyml_core.pipeline import parse_stats, parse_iqtree_stats +from phyml_core.engines.phyml import parse_phyml_stats +from phyml_core.pipeline import parse_iqtree_stats def test_parse_phyml_stats(): - s = parse_stats(str(DATA / 'x_phyml_stats.txt')) + s = parse_phyml_stats(str(DATA / 'x_phyml_stats.txt')) assert s['model'] == 'HKY85' and s['taxa'] == '8' and s['lnL'] == '-1811.15467' assert s['gamma'] == '881.743' and s['tstv'] == '0.980317' From 42fcd73e9923666942e695b1ab51e1cb85236687 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:22:50 +0300 Subject: [PATCH 23/36] Add the IQ-TREE engine and an engine registry; the model list comes from engines IqTree(command, validate, normalize, executable, outputs, launcher, models) with parse_iqtree_stats and ufboot_replicates moved from pipeline.py, which imports them. ENGINES now holds phyml and iqtree and the window asks it for the model list. format_issue shows UI labels for parameter names in val_not_number, parse_decimal accepts exponent notation, and param_text moves to engines/base.py. Co-Authored-By: Claude Sonnet 5.5 --- phyml_core/engines/__init__.py | 3 +- phyml_core/engines/base.py | 39 ++- phyml_core/engines/iqtree.py | 115 +++++++++ phyml_core/engines/phyml.py | 43 ++-- phyml_core/pipeline.py | 33 +-- phyml_ui/main_window.py | 13 +- tests/e2e/test_iqtree.py | 12 +- tests/gui/test_main_window.py | 21 ++ tests/unit/test_engine_iqtree.py | 323 ++++++++++++++++++++++++ tests/unit/test_engine_phyml.py | 49 +++- tests/unit/test_pipeline_regressions.py | 11 +- tests/unit/test_stats_parsers.py | 2 +- 12 files changed, 568 insertions(+), 96 deletions(-) create mode 100644 phyml_core/engines/iqtree.py create mode 100644 tests/unit/test_engine_iqtree.py diff --git a/phyml_core/engines/__init__.py b/phyml_core/engines/__init__.py index be1d7e4..99e6fe2 100644 --- a/phyml_core/engines/__init__.py +++ b/phyml_core/engines/__init__.py @@ -2,6 +2,7 @@ from typing import Dict from .base import Engine +from .iqtree import IqTree from .phyml import PhyML -ENGINES: Dict[str, Engine] = {'phyml': PhyML()} +ENGINES: Dict[str, Engine] = {'phyml': PhyML(), 'iqtree': IqTree()} diff --git a/phyml_core/engines/base.py b/phyml_core/engines/base.py index 7d575bc..d5db255 100644 --- a/phyml_core/engines/base.py +++ b/phyml_core/engines/base.py @@ -1,4 +1,5 @@ """The Engine interface every phylogeny program implements (no Qt).""" +import math import re from abc import ABC, abstractmethod from dataclasses import dataclass @@ -26,22 +27,48 @@ class EngineOutputs: log: Optional[str] +# settings key -> localization key of the label the user sees in the window +_PARAM_LABELS = { + 'bootstrap_replicates': 'lbl_boot', + 'mpi_np': 'lbl_mpi_np', + 'pinv': 'lbl_pinv', + 'classes': 'lbl_classes', + 'tstv': 'lbl_tstv', + 'gamma': 'lbl_gamma', +} + + def format_issue(issue: Issue) -> str: - return _t(issue.key).format(*issue.args) + """The localized message; a parameter named in val_not_number is shown by its UI label.""" + args = issue.args + if issue.key == 'val_not_number' and args and args[0] in _PARAM_LABELS: + label = _t(_PARAM_LABELS[args[0]]).rstrip().rstrip(':').rstrip() + args = (label,) + tuple(args[1:]) + return _t(issue.key).format(*args) + + +def param_text(params: dict, key: str, default: str = '') -> str: + """A parameter as trimmed text; a missing or None value gives the default.""" + value = params.get(key) + return default if value is None else str(value).strip() -_DECIMAL = re.compile(r'[0-9]+(?:\.[0-9]*)?|\.[0-9]+') +_DECIMAL = re.compile(r'(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?') _INTEGER = re.compile(r'[0-9]+') def parse_decimal(value) -> Optional[float]: - """A plain non-negative decimal ('3', '0.5', '.5', '2.') as a float, else None. + """A plain non-negative decimal ('3', '0.5', '.5', '2.', '1e-3') as a float, else None. - Unlike float() this rejects signs, exponents, 'nan'/'inf', digit separators and - non-ASCII digits, i.e. everything an engine's own argument parser may not accept. + Unlike float() this rejects signs, 'nan'/'inf' (and exponents that overflow to inf), + digit separators and non-ASCII digits, i.e. everything an engine's own argument parser + may not accept. """ text = str(value).strip() - return float(text) if _DECIMAL.fullmatch(text) else None + if not _DECIMAL.fullmatch(text): + return None + number = float(text) + return number if math.isfinite(number) else None def positive_number(value) -> bool: diff --git a/phyml_core/engines/iqtree.py b/phyml_core/engines/iqtree.py new file mode 100644 index 0000000..c800561 --- /dev/null +++ b/phyml_core/engines/iqtree.py @@ -0,0 +1,115 @@ +"""The IQ-TREE engine: argv, validation, outputs and the stats parser (no Qt).""" +import os +import re +from typing import Dict, List, Tuple, Union + +from ..launchers import Mpi, Native +from ..paths import AppPaths +from .base import Engine, EngineOutputs, Issue, param_text, positive_int + +NT_MODELS = ("MFP", "GTR+F+G4", "HKY+F+G4", "TIM2+F+G4", "TVM+F+G4") +AA_MODELS = ("MFP", "LG+G4", "WAG+G4", "JTT+G4", "mtREV+G4") + +EXECUTABLE_NAMES = ('iqtree2.exe',) + +UFBOOT_MIN_REPLICATES = 1000 + + +def parse_iqtree_stats(stats_path: str) -> dict: + """Parse key values from an IQ-TREE .iqtree report into a dict.""" + try: + with open(stats_path, encoding='utf-8', errors='replace') as f: + content = f.read() + except OSError: + return {} + def grab(pattern): + m = re.search(pattern, content) + return m.group(1) if m else "" + return { + 'model': grab(r'Best-fit model according to \w+:\s*(\S+)') or grab(r'Model of substitution:\s*(\S+)'), + 'taxa': grab(r'Input data:\s*(\d+) sequences'), + 'lnL': grab(r'Log-likelihood of the tree:\s*(-?[\d.]+)'), + 'gamma': grab(r'Gamma shape alpha:\s*([\d.]+)'), + 'tstv': '', + 'time': (grab(r'Total wall-clock time used:\s*([\d.]+)') + ' s').strip(), + } + + +def ufboot_replicates(value) -> str: + """IQ-TREE -B value: UFBoot refuses fewer than 1000 replicates.""" + try: + n = int(str(value).strip()) + except ValueError: + n = 0 + return str(max(n, UFBOOT_MIN_REPLICATES)) + + +def _prefix(phylip: str) -> str: + """IQ-TREE --prefix: every output file is this plus an extension.""" + return phylip.rsplit('.', 1)[0] + '_iqtree' + + +class IqTree(Engine): + key = 'iqtree' + name = 'IQ-TREE' + label_key = 'engine_iqtree' + + def models(self, datatype: str) -> List[str]: + return list(NT_MODELS if datatype == 'nt' else AA_MODELS) + + def validate(self, params: dict, paths: AppPaths) -> List[Issue]: + issues: List[Issue] = [] + exe = self.executable(params, paths) + if not exe or not os.path.isfile(exe): + issues.append(Issue('error', 'val_no_exe', (self.name, exe))) + + def check(name: str, value: str, ok: bool) -> None: + if not ok: + issues.append(Issue('error', 'val_not_number', (name, value))) + + if param_text(params, 'support_method', 'bootstrap') == 'bootstrap': + # below 1000 is valid input: normalize() raises it + value = param_text(params, 'bootstrap_replicates', '1000') + check('bootstrap_replicates', value, positive_int(value)) + # IQ-TREE has no MPI mode: mpi_np is its thread count (-T), whatever use_mpi says + value = param_text(params, 'mpi_np', '2') + check('mpi_np', value, positive_int(value)) + return issues + + def normalize(self, params: dict) -> Tuple[dict, List[Issue]]: + normalized, notes = dict(params), [] + if param_text(params, 'support_method', 'bootstrap') == 'bootstrap': + requested = param_text(params, 'bootstrap_replicates', '1000') + replicates = ufboot_replicates(requested) + if replicates != requested: + normalized['bootstrap_replicates'] = replicates + notes.append(Issue('warning', 'iqtree_ufboot_min', (replicates, requested))) + return normalized, notes + + def executable(self, params: dict, paths: AppPaths) -> str: + return paths.find_tool(params.get('iqtree_path', ''), EXECUTABLE_NAMES) + + def command(self, exe: str, phylip: str, params: dict) -> List[str]: + argv = [exe, '-s', phylip, '--prefix', _prefix(phylip), '-redo', + '-m', param_text(params, 'model', 'MFP'), + '-T', param_text(params, 'mpi_np', '2')] + support = param_text(params, 'support_method', 'bootstrap') + if support == 'bootstrap': + # normalize() reports a raised count; this keeps the argv valid even without it + argv += ['-B', ufboot_replicates(param_text(params, 'bootstrap_replicates', '1000'))] + elif support.startswith('-'): + argv += ['--alrt', '1000'] # aLRT branch support + return argv + + def outputs(self, phylip: str) -> EngineOutputs: + prefix = _prefix(phylip) + return EngineOutputs(tree=prefix + '.treefile', + stats=prefix + '.iqtree', + boot_trees=None, + log=prefix + '.log') + + def parse_stats(self, outputs: EngineOutputs) -> Dict[str, str]: + return parse_iqtree_stats(outputs.stats) + + def launcher(self, params: dict) -> Union[Native, Mpi]: + return Native() diff --git a/phyml_core/engines/phyml.py b/phyml_core/engines/phyml.py index a419395..4711a3a 100644 --- a/phyml_core/engines/phyml.py +++ b/phyml_core/engines/phyml.py @@ -7,7 +7,8 @@ from ..launchers import Mpi, Native from ..paths import AppPaths from ..utils import is_mpi_binary -from .base import Engine, EngineOutputs, Issue, parse_decimal, positive_int, positive_number +from .base import (Engine, EngineOutputs, Issue, param_text, parse_decimal, positive_int, + positive_number) NT_MODELS = ("HKY85", "JC69", "K80", "F81", "F84", "TN93", "GTR", "custom") AA_MODELS = ("LG", "WAG", "JTT", "MtREV", "Dayhoff", "DCMut", "RtREV", "CPREV", "VT", "Blosum62", @@ -36,12 +37,6 @@ def grab(pattern): } -def _text(params: dict, key: str, default: str = '') -> str: - """A parameter as trimmed text; a missing or None value gives the default.""" - value = params.get(key) - return default if value is None else str(value).strip() - - def _estimate_or_positive(value: str) -> bool: return value == 'e' or positive_number(value) @@ -69,7 +64,7 @@ def models(self, datatype: str) -> List[str]: def validate(self, params: dict, paths: AppPaths) -> List[Issue]: issues: List[Issue] = [] use_mpi = bool(params.get('use_mpi', False)) - bootstrap = _text(params, 'support_method', 'bootstrap') == 'bootstrap' + bootstrap = param_text(params, 'support_method', 'bootstrap') == 'bootstrap' exe = self.executable(params, paths) if not exe or not os.path.isfile(exe): @@ -78,7 +73,7 @@ def validate(self, params: dict, paths: AppPaths) -> List[Issue]: issues.append(Issue('error', 'mpi_boot_msg')) if use_mpi: - mpiexec = _text(params, 'mpirun_path', 'mpiexec') + mpiexec = param_text(params, 'mpirun_path', 'mpiexec') if not _mpiexec_found(mpiexec): issues.append(Issue('error', 'val_no_mpiexec', (mpiexec,))) @@ -87,19 +82,19 @@ def check(name: str, value: str, ok: bool) -> None: issues.append(Issue('error', 'val_not_number', (name, value))) if bootstrap: - value = _text(params, 'bootstrap_replicates', '100') + value = param_text(params, 'bootstrap_replicates', '100') check('bootstrap_replicates', value, positive_int(value)) # empty tstv / pinv / classes / gamma are legal: they are left out of the argv - value = _text(params, 'tstv') + value = param_text(params, 'tstv') check('tstv', value, not value or _estimate_or_positive(value)) - value = _text(params, 'pinv') + value = param_text(params, 'pinv') check('pinv', value, not value or _estimate_or_fraction(value)) - value = _text(params, 'classes') + value = param_text(params, 'classes') check('classes', value, not value or positive_int(value)) - value = _text(params, 'gamma') + value = param_text(params, 'gamma') check('gamma', value, not value or _estimate_or_positive(value)) if use_mpi: - value = _text(params, 'mpi_np', '2') + value = param_text(params, 'mpi_np', '2') check('mpi_np', value, positive_int(value)) return issues @@ -111,18 +106,18 @@ def executable(self, params: dict, paths: AppPaths) -> str: def command(self, exe: str, phylip: str, params: dict) -> List[str]: argv = [exe, '-i', phylip, - '-d', _text(params, 'datatype', 'nt'), - '-m', _text(params, 'model', 'HKY85')] - support = _text(params, 'support_method', 'bootstrap') - argv += ['-b', _text(params, 'bootstrap_replicates', '100') if support == 'bootstrap' else support] - freq = _text(params, 'freq') + '-d', param_text(params, 'datatype', 'nt'), + '-m', param_text(params, 'model', 'HKY85')] + support = param_text(params, 'support_method', 'bootstrap') + argv += ['-b', param_text(params, 'bootstrap_replicates', '100') if support == 'bootstrap' else support] + freq = param_text(params, 'freq') if freq and freq != 'default': argv += ['-f', freq] for flag, key in (('-t', 'tstv'), ('-v', 'pinv'), ('-c', 'classes'), ('-a', 'gamma')): - value = _text(params, key) + value = param_text(params, key) if value: argv += [flag, value] - argv += ['-s', _text(params, 'search', 'NNI'), '-o', _text(params, 'optimize', 'tlr')] + argv += ['-s', param_text(params, 'search', 'NNI'), '-o', param_text(params, 'optimize', 'tlr')] if params.get('parsimony'): argv.append('-p') return argv @@ -140,5 +135,5 @@ def launcher(self, params: dict) -> Union[Native, Mpi]: """Call validate() first: a bad mpi_np raises ValueError here.""" if not params.get('use_mpi', False): return Native() - mpiexec = _text(params, 'mpirun_path', 'mpiexec') - return Mpi((mpiexec and shutil.which(mpiexec)) or mpiexec, int(_text(params, 'mpi_np', '2'))) + mpiexec = param_text(params, 'mpirun_path', 'mpiexec') + return Mpi((mpiexec and shutil.which(mpiexec)) or mpiexec, int(param_text(params, 'mpi_np', '2'))) diff --git a/phyml_core/pipeline.py b/phyml_core/pipeline.py index 7e75852..e89ca1d 100644 --- a/phyml_core/pipeline.py +++ b/phyml_core/pipeline.py @@ -6,42 +6,11 @@ from Bio import AlignIO, SeqIO from Bio.Seq import Seq +from .engines.iqtree import parse_iqtree_stats, ufboot_replicates from .engines.phyml import parse_phyml_stats from .i18n import _t -def parse_iqtree_stats(stats_path): - """Parse key values from an IQ-TREE .iqtree report into a dict.""" - try: - with open(stats_path, encoding='utf-8', errors='replace') as f: - content = f.read() - except OSError: - return {} - def grab(pattern): - m = re.search(pattern, content) - return m.group(1) if m else "" - return { - 'model': grab(r'Best-fit model according to \w+:\s*(\S+)') or grab(r'Model of substitution:\s*(\S+)'), - 'taxa': grab(r'Input data:\s*(\d+) sequences'), - 'lnL': grab(r'Log-likelihood of the tree:\s*(-?[\d.]+)'), - 'gamma': grab(r'Gamma shape alpha:\s*([\d.]+)'), - 'tstv': '', - 'time': (grab(r'Total wall-clock time used:\s*([\d.]+)') + ' s').strip(), - } - - -UFBOOT_MIN_REPLICATES = 1000 - - -def ufboot_replicates(value): - """IQ-TREE -B value: UFBoot refuses fewer than 1000 replicates.""" - try: - n = int(str(value).strip()) - except ValueError: - n = 0 - return str(max(n, UFBOOT_MIN_REPLICATES)) - - # --- Background Worker Thread --- class PipelineWorker(QThread): log_signal = Signal(str) diff --git a/phyml_ui/main_window.py b/phyml_ui/main_window.py index 0936a43..6b53849 100644 --- a/phyml_ui/main_window.py +++ b/phyml_ui/main_window.py @@ -18,6 +18,7 @@ from PySide6.QtGui import QIcon, QFont, QShortcut, QKeySequence from phyml_core import __version__, presets, settings +from phyml_core.engines import ENGINES from phyml_core.history import HistoryStore, best_tree, display_input, new_record from phyml_core.i18n import _t from phyml_core.paths import AppPaths @@ -543,14 +544,10 @@ def toggle_bootstrap_input(self): def update_models(self): self.cb_model.clear() - if self.engine_setting == 'iqtree': - models = ["MFP", "GTR+F+G4", "HKY+F+G4", "TIM2+F+G4", "TVM+F+G4"] \ - if self.cb_datatype.currentData() == "nt" else \ - ["MFP", "LG+G4", "WAG+G4", "JTT+G4", "mtREV+G4"] - else: - models = ["HKY85", "JC69", "K80", "F81", "F84", "TN93", "GTR", "custom"] if self.cb_datatype.currentData() == "nt" else \ - ["LG", "WAG", "JTT", "MtREV", "Dayhoff", "DCMut", "RtREV", "CPREV", "VT", "Blosum62", "MtMam", "MtArt", "HIVw", "HIVb", "custom"] - for m in models: self.cb_model.addItem(m, m) + # an unknown engine in a hand-edited config gets PhyML's list, as the pipeline runs PhyML for it + engine = ENGINES.get(self.engine_setting, ENGINES['phyml']) + for m in engine.models(self.cb_datatype.currentData()): + self.cb_model.addItem(m, m) if not self._is_loading: self.auto_save() def _update_fasta_info(self, path): diff --git a/tests/e2e/test_iqtree.py b/tests/e2e/test_iqtree.py index 1175d53..60ff953 100644 --- a/tests/e2e/test_iqtree.py +++ b/tests/e2e/test_iqtree.py @@ -9,6 +9,7 @@ from Bio import AlignIO, Phylo from datasets import synthetic # noqa: F401 (fixture) +from phyml_core.engines import ENGINES from phyml_core.pipeline import PipelineWorker pytestmark = pytest.mark.needs_iqtree @@ -96,11 +97,8 @@ def test_gui_validation_and_run_with_xlsx(window, warned, qtbot, binaries, run_d assert wb.active.max_row == 2 -def test_every_listed_model_accepted(window, binaries, run_dir): - """Every model offered in the IQ-TREE model list is accepted by IQ-TREE.""" - window.engine_setting = 'iqtree' - window.update_models() - +def test_every_listed_model_accepted(binaries, run_dir): + """Every model the IQ-TREE engine offers is accepted by IQ-TREE.""" nt_phy = run_dir / 'test_prealigned.phy' AlignIO.convert(str(run_dir / 'test_prealigned.fasta'), 'fasta', str(nt_phy), 'phylip-relaxed') @@ -116,9 +114,7 @@ def test_every_listed_model_accepted(window, binaries, run_dir): rejected = [] for dt, phy in {'nt': nt_phy, 'aa': aa_phy}.items(): - window.set_combo_by_data(window.cb_datatype, dt) - models = [window.cb_model.itemData(i) for i in range(window.cb_model.count())] - for m in models: + for m in ENGINES['iqtree'].models(dt): if m == 'MFP': continue # ModelFinder is exercised by the runs above r = subprocess.run([binaries['iqtree'], '-s', str(phy), '-m', m, '-n', '0', diff --git a/tests/gui/test_main_window.py b/tests/gui/test_main_window.py index 524e3c6..781db5e 100644 --- a/tests/gui/test_main_window.py +++ b/tests/gui/test_main_window.py @@ -123,3 +123,24 @@ def test_empty_folder_blocked(window, warned, tmp_path): assert warned, 'empty folder must trigger a warning' assert warned[0][2] == _t('batch_empty'), 'blocked for the wrong reason' assert window.worker is None or not window.worker.isRunning() + + +@pytest.mark.parametrize('engine', ['phyml', 'iqtree']) +@pytest.mark.parametrize('datatype', ['nt', 'aa']) +def test_model_combo_lists_the_models_of_the_engine(window, engine, datatype): + """The model list comes from the engine registry (data and text both).""" + from phyml_core.engines import ENGINES + window.engine_setting = engine + window.set_combo_by_data(window.cb_datatype, datatype) + window.update_models() + shown = [(window.cb_model.itemText(i), window.cb_model.itemData(i)) for i in range(window.cb_model.count())] + expected = ENGINES[engine].models(datatype) + assert shown == [(m, m) for m in expected] + assert window.cb_model.currentData() == expected[0] + + +def test_unknown_engine_in_the_settings_shows_the_phyml_models(window): + from phyml_core.engines import ENGINES + window.engine_setting = 'no-such-engine' + window.update_models() + assert [window.cb_model.itemData(i) for i in range(window.cb_model.count())] == ENGINES['phyml'].models('nt') diff --git a/tests/unit/test_engine_iqtree.py b/tests/unit/test_engine_iqtree.py new file mode 100644 index 0000000..c9710f1 --- /dev/null +++ b/tests/unit/test_engine_iqtree.py @@ -0,0 +1,323 @@ +import json +import os +import shutil + +import pytest +from conftest import DATA, ROOT + +from phyml_core.engines import ENGINES +from phyml_core.engines.base import Engine, EngineOutputs, Issue, format_issue +from phyml_core.engines.iqtree import (UFBOOT_MIN_REPLICATES, IqTree, parse_iqtree_stats, + ufboot_replicates) +from phyml_core.engines.phyml import PhyML +from phyml_core.launchers import Native +from phyml_core.paths import AppPaths + +Q = {'model': 'MFP', 'mpi_np': '2', 'support_method': 'bootstrap', 'bootstrap_replicates': '1000'} + +NT_MODELS = ["MFP", "GTR+F+G4", "HKY+F+G4", "TIM2+F+G4", "TVM+F+G4"] +AA_MODELS = ["MFP", "LG+G4", "WAG+G4", "JTT+G4", "mtREV+G4"] + + +@pytest.fixture +def paths(tmp_path): + return AppPaths(app_dir=str(tmp_path / 'app'), data_dir=str(tmp_path / 'data')) + + +@pytest.fixture +def exe(tmp_path): + """An existing IQ-TREE-looking file outside bin/.""" + path = tmp_path / 'iqtree2.exe' + path.write_bytes(b'') + return str(path) + + +def keys(issues): + return [i.key for i in issues] + + +# --- identity / registry ------------------------------------------------------------------ + +def test_identity(): + engine = IqTree() + assert isinstance(engine, Engine) + assert (engine.key, engine.name, engine.label_key) == ('iqtree', 'IQ-TREE', 'engine_iqtree') + + +def test_registry_keys(): + assert list(ENGINES) == ['phyml', 'iqtree'] + assert isinstance(ENGINES['phyml'], PhyML) and isinstance(ENGINES['iqtree'], IqTree) + assert all(engine.key == key for key, engine in ENGINES.items()) + + +@pytest.mark.parametrize('key', ['engine_iqtree', 'iqtree_ufboot_min']) +def test_texts_exist_in_both_languages(key): + loc = json.loads((ROOT / 'locales.json').read_text(encoding='utf-8')) + assert loc['en'][key] and loc['ru'][key] + assert loc['en'][key].count('{}') == loc['ru'][key].count('{}') + + +def test_ufboot_warning_text_takes_the_new_and_the_requested_count(): + assert '1000' in format_issue(Issue('warning', 'iqtree_ufboot_min', ('1000', '100'))) + + +# --- command ------------------------------------------------------------------------------ + +def test_command_ufboot(): + assert IqTree().command('C:/b/iqtree2.exe', 'D:/d/a.phy', Q) == [ + 'C:/b/iqtree2.exe', '-s', 'D:/d/a.phy', '--prefix', 'D:/d/a_iqtree', '-redo', + '-m', 'MFP', '-T', '2', '-B', '1000'] + + +def test_command_alrt_and_none(): + alrt = IqTree().command('x', 'D:/d/a.phy', dict(Q, support_method='-4')) + assert alrt == ['x', '-s', 'D:/d/a.phy', '--prefix', 'D:/d/a_iqtree', '-redo', + '-m', 'MFP', '-T', '2', '--alrt', '1000'] + none = IqTree().command('x', 'D:/d/a.phy', dict(Q, support_method='0')) + assert none == ['x', '-s', 'D:/d/a.phy', '--prefix', 'D:/d/a_iqtree', '-redo', '-m', 'MFP', '-T', '2'] + assert none[-2:] == ['-T', '2'] + + +@pytest.mark.parametrize('method', ['-1', '-2', '-3', '-4', '-5']) +def test_command_any_negative_support_method_is_alrt(method): + assert IqTree().command('x', 'a.phy', dict(Q, support_method=method))[-2:] == ['--alrt', '1000'] + + +def test_command_uses_the_documented_defaults_for_a_bare_dict(): + assert IqTree().command('x', 'a.phy', {}) == [ + 'x', '-s', 'a.phy', '--prefix', 'a_iqtree', '-redo', '-m', 'MFP', '-T', '2', '-B', '1000'] + + +def test_command_never_emits_fewer_than_1000_ufboot_replicates(): + argv = IqTree().command('x', 'a.phy', dict(Q, bootstrap_replicates='100')) + assert argv[argv.index('-B') + 1] == '1000' + assert IqTree().command('x', 'a.phy', dict(Q, bootstrap_replicates='5000'))[-1] == '5000' + + +def test_command_passes_model_and_threads(): + argv = IqTree().command('x', 'a.phy', dict(Q, model='HKY+F+G4', mpi_np='8')) + assert argv[argv.index('-m') + 1] == 'HKY+F+G4' and argv[argv.index('-T') + 1] == '8' + + +def test_command_ignores_phyml_only_and_mpi_settings(): + params = dict(Q, use_mpi=True, mpirun_path='mpiexec', datatype='aa', freq='m', tstv='e', pinv='0.1', + classes='4', gamma='e', search='SPR', optimize='tlr', parsimony=True) + assert IqTree().command('x', 'a.phy', params) == IqTree().command('x', 'a.phy', Q) + + +def test_command_prefix_is_the_phylip_name_without_its_last_extension(): + argv = IqTree().command('x', 'D:/d/a.b.phy', Q) + assert argv[argv.index('--prefix') + 1] == 'D:/d/a.b_iqtree' + + +def test_command_survives_none_values(): + argv = IqTree().command('x', 'a.phy', {'model': None, 'mpi_np': None, 'support_method': None, + 'bootstrap_replicates': None}) + assert None not in argv and argv[argv.index('-m') + 1] == 'MFP' and argv[argv.index('-T') + 1] == '2' + + +# --- normalize ---------------------------------------------------------------------------- + +def test_normalize_raises_ufboot(): + p, notes = IqTree().normalize(dict(Q, bootstrap_replicates='100')) + assert p['bootstrap_replicates'] == '1000' and notes == [Issue('warning', 'iqtree_ufboot_min', ('1000', '100'))] + assert IqTree().normalize(dict(Q, bootstrap_replicates='5000')) == (dict(Q, bootstrap_replicates='5000'), []) + + +def test_normalize_keeps_1000_and_does_not_mutate_the_input(): + params = dict(Q, bootstrap_replicates='100') + normalized, _ = IqTree().normalize(params) + assert params['bootstrap_replicates'] == '100' and normalized is not params + assert IqTree().normalize(Q) == (Q, []) + + +def test_normalize_reports_the_value_the_user_typed(): + _, notes = IqTree().normalize(dict(Q, bootstrap_replicates=' 100 ')) + assert notes == [Issue('warning', 'iqtree_ufboot_min', ('1000', '100'))] + p, notes = IqTree().normalize(dict(Q, bootstrap_replicates='')) + assert p['bootstrap_replicates'] == '1000' and notes == [Issue('warning', 'iqtree_ufboot_min', ('1000', ''))] + + +def test_normalize_leaves_an_absent_replicate_count_absent(): + bare = {k: v for k, v in Q.items() if k != 'bootstrap_replicates'} + assert IqTree().normalize(bare) == (bare, []) + + +@pytest.mark.parametrize('method', ['-4', '0']) +def test_normalize_ignores_replicates_unless_bootstrap(method): + params = dict(Q, support_method=method, bootstrap_replicates='100') + assert IqTree().normalize(params) == (params, []) + + +def test_ufboot_replicates_raised_to_1000(): + """IQ-TREE UFBoot refuses < 1000 replicates: the PhyML-sized value is raised.""" + assert UFBOOT_MIN_REPLICATES == 1000 + assert ufboot_replicates('100') == '1000' + assert ufboot_replicates('1000') == '1000' + assert ufboot_replicates('5000') == '5000' + assert ufboot_replicates('') == '1000' + assert ufboot_replicates('abc') == '1000' + + +# --- executable --------------------------------------------------------------------------- + +def test_executable_prefers_the_configured_file(paths, exe): + assert IqTree().executable({'iqtree_path': exe}, paths) == exe + + +def test_executable_falls_back_to_bin(paths): + bin_exe = os.path.join(paths.bin_dir, 'iqtree2.exe') + os.makedirs(paths.bin_dir) + open(bin_exe, 'wb').close() + assert IqTree().executable({'iqtree_path': ''}, paths) == bin_exe + assert IqTree().executable({}, paths) == bin_exe + + +def test_executable_ignores_the_phyml_path(paths, exe): + assert IqTree().executable({'phyml_path': exe}, paths) != exe + + +def test_executable_returns_the_configured_path_when_nothing_exists(paths, tmp_path): + missing = str(tmp_path / 'gone.exe') + assert IqTree().executable({'iqtree_path': missing}, paths) == missing + + +# --- validate ----------------------------------------------------------------------------- + +def test_validate_ok(paths, exe): + assert IqTree().validate(dict(Q, iqtree_path=exe), paths) == [] + + +def test_validate(paths, tmp_path, exe): + missing = str(tmp_path / 'missing.exe') + assert IqTree().validate(dict(Q, iqtree_path=missing), paths) == [ + Issue('error', 'val_no_exe', ('IQ-TREE', missing))] + assert IqTree().validate(dict(Q, iqtree_path=exe, mpi_np='x'), paths) == [ + Issue('error', 'val_not_number', ('mpi_np', 'x'))] + assert IqTree().validate(dict(Q, iqtree_path=exe), paths) == [] + + +def test_validate_a_directory_is_not_an_exe(paths, tmp_path): + assert keys(IqTree().validate(dict(Q, iqtree_path=str(tmp_path)), paths)) == ['val_no_exe'] + + +def test_validate_empty_path_without_bin_reports_the_missing_exe(paths): + issues = IqTree().validate(dict(Q, iqtree_path=''), paths) + assert keys(issues) == ['val_no_exe'] and issues[0].args[0] == 'IQ-TREE' + + +def test_validate_uses_the_bin_fallback(paths): + os.makedirs(paths.bin_dir) + open(os.path.join(paths.bin_dir, 'iqtree2.exe'), 'wb').close() + assert IqTree().validate(dict(Q, iqtree_path=''), paths) == [] + + +@pytest.mark.parametrize('value, ok', [('1', True), ('2', True), ('16', True), (' 4 ', True), ('', False), + ('0', False), ('-2', False), ('1.5', False), ('x', False), ('AUTO', False)]) +def test_validate_threads_are_always_a_positive_integer(paths, exe, value, ok): + for use_mpi in (False, True): # IQ-TREE has no MPI mode: -T is checked either way + issues = IqTree().validate(dict(Q, iqtree_path=exe, mpi_np=value, use_mpi=use_mpi), paths) + assert (issues == []) is ok + if not ok: + assert issues == [Issue('error', 'val_not_number', ('mpi_np', value))] + + +def test_validate_an_absent_thread_count_means_the_default(paths, exe): + params = {k: v for k, v in Q.items() if k != 'mpi_np'} + assert IqTree().validate(dict(params, iqtree_path=exe), paths) == [] + + +@pytest.mark.parametrize('value, ok', [('1000', True), ('100', True), ('1', True), ('5000', True), ('', False), + ('0', False), ('10.5', False), ('abc', False), ('-5', False)]) +def test_validate_replicates_when_bootstrap(paths, exe, value, ok): + """Anything below 1000 is valid input: normalize() raises it.""" + issues = IqTree().validate(dict(Q, iqtree_path=exe, bootstrap_replicates=value), paths) + assert (issues == []) is ok + if not ok: + assert issues == [Issue('error', 'val_not_number', ('bootstrap_replicates', value))] + + +@pytest.mark.parametrize('method', ['-4', '0']) +def test_validate_ignores_replicates_unless_bootstrap(paths, exe, method): + assert IqTree().validate(dict(Q, iqtree_path=exe, support_method=method, bootstrap_replicates='abc'), paths) == [] + + +def test_validate_ignores_phyml_only_parameters(paths, exe): + params = dict(Q, iqtree_path=exe, tstv='x', pinv='9', classes='0', gamma='-1', use_mpi=True, + mpirun_path='nowhere') + assert IqTree().validate(params, paths) == [] + + +def test_validate_collects_all_problems_at_once(paths, tmp_path): + issues = IqTree().validate(dict(Q, iqtree_path=str(tmp_path / 'no.exe'), mpi_np='x', + bootstrap_replicates='y'), paths) + assert keys(issues) == ['val_no_exe', 'val_not_number', 'val_not_number'] + assert [i.args[0] for i in issues[1:]] == ['bootstrap_replicates', 'mpi_np'] + + +def test_validate_survives_none_values(paths, exe): + params = dict(Q, iqtree_path=exe, mpi_np=None, bootstrap_replicates=None, support_method=None) + assert IqTree().validate(params, paths) == [] + + +def test_validate_messages_name_the_ui_label_not_the_settings_key(paths, exe): + issues = IqTree().validate(dict(Q, iqtree_path=exe, mpi_np='x'), paths) + assert 'mpi_np' not in format_issue(issues[0]) + + +# --- outputs / stats / launcher / models -------------------------------------------------- + +def test_outputs(): + out = IqTree().outputs('D:/d/a.phy') + assert out == EngineOutputs(tree='D:/d/a_iqtree.treefile', stats='D:/d/a_iqtree.iqtree', + boot_trees=None, log='D:/d/a_iqtree.log') + assert out.boot_trees is None + + +def test_outputs_share_the_prefix_of_the_command(): + argv = IqTree().command('x', 'D:/d/a.phy', Q) + prefix = argv[argv.index('--prefix') + 1] + out = IqTree().outputs('D:/d/a.phy') + assert out.tree == prefix + '.treefile' and out.stats == prefix + '.iqtree' and out.log == prefix + '.log' + + +def test_parse_iqtree_stats_reads_the_real_report(): + s = parse_iqtree_stats(str(DATA / 'x.iqtree')) + assert s['model'] == 'JC' and s['taxa'] == '8' and s['lnL'] == '-1828.7117' + + +def test_engine_parse_stats_reads_the_stats_output(tmp_path): + phylip = tmp_path / 'a.phy' + shutil.copy(DATA / 'x.iqtree', tmp_path / 'a_iqtree.iqtree') + stats = IqTree().parse_stats(IqTree().outputs(str(phylip))) + assert stats['model'] == 'JC' and stats['lnL'] == '-1828.7117' and stats['time'] + + +def test_parse_stats_of_a_missing_file_is_empty(tmp_path): + assert IqTree().parse_stats(IqTree().outputs(str(tmp_path / 'nope.phy'))) == {} + assert parse_iqtree_stats(str(tmp_path / 'nope.iqtree')) == {} + + +def test_launcher_is_native(): + assert IqTree().launcher(Q) == Native() + assert IqTree().launcher(dict(Q, use_mpi=True, mpirun_path='mpiexec', mpi_np='4')) == Native() + assert IqTree().launcher({}) == Native() + + +def test_models_list_has_hky_not_hky2(): + assert 'HKY+F+G4' in IqTree().models('nt') and 'HKY2+F+G4' not in IqTree().models('nt') + + +def test_models_unchanged(): + assert IqTree().models('nt') == NT_MODELS + assert IqTree().models('aa') == AA_MODELS + + +def test_models_returns_a_fresh_list(): + IqTree().models('nt').append('junk') + assert IqTree().models('nt') == NT_MODELS + + +def test_both_engines_offer_a_model_list_per_datatype(): + for engine in ENGINES.values(): + assert engine.models('nt') and engine.models('aa') and engine.models('nt') != engine.models('aa') diff --git a/tests/unit/test_engine_phyml.py b/tests/unit/test_engine_phyml.py index 523348b..f6ec498 100644 --- a/tests/unit/test_engine_phyml.py +++ b/tests/unit/test_engine_phyml.py @@ -7,7 +7,7 @@ from phyml_core import settings from phyml_core.engines import ENGINES -from phyml_core.engines.base import (Engine, EngineOutputs, Issue, format_issue, +from phyml_core.engines.base import (Engine, EngineOutputs, Issue, format_issue, parse_decimal, positive_int, positive_number) from phyml_core.engines.phyml import PhyML, parse_phyml_stats from phyml_core.launchers import Mpi, Native @@ -62,9 +62,31 @@ def test_issue_and_outputs_are_frozen_values(): def test_format_issue_fills_the_localized_template(): from phyml_core.i18n import _t - issue = Issue('error', 'val_not_number', ('classes', '0')) - assert format_issue(issue) == _t('val_not_number').format('classes', '0') - assert 'classes' in format_issue(issue) and '0' in format_issue(issue) + issue = Issue('error', 'val_not_number', ('threshold', '0')) + assert format_issue(issue) == _t('val_not_number').format('threshold', '0') + assert 'threshold' in format_issue(issue) and '0' in format_issue(issue) + + +@pytest.mark.parametrize('lang', ['en', 'ru']) +@pytest.mark.parametrize('name, label_key', [ + ('bootstrap_replicates', 'lbl_boot'), ('mpi_np', 'lbl_mpi_np'), ('pinv', 'lbl_pinv'), + ('classes', 'lbl_classes'), ('tstv', 'lbl_tstv'), ('gamma', 'lbl_gamma')]) +def test_format_issue_uses_ui_labels_for_parameter_names(monkeypatch, lang, name, label_key): + from phyml_core import i18n + monkeypatch.setattr(i18n, 'LANG', lang) + label = i18n._t(label_key) + assert label.endswith(':') # the UI label carries a trailing colon... + shown = label[:-1].strip() # ...which the message drops + text = format_issue(Issue('error', 'val_not_number', (name, 'x'))) + assert text == i18n._t('val_not_number').format(shown, 'x') + assert shown in text and name not in text + + +def test_format_issue_labels_only_parameter_names_of_val_not_number(): + from phyml_core.i18n import _t + issue = Issue('error', 'val_no_exe', ('mpi_np', 'C:/x')) + assert format_issue(issue) == _t('val_no_exe').format('mpi_np', 'C:/x') + assert 'mpi_np_typo' in format_issue(Issue('error', 'val_not_number', ('mpi_np_typo', 'x'))) @pytest.mark.parametrize('key', ['val_not_number', 'val_no_exe', 'val_no_mpiexec', 'mpi_boot_msg']) @@ -79,11 +101,25 @@ def test_positive_number_accepts_plain_positive_decimals(value): assert positive_number(value) -@pytest.mark.parametrize('value', ['', '0', '0.0', '-1', 'abc', 'e', '1e3', 'nan', 'inf', '1_0', '1,5', '\u0661\u0662']) +@pytest.mark.parametrize('value', ['', '0', '0.0', '-1', 'abc', 'e', 'nan', 'inf', '1_0', '1,5', '\u0661\u0662', + '+1', '-1e3', '1e', 'e3', '1e+', '1e-', '1e3.5', '1e_3', '1 e3', 'infinity', + '1e999', '0e0', '1e-999', '1e\u0663', '\u0661e3']) def test_positive_number_rejects_everything_else(value): assert not positive_number(value) +@pytest.mark.parametrize('value, expected', [('1e-3', 0.001), ('2E2', 200.0), ('1.5e+1', 15.0), ('.5e1', 5.0), + ('2.e1', 20.0), (' 1e3 ', 1000.0), ('0.5', 0.5), ('3', 3.0)]) +def test_parse_decimal_accepts_exponent_notation(value, expected): + assert parse_decimal(value) == expected + assert positive_number(value) + + +@pytest.mark.parametrize('value', ['', '-1', '+1', 'nan', 'inf', '1e999', '1_0', '\u0661\u0662', 'e3', '1e']) +def test_parse_decimal_rejects_what_float_would_misread(value): + assert parse_decimal(value) is None + + @pytest.mark.parametrize('value, ok', [('1', True), ('100', True), (' 4 ', True), ('0', False), ('-2', False), ('2.5', False), ('', False), ('x', False)]) def test_positive_int(value, ok): @@ -305,7 +341,8 @@ def test_validate_pinv_is_e_or_a_fraction_below_one(paths, exe, pinv, ok): @pytest.mark.parametrize('field', ['tstv', 'gamma']) @pytest.mark.parametrize('value, ok', [('e', True), ('', True), ('4.0', True), ('2', True), ('0.5', True), - ('0', False), ('-1', False), ('x', False), ('1e3', False)]) + ('1e-3', True), ('2E2', True), + ('0', False), ('-1', False), ('x', False), ('1e', False), ('nan', False)]) def test_validate_tstv_and_gamma_are_e_or_positive(paths, exe, field, value, ok): issues = PhyML().validate(dict(P, phyml_path=exe, **{field: value}), paths) assert (issues == []) is ok diff --git a/tests/unit/test_pipeline_regressions.py b/tests/unit/test_pipeline_regressions.py index 1541942..c159da2 100644 --- a/tests/unit/test_pipeline_regressions.py +++ b/tests/unit/test_pipeline_regressions.py @@ -1,4 +1,4 @@ -from phyml_core.pipeline import PipelineWorker, ufboot_replicates +from phyml_core.pipeline import PipelineWorker def test_empty_fasta_fails_cleanly(qapp, tmp_path): @@ -14,12 +14,3 @@ def test_empty_fasta_fails_cleanly(qapp, tmp_path): raise AssertionError(f'worker crashed on an empty FASTA: {e!r}') assert done == [(False, '')], done assert worker.results[0]['status'] == 'FAIL', worker.results - - -def test_ufboot_replicates_raised_to_1000(): - """IQ-TREE UFBoot refuses < 1000 replicates: the PhyML-sized value is raised.""" - assert ufboot_replicates('100') == '1000' - assert ufboot_replicates('1000') == '1000' - assert ufboot_replicates('5000') == '5000' - assert ufboot_replicates('') == '1000' - assert ufboot_replicates('abc') == '1000' diff --git a/tests/unit/test_stats_parsers.py b/tests/unit/test_stats_parsers.py index 33a61c3..6790bb9 100644 --- a/tests/unit/test_stats_parsers.py +++ b/tests/unit/test_stats_parsers.py @@ -1,6 +1,6 @@ from conftest import DATA +from phyml_core.engines.iqtree import parse_iqtree_stats from phyml_core.engines.phyml import parse_phyml_stats -from phyml_core.pipeline import parse_iqtree_stats def test_parse_phyml_stats(): From 79cbba38cd9deef4c86d239172ca589b1e777e1f Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:31:01 +0300 Subject: [PATCH 24/36] Add alignment helpers with BOM-tolerant reading and ID collision detection phyml_core/alignment.py: read_fasta opens the file itself (utf-8-sig, CRLF ok, no leaked handle) and raises AlignmentError for empty or non-FASTA input; is_aligned, clean_id, find_collisions, write_phylip (cleaned IDs, '.' -> '-', phylip-relaxed, refuses colliding IDs with err_id_collision), summarize and the aligned/phylip path helpers. The window's FASTA summary label now uses summarize. pipeline.py is untouched until the new runner replaces it. Co-Authored-By: Claude Sonnet 5.5 --- locales.json | 2 + phyml_core/alignment.py | 100 +++++++++++++++++++++ phyml_ui/main_window.py | 17 +--- tests/gui/test_main_window.py | 16 ++++ tests/unit/test_alignment.py | 161 ++++++++++++++++++++++++++++++++++ 5 files changed, 283 insertions(+), 13 deletions(-) create mode 100644 phyml_core/alignment.py create mode 100644 tests/unit/test_alignment.py diff --git a/locales.json b/locales.json index b2e2bb8..8a86218 100644 --- a/locales.json +++ b/locales.json @@ -15,6 +15,7 @@ "cleaning": "[*] Cleaning characters and converting (FASTA -> PHYLIP)...", "conv_success": "[+] Conversion complete: {}", "err_conv": "[ERROR] Conversion failed: {}", + "err_id_collision": "[ERROR] Sequence IDs collide after cleaning: {}", "err_phyml": "[ERROR] Invalid path to PhyML.", "run_phyml": "[*] Running Command: {}", "cancel_analysis": "\n[-] Analysis cancelled by user.", @@ -209,6 +210,7 @@ "cleaning": "[*] Очистка символов и конвертация (FASTA -> PHYLIP)...", "conv_success": "[+] Конвертация завершена: {}", "err_conv": "[ОШИБКА] Сбой при конвертации: {}", + "err_id_collision": "[ОШИБКА] После очистки совпали ID последовательностей: {}", "err_phyml": "[ОШИБКА] Некорректный путь к PhyML.", "run_phyml": "[*] Выполняемая команда: {}", "cancel_analysis": "\n[-] Анализ прерван пользователем.", diff --git a/phyml_core/alignment.py b/phyml_core/alignment.py new file mode 100644 index 0000000..3467df7 --- /dev/null +++ b/phyml_core/alignment.py @@ -0,0 +1,100 @@ +"""FASTA reading, alignment checks and the FASTA -> PHYLIP conversion (no Qt).""" +import io +import re + +from Bio import AlignIO, SeqIO +from Bio.Align import MultipleSeqAlignment +from Bio.Seq import Seq +from Bio.SeqRecord import SeqRecord + +from .i18n import _t + + +class AlignmentError(ValueError): + """A problem with the input sequences; the message is already localized.""" + + +def read_fasta(path): + """Records of a FASTA file; an empty file, or one that is not FASTA, is an AlignmentError. + + The file is opened here (not by SeqIO) so the handle is always closed, an UTF-8 BOM is + skipped and CRLF line ends are accepted. + """ + try: + with open(path, encoding='utf-8-sig') as handle: + records = list(SeqIO.parse(handle, 'fasta')) + except UnicodeDecodeError: + raise + except ValueError as e: + # Biopython refuses text that comes before the first '>' + raise AlignmentError(_t('err_empty')) from e + if not records: + raise AlignmentError(_t('err_empty')) + return records + + +def is_aligned(records): + """True when all the sequences have the same length.""" + if not records: + raise AlignmentError(_t('err_empty')) + first_len = len(records[0].seq) + return all(len(rec.seq) == first_len for rec in records) + + +def clean_id(raw): + """A sequence ID safe for PHYLIP and Newick: the separators become underscores.""" + return re.sub(r'[.|:;(),\s]', '_', raw) + + +def find_collisions(ids): + """Cleaned ID -> the original IDs that give it, only where there is more than one. + + Exact duplicates count: ['X', 'X'] gives {'X': ['X', 'X']}. + """ + groups = {} + for raw in ids: + groups.setdefault(clean_id(raw), []).append(raw) + return {clean: originals for clean, originals in groups.items() if len(originals) > 1} + + +def write_phylip(records, out_path): + """Write the records as relaxed PHYLIP: cleaned IDs, '.' gaps turned into '-'. + + The records are not changed. Nothing is written when IDs collide after cleaning, as + PhyML and IQ-TREE would then merge or reject the taxa. The file is UTF-8, like the + FASTA reading. + """ + collisions = find_collisions([rec.id for rec in records]) + if collisions: + details = '; '.join(f"{', '.join(originals)} → {clean}" + for clean, originals in collisions.items()) + raise AlignmentError(_t('err_id_collision').format(details)) + cleaned = [] + for rec in records: + new_id = clean_id(rec.id) + cleaned.append(SeqRecord(Seq(str(rec.seq).replace('.', '-')), + id=new_id, name=new_id, description='')) + buffer = io.StringIO() + AlignIO.write(MultipleSeqAlignment(cleaned), buffer, 'phylip-relaxed') + with open(out_path, 'w', encoding='utf-8') as handle: + handle.write(buffer.getvalue()) + + +def summarize(path): + """Taxa count, shortest and longest sequence and the share of '-' as a percentage.""" + records = read_fasta(path) + lengths = [len(rec.seq) for rec in records] + total = sum(lengths) + gaps = sum(rec.seq.count('-') for rec in records) + return {'n': len(records), 'min_len': min(lengths), 'max_len': max(lengths), + 'gap_pct': round(gaps / total * 100, 1) if total else 0} + + +def aligned_path(fasta): + """Where MUSCLE puts the alignment of a FASTA file.""" + return fasta.rsplit('.', 1)[0] + '_aligned.fasta' + + +def phylip_path(fasta): + """Where the PHYLIP conversion of a FASTA file goes.""" + return fasta.rsplit('.', 1)[0] + '.phy' diff --git a/phyml_ui/main_window.py b/phyml_ui/main_window.py index 6b53849..cb5e379 100644 --- a/phyml_ui/main_window.py +++ b/phyml_ui/main_window.py @@ -17,7 +17,7 @@ from PySide6.QtCore import QTimer, Qt from PySide6.QtGui import QIcon, QFont, QShortcut, QKeySequence -from phyml_core import __version__, presets, settings +from phyml_core import __version__, alignment, presets, settings from phyml_core.engines import ENGINES from phyml_core.history import HistoryStore, best_tree, display_input, new_record from phyml_core.i18n import _t @@ -562,18 +562,9 @@ def _update_fasta_info(self, path): and not f.lower().endswith('_aligned.fasta')) self.fasta_info_label.setText(_t('fasta_info').format(n, '—', '—', '—')) return - from Bio import SeqIO - lengths = [len(r.seq) for r in SeqIO.parse(path, 'fasta')] - if not lengths: - self.fasta_info_label.setText(' ') - return - gaps = 0 - total = sum(lengths) - for r in SeqIO.parse(path, 'fasta'): - gaps += str(r.seq).count('-') - pct = round(gaps / total * 100, 1) if total else 0 - self.fasta_info_label.setText( - _t('fasta_info').format(len(lengths), min(lengths), max(lengths), pct)) + info = alignment.summarize(path) # an empty or broken file raises: no label + self.fasta_info_label.setText(_t('fasta_info').format( + info['n'], info['min_len'], info['max_len'], info['gap_pct'])) except Exception: self.fasta_info_label.setText(' ') diff --git a/tests/gui/test_main_window.py b/tests/gui/test_main_window.py index 781db5e..e5ed61b 100644 --- a/tests/gui/test_main_window.py +++ b/tests/gui/test_main_window.py @@ -144,3 +144,19 @@ def test_unknown_engine_in_the_settings_shows_the_phyml_models(window): window.engine_setting = 'no-such-engine' window.update_models() assert [window.cb_model.itemData(i) for i in range(window.cb_model.count())] == ENGINES['phyml'].models('nt') + + +def test_fasta_info_label_summarizes_the_alignment(window, tmp_path): + fasta = tmp_path / 'выравнивание 1.fasta' + fasta.write_text('>A\nACGT--\n>B\nAC\n>C\nACGTA\n', encoding='utf-8') + window.fasta_path_input.setText(str(fasta)) + assert window.fasta_info_label.text() == _t('fasta_info').format(3, 2, 6, 15.4) + + +def test_fasta_info_label_is_blank_for_an_empty_file_and_counts_a_folder(window, tmp_path): + empty = tmp_path / 'empty.fasta' + empty.write_text('') + window.fasta_path_input.setText(str(empty)) + assert window.fasta_info_label.text() == ' ' + window.fasta_path_input.setText(str(tmp_path)) + assert window.fasta_info_label.text() == _t('fasta_info').format(1, '—', '—', '—') diff --git a/tests/unit/test_alignment.py b/tests/unit/test_alignment.py new file mode 100644 index 0000000..4d07f3c --- /dev/null +++ b/tests/unit/test_alignment.py @@ -0,0 +1,161 @@ +import gc + +import pytest +from Bio import AlignIO +from Bio.Seq import Seq +from Bio.SeqRecord import SeqRecord + +from phyml_core import alignment +from phyml_core.alignment import AlignmentError +from phyml_core.i18n import _t + + +def rec(rid, seq): + return SeqRecord(Seq(seq), id=rid, name=rid, description='') + + +def write_fasta(path, records): + path.write_text(''.join(f'>{rid}\n{seq}\n' for rid, seq in records), encoding='utf-8') + return str(path) + + +def test_alignment_error_is_a_value_error(): + assert issubclass(AlignmentError, ValueError) + + +@pytest.mark.parametrize('content', ['', '\n\n', ' \n', 'just some text\n>A\nACGT\n']) +def test_read_fasta_empty_raises(tmp_path, content): + path = tmp_path / 'empty.fasta' + path.write_text(content) + with pytest.raises(AlignmentError) as exc: + alignment.read_fasta(str(path)) + assert str(exc.value) == _t('err_empty') + + +def test_read_fasta_does_not_hide_an_encoding_error(tmp_path): + path = tmp_path / 'cp1251.fasta' + path.write_bytes('>Волк\nACGT\n'.encode('cp1251')) + with pytest.raises(UnicodeDecodeError): + alignment.read_fasta(str(path)) + + +def test_read_fasta_closes_the_file_when_it_is_empty(tmp_path, recwarn): + """An empty file must not leave an open handle behind (a ResourceWarning at collection).""" + path = tmp_path / 'empty.fasta' + path.write_text('') + with pytest.raises(AlignmentError): + alignment.read_fasta(str(path)) + gc.collect() + leaked = [w for w in recwarn if issubclass(w.category, ResourceWarning)] + assert leaked == [] + + +def test_read_fasta_tolerates_bom_and_crlf(tmp_path): + path = tmp_path / 'выравнивание с пробелом.fasta' + path.write_bytes(b'\xef\xbb\xbf>A\r\nACGT\r\n>B\r\nACGA\r\n') + records = alignment.read_fasta(str(path)) + assert [r.id for r in records] == ['A', 'B'] + assert str(records[0].seq) == 'ACGT' + assert str(records[1].seq) == 'ACGA' + + +def test_is_aligned(): + assert alignment.is_aligned([rec('A', 'ACGT'), rec('B', 'AC-T')]) + assert alignment.is_aligned([rec('A', 'ACGT')]) + assert not alignment.is_aligned([rec('A', 'ACGT'), rec('B', 'ACG')]) + + +def test_is_aligned_rejects_an_empty_list(): + with pytest.raises(AlignmentError) as exc: + alignment.is_aligned([]) + assert str(exc.value) == _t('err_empty') + + +def test_clean_id(): + assert alignment.clean_id('Homo sapiens|x:1') == 'Homo_sapiens_x_1' + assert alignment.clean_id('a.b;c(d),e\tf') == 'a_b_c_d__e_f' + assert alignment.clean_id('plain-id_1') == 'plain-id_1' + + +def test_find_collisions(): + assert alignment.find_collisions(['A.1', 'A_1', 'B']) == {'A_1': ['A.1', 'A_1']} + + +def test_find_collisions_reports_exact_duplicates(): + assert alignment.find_collisions(['X', 'X']) == {'X': ['X', 'X']} + + +def test_find_collisions_empty_when_ids_are_distinct(): + assert alignment.find_collisions(['A.1', 'A.2', 'B']) == {} + assert alignment.find_collisions([]) == {} + + +def test_write_phylip_cleans_ids_and_dots(tmp_path): + out = tmp_path / 'вывод с пробелом.phy' + records = [rec('Homo sapiens|x:1', 'AC.GT'), rec('Pan.troglodytes', 'ACGG.')] + alignment.write_phylip(records, str(out)) + back = AlignIO.read(str(out), 'phylip-relaxed') + assert [r.id for r in back] == ['Homo_sapiens_x_1', 'Pan_troglodytes'] + assert [str(r.seq) for r in back] == ['AC-GT', 'ACGG-'] + # the caller's records are left as they were + assert records[0].id == 'Homo sapiens|x:1' + assert str(records[0].seq) == 'AC.GT' + + +def test_write_phylip_raises_on_collision(tmp_path): + out = tmp_path / 'clash.phy' + with pytest.raises(AlignmentError) as exc: + alignment.write_phylip([rec('A.1', 'ACGT'), rec('A_1', 'ACGA'), rec('B', 'ACGG')], str(out)) + text = str(exc.value) + assert 'A.1' in text and 'A_1' in text + assert text == _t('err_id_collision').format('A.1, A_1 → A_1') + assert not out.exists() + + +def test_write_phylip_raises_on_exact_duplicates(tmp_path): + out = tmp_path / 'dup.phy' + with pytest.raises(AlignmentError): + alignment.write_phylip([rec('X', 'ACGT'), rec('X', 'ACGA')], str(out)) + assert not out.exists() + + +def test_write_phylip_writes_non_ascii_ids_as_utf8(tmp_path): + """read_fasta decodes UTF-8, so the PHYLIP file is UTF-8 too, whatever the locale is.""" + out = tmp_path / 'ru.phy' + alignment.write_phylip([rec('Волк обыкновенный', 'ACGT'), rec('Лиса', 'ACGA')], str(out)) + with open(out, encoding='utf-8') as fh: + back = AlignIO.read(fh, 'phylip-relaxed') + assert [r.id for r in back] == ['Волк_обыкновенный', 'Лиса'] + + +def test_summarize(tmp_path): + path = tmp_path / 'набор данных 1.fasta' + write_fasta(path, [('A', 'ACGT--'), ('B', 'AC'), ('C', 'ACGTA')]) + # 13 sites, 2 gaps -> 15.4 % + assert alignment.summarize(str(path)) == {'n': 3, 'min_len': 2, 'max_len': 6, 'gap_pct': 15.4} + + +def test_summarize_without_gaps(tmp_path): + path = tmp_path / 'a.fasta' + write_fasta(path, [('A', 'ACGT'), ('B', 'ACGA')]) + assert alignment.summarize(str(path)) == {'n': 2, 'min_len': 4, 'max_len': 4, 'gap_pct': 0.0} + + +def test_summarize_of_empty_sequences_has_no_division_by_zero(tmp_path): + path = tmp_path / 'blank.fasta' + path.write_text('>A\n>B\n') + assert alignment.summarize(str(path)) == {'n': 2, 'min_len': 0, 'max_len': 0, 'gap_pct': 0} + + +def test_summarize_of_an_empty_file_raises(tmp_path): + path = tmp_path / 'empty.fasta' + path.write_text('') + with pytest.raises(AlignmentError): + alignment.summarize(str(path)) + + +def test_path_helpers(): + assert alignment.aligned_path('/d/run 1/in.fasta') == '/d/run 1/in_aligned.fasta' + assert alignment.phylip_path('/d/run 1/in.fasta') == '/d/run 1/in.phy' + assert alignment.aligned_path('in.fas') == 'in_aligned.fasta' + assert alignment.phylip_path('in.fas') == 'in.phy' From 6b5bb5c7b68b5d50d351d4db93e4025941e9abca Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:34:25 +0300 Subject: [PATCH 25/36] Read FASTA with a locale-encoding fallback and write PHYLIP in the locale encoding read_fasta tries utf-8-sig first and falls back to the locale encoding for ANSI files (cp1251 headers), as the old pipeline read every file; a file neither decodes gives an AlignmentError with the decoder's message. write_phylip writes in the locale encoding like the old conversion, so the tree viewer reads the resulting trees consistently; an ID the codec cannot hold is an AlignmentError raised before the file is created. The write_phylip docstring says the sequences must have equal lengths. Co-Authored-By: Claude Sonnet 5.5 --- phyml_core/alignment.py | 41 +++++++++++++++++++------- tests/unit/test_alignment.py | 56 ++++++++++++++++++++++++++++++++---- 2 files changed, 80 insertions(+), 17 deletions(-) diff --git a/phyml_core/alignment.py b/phyml_core/alignment.py index 3467df7..8f93a60 100644 --- a/phyml_core/alignment.py +++ b/phyml_core/alignment.py @@ -1,5 +1,6 @@ """FASTA reading, alignment checks and the FASTA -> PHYLIP conversion (no Qt).""" import io +import locale import re from Bio import AlignIO, SeqIO @@ -14,17 +15,27 @@ class AlignmentError(ValueError): """A problem with the input sequences; the message is already localized.""" +def _parse_fasta(path, encoding): + with open(path, encoding=encoding) as handle: + return list(SeqIO.parse(handle, 'fasta')) + + def read_fasta(path): """Records of a FASTA file; an empty file, or one that is not FASTA, is an AlignmentError. - The file is opened here (not by SeqIO) so the handle is always closed, an UTF-8 BOM is - skipped and CRLF line ends are accepted. + The file is opened here (not by SeqIO) so the handle is always closed, a BOM is + skipped and CRLF line ends are accepted. It is read as UTF-8 first; a file that is not + valid UTF-8 (ANSI, for example cp1251 headers) is read again with the locale encoding, + as the old pipeline read every file. When neither decodes it, the AlignmentError + carries the decoder's message. """ try: - with open(path, encoding='utf-8-sig') as handle: - records = list(SeqIO.parse(handle, 'fasta')) - except UnicodeDecodeError: - raise + try: + records = _parse_fasta(path, 'utf-8-sig') + except UnicodeDecodeError: + records = _parse_fasta(path, locale.getpreferredencoding(False)) + except UnicodeDecodeError as e: + raise AlignmentError(str(e)) from e except ValueError as e: # Biopython refuses text that comes before the first '>' raise AlignmentError(_t('err_empty')) from e @@ -60,9 +71,11 @@ def find_collisions(ids): def write_phylip(records, out_path): """Write the records as relaxed PHYLIP: cleaned IDs, '.' gaps turned into '-'. - The records are not changed. Nothing is written when IDs collide after cleaning, as - PhyML and IQ-TREE would then merge or reject the taxa. The file is UTF-8, like the - FASTA reading. + The sequences must have the same length (check with is_aligned first). The records + are not changed. Nothing is written when IDs collide after cleaning, as PhyML and + IQ-TREE would then merge or reject the taxa. The file is written in the locale + encoding, as the old conversion did, so the tree files that come back are read the + same way by the tree viewer; an ID the locale encoding cannot hold is an AlignmentError. """ collisions = find_collisions([rec.id for rec in records]) if collisions: @@ -76,8 +89,14 @@ def write_phylip(records, out_path): id=new_id, name=new_id, description='')) buffer = io.StringIO() AlignIO.write(MultipleSeqAlignment(cleaned), buffer, 'phylip-relaxed') - with open(out_path, 'w', encoding='utf-8') as handle: - handle.write(buffer.getvalue()) + text = buffer.getvalue() + encoding = locale.getpreferredencoding(False) + try: + text.encode(encoding) # fail before the file is created, not halfway through it + except UnicodeEncodeError as e: + raise AlignmentError(str(e)) from e + with open(out_path, 'w', encoding=encoding) as handle: + handle.write(text) def summarize(path): diff --git a/tests/unit/test_alignment.py b/tests/unit/test_alignment.py index 4d07f3c..44c9b2e 100644 --- a/tests/unit/test_alignment.py +++ b/tests/unit/test_alignment.py @@ -1,4 +1,5 @@ import gc +import locale import pytest from Bio import AlignIO @@ -10,6 +11,14 @@ from phyml_core.i18n import _t +@pytest.fixture +def locale_codec(monkeypatch): + """locale_codec('cp1251'): make the preferred (ANSI) encoding the same on every machine.""" + def use(name): + monkeypatch.setattr(locale, 'getpreferredencoding', lambda *args, **kwargs: name) + return use + + def rec(rid, seq): return SeqRecord(Seq(seq), id=rid, name=rid, description='') @@ -32,11 +41,32 @@ def test_read_fasta_empty_raises(tmp_path, content): assert str(exc.value) == _t('err_empty') -def test_read_fasta_does_not_hide_an_encoding_error(tmp_path): - path = tmp_path / 'cp1251.fasta' +def test_read_fasta_falls_back_to_locale_encoding(tmp_path, locale_codec, recwarn): + """An ANSI file (cp1251 headers) still reads, as it did with the locale codec.""" + locale_codec('cp1251') + path = tmp_path / 'ansi.fasta' + path.write_bytes('>Волк_1 обыкновенный\nACGT\n>Лиса\nACGA\n'.encode('cp1251')) + records = alignment.read_fasta(str(path)) + assert [r.id for r in records] == ['Волк_1', 'Лиса'] + assert records[0].description.endswith('обыкновенный') + gc.collect() + assert [w for w in recwarn if issubclass(w.category, ResourceWarning)] == [] + + +def test_read_fasta_prefers_utf8_over_the_locale_encoding(tmp_path, locale_codec): + locale_codec('cp1251') + path = tmp_path / 'utf8.fasta' + path.write_bytes('>Волк\nACGT\n'.encode('utf-8')) + assert [r.id for r in alignment.read_fasta(str(path))] == ['Волк'] + + +def test_read_fasta_reports_a_file_no_encoding_decodes(tmp_path, locale_codec): + locale_codec('ascii') + path = tmp_path / 'ansi.fasta' path.write_bytes('>Волк\nACGT\n'.encode('cp1251')) - with pytest.raises(UnicodeDecodeError): + with pytest.raises(AlignmentError) as exc: alignment.read_fasta(str(path)) + assert 'ascii' in str(exc.value) and 'decode' in str(exc.value) def test_read_fasta_closes_the_file_when_it_is_empty(tmp_path, recwarn): @@ -119,15 +149,29 @@ def test_write_phylip_raises_on_exact_duplicates(tmp_path): assert not out.exists() -def test_write_phylip_writes_non_ascii_ids_as_utf8(tmp_path): - """read_fasta decodes UTF-8, so the PHYLIP file is UTF-8 too, whatever the locale is.""" +def test_write_phylip_uses_locale_encoding(tmp_path, locale_codec): + """The bytes are the locale codec's, as with the old conversion; the tree viewer reads them back so.""" + locale_codec('cp1251') out = tmp_path / 'ru.phy' alignment.write_phylip([rec('Волк обыкновенный', 'ACGT'), rec('Лиса', 'ACGA')], str(out)) - with open(out, encoding='utf-8') as fh: + raw = out.read_bytes() + assert 'Волк_обыкновенный'.encode('cp1251') in raw + assert 'Лиса'.encode('cp1251') in raw + assert 'Лиса'.encode('utf-8') not in raw + with open(out, encoding='cp1251') as fh: back = AlignIO.read(fh, 'phylip-relaxed') assert [r.id for r in back] == ['Волк_обыкновенный', 'Лиса'] +def test_write_phylip_reports_an_id_the_locale_encoding_cannot_hold(tmp_path, locale_codec): + locale_codec('cp1251') + out = tmp_path / 'bad.phy' + with pytest.raises(AlignmentError) as exc: + alignment.write_phylip([rec('café', 'ACGT'), rec('B', 'ACGA')], str(out)) + assert 'encode' in str(exc.value) + assert not out.exists() + + def test_summarize(tmp_path): path = tmp_path / 'набор данных 1.fasta' write_fasta(path, [('A', 'ACGT--'), ('B', 'AC'), ('C', 'ACGTA')]) From 78d79be44761acd34505f21144357d4f8a9b7b2c Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:49:31 +0300 Subject: [PATCH 26/36] Add a Qt-free PipelineRunner that never raises and cancels whole process trees Co-Authored-By: Claude Opus 5.5 --- locales.json | 4 + phyml_core/runner.py | 319 ++++++++++++++++ tests/README.md | 2 + tests/unit/fake_tool.py | 102 +++++ tests/unit/fakes.py | 71 ++++ tests/unit/test_pipeline_regressions.py | 16 - tests/unit/test_runner.py | 485 ++++++++++++++++++++++++ 7 files changed, 983 insertions(+), 16 deletions(-) create mode 100644 phyml_core/runner.py create mode 100644 tests/unit/fake_tool.py create mode 100644 tests/unit/fakes.py delete mode 100644 tests/unit/test_pipeline_regressions.py create mode 100644 tests/unit/test_runner.py diff --git a/locales.json b/locales.json index 8a86218..bc518ca 100644 --- a/locales.json +++ b/locales.json @@ -20,6 +20,8 @@ "run_phyml": "[*] Running Command: {}", "cancel_analysis": "\n[-] Analysis cancelled by user.", "err_crit_phyml": "\n[CRITICAL ERROR] PhyML exited with code {}.", + "err_engine_exit": "[CRITICAL ERROR] {} exited with code {}.", + "err_no_tree": "[ERROR] {} finished but the tree file is missing: {}", "success_pipe": "\n[+] PIPELINE COMPLETED SUCCESSFULLY.", "err_crit": "\n[CRITICAL ERROR]: {}", "err_empty": "Input file is empty or has invalid FASTA format.", @@ -215,6 +217,8 @@ "run_phyml": "[*] Выполняемая команда: {}", "cancel_analysis": "\n[-] Анализ прерван пользователем.", "err_crit_phyml": "\n[КРИТИЧЕСКАЯ ОШИБКА] PhyML завершил работу с кодом {}.", + "err_engine_exit": "[КРИТИЧЕСКАЯ ОШИБКА] {} завершил работу с кодом {}.", + "err_no_tree": "[ОШИБКА] {} завершился, но файл дерева не найден: {}", "success_pipe": "\n[+] ПАЙПЛАЙН УСПЕШНО ЗАВЕРШЕН.", "err_crit": "\n[КРИТИЧЕСКАЯ ОШИБКА]: {}", "err_empty": "Входной файл пуст или имеет некорректный формат FASTA.", diff --git a/phyml_core/runner.py b/phyml_core/runner.py new file mode 100644 index 0000000..63446d6 --- /dev/null +++ b/phyml_core/runner.py @@ -0,0 +1,319 @@ +"""The analysis pipeline over FASTA files, with no Qt: MUSCLE, PHYLIP conversion, the engine. + +A PipelineRunner is used once. run() goes through the files in order, reports progress via +two optional callbacks (called on the thread that runs it) and always returns a RunResult; +nothing it does raises out of it. cancel() may be called from any thread at any time: it +stops the whole process tree of the current step, and the files not finished yet end as +'cancelled'. +""" +import os +import signal +import subprocess +import threading +from dataclasses import dataclass, field +from typing import Callable, List, Literal, Mapping, Optional, Tuple + +from . import alignment +from .alignment import AlignmentError +from .engines import ENGINES +from .engines.base import Engine, format_issue, param_text +from .i18n import _t +from .launchers import Native +from .paths import AppPaths + +try: + import psutil +except ImportError: # a hard dependency (requirements.txt); kill_process_tree has a fallback + psutil = None + +LEVELS = ('error', 'success', 'system', 'out') +STATUSES = ('pending', 'running', 'ok', 'fail', 'cancelled') +STAT_KEYS = ('model', 'taxa', 'lnL', 'gamma', 'tstv', 'time') +MUSCLE_NAMES = ('muscle.exe',) +KILL_TIMEOUT = 3.0 # seconds between terminate and kill + +_CREATION_FLAGS = subprocess.CREATE_NO_WINDOW if os.name == 'nt' else 0 + + +@dataclass +class FileResult: + file: str # the input path as given + status: Literal['ok', 'fail', 'cancelled'] + tree: str = '' + stats: dict = field(default_factory=dict) + error: str = '' + + +@dataclass +class RunResult: + files: List[FileResult] + cancelled: bool + + @property + def ok_count(self) -> int: + return sum(1 for f in self.files if f.status == 'ok') + + @property + def success(self) -> bool: + """There were files, all of them are 'ok' and the run was not cancelled.""" + return bool(self.files) and not self.cancelled and all(f.status == 'ok' for f in self.files) + + +def kill_process_tree(pid: int) -> None: + """Stop a process and every process it started: terminate, wait up to 3 s, then kill. + + The parent goes first, so it cannot start a replacement child. Processes that are + already gone are ignored. Without psutil only the process itself is signalled. + """ + if psutil is None: + try: + os.kill(pid, signal.SIGTERM) # TerminateProcess on Windows + except OSError: + pass + return + try: + root = psutil.Process(pid) + procs = [root] + root.children(recursive=True) + except psutil.Error: + return + for proc in procs: + try: + proc.terminate() + except psutil.Error: + pass + try: + _, alive = psutil.wait_procs(procs, timeout=KILL_TIMEOUT) + except psutil.Error: + alive = procs + for proc in alive: + try: + proc.kill() + except psutil.Error: + pass + + +class _FileFailed(Exception): + """Ends the current file as 'fail'; the argument is the localized message.""" + + +class _Cancelled(Exception): + """Ends the current file as 'cancelled'; the argument is the key of the log message.""" + + +class PipelineRunner: + def __init__(self, params: dict, files: List[str], paths: AppPaths, + on_log: Optional[Callable[[str, str], None]] = None, + on_status: Optional[Callable[[str, str], None]] = None, + engines: Optional[Mapping[str, Engine]] = None): + """on_log(level, text) with a level from LEVELS; on_status(file name, status) with a + status from STATUSES. engines replaces the registry (tests pass fakes).""" + self._params = dict(params) + self._files = [files] if isinstance(files, str) else list(files) + self._paths = paths + self._on_log = on_log + self._on_status = on_status + self._engines = ENGINES if engines is None else engines + self._cancelled = threading.Event() + # guards _proc: a process is started and published, or killed, as one step + self._lock = threading.Lock() + self._proc: Optional[subprocess.Popen] = None + + @property + def active_pid(self) -> Optional[int]: + """PID of the program the current step runs (MUSCLE or the engine), else None.""" + proc = self._proc + return proc.pid if proc is not None else None + + def cancel(self) -> None: + """Stop the run: no new step starts, the running one loses its process tree.""" + self._cancelled.set() + try: + with self._lock: + proc = self._proc + # poll() is None: not reaped yet, so the pid still belongs to this process + if proc is not None and proc.poll() is None: + kill_process_tree(proc.pid) + except Exception: + pass + + def run(self) -> RunResult: + results: List[FileResult] = [] + error = '' + try: + self._run_files(results) + except Exception as e: # a bug outside the per-file guard: still return a result + error = _t('err_crit').format(e) + self._log('error', error) + cancelled = self._cancelled.is_set() + for fasta in self._files[len(results):]: + status = 'cancelled' if cancelled else 'fail' + results.append(FileResult(fasta, status, error='' if cancelled else error.strip())) + self._status(fasta, status) + return RunResult(results, cancelled) + + # --- the run -------------------------------------------------------------------------- + + def _run_files(self, results: List[FileResult]) -> None: + total = len(self._files) + batch = total > 1 + if batch: + self._log('system', _t('batch_found').format(total)) + for fasta in self._files: + self._status(fasta, 'pending') + for i, fasta in enumerate(self._files, 1): + if self._cancelled.is_set(): + return + if batch: + self._log('system', _t('batch_file_header').format(i, total, os.path.basename(fasta))) + self._status(fasta, 'running') + result = self._run_file(fasta) + results.append(result) + self._status(fasta, result.status) + if batch and not self._cancelled.is_set(): + ok = sum(1 for r in results if r.status == 'ok') + self._log('success' if ok == total else 'system', _t('batch_summary').format(ok, total)) + + def _run_file(self, fasta: str) -> FileResult: + """One file, never raising: every failure becomes a FileResult.""" + try: + tree, stats = self._process(fasta) + return FileResult(fasta, 'ok', tree=tree, stats=stats) + except _Cancelled as c: + self._log('system', _t(c.args[0])) + return FileResult(fasta, 'cancelled') + except Exception as e: + if self._cancelled.is_set(): # most likely caused by the cancel itself + self._log('system', _t('cancel_analysis')) + return FileResult(fasta, 'cancelled') + message = e.args[0] if isinstance(e, _FileFailed) else _t('err_crit').format(e) + self._log('error', message) + return FileResult(fasta, 'fail', error=message.strip()) + + def _process(self, fasta: str) -> Tuple[str, dict]: + key = param_text(self._params, 'engine', 'phyml') + engine = self._engines.get(key) + if engine is None: + raise _FileFailed(_t('err_crit').format(f'unknown engine: {key}')) + self._log('system', _t('init_analysis').format(fasta)) + records = alignment.read_fasta(fasta) # AlignmentError: empty or not FASTA + if all(len(rec.seq) == 0 for rec in records): # headers only + raise _FileFailed(_t('err_crit').format(_t('err_empty'))) + source = fasta + if alignment.is_aligned(records): + self._log('success', _t('seq_aligned')) + else: + self._log('system', _t('diff_len')) + source = self._align(fasta) + records = alignment.read_fasta(source) + if self._cancelled.is_set(): + raise _Cancelled('cancel_analysis') + self._log('system', _t('cleaning')) + phylip = alignment.phylip_path(source) + self._convert(records, phylip) + self._log('success', _t('conv_success').format(phylip)) + return self._analyse(engine, phylip) + + def _align(self, fasta: str) -> str: + """Run MUSCLE (version 5 syntax, then version 3); returns the aligned file.""" + muscle = self._paths.find_tool(self._params.get('muscle_path') or '', MUSCLE_NAMES) + if not muscle or not os.path.isfile(muscle): + raise _FileFailed(_t('err_muscle')) + aligned = alignment.aligned_path(fasta) + if os.path.exists(aligned): + try: + os.remove(aligned) + except OSError as e: + raise _FileFailed(_t('err_del').format(e)) from e + self._log('system', _t('del_old')) + launcher = Native() + code = self._execute(launcher.wrap([muscle, '-align', fasta, '-output', aligned]), + 'cancel_align', show_output=False) + if code != 0: + code = self._execute(launcher.wrap([muscle, '-in', fasta, '-out', aligned]), + 'cancel_align', show_output=False) + if code != 0 or not os.path.isfile(aligned): + raise _FileFailed(_t('err_muscle_fail')) + self._log('success', _t('align_success').format(aligned)) + return aligned + + def _convert(self, records, phylip: str) -> None: + collide = bool(alignment.find_collisions([rec.id for rec in records])) + try: + alignment.write_phylip(records, phylip) + except AlignmentError as e: # the collision message is complete as it is + raise _FileFailed(str(e) if collide else _t('err_conv').format(e)) from e + except Exception as e: # Biopython refusing the records, a write error + raise _FileFailed(_t('err_conv').format(e)) from e + + def _analyse(self, engine: Engine, phylip: str) -> Tuple[str, dict]: + issues = engine.validate(self._params, self._paths) + for issue in issues: + if issue.level != 'error': + self._log('system', format_issue(issue)) + errors = [format_issue(issue) for issue in issues if issue.level == 'error'] + if errors: + raise _FileFailed('\n'.join(errors)) + params, notes = engine.normalize(self._params) + for note in notes: + self._log('system', format_issue(note)) + exe = engine.executable(params, self._paths) + argv = engine.launcher(params).wrap(engine.command(exe, phylip, params)) + outputs = engine.outputs(phylip) + self._log('system', _t('run_phyml').format(subprocess.list2cmdline(argv))) + code = self._execute(argv, 'cancel_analysis', show_output=True) + if code != 0: + raise _FileFailed(_t('err_engine_exit').format(engine.name, code)) + if not os.path.isfile(outputs.tree): + raise _FileFailed(_t('err_no_tree').format(engine.name, outputs.tree)) + self._log('success', _t('success_pipe')) + parsed = engine.parse_stats(outputs) or {} + stats = {key: parsed.get(key, '') for key in STAT_KEYS} + self._log('success', _t('stats_summary').format(*(stats[k] or 'n/a' for k in STAT_KEYS))) + return outputs.tree, stats + + def _execute(self, argv: List[str], cancel_key: str, show_output: bool) -> int: + """Run a program to its end and return its exit code; its output goes to the log + ('out') when show_output is set. Raises _Cancelled when the run is cancelled.""" + with self._lock: + if self._cancelled.is_set(): + raise _Cancelled(cancel_key) + proc = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, errors='replace', + creationflags=_CREATION_FLAGS) + self._proc = proc + try: + for line in proc.stdout: + if show_output: + self._log('out', line.rstrip()) + proc.wait() + finally: + if proc.poll() is None: # left early by an exception: do not leave it running + kill_process_tree(proc.pid) + try: + proc.wait(KILL_TIMEOUT) + except subprocess.TimeoutExpired: + pass + proc.stdout.close() + with self._lock: + self._proc = None + if self._cancelled.is_set(): + raise _Cancelled(cancel_key) + return proc.returncode + + # --- callbacks ------------------------------------------------------------------------ + + def _log(self, level: str, text: str) -> None: + if self._on_log is None: + return + try: + self._on_log(level, str(text).rstrip('\r\n')) + except Exception: + pass + + def _status(self, fasta: str, status: str) -> None: + if self._on_status is None: + return + try: + self._on_status(os.path.basename(fasta), status) + except Exception: + pass diff --git a/tests/README.md b/tests/README.md index 064b966..a5fab63 100644 --- a/tests/README.md +++ b/tests/README.md @@ -10,6 +10,8 @@ All suites run under pytest (`pip install -r requirements-dev.txt`): Layout: - `unit/`, `gui/` - fast checks. `gui/` builds `PhyMLApp` under `QT_QPA_PLATFORM=offscreen`. +- `unit/fakes.py`, `unit/fake_tool.py` - `FakeEngine` and `fake_muscle()`: stand-ins for an + analysis program and MUSCLE, so the `PipelineRunner` tests need no binaries. - `e2e/` - end-to-end runs of the pipeline, batch mode, CPU monitor and IQ-TREE engine. `e2e/datasets.py` holds the synthetic two-clade dataset (fixture `synthetic`). - `data/` - static fixtures (stats files, trees). diff --git a/tests/unit/fake_tool.py b/tests/unit/fake_tool.py new file mode 100644 index 0000000..69f705e --- /dev/null +++ b/tests/unit/fake_tool.py @@ -0,0 +1,102 @@ +"""Stand-in for an analysis program or MUSCLE in the runner tests (no real binaries). + + python fake_tool.py MODE ARGS... + +Engine modes, ARGS = the PHYLIP file: + ok prints a few lines (one with bytes no codec decodes cleanly), writes the tree and + the stats next to the PHYLIP file, exits 0 + exit3 prints a line and exits with code 3 + notree exits 0 without writing anything + hang starts a child `python -c "import time; time.sleep(60)"` and sleeps itself + +MUSCLE modes, ARGS = MUSCLE's options: + copy `-align IN -output OUT` (MUSCLE 5): copies IN to OUT, padding the short + sequences with '-' so the output is aligned; any other syntax exits 1 + v3only `-in IN -out OUT` (MUSCLE 3) as copy; the MUSCLE 5 syntax exits 1 + hang as the engine's hang +""" +import subprocess +import sys +import time + +# FakeEngine.outputs() in fakes.py uses the same suffixes +TREE_SUFFIX = '_fake_tree.txt' +STATS_SUFFIX = '_fake_stats.txt' + +SLEEPER = 'import time; time.sleep(60)' + + +def say(text): + sys.stdout.write(text + '\n') + sys.stdout.flush() + + +def hang(): + say('fake tool: working') + subprocess.Popen([sys.executable, '-c', SLEEPER]) + time.sleep(60) + return 0 + + +def engine_ok(phylip): + say('fake engine: reading ' + phylip) + # 0x81 is undefined in cp1252, 0x98 in cp1251, 0xff is never valid UTF-8 + sys.stdout.buffer.write(b'progress \x81\x98\xff\n') + sys.stdout.flush() + with open(phylip, encoding='latin-1') as handle: + ids = [line.split()[0] for line in handle.read().splitlines()[1:] if line.strip()] + with open(phylip + TREE_SUFFIX, 'w', encoding='utf-8') as handle: + handle.write('(' + ','.join(ids) + ');\n') + with open(phylip + STATS_SUFFIX, 'w', encoding='utf-8') as handle: + handle.write(f'model=FAKE\ntaxa={len(ids)}\nlnL=-123.45\ngamma=0.5\ntstv=2.0\ntime=1s\n') + say('fake engine: done') + return 0 + + +def read_fasta(path): + records, name = [], None + with open(path, encoding='latin-1') as handle: + for line in handle.read().splitlines(): + if line.startswith('>'): + name = line + records.append([name, '']) + elif name is not None: + records[-1][1] += line.strip() + return records + + +def muscle(args, source_flag, out_flag): + if source_flag not in args or out_flag not in args: + say('fake muscle: unknown syntax ' + ' '.join(args)) + return 1 + source = args[args.index(source_flag) + 1] + target = args[args.index(out_flag) + 1] + records = read_fasta(source) + width = max(len(seq) for _, seq in records) + with open(target, 'w', encoding='latin-1') as handle: + for header, seq in records: + handle.write(f'{header}\n{seq.ljust(width, "-")}\n') + return 0 + + +def main(argv): + mode, args = argv[1], argv[2:] + if mode == 'hang': + return hang() + if mode == 'ok': + return engine_ok(args[0]) + if mode == 'exit3': + say('fake engine: failing') + return 3 + if mode == 'notree': + return 0 + if mode == 'copy': + return muscle(args, '-align', '-output') + if mode == 'v3only': + return muscle(args, '-in', '-out') + say('fake tool: unknown mode ' + mode) + return 2 + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/tests/unit/fakes.py b/tests/unit/fakes.py new file mode 100644 index 0000000..cd0d819 --- /dev/null +++ b/tests/unit/fakes.py @@ -0,0 +1,71 @@ +"""Test doubles for PipelineRunner: an engine and a MUSCLE that run fake_tool.py.""" +import os +import sys +from typing import Dict, List, Tuple + +from phyml_core.engines.base import Engine, EngineOutputs, Issue +from phyml_core.launchers import Native + +FAKE_TOOL = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'fake_tool.py') + +# the same suffixes as TREE_SUFFIX / STATS_SUFFIX in fake_tool.py +_TREE_SUFFIX = '_fake_tree.txt' +_STATS_SUFFIX = '_fake_stats.txt' + + +class FakeEngine(Engine): + """An engine whose program is fake_tool.py. + + params['fake_mode']: 'ok' (default), 'exit3', 'notree', 'hang' or 'raise' (command() + raises RuntimeError('boom')); a dict picks the mode per input by the PHYLIP file's stem. + params['fake_issues'] is what validate() returns, params['fake_notes'] the notes of + normalize(). + """ + key = 'fake' + name = 'Fake' + label_key = 'engine_fake' + + def models(self, datatype: str) -> List[str]: + return ['FAKE'] + + def validate(self, params: dict, paths) -> List[Issue]: + return list(params.get('fake_issues', [])) + + def normalize(self, params: dict) -> Tuple[dict, List[Issue]]: + return dict(params), list(params.get('fake_notes', [])) + + def executable(self, params: dict, paths) -> str: + return sys.executable + + def command(self, exe: str, phylip: str, params: dict) -> List[str]: + mode = params.get('fake_mode', 'ok') + if isinstance(mode, dict): + mode = mode.get(os.path.splitext(os.path.basename(phylip))[0], 'ok') + if mode == 'raise': + raise RuntimeError('boom') + return [sys.executable, FAKE_TOOL, mode, phylip] + + def outputs(self, phylip: str) -> EngineOutputs: + return EngineOutputs(tree=phylip + _TREE_SUFFIX, stats=phylip + _STATS_SUFFIX, + boot_trees=None, log=None) + + def parse_stats(self, outputs: EngineOutputs) -> Dict[str, str]: + try: + with open(outputs.stats, encoding='utf-8') as handle: + lines = handle.read().splitlines() + except OSError: + return {} + return dict(line.split('=', 1) for line in lines if '=' in line) + + def launcher(self, params: dict) -> Native: + return Native() + + +def fake_muscle(tmp_path, mode: str) -> str: + """A muscle-.cmd in tmp_path that runs fake_tool.py in that MUSCLE mode.""" + script = os.path.join(str(tmp_path), f'muscle-{mode}.cmd') + with open(script, 'w', encoding='oem', newline='\r\n') as handle: + handle.write('@echo off\n') + handle.write(f'"{sys.executable}" "{FAKE_TOOL}" {mode} %*\n') + handle.write('exit /b %errorlevel%\n') + return script diff --git a/tests/unit/test_pipeline_regressions.py b/tests/unit/test_pipeline_regressions.py deleted file mode 100644 index c159da2..0000000 --- a/tests/unit/test_pipeline_regressions.py +++ /dev/null @@ -1,16 +0,0 @@ -from phyml_core.pipeline import PipelineWorker - - -def test_empty_fasta_fails_cleanly(qapp, tmp_path): - """A broken input fails the run cleanly instead of killing the worker thread.""" - empty = tmp_path / 'empty.fasta' - empty.write_text('') - worker = PipelineWorker({'engine': 'phyml'}, [str(empty)]) - done = [] - worker.finished_signal.connect(lambda ok, tree: done.append((ok, tree))) - try: - worker.run() - except Exception as e: - raise AssertionError(f'worker crashed on an empty FASTA: {e!r}') - assert done == [(False, '')], done - assert worker.results[0]['status'] == 'FAIL', worker.results diff --git a/tests/unit/test_runner.py b/tests/unit/test_runner.py new file mode 100644 index 0000000..aac5ded --- /dev/null +++ b/tests/unit/test_runner.py @@ -0,0 +1,485 @@ +import os +import subprocess +import sys +import threading +import time + +import psutil +import pytest + +from fakes import FakeEngine, fake_muscle +from fake_tool import SLEEPER +from phyml_core import runner as runner_module +from phyml_core.engines.base import Issue, format_issue +from phyml_core.i18n import _t +from phyml_core.paths import AppPaths +from phyml_core.runner import LEVELS, FileResult, PipelineRunner, RunResult, kill_process_tree + +ALIGNED = [('A', 'ACGTACGTAC'), ('B', 'ACGTACGAAC'), ('C', 'ACGAACGTAC')] +UNALIGNED = [('A', 'ACGTACGTAC'), ('B', 'ACGTAC'), ('C', 'ACGAACGTACGT')] + +WAIT = 15 # seconds: the upper bound of every wait, so a bug cannot hang the suite + + +def write_fasta(path, records): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(''.join(f'>{rid}\n{seq}\n' for rid, seq in records), encoding='utf-8') + return str(path) + + +@pytest.fixture +def paths(tmp_path): + return AppPaths(app_dir=str(tmp_path / 'app'), data_dir=str(tmp_path / 'data')) + + +class Recorder: + """on_log / on_status that keep what they got; `running` is set by the first 'running'.""" + + def __init__(self): + self.logs = [] + self.statuses = [] + self.running = threading.Event() + + def on_log(self, level, text): + self.logs.append((level, text)) + + def on_status(self, name, status): + self.statuses.append((name, status)) + if status == 'running': + self.running.set() + + def texts(self, level=None): + return [text for lvl, text in self.logs if level is None or lvl == level] + + def statuses_of(self, name): + return [status for n, status in self.statuses if n == name] + + +def make_runner(files, paths, rec=None, **params): + rec = rec or Recorder() + params.setdefault('engine', 'fake') + return PipelineRunner(params, files, paths, on_log=rec.on_log, on_status=rec.on_status, + engines={'fake': FakeEngine()}) + + +def start(runner): + """run() in a daemon thread; box['result'] gets the RunResult.""" + box = {} + thread = threading.Thread(target=lambda: box.update(result=runner.run()), daemon=True) + thread.start() + return thread, box + + +def wait_for_sleeper(runner): + """The process tree of the current step once its sleeping grandchild is up.""" + deadline = time.monotonic() + WAIT + while time.monotonic() < deadline: + pid = runner.active_pid + if pid is not None: + try: + root = psutil.Process(pid) + descendants = root.children(recursive=True) + if any(SLEEPER in ' '.join(_cmdline(p)) for p in descendants): + return [root, *descendants] + except psutil.Error: + pass + time.sleep(0.05) + raise AssertionError('the fake tool never started its sleeping child') + + +def _cmdline(proc): + try: + return proc.cmdline() + except psutil.Error: + return [] + + +def alive(proc): + try: + return proc.is_running() and proc.status() != psutil.STATUS_ZOMBIE + except psutil.Error: + return False + + +def kill_all(procs): + for proc in procs: + try: + proc.kill() + except psutil.Error: + pass + + +# --- results ---------------------------------------------------------------------------- + +def test_run_result_counts_and_success(): + ok, fail = FileResult('a.fasta', 'ok'), FileResult('b.fasta', 'fail') + assert RunResult([ok, ok], cancelled=False).success + assert RunResult([ok, fail], cancelled=False).ok_count == 1 + assert not RunResult([ok, fail], cancelled=False).success + assert not RunResult([ok], cancelled=True).success + assert not RunResult([], cancelled=False).success + + +def test_ok_run_returns_tree_and_stats(tmp_path, paths): + fasta = write_fasta(tmp_path / 'a.fasta', ALIGNED) + rec = Recorder() + result = make_runner([fasta], paths, rec).run() + assert isinstance(result, RunResult) + assert not result.cancelled and result.success and result.ok_count == 1 + file = result.files[0] + assert file.file == fasta + assert file.status == 'ok' and file.error == '' + assert file.tree == str(tmp_path / 'a.phy') + '_fake_tree.txt' + assert os.path.isfile(file.tree) + assert file.stats == {'model': 'FAKE', 'taxa': '3', 'lnL': '-123.45', 'gamma': '0.5', + 'tstv': '2.0', 'time': '1s'} + assert rec.statuses == [('a.fasta', 'pending'), ('a.fasta', 'running'), ('a.fasta', 'ok')] + assert any('fake engine: done' in text for text in rec.texts('out')) + assert _t('seq_aligned') in rec.texts('success') + + +def test_empty_fasta_fails_cleanly(tmp_path, paths): + """A broken input fails the file instead of raising out of run().""" + empty = tmp_path / 'empty.fasta' + empty.write_text('') + rec = Recorder() + result = make_runner([str(empty)], paths, rec).run() + assert not result.success + assert result.files[0].status == 'fail' + assert _t('err_empty') in result.files[0].error + assert rec.statuses_of('empty.fasta') == ['pending', 'running', 'fail'] + assert any(_t('err_empty') in text for text in rec.texts('error')) + + +def test_header_only_fasta_fails_with_err_empty(tmp_path, paths): + fasta = tmp_path / 'headers.fasta' + fasta.write_text('>A\n>B\n') + result = make_runner([str(fasta)], paths).run() + assert result.files[0].status == 'fail' + assert _t('err_empty') in result.files[0].error + + +def test_engine_exit_code_reported(tmp_path, paths): + fasta = write_fasta(tmp_path / 'a.fasta', ALIGNED) + rec = Recorder() + result = make_runner([fasta], paths, rec, fake_mode='exit3').run() + assert result.files[0].status == 'fail' + assert result.files[0].error == _t('err_engine_exit').format('Fake', 3) + assert _t('err_engine_exit').format('Fake', 3) in rec.texts('error') + + +def test_missing_tree_reported(tmp_path, paths): + fasta = write_fasta(tmp_path / 'a.fasta', ALIGNED) + result = make_runner([fasta], paths, fake_mode='notree').run() + tree = str(tmp_path / 'a.phy') + '_fake_tree.txt' + assert result.files[0].status == 'fail' + assert result.files[0].error == _t('err_no_tree').format('Fake', tree) + + +def test_engine_exception_never_escapes(tmp_path, paths): + bad = write_fasta(tmp_path / 'bad.fasta', ALIGNED) + good = write_fasta(tmp_path / 'good.fasta', ALIGNED) + rec = Recorder() + result = make_runner([bad, good], paths, rec, fake_mode={'bad': 'raise'}).run() + assert [f.status for f in result.files] == ['fail', 'ok'] + assert 'boom' in result.files[0].error + assert rec.statuses_of('good.fasta') == ['pending', 'running', 'ok'] + assert any('boom' in text for text in rec.texts('error')) + + +def test_validate_error_stops_file_and_warning_is_logged(tmp_path, paths): + fasta = write_fasta(tmp_path / 'a.fasta', ALIGNED) + warning = Issue('warning', 'iqtree_ufboot_min', ('1000', '10')) + error = Issue('error', 'val_no_exe', ('Fake', 'C:/nowhere/fake.exe')) + rec = Recorder() + result = make_runner([fasta], paths, rec, fake_issues=[warning, error]).run() + file = result.files[0] + assert file.status == 'fail' + assert file.error == format_issue(error) + assert format_issue(warning) in rec.texts('system') + assert format_issue(error) in rec.texts('error') + assert not os.path.exists(str(tmp_path / 'a.phy') + '_fake_tree.txt') + assert rec.texts('out') == [] # the engine never started + + +def test_warnings_and_notes_do_not_stop_the_file(tmp_path, paths): + fasta = write_fasta(tmp_path / 'a.fasta', ALIGNED) + warning = Issue('warning', 'val_no_mpiexec', ('mpiexec',)) + note = Issue('warning', 'iqtree_ufboot_min', ('1000', '10')) + rec = Recorder() + result = make_runner([fasta], paths, rec, fake_issues=[warning], fake_notes=[note]).run() + assert result.files[0].status == 'ok' + assert format_issue(warning) in rec.texts('system') + assert format_issue(note) in rec.texts('system') + + +def test_id_collision_fails_with_listing(tmp_path, paths): + fasta = write_fasta(tmp_path / 'a.fasta', [('A.1', 'ACGT'), ('A_1', 'ACGA'), ('B', 'ACGG')]) + result = make_runner([fasta], paths).run() + file = result.files[0] + assert file.status == 'fail' + assert 'A.1' in file.error and 'A_1' in file.error + assert file.error.startswith(_t('err_id_collision').split('{}')[0]) + assert not (tmp_path / 'a.phy').exists() + + +def test_unaligned_input_runs_muscle(tmp_path, paths): + fasta = write_fasta(tmp_path / 'a.fasta', UNALIGNED) + rec = Recorder() + result = make_runner([fasta], paths, rec, muscle_path=fake_muscle(tmp_path, 'copy')).run() + aligned = tmp_path / 'a_aligned.fasta' + assert aligned.is_file() + assert result.files[0].status == 'ok', result.files[0].error + assert result.files[0].tree == str(tmp_path / 'a_aligned.phy') + '_fake_tree.txt' + assert _t('diff_len') in rec.texts('system') + assert _t('align_success').format(str(aligned)) in rec.texts('success') + + +def test_muscle_falls_back_to_v3_syntax(tmp_path, paths): + fasta = write_fasta(tmp_path / 'a.fasta', UNALIGNED) + result = make_runner([fasta], paths, muscle_path=fake_muscle(tmp_path, 'v3only')).run() + assert result.files[0].status == 'ok', result.files[0].error + assert (tmp_path / 'a_aligned.fasta').is_file() + + +def test_old_alignment_is_replaced(tmp_path, paths): + fasta = write_fasta(tmp_path / 'a.fasta', UNALIGNED) + (tmp_path / 'a_aligned.fasta').write_text('stale') + rec = Recorder() + result = make_runner([fasta], paths, rec, muscle_path=fake_muscle(tmp_path, 'copy')).run() + assert result.files[0].status == 'ok', result.files[0].error + assert _t('del_old') in rec.texts('system') + assert (tmp_path / 'a_aligned.fasta').read_text().startswith('>A') + + +def test_missing_muscle_fails_with_err_muscle(tmp_path, paths): + fasta = write_fasta(tmp_path / 'a.fasta', UNALIGNED) + rec = Recorder() + result = make_runner([fasta], paths, rec, muscle_path=str(tmp_path / 'no-muscle.exe')).run() + assert result.files[0].status == 'fail' + assert result.files[0].error == _t('err_muscle') + assert _t('err_muscle') in rec.texts('error') + + +def test_unknown_engine_fails_the_files(tmp_path, paths): + fasta = write_fasta(tmp_path / 'a.fasta', ALIGNED) + result = make_runner([fasta], paths, engine='raxml').run() + assert result.files[0].status == 'fail' + assert 'raxml' in result.files[0].error + + +def test_batch_logs_found_header_and_summary(tmp_path, paths): + a = write_fasta(tmp_path / 'a.fasta', ALIGNED) + b = write_fasta(tmp_path / 'b.fasta', ALIGNED) + rec = Recorder() + result = make_runner([a, b], paths, rec, fake_mode={'b': 'exit3'}).run() + assert [f.status for f in result.files] == ['ok', 'fail'] + assert [f.file for f in result.files] == [a, b] + texts = rec.texts() + assert _t('batch_found').format(2) in texts + assert _t('batch_file_header').format(1, 2, 'a.fasta').rstrip('\n') in texts + assert _t('batch_file_header').format(2, 2, 'b.fasta').rstrip('\n') in texts + assert _t('batch_summary').format(1, 2).rstrip('\n') in texts + assert rec.statuses[:2] == [('a.fasta', 'pending'), ('b.fasta', 'pending')] + + +def test_log_levels_are_from_levels(tmp_path, paths): + files = [write_fasta(tmp_path / 'ok.fasta', ALIGNED), + write_fasta(tmp_path / 'msa.fasta', UNALIGNED), + write_fasta(tmp_path / 'bad.fasta', [('A.1', 'ACGT'), ('A_1', 'ACGA')]), + write_fasta(tmp_path / 'exit.fasta', ALIGNED)] + rec = Recorder() + make_runner(files, paths, rec, muscle_path=fake_muscle(tmp_path, 'copy'), + fake_mode={'exit': 'exit3'}, + fake_notes=[Issue('warning', 'iqtree_ufboot_min', ('1000', '10'))]).run() + levels = {level for level, _ in rec.logs} + assert levels <= set(LEVELS) + assert levels == set(LEVELS) # the scenario produces every level + assert all(not text.endswith('\n') for text in rec.texts()) + + +def test_undecodable_engine_output_is_replaced(tmp_path, paths): + """fake_tool 'ok' prints bytes that no single codec decodes cleanly.""" + fasta = write_fasta(tmp_path / 'a.fasta', ALIGNED) + rec = Recorder() + result = make_runner([fasta], paths, rec).run() + assert result.files[0].status == 'ok', result.files[0].error + assert any(text.startswith('progress ') for text in rec.texts('out')) + + +def test_run_in_cyrillic_dir_with_spaces(tmp_path, paths): + folder = tmp_path / 'Мои данные' / 'проект 1' + aligned = write_fasta(folder / 'выборка 1.fasta', ALIGNED) + unaligned = write_fasta(folder / 'выборка 2.fasta', UNALIGNED) + rec = Recorder() + result = make_runner([aligned, unaligned], paths, rec, + muscle_path=fake_muscle(tmp_path, 'copy')).run() + assert [f.status for f in result.files] == ['ok', 'ok'], [f.error for f in result.files] + assert result.files[0].tree == str(folder / 'выборка 1.phy') + '_fake_tree.txt' + assert os.path.isfile(result.files[0].tree) + assert (folder / 'выборка 2_aligned.fasta').is_file() + assert os.path.isfile(result.files[1].tree) + assert rec.statuses_of('выборка 1.fasta') == ['pending', 'running', 'ok'] + + +# --- cancel ------------------------------------------------------------------------------- + +def test_cancel_kills_process_tree(tmp_path, paths): + fasta = write_fasta(tmp_path / 'a.fasta', ALIGNED) + rec = Recorder() + runner = make_runner([fasta], paths, rec, fake_mode='hang') + thread, box = start(runner) + procs = [] + try: + assert rec.running.wait(WAIT) + procs = wait_for_sleeper(runner) + assert procs[0].pid == runner.active_pid + started = time.monotonic() + canceller = threading.Thread(target=runner.cancel, daemon=True) + canceller.start() + canceller.join(WAIT) + thread.join(WAIT) + assert not thread.is_alive(), 'run() did not return after cancel()' + assert time.monotonic() - started < 10 + result = box['result'] + assert result.cancelled and not result.success + assert result.files[0].status == 'cancelled' + assert rec.statuses_of('a.fasta') == ['pending', 'running', 'cancelled'] + assert [p.pid for p in procs if alive(p)] == [] + assert runner.active_pid is None + assert _t('cancel_analysis').rstrip('\n') in rec.texts('system') + finally: + runner.cancel() + kill_all(procs) + + +def test_cancel_during_alignment_marks_rest_cancelled(tmp_path, paths): + first = write_fasta(tmp_path / 'a.fasta', UNALIGNED) + second = write_fasta(tmp_path / 'b.fasta', UNALIGNED) + rec = Recorder() + runner = make_runner([first, second], paths, rec, muscle_path=fake_muscle(tmp_path, 'hang')) + thread, box = start(runner) + procs = [] + try: + procs = wait_for_sleeper(runner) + canceller = threading.Thread(target=runner.cancel, daemon=True) + canceller.start() + canceller.join(WAIT) + thread.join(WAIT) + assert not thread.is_alive(), 'run() did not return after cancel()' + result = box['result'] + assert result.cancelled + assert [f.status for f in result.files] == ['cancelled', 'cancelled'] + assert [f.file for f in result.files] == [first, second] + assert rec.statuses_of('a.fasta') == ['pending', 'running', 'cancelled'] + assert rec.statuses_of('b.fasta') == ['pending', 'cancelled'] + assert [p.pid for p in procs if alive(p)] == [] + assert _t('cancel_align') in rec.texts('system') + assert not (tmp_path / 'a_aligned.phy').exists() + finally: + runner.cancel() + kill_all(procs) + + +def test_cancel_before_run_cancels_every_file(tmp_path, paths): + files = [write_fasta(tmp_path / 'a.fasta', ALIGNED), write_fasta(tmp_path / 'b.fasta', ALIGNED)] + rec = Recorder() + runner = make_runner(files, paths, rec) + runner.cancel() + result = runner.run() + assert result.cancelled + assert [f.status for f in result.files] == ['cancelled', 'cancelled'] + assert rec.statuses_of('a.fasta') == ['pending', 'cancelled'] + assert not (tmp_path / 'a.phy').exists() + + +def test_cancel_from_a_callback_and_after_finish_is_safe(tmp_path, paths): + files = [write_fasta(tmp_path / 'a.fasta', ALIGNED), write_fasta(tmp_path / 'b.fasta', ALIGNED)] + rec = Recorder() + holder = {} + + def on_status(name, status): + rec.on_status(name, status) + if status == 'running': + holder['runner'].cancel() # on the runner's own thread, between steps + + runner = PipelineRunner({'engine': 'fake'}, files, paths, on_log=rec.on_log, + on_status=on_status, engines={'fake': FakeEngine()}) + holder['runner'] = runner + thread, box = start(runner) + thread.join(WAIT) + assert not thread.is_alive() + assert [f.status for f in box['result'].files] == ['cancelled', 'cancelled'] + runner.cancel() # after run() returned + runner.cancel() + assert runner.active_pid is None + + +def test_callbacks_that_raise_do_not_escape(tmp_path, paths): + fasta = write_fasta(tmp_path / 'a.fasta', ALIGNED) + + def broken(*args): + raise RuntimeError('callback') + + runner = PipelineRunner({'engine': 'fake'}, [fasta], paths, on_log=broken, on_status=broken, + engines={'fake': FakeEngine()}) + assert runner.run().files[0].status == 'ok' + + +def test_runner_without_callbacks(tmp_path, paths): + fasta = write_fasta(tmp_path / 'a.fasta', ALIGNED) + runner = PipelineRunner({'engine': 'fake'}, [fasta], paths, engines={'fake': FakeEngine()}) + assert runner.run().success + + +# --- kill_process_tree -------------------------------------------------------------------- + +def _spawn_tree(): + """A python that starts a sleeping child, once the child is up: [parent, child].""" + script = ('import subprocess, sys, time; ' + f'subprocess.Popen([sys.executable, "-c", {SLEEPER!r}]); time.sleep(60)') + proc = subprocess.Popen([sys.executable, '-c', script]) + parent = psutil.Process(proc.pid) + deadline = time.monotonic() + WAIT + while time.monotonic() < deadline: + children = parent.children(recursive=True) + if children: + return proc, [parent, *children] + time.sleep(0.05) + proc.kill() + raise AssertionError('the child never started') + + +def test_kill_process_tree_kills_children(): + proc, procs = _spawn_tree() + try: + kill_process_tree(proc.pid) + proc.wait(WAIT) + assert [p.pid for p in procs if alive(p)] == [] + finally: + kill_all(procs) + + +def test_kill_process_tree_ignores_a_finished_process(): + proc = subprocess.Popen([sys.executable, '-c', 'pass']) + proc.wait(WAIT) + kill_process_tree(proc.pid) # must not raise + + +def test_kill_process_tree_without_psutil_terminates_the_process(monkeypatch): + monkeypatch.setattr(runner_module, 'psutil', None) + proc = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)']) + try: + kill_process_tree(proc.pid) + proc.wait(WAIT) + kill_process_tree(proc.pid) # gone: ignored + finally: + if proc.poll() is None: + proc.kill() + + +def test_runner_module_imports_no_qt(): + code = ('import sys; import phyml_core.runner; ' + 'sys.exit(1 if any(m.startswith("PySide6") for m in sys.modules) else 0)') + root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + assert subprocess.run([sys.executable, '-c', code], cwd=root, timeout=60).returncode == 0 From b09caf47ace231b2ca215ad5903a9c967aea5ee9 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:00:17 +0300 Subject: [PATCH 27/36] Keep the error of a failure that coincides with a cancel and kill outside the runner lock Co-Authored-By: Claude Opus 5.5 --- phyml_core/runner.py | 17 +++++++----- tests/unit/test_runner.py | 57 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/phyml_core/runner.py b/phyml_core/runner.py index 63446d6..ca48122 100644 --- a/phyml_core/runner.py +++ b/phyml_core/runner.py @@ -128,11 +128,15 @@ def cancel(self) -> None: """Stop the run: no new step starts, the running one loses its process tree.""" self._cancelled.set() try: + # only the snapshot is taken under the lock: the kill can take a while and the + # caller may be the GUI thread. No process starts after this point (_execute + # checks the flag under the lock), and holding `proc` keeps its handle open, so + # on Windows its pid cannot be reused while we kill it. with self._lock: proc = self._proc - # poll() is None: not reaped yet, so the pid still belongs to this process - if proc is not None and proc.poll() is None: - kill_process_tree(proc.pid) + running = proc is not None and proc.poll() is None + if running: + kill_process_tree(proc.pid) except Exception: pass @@ -182,12 +186,11 @@ def _run_file(self, fasta: str) -> FileResult: self._log('system', _t(c.args[0])) return FileResult(fasta, 'cancelled') except Exception as e: - if self._cancelled.is_set(): # most likely caused by the cancel itself - self._log('system', _t('cancel_analysis')) - return FileResult(fasta, 'cancelled') + # whatever a cancel itself causes arrives as _Cancelled; this is a real failure message = e.args[0] if isinstance(e, _FileFailed) else _t('err_crit').format(e) self._log('error', message) - return FileResult(fasta, 'fail', error=message.strip()) + status = 'cancelled' if self._cancelled.is_set() else 'fail' + return FileResult(fasta, status, error=message.strip()) def _process(self, fasta: str) -> Tuple[str, dict]: key = param_text(self._params, 'engine', 'phyml') diff --git a/tests/unit/test_runner.py b/tests/unit/test_runner.py index aac5ded..b04ff38 100644 --- a/tests/unit/test_runner.py +++ b/tests/unit/test_runner.py @@ -415,6 +415,63 @@ def on_status(name, status): assert runner.active_pid is None +def test_error_while_cancelled_keeps_its_message(tmp_path, paths): + """A real failure that coincides with a cancel is 'cancelled' but not silent.""" + fasta = write_fasta(tmp_path / 'a.fasta', ALIGNED) + rec = Recorder() + holder = {} + converted = _t('conv_success').split('{}')[0] + + def on_log(level, text): + rec.on_log(level, text) + if text.startswith(converted): # past the last checkpoint before the engine step + holder['runner'].cancel() + + runner = PipelineRunner({'engine': 'fake', 'fake_mode': 'raise'}, [fasta], paths, + on_log=on_log, on_status=rec.on_status, + engines={'fake': FakeEngine()}) + holder['runner'] = runner + result = runner.run() + file = result.files[0] + assert result.cancelled + assert file.status == 'cancelled' + assert 'boom' in file.error + assert any('boom' in text for text in rec.texts('error')) + + +def test_cancel_does_not_hold_the_lock_while_killing(tmp_path, paths, monkeypatch): + """cancel() kills outside the runner's lock: a slow kill must not block other threads.""" + fasta = write_fasta(tmp_path / 'a.fasta', ALIGNED) + rec = Recorder() + runner = make_runner([fasta], paths, rec, fake_mode='hang') + real_kill = runner_module.kill_process_tree + seen = {} + + def slow_kill(pid): + acquired = runner._lock.acquire(timeout=2) # same thread: fails if cancel holds it + if acquired: + runner._lock.release() + seen['lock_free'] = acquired + time.sleep(0.5) + real_kill(pid) + + monkeypatch.setattr(runner_module, 'kill_process_tree', slow_kill) + thread, box = start(runner) + procs = [] + try: + procs = wait_for_sleeper(runner) + canceller = threading.Thread(target=runner.cancel, daemon=True) + canceller.start() + canceller.join(WAIT) + thread.join(WAIT) + assert not thread.is_alive(), 'run() did not return after cancel()' + assert seen == {'lock_free': True} + assert box['result'].files[0].status == 'cancelled' + finally: + runner.cancel() + kill_all(procs) + + def test_callbacks_that_raise_do_not_escape(tmp_path, paths): fasta = write_fasta(tmp_path / 'a.fasta', ALIGNED) From b9a790a2018dd9674d71161ba8ae295cc229a29b Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:02:56 +0300 Subject: [PATCH 28/36] Move the CSV/XLSX run summary into phyml_core.report Co-Authored-By: Claude Opus 5.5 --- phyml_core/report.py | 54 ++++++++++++++++++++++++++++ tests/unit/test_report.py | 76 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 phyml_core/report.py create mode 100644 tests/unit/test_report.py diff --git a/phyml_core/report.py b/phyml_core/report.py new file mode 100644 index 0000000..dc86cc1 --- /dev/null +++ b/phyml_core/report.py @@ -0,0 +1,54 @@ +"""CSV/XLSX run summary (Qt-free).""" +import csv +import os +from typing import List + +from .i18n import _t +from .runner import FileResult + +SUMMARY_COLUMNS = ('file', 'status', 'model', 'taxa', 'lnL', 'gamma', 'tstv', 'time', 'tree') +_STATUS_TEXT = {'ok': 'OK', 'fail': 'FAIL', 'cancelled': 'CANCELLED'} +_XLSX_WIDTHS = (28, 8, 16, 8, 12, 10, 10, 12) +CSV_NAME = 'phymlgui_summary.csv' +XLSX_NAME = 'phymlgui_summary.xlsx' + + +def summary_rows(files: List[FileResult]) -> List[dict]: + rows = [] + for f in files: + row = {c: (f.stats or {}).get(c) or '' for c in SUMMARY_COLUMNS} + row['file'] = os.path.basename(f.file) + row['status'] = _STATUS_TEXT.get(f.status, str(f.status).upper()) + row['tree'] = f.tree or '' + rows.append(row) + return rows + + +def write_summary(rows: List[dict], out_dir: str) -> List[str]: + """Write the CSV (and the XLSX if openpyxl is installed); return the written paths. + + OSError is propagated to the caller. + """ + csv_path = os.path.join(out_dir, CSV_NAME) + with open(csv_path, 'w', newline='', encoding='utf-8-sig') as f: + wr = csv.DictWriter(f, fieldnames=SUMMARY_COLUMNS, delimiter=';', extrasaction='ignore') + wr.writeheader() + wr.writerows(rows) + written = [csv_path] + try: + from openpyxl import Workbook + except ImportError: + return written + cols = SUMMARY_COLUMNS[:-1] + wb = Workbook() + ws = wb.active + ws.title = 'Summary' + ws.append([_t('sum_col_' + c) for c in cols]) + for r in rows: + ws.append([r.get(c) or '' for c in cols]) + for col, width in zip('ABCDEFGHI', _XLSX_WIDTHS): + ws.column_dimensions[col].width = width + xlsx_path = os.path.join(out_dir, XLSX_NAME) + wb.save(xlsx_path) + written.append(xlsx_path) + return written diff --git a/tests/unit/test_report.py b/tests/unit/test_report.py new file mode 100644 index 0000000..2dcd02b --- /dev/null +++ b/tests/unit/test_report.py @@ -0,0 +1,76 @@ +import os + +import pytest + +from phyml_core import report +from phyml_core.runner import FileResult + + +def _results(): + return [ + FileResult(file=os.path.join('some', 'dir', 'a.fasta'), status='ok', tree='a.tre', + stats={'model': 'GTR', 'taxa': '5', 'lnL': '-100.5', 'gamma': '0.5', + 'tstv': '2.0', 'time': '1s'}), + FileResult(file='b.fasta', status='fail', error='boom'), + FileResult(file='c.fasta', status='cancelled'), + ] + + +def test_rows_from_results(): + rows = report.summary_rows(_results()) + assert [r['status'] for r in rows] == ['OK', 'FAIL', 'CANCELLED'] + assert [r['file'] for r in rows] == ['a.fasta', 'b.fasta', 'c.fasta'] + assert rows[0]['model'] == 'GTR' and rows[0]['lnL'] == '-100.5' + assert rows[0]['tree'] == 'a.tre' + assert rows[1]['model'] == '' and rows[1]['tree'] == '' + assert set(rows[0]) == set(report.SUMMARY_COLUMNS) + + +def test_csv_matches_current_format(tmp_path): + rows = report.summary_rows(_results()) + paths = report.write_summary(rows, str(tmp_path)) + csv_path = tmp_path / 'phymlgui_summary.csv' + assert str(csv_path) in paths + raw = csv_path.read_bytes() + assert raw.startswith(b'\xef\xbb\xbf') + lines = raw.decode('utf-8-sig').splitlines() + assert lines[0] == 'file;status;model;taxa;lnL;gamma;tstv;time;tree' + assert lines[1].startswith('a.fasta;OK;GTR;5;') + assert len(lines) == 4 + + +def test_xlsx_written_when_openpyxl_available(tmp_path): + openpyxl = pytest.importorskip('openpyxl') + rows = report.summary_rows(_results()) + paths = report.write_summary(rows, str(tmp_path)) + xlsx = tmp_path / 'phymlgui_summary.xlsx' + assert str(xlsx) in paths + ws = openpyxl.load_workbook(str(xlsx))['Summary'] + assert ws.max_row == len(rows) + 1 + assert ws.max_column == len(report.SUMMARY_COLUMNS) - 1 + assert ws.column_dimensions['A'].width == 28 + + +def test_csv_only_without_openpyxl(tmp_path, monkeypatch): + import builtins + real = builtins.__import__ + + def fake(name, *a, **k): + if name.startswith('openpyxl'): + raise ImportError(name) + return real(name, *a, **k) + monkeypatch.setattr(builtins, '__import__', fake) + paths = report.write_summary(report.summary_rows(_results()), str(tmp_path)) + assert paths == [str(tmp_path / 'phymlgui_summary.csv')] + + +def test_cyrillic_and_space_folder(tmp_path): + out = tmp_path / 'Мои результаты 1' + out.mkdir() + paths = report.write_summary(report.summary_rows(_results()), str(out)) + assert paths and all(os.path.isfile(p) for p in paths) + + +def test_oserror_propagates(tmp_path): + with pytest.raises(OSError): + report.write_summary([], str(tmp_path / 'missing')) From c9156bc5e3f71759c5eb3265a32817296d24dcb3 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:12:07 +0300 Subject: [PATCH 29/36] Drive the window through PipelineRunner and RunWorker; log levels come from the core Co-Authored-By: Claude Opus 5.5 --- phyml_core/cli.py | 25 ++- phyml_core/pipeline.py | 259 ------------------------- phyml_ui/main_window.py | 264 ++++++++++++-------------- phyml_ui/worker.py | 55 ++++++ phymlgui.py | 1 - tests/e2e/datasets.py | 2 +- tests/e2e/test_batch.py | 24 +-- tests/e2e/test_cpu_monitor.py | 12 +- tests/e2e/test_iqtree.py | 24 +-- tests/e2e/test_phyml_pipeline.py | 15 +- tests/gui/test_log_console.py | 33 +++- tests/gui/test_main_window.py | 178 ++++++++++++++++- tests/gui/test_main_window_storage.py | 48 ++--- tests/gui/test_worker.py | 65 +++++++ 14 files changed, 507 insertions(+), 498 deletions(-) delete mode 100644 phyml_core/pipeline.py create mode 100644 phyml_ui/worker.py create mode 100644 tests/gui/test_worker.py diff --git a/phyml_core/cli.py b/phyml_core/cli.py index 356a433..6e28834 100644 --- a/phyml_core/cli.py +++ b/phyml_core/cli.py @@ -8,8 +8,6 @@ import os import sys -from PySide6.QtCore import QCoreApplication - def main(argv=None): parser = argparse.ArgumentParser(prog='phymlgui', description='phymlGUI headless pipeline') @@ -22,8 +20,9 @@ def main(argv=None): if not args.file and not args.batch: parser.error('specify --file or --batch') - from phyml_core.pipeline import PipelineWorker from phyml_core.i18n import _t + from phyml_core.paths import AppPaths + from phyml_core.runner import PipelineRunner config = {} if os.path.exists(args.config): @@ -40,17 +39,15 @@ def main(argv=None): else: files = [args.file] - app = QCoreApplication.instance() or QCoreApplication([]) - worker = PipelineWorker(config, files) - worker.log_signal.connect(print) - result = {} - worker.finished_signal.connect(lambda ok, tree: result.update(ok=ok, tree=tree)) - worker.run() # synchronous, no event loop needed - - print('OK' if result.get('ok') else 'FAILED') - if result.get('tree'): - print('tree:', result['tree']) - return 0 if result.get('ok') else 1 + runner = PipelineRunner(config, files, AppPaths.detect(), + on_log=lambda level, text: print(text)) + result = runner.run() # synchronous, never raises + trees = [f.tree for f in result.files if f.status == 'ok' and f.tree] + + print('OK' if result.success else 'FAILED') + if trees: + print('tree:', trees[-1]) + return 0 if result.success else 1 if __name__ == '__main__': diff --git a/phyml_core/pipeline.py b/phyml_core/pipeline.py deleted file mode 100644 index e89ca1d..0000000 --- a/phyml_core/pipeline.py +++ /dev/null @@ -1,259 +0,0 @@ -import os -import re -import subprocess - -from PySide6.QtCore import QThread, Signal -from Bio import AlignIO, SeqIO -from Bio.Seq import Seq - -from .engines.iqtree import parse_iqtree_stats, ufboot_replicates -from .engines.phyml import parse_phyml_stats -from .i18n import _t - - -# --- Background Worker Thread --- -class PipelineWorker(QThread): - log_signal = Signal(str) - file_status_signal = Signal(str, str) # (file name, pending|running|ok|fail) - finished_signal = Signal(bool, str) - - def __init__(self, config, fasta_files): - super().__init__() - self.config = config - if isinstance(fasta_files, str): - fasta_files = [fasta_files] - self.fasta_files = list(fasta_files) - self.active_process = None - self._is_cancelled = False - self.results = [] # one row per input file, for the CSV summary - - def run(self): - n_ok, last_tree = 0, "" - batch = len(self.fasta_files) > 1 - if batch: - self.log_signal.emit(_t("batch_found").format(len(self.fasta_files))) - for i, fasta in enumerate(self.fasta_files, 1): - if self._is_cancelled: - break - self.file_status_signal.emit(os.path.basename(fasta), 'pending') - for i, fasta in enumerate(self.fasta_files, 1): - if self._is_cancelled: - break - if batch: - self.log_signal.emit(_t("batch_file_header").format(i, len(self.fasta_files), os.path.basename(fasta))) - self.file_status_signal.emit(os.path.basename(fasta), 'running') - ok, tree, stats = self._process_one(fasta) - self.file_status_signal.emit(os.path.basename(fasta), 'ok' if ok else 'fail') - row = {'file': os.path.basename(fasta), 'status': 'OK' if ok else 'FAIL', 'tree': tree if ok else ''} - row.update({k: stats.get(k, '') for k in ('model', 'taxa', 'lnL', 'gamma', 'tstv', 'time')}) - self.results.append(row) - if ok: - n_ok += 1 - last_tree = tree - if self._is_cancelled: - break - if batch and not self._is_cancelled: - self.log_signal.emit(_t("batch_summary").format(n_ok, len(self.fasta_files))) - success = n_ok == len(self.fasta_files) and n_ok > 0 and not self._is_cancelled - self.finished_signal.emit(success, last_tree) - - def _process_one(self, current_fasta): - """Full pipeline for a single FASTA file: returns (success, tree_path, stats).""" - creation_flags = 0 - if os.name == 'nt': - creation_flags = subprocess.CREATE_NO_WINDOW - - try: - self.log_signal.emit(_t("init_analysis").format(current_fasta)) - - # 1. Alignment check - if not self.is_aligned(current_fasta): - self.log_signal.emit(_t("diff_len")) - muscle_exe = self.config.get("muscle_path", "") - - if not muscle_exe or not os.path.exists(muscle_exe): - self.log_signal.emit(_t("err_muscle")) - return False, "", {} - - aligned_fasta = current_fasta.rsplit('.', 1)[0] + '_aligned.fasta' - if os.path.exists(aligned_fasta): - try: - os.remove(aligned_fasta) - self.log_signal.emit(_t("del_old")) - except Exception as e: - self.log_signal.emit(_t("err_del").format(e)) - return False, "", {} - - cmd_muscle = [muscle_exe, "-align", current_fasta, "-output", aligned_fasta] - self.active_process = subprocess.Popen(cmd_muscle, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, creationflags=creation_flags) - for line in iter(self.active_process.stdout.readline, ''): - if self._is_cancelled: break - self.active_process.stdout.close() - self.active_process.wait() - - if self.active_process.returncode != 0 and not self._is_cancelled: - cmd_muscle_v3 = [muscle_exe, "-in", current_fasta, "-out", aligned_fasta] - self.active_process = subprocess.Popen(cmd_muscle_v3, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, creationflags=creation_flags) - for line in iter(self.active_process.stdout.readline, ''): - if self._is_cancelled: break - self.active_process.stdout.close() - self.active_process.wait() - - if self._is_cancelled: - self.log_signal.emit(_t("cancel_align")) - return False, "", {} - - if self.active_process.returncode == 0 and os.path.exists(aligned_fasta): - self.log_signal.emit(_t("align_success").format(aligned_fasta)) - current_fasta = aligned_fasta - else: - self.log_signal.emit(_t("err_muscle_fail")) - return False, "", {} - else: - self.log_signal.emit(_t("seq_aligned")) - - # 2. Conversion (FASTA -> PHYLIP) - self.log_signal.emit(_t("cleaning")) - phylip_path = current_fasta.rsplit('.', 1)[0] + '.phy' - try: - alignment = AlignIO.read(current_fasta, "fasta") - for record in alignment: - record.seq = Seq(str(record.seq).replace('.', '-')) - clean_id = re.sub(r'[.|:;(),\s]', '_', record.id) - record.id = clean_id - record.name = clean_id - record.description = "" - AlignIO.write(alignment, phylip_path, "phylip-relaxed") - self.log_signal.emit(_t("conv_success").format(phylip_path)) - except Exception as e: - self.log_signal.emit(_t("err_conv").format(e)); return False, "", {} - - if self._is_cancelled: return False, "", {} - - # 3. Run the configured engine - engine = self.config.get('engine', 'phyml') - if engine == 'iqtree': - return self._run_iqtree(phylip_path, creation_flags) - return self._run_phyml(phylip_path, creation_flags) - - except Exception as e: - self.log_signal.emit(_t("err_crit").format(str(e))) - return False, "", {} - - def _run_phyml(self, phylip_path, creation_flags): - try: - phyml_exe = self.config.get("phyml_path", "") - if not phyml_exe or not os.path.exists(phyml_exe): - self.log_signal.emit(_t("err_phyml")); return False, "", {} - - cmd_phyml = [] - if self.config.get("use_mpi", False): - mpirun_exe = self.config.get("mpirun_path", "mpiexec") - np = self.config.get("mpi_np", "2") - cmd_phyml.extend([mpirun_exe, "-np", np]) - - cmd_phyml.append(phyml_exe) - cmd_phyml.extend(['-i', phylip_path]) - cmd_phyml.extend(['-d', self.config.get("datatype", "nt")]) - cmd_phyml.extend(['-m', self.config.get("model", "HKY85")]) - - support_method = self.config.get("support_method", "bootstrap") - if support_method == "bootstrap": - cmd_phyml.extend(['-b', self.config.get("bootstrap_replicates", "100")]) - else: - cmd_phyml.extend(['-b', support_method]) - - if self.config.get("freq") != "default": cmd_phyml.extend(['-f', self.config.get("freq")]) - if self.config.get("tstv"): cmd_phyml.extend(['-t', self.config.get("tstv")]) - if self.config.get("pinv"): cmd_phyml.extend(['-v', self.config.get("pinv")]) - if self.config.get("classes"): cmd_phyml.extend(['-c', self.config.get("classes")]) - if self.config.get("gamma"): cmd_phyml.extend(['-a', self.config.get("gamma")]) - cmd_phyml.extend(['-s', self.config.get("search", "NNI")]) - cmd_phyml.extend(['-o', self.config.get("optimize", "tlr")]) - if self.config.get("parsimony"): cmd_phyml.append('-p') - - self.log_signal.emit(_t("run_phyml").format(' '.join(cmd_phyml))) - self.active_process = subprocess.Popen(cmd_phyml, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, creationflags=creation_flags) - - for line in iter(self.active_process.stdout.readline, ''): - if line: self.log_signal.emit(line.strip()) - if self._is_cancelled: break - - self.active_process.stdout.close() - self.active_process.wait() - - if self._is_cancelled: - self.log_signal.emit(_t("cancel_analysis")) - return False, "", {} - elif self.active_process.returncode != 0: - self.log_signal.emit(_t("err_crit_phyml").format(self.active_process.returncode)) - return False, "", {} - else: - self.log_signal.emit(_t("success_pipe")) - tree_path = f"{phylip_path}_phyml_tree.txt" - stats = parse_phyml_stats(tree_path.replace('_phyml_tree.txt', '_phyml_stats.txt')) - self.log_signal.emit(_t("stats_summary").format( - *(stats.get(k) or "n/a" for k in ('model', 'taxa', 'lnL', 'gamma', 'tstv', 'time')))) - return True, tree_path, stats - - except Exception as e: - self.log_signal.emit(_t("err_crit").format(str(e))) - return False, "", {} - - def _run_iqtree(self, phylip_path, creation_flags): - try: - iqtree_exe = self.config.get("iqtree_path", "") - if not iqtree_exe or not os.path.exists(iqtree_exe): - self.log_signal.emit(_t("err_iqtree")); return False, "", {} - - prefix = phylip_path.rsplit('.', 1)[0] + '_iqtree' - cmd_iqtree = [iqtree_exe, '-s', phylip_path, '--prefix', prefix, - '-redo', '-m', self.config.get("model", "MFP"), - '-T', self.config.get("mpi_np", "2")] - support_method = self.config.get("support_method", "bootstrap") - if support_method == "bootstrap": - requested = self.config.get("bootstrap_replicates", "1000") - replicates = ufboot_replicates(requested) - if replicates != str(requested).strip(): - self.log_signal.emit(_t("iqtree_ufboot_min").format(replicates, requested)) - cmd_iqtree.extend(['-B', replicates]) - elif support_method.startswith('-'): - cmd_iqtree.extend(['--alrt', '1000']) # aLRT branch support - - self.log_signal.emit(_t("run_phyml").format(' '.join(cmd_iqtree))) - self.active_process = subprocess.Popen(cmd_iqtree, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, text=True, - creationflags=creation_flags) - for line in iter(self.active_process.stdout.readline, ''): - if line: self.log_signal.emit(line.strip()) - if self._is_cancelled: break - self.active_process.stdout.close() - self.active_process.wait() - - if self._is_cancelled: - self.log_signal.emit(_t("cancel_analysis")) - return False, "", {} - tree_path = prefix + '.treefile' - if self.active_process.returncode != 0 or not os.path.exists(tree_path): - self.log_signal.emit(_t("err_crit_phyml").format(self.active_process.returncode)) - return False, "", {} - self.log_signal.emit(_t("success_pipe")) - stats = parse_iqtree_stats(prefix + '.iqtree') - self.log_signal.emit(_t("stats_summary").format( - *(stats.get(k) or "n/a" for k in ('model', 'taxa', 'lnL', 'gamma', 'tstv', 'time')))) - return True, tree_path, stats - except Exception as e: - self.log_signal.emit(_t("err_crit").format(str(e))) - return False, "", {} - - def cancel(self): - self._is_cancelled = True - if self.active_process and self.active_process.poll() is None: - try: self.active_process.terminate() - except Exception: self.active_process.kill() - - def is_aligned(self, file_path): - records = list(SeqIO.parse(file_path, "fasta")) - if not records: raise ValueError(_t("err_empty")) - first_len = len(records[0].seq) - return all(len(rec.seq) == first_len for rec in records) diff --git a/phyml_ui/main_window.py b/phyml_ui/main_window.py index cb5e379..7efbb0a 100644 --- a/phyml_ui/main_window.py +++ b/phyml_ui/main_window.py @@ -1,7 +1,7 @@ import copy -import csv import os import shutil +from dataclasses import asdict from datetime import datetime from html import escape @@ -17,28 +17,30 @@ from PySide6.QtCore import QTimer, Qt from PySide6.QtGui import QIcon, QFont, QShortcut, QKeySequence -from phyml_core import __version__, alignment, presets, settings +from phyml_core import __version__, alignment, presets, report, settings from phyml_core.engines import ENGINES +from phyml_core.engines.base import format_issue from phyml_core.history import HistoryStore, best_tree, display_input, new_record from phyml_core.i18n import _t from phyml_core.paths import AppPaths from phyml_core.utils import psutil, detect_cores, is_mpi_binary, resource_path -from phyml_core.pipeline import PipelineWorker from phyml_ui.tree_viewer import TreeViewerWidget from phyml_ui.themes import resolve, build_qss, role_colors, ACCENTS, PALETTES +from phyml_ui.worker import RunWorker -# per-file values of a worker row that go into the history -_STAT_KEYS = ('model', 'taxa', 'lnL', 'gamma', 'tstv', 'time') +# queue marker per file status of the runner +_STATUS_MARKERS = {'pending': '…', 'running': '▶', 'ok': '✓', 'fail': '✗', + 'cancelled': '⊘'} # settings that say nothing about how a run was made: left out of a history record _NOT_RUN_PARAMS = ('presets', 'recent', 'window') class LogConsole(QWidget): - """Structured log: colored roles, severity filter, substring search. + """Structured log: colored levels, severity filter, substring search. - Keeps the QTextEdit-style API used by the pipeline (append/clear) and - by tests (toPlainText); entries are always stored in full, filtering - only affects what is displayed. + Every entry carries a level from phyml_core.runner.LEVELS ('error', 'success', + 'system', 'out'), given by whoever writes it. Entries are always stored in full; + filtering only affects what is displayed. """ def __init__(self, parent=None): @@ -69,22 +71,13 @@ def __init__(self, parent=None): self.view.setFont(QFont('Consolas', 9)) layout.addWidget(self.view) - @staticmethod - def _classify(text): - s = text.strip() - if ('[ERROR]' in s or '[ОШИБКА]' in s or '[CRITICAL' in s - or '[КРИТИЧ' in s or s.startswith('[!')): - return 'error' - if s.startswith('[+]') or 'SUCCESSFULLY' in s or 'УСПЕШНО' in s: - return 'success' - if s.startswith('['): - return 'system' - return 'out' - - def append(self, text): - self._entries.append((self._classify(text), text)) - if self._passes(*self._entries[-1]): - self._append_html(text, self._entries[-1][0]) + def append(self, level: str, text: str) -> None: + """Add an entry; a level this console has no color for is shown as 'out'.""" + if level not in self._role_colors: + level = 'out' + self._entries.append((level, text)) + if self._passes(level, text): + self._append_html(text, level) def clear(self): self._entries.clear() @@ -210,9 +203,10 @@ def accept(self): # --- Main Application Window --- class PhyMLApp(QWidget): - def __init__(self, paths=None): + def __init__(self, paths=None, engines=None): super().__init__() self.paths = paths if paths is not None else AppPaths.detect() + self.engines = ENGINES if engines is None else engines # tests pass fakes self.history_store = HistoryStore(self.paths.history_file) self._is_loading = True self._autosave_blocked = False # set when the settings file could not be read @@ -232,7 +226,11 @@ def __init__(self, paths=None): self._user_presets = {} self._failed_files = [] self._run_started = None + self._run_config = None # the config the current/last run was started with + self._run_inputs = [] # what that run was asked to process (history 'inputs') + self._run_out_dir = None # where its summary goes self._last_out_dir = None + self.last_result = None # RunResult of the last finished run self.phys_cores, self.logi_cores = detect_cores() self.initUI() self.load_config() @@ -544,8 +542,8 @@ def toggle_bootstrap_input(self): def update_models(self): self.cb_model.clear() - # an unknown engine in a hand-edited config gets PhyML's list, as the pipeline runs PhyML for it - engine = ENGINES.get(self.engine_setting, ENGINES['phyml']) + # an unknown engine in a hand-edited config gets PhyML's list, as a run uses PhyML for it + engine = self.engines.get(self.engine_setting) or ENGINES['phyml'] for m in engine.models(self.cb_datatype.currentData()): self.cb_model.addItem(m, m) if not self._is_loading: self.auto_save() @@ -623,9 +621,9 @@ def get_current_config(self): } } - def _log(self, text): - """A system line for the log; messages from the core modules may end with a newline.""" - self.log_console.append(text.rstrip('\n')) + def _log(self, level, text): + """A log line at the given level; messages from the core modules may end with a newline.""" + self.log_console.append(level, text.rstrip('\n')) def auto_save(self): if self._autosave_blocked: @@ -638,7 +636,7 @@ def auto_save(self): self._save_failed = False except OSError as e: if not self._save_failed: # every field change saves: report a failure once - self._log(_t('err_settings_save').format(e)) + self._log('error', _t('err_settings_save').format(e)) self._save_failed = True def _migrate_legacy_files(self): @@ -647,14 +645,14 @@ def _migrate_legacy_files(self): cwd = os.getcwd() copied = settings.migrate_legacy(self.paths, cwd) except OSError as e: - self._log(_t('err_load').format(e)) + self._log('error', _t('err_load').format(e)) if not os.path.exists(self.paths.config_file): # Saving the defaults now would create the config that closes the migration # gate, and the legacy settings would never be copied. self._autosave_blocked = True return for name in copied: - self._log(_t('settings_migrated').format( + self._log('system', _t('settings_migrated').format( os.path.join(cwd, name), os.path.join(self.paths.data_dir, name))) def load_config(self): @@ -670,14 +668,14 @@ def load_config(self): problems.append(_t('err_load').format(e)) raw = copy.deepcopy(settings.DEFAULTS) for message in problems: - self._log(message) + self._log('error', message) self._extra_settings = {k: v for k, v in raw.items() if k not in settings.DEFAULTS} try: self._apply_config(settings.from_disk(raw, self.paths)) if not problems and os.path.isfile(self.paths.config_file): - self._log(_t("sys_load")) + self._log('system', _t("sys_load")) except Exception as e: - self._log(_t("err_load").format(e)) + self._log('error', _t("err_load").format(e)) # lists must be populated even without a config file self._refresh_presets() self._refresh_history() @@ -744,19 +742,29 @@ def start_pipeline(self, files_override=None): if not files: QMessageBox.warning(self, _t("err_box_title"), _t("batch_empty")); return files = [os.path.join(target, f) for f in files] + out_dir = target else: files = [target] + out_dir = os.path.dirname(os.path.abspath(target)) + inputs = [target] else: files = list(files_override) - if self.engine_setting == 'iqtree': - if not self.iqtree_path_input.text() or not os.path.exists(self.iqtree_path_input.text()): - QMessageBox.warning(self, _t("err_box_title"), _t("err_iqtree")); return - elif (is_mpi_binary(self.phyml_path_input.text()) - and not self.chk_use_mpi.isChecked() - and self.cb_support_method.currentData() == "bootstrap"): - QMessageBox.warning(self, _t("mpi_boot_title"), _t("mpi_boot_msg")); return + if not files: + return + target = os.path.dirname(files[0]) + out_dir = os.path.dirname(os.path.abspath(files[0])) + inputs = list(files) + config = self.get_current_config() + if config.get('engine') not in self.engines: + config['engine'] = 'phyml' # a hand-edited unknown engine: PhyML, as its model list + errors = [issue for issue in self.engines[config['engine']].validate(config, self.paths) + if issue.level == 'error'] + if errors: + QMessageBox.warning(self, _t('err_box_title'), + '\n'.join(format_issue(issue) for issue in errors)) + return self._batch_mode = len(files) > 1 - self._remember_recent(target if files_override is None else os.path.dirname(files[0])) + self._remember_recent(target) if not self._is_loading: self.auto_save() self.btn_run.setEnabled(False); self.btn_cancel.setEnabled(True); self.log_console.clear() self.result_tabs.setCurrentWidget(self.log_console) @@ -766,42 +774,52 @@ def start_pipeline(self, files_override=None): self._cpu_samples = [] self._mon_procs = {} self._run_started = datetime.now() - config = self.get_current_config() - self.worker = PipelineWorker(config, files) - self.worker.log_signal.connect(self.log_console.append) - self.worker.file_status_signal.connect(self._on_file_status) - self.worker.finished_signal.connect(self.pipeline_finished) + self._run_config = config + self._run_inputs = inputs + self._run_out_dir = out_dir + self.worker = RunWorker.for_run(config, files, self.paths, engines=self.engines) + self.worker.log.connect(self._log) + self.worker.status.connect(self._on_file_status) + self.worker.done.connect(self.pipeline_finished) self.worker.start() def cancel_pipeline(self): if self.worker and self.worker.isRunning(): - self.worker.cancel() + self.worker.cancel() # returns at once; the kill runs on its own thread self.btn_cancel.setEnabled(False) - self.log_console.append(_t("sys_interrupt")) + self._log('error', _t("sys_interrupt")) - def pipeline_finished(self, success, tree_path): + def pipeline_finished(self, result): + """Show a finished run (a RunResult): tables, summary files, history, retry, tree.""" + self.last_result = result self.btn_run.setEnabled(True); self.btn_cancel.setEnabled(False) avg = sum(self._cpu_samples) / len(self._cpu_samples) if self._cpu_samples else 0.0 self.cpu_run_label.setText(_t("cpu_analysis_done").format(int(avg), round(avg / 100, 1))) - if self.worker is not None and getattr(self.worker, 'results', None): - self._populate_stats() - self._populate_summary() - self._save_summary_csv() - n_ok = sum(1 for r in getattr(self.worker, 'results', []) if r['status'] == 'OK') - n_all = len(getattr(self.worker, 'results', [])) - self._append_history(n_ok, n_all, tree_path) - self._failed_files = [f for f, r in zip(getattr(self.worker, 'fasta_files', []), - getattr(self.worker, 'results', [])) - if r.get('status') == 'FAIL'] + if result.files: + rows = report.summary_rows(result.files) + self._populate_stats(result) + self._populate_summary(rows) + self._write_summary(rows) + self._append_history(result) + self._failed_files = [f.file for f in result.files if f.status == 'fail'] self.retry_btn.setVisible(bool(self._failed_files)) - target = self.fasta_path_input.text() or self.fasta_file - if target and os.path.isdir(target): - self._last_out_dir = target - elif getattr(self.worker, 'fasta_files', None): - self._last_out_dir = os.path.dirname(os.path.abspath(self.worker.fasta_files[0])) + self._last_out_dir = self._run_out_dir self.open_dir_btn.setEnabled(self._last_out_dir is not None) - if success and tree_path and os.path.exists(tree_path) and self.chk_show_tree.isChecked(): - self.tree_view.load_tree(tree_path) + if result.success and self.chk_show_tree.isChecked(): + tree = result.files[-1].tree + if tree and os.path.exists(tree): + self.tree_view.load_tree(tree) + + def _write_summary(self, rows): + """phymlgui_summary.csv (and .xlsx) next to the inputs; a failure is only logged.""" + try: + written = report.write_summary(rows, self._run_out_dir) + except Exception as e: # OSError, and whatever openpyxl raises after the CSV + self._log('error', _t("err_summary_save").format(e)) + return + for path in written: + key = 'xlsx_saved' if path.lower().endswith('.xlsx') else 'summary_saved' + self._log('success', _t(key).format(path)) # ---------- queue ---------- @@ -811,7 +829,7 @@ def _populate_queue(self, files): self.queue_list.addItem(os.path.basename(f) + ' …') def _on_file_status(self, name, status): - marker = {'pending': '…', 'running': '▶', 'ok': '✓', 'fail': '✗'}.get(status, '?') + marker = _STATUS_MARKERS.get(status, '?') for i in range(self.queue_list.count()): item = self.queue_list.item(i) if item.text().split(' ')[0] == name: @@ -828,43 +846,30 @@ def _open_output_dir(self): # ---------- history ---------- - def _append_history(self, n_ok, n_all, tree_path): - """Add the finished run to the history, as the worker's rows describe it. - - The counts and the best tree follow from those rows, so n_ok, n_all and tree_path - only keep the signature of the call. - """ - worker = self.worker - fasta_files = list(getattr(worker, 'fasta_files', None) or []) - rows = list(getattr(worker, 'results', None) or []) - config = getattr(worker, 'config', None) # what the run used, not what the form shows now - if not isinstance(config, dict): - config = self.get_current_config() + def _append_history(self, result): + """Add a finished run (a RunResult) to the history: one entry per file, with the + config the run was started with, not what the form shows now.""" + config = self._run_config if isinstance(self._run_config, dict) else self.get_current_config() params = {k: v for k, v in config.items() if k not in _NOT_RUN_PARAMS} - files = [{'file': path, - 'status': 'ok' if row.get('status') == 'OK' else 'fail', - 'tree': row.get('tree', ''), - 'stats': {k: row.get(k, '') for k in _STAT_KEYS}, - 'error': row.get('error', '')} - for path, row in zip(fasta_files, rows)] - target = self.fasta_path_input.text() or self.fasta_file or (fasta_files[0] if fasta_files else '') duration = (datetime.now() - self._run_started).total_seconds() if self._run_started else 0.0 - rec = new_record(inputs=[target] if target else [], + rec = new_record(inputs=self._run_inputs, engine=params.get('engine') or self.engine_setting, - params=params, files=files, duration_s=duration) + params=params, files=[asdict(f) for f in result.files], + duration_s=duration) try: self.history_store.append(rec) except OSError as e: - self._log(_t('err_settings_save').format(e)) + self._log('error', _t('err_settings_save').format(e)) self._refresh_history() def _refresh_history(self): self.history_list.clear() try: - self._history_records = self.history_store.load(on_error=self._log)[:50] + self._history_records = self.history_store.load( + on_error=lambda message: self._log('error', message))[:50] except OSError as e: self._history_records = [] - self._log(_t('err_load').format(e)) + self._log('error', _t('err_load').format(e)) for rec in self._history_records: self.history_list.addItem(_t('hist_entry').format( rec.get('ts', ''), display_input(rec), rec.get('ok', 0), rec.get('n', 0))) @@ -877,7 +882,7 @@ def _open_history_entry(self, item): if tree and os.path.exists(tree): self.tree_view.load_tree(tree) else: - self.log_console.append(_t('hist_no_tree')) + self._log('error', _t('hist_no_tree')) # ---------- presets ---------- @@ -903,7 +908,7 @@ def _preset_activated(self, item): kind, key = data preset = presets.BUILTIN[key] if kind == 'builtin' else self._user_presets.get(key, {}) self._apply_preset(preset) - self.log_console.append(_t('preset_applied').format(item.text())) + self._log('success', _t('preset_applied').format(item.text())) def _apply_preset(self, preset): params = presets.apply(preset, self.get_current_config()) @@ -924,7 +929,7 @@ def _save_preset(self): self._user_presets[name] = presets.extract(self.get_current_config()) self.auto_save() self._refresh_presets() - self.log_console.append(_t('preset_saved').format(name)) + self._log('success', _t('preset_saved').format(name)) def _preset_menu(self, pos): item = self.presets_list.itemAt(pos) @@ -940,70 +945,38 @@ def _preset_menu(self, pos): self.auto_save() self._refresh_presets() - def _populate_stats(self): - rows = [r for r in self.worker.results if r['status'] == 'OK'] - if not rows: + def _populate_stats(self, result): + """The statistics of the last file that finished 'ok'.""" + done = [f for f in result.files if f.status == 'ok'] + if not done: return - r = rows[-1] - self.stats_file_label.setText(_t("stats_field_file") + ': ' + r['file']) + last = done[-1] + self.stats_file_label.setText(_t("stats_field_file") + ': ' + os.path.basename(last.file)) fields = [('stats_field_model', 'model'), ('stats_field_taxa', 'taxa'), ('stats_field_lnl', 'lnL'), ('stats_field_gamma', 'gamma'), ('stats_field_tstv', 'tstv'), ('stats_field_time', 'time')] self.stats_table.setRowCount(len(fields)) for i, (key, col) in enumerate(fields): self.stats_table.setItem(i, 0, QTableWidgetItem(_t(key))) - self.stats_table.setItem(i, 1, QTableWidgetItem(r.get(col) or 'n/a')) + self.stats_table.setItem(i, 1, QTableWidgetItem((last.stats or {}).get(col) or 'n/a')) - def _populate_summary(self): + def _populate_summary(self, rows): cols = ['file', 'status', 'model', 'taxa', 'lnL', 'gamma', 'tstv', 'time'] heads = [_t("sum_col_" + c) for c in cols] self.summary_table.setColumnCount(len(cols)) self.summary_table.setHorizontalHeaderLabels(heads) - self.summary_table.setRowCount(len(self.worker.results)) - for i, r in enumerate(self.worker.results): + self.summary_table.setRowCount(len(rows)) + for i, r in enumerate(rows): for j, c in enumerate(cols): self.summary_table.setItem(i, j, QTableWidgetItem(r.get(c) or '')) - def _save_summary_csv(self): - """Write per-file results (status, model, lnL, ...) to phymlgui_summary.csv.""" - target = self.fasta_path_input.text() or self.fasta_file - out_dir = target if os.path.isdir(target) else os.path.dirname(os.path.abspath(target)) - out_path = os.path.join(out_dir, 'phymlgui_summary.csv') - fields = ['file', 'status', 'model', 'taxa', 'lnL', 'gamma', 'tstv', 'time', 'tree'] - try: - with open(out_path, 'w', newline='', encoding='utf-8-sig') as f: - wr = csv.DictWriter(f, fieldnames=fields, delimiter=';', extrasaction='ignore') - wr.writeheader() - wr.writerows(self.worker.results) - self.log_console.append(_t("summary_saved").format(out_path)) - except OSError as e: - self.log_console.append(_t("err_summary_save").format(e)) - # optional Excel workbook with the same data (needs openpyxl) - try: - from openpyxl import Workbook - wb = Workbook() - ws = wb.active - ws.title = 'Summary' - ws.append([_t("sum_col_" + c) for c in fields[:-1]]) - for r in self.worker.results: - ws.append([r.get(c) or '' for c in fields[:-1]]) - for col, width in zip('ABCDEFGHI', (28, 8, 16, 8, 12, 10, 10, 12)): - ws.column_dimensions[col].width = width - xlsx_path = out_path[:-4] + '.xlsx' - wb.save(xlsx_path) - self.log_console.append(_t("xlsx_saved").format(xlsx_path)) - except ImportError: - pass - except Exception as e: - self.log_console.append(_t("err_summary_save").format(e)) - def on_phyml_path_changed(self, path): if is_mpi_binary(path): if not self.chk_use_mpi.isChecked() and self._mpi_auto_path != path: self._mpi_auto_path = path self.chk_use_mpi.setChecked(True) if not self._is_loading: - self.log_console.append(_t("mpi_detected")) + self._log('system', _t("mpi_detected")) else: self._mpi_auto_path = "" @@ -1013,12 +986,11 @@ def update_cpu_status(self): sys_pct = psutil.cpu_percent(interval=None) self.cpu_sys_bar.setValue(int(sys_pct)) self.cpu_sys_label.setText(_t("cpu_sys").format(int(sys_pct))) - proc = self.worker.active_process if self.worker is not None else None - if (self.worker is not None and self.worker.isRunning() - and proc is not None and proc.poll() is None): + pid = self.worker.active_pid if self.worker is not None else None + if pid is not None and self.worker.isRunning(): try: - root = psutil.Process(proc.pid) - pids = {proc.pid} | {c.pid for c in root.children(recursive=True)} + root = psutil.Process(pid) + pids = {pid} | {c.pid for c in root.children(recursive=True)} except psutil.Error: return # cpu_percent() keeps per-call state inside each Process object, diff --git a/phyml_ui/worker.py b/phyml_ui/worker.py new file mode 100644 index 0000000..74f3223 --- /dev/null +++ b/phyml_ui/worker.py @@ -0,0 +1,55 @@ +"""RunWorker: a PipelineRunner on a QThread, its callbacks turned into Qt signals.""" +import threading +from typing import List, Mapping, Optional + +from PySide6.QtCore import QThread, Signal + +from phyml_core.engines.base import Engine +from phyml_core.paths import AppPaths +from phyml_core.runner import PipelineRunner + + +class RunWorker(QThread): + """Runs one PipelineRunner. The signals are emitted on the worker thread, so slots of + objects living in the GUI thread get them queued. + + log(level, text) and status(file name, status) carry the runner's callbacks when the + runner was built by for_run(); done(RunResult) is emitted once run() is over. + """ + log = Signal(str, str) + status = Signal(str, str) + done = Signal(object) + + def __init__(self, runner: PipelineRunner): + super().__init__() + self._runner = runner + + @classmethod + def for_run(cls, params: dict, files: List[str], paths: AppPaths, + engines: Optional[Mapping[str, Engine]] = None) -> 'RunWorker': + """A worker whose runner reports through the log and status signals.""" + worker = None + + def on_log(level: str, text: str) -> None: + worker.log.emit(level, text) + + def on_status(name: str, status: str) -> None: + worker.status.emit(name, status) + + runner = PipelineRunner(params, files, paths, on_log=on_log, on_status=on_status, + engines=engines) + worker = cls(runner) + return worker + + @property + def active_pid(self) -> Optional[int]: + """PID of the program the current step runs, else None.""" + return self._runner.active_pid + + def run(self) -> None: + self.done.emit(self._runner.run()) # run() never raises + + def cancel(self) -> None: + """Ask the runner to stop and return at once: killing the process tree can take + seconds, which must not be spent on the GUI thread.""" + threading.Thread(target=self._runner.cancel, name='run-cancel', daemon=True).start() diff --git a/phymlgui.py b/phymlgui.py index b821b91..20eb128 100644 --- a/phymlgui.py +++ b/phymlgui.py @@ -5,7 +5,6 @@ from phyml_core import __version__ # noqa: F401 from phyml_core.i18n import _t, LANG, STRINGS # noqa: F401 -from phyml_core.pipeline import PipelineWorker # noqa: F401 from phyml_core.utils import psutil, detect_cores, is_mpi_binary, resource_path # noqa: F401 from phyml_ui.main_window import PhyMLApp # noqa: F401 from phyml_ui.tree_viewer import TreeViewerDialog # noqa: F401 diff --git a/tests/e2e/datasets.py b/tests/e2e/datasets.py index 77a0a58..8fd7dfd 100644 --- a/tests/e2e/datasets.py +++ b/tests/e2e/datasets.py @@ -54,7 +54,7 @@ def synthetic(tmp_path): def phyml_config(binaries, **overrides): - """PipelineWorker config for a small PhyML run (MPI x2, HKY85, 20 bootstraps).""" + """PipelineRunner params for a small PhyML run (MPI x2, HKY85, 20 bootstraps).""" config = { 'phyml_path': binaries['phyml'], 'muscle_path': binaries['muscle'], 'mpirun_path': binaries['mpiexec'], 'mpi_np': '2', 'use_mpi': True, diff --git a/tests/e2e/test_batch.py b/tests/e2e/test_batch.py index 71eb962..c00114e 100644 --- a/tests/e2e/test_batch.py +++ b/tests/e2e/test_batch.py @@ -6,7 +6,8 @@ from datasets import phyml_config, synthetic # noqa: F401 (fixture) from phyml_core.history import HistoryStore -from phyml_core.pipeline import PipelineWorker +from phyml_core.paths import AppPaths +from phyml_core.runner import PipelineRunner pytestmark = [pytest.mark.needs_phyml, pytest.mark.needs_mpi, pytest.mark.needs_muscle] @@ -41,21 +42,16 @@ def gui_batch(window, warned, qtbot, binaries, batch_dir, synthetic): return window, batch_dir -def test_two_file_batch(qapp, qtbot, binaries, batch_dir): - """Batch of two files through the real worker (MPI + bootstrap).""" +def test_two_file_batch(tmp_path, binaries, batch_dir): + """Batch of two files through the real runner (MPI + bootstrap).""" files = [str(batch_dir / 'batchA.fasta'), str(batch_dir / 'batchB.fasta')] - done, logs = [], [] - worker = PipelineWorker(phyml_config(binaries), files) - worker.log_signal.connect(logs.append) - worker.finished_signal.connect(lambda ok, tree: done.append((ok, tree))) - worker.start() - try: - qtbot.waitUntil(lambda: bool(done), timeout=180_000) - finally: - worker.cancel() - worker.wait() + logs = [] + paths = AppPaths(app_dir=str(tmp_path / 'app'), data_dir=str(tmp_path / 'data')) + runner = PipelineRunner(phyml_config(binaries), files, paths, + on_log=lambda level, text: logs.append(text)) + result = runner.run() - assert done[0][0], 'batch failed:\n' + '\n'.join(logs[-10:]) + assert result.success, 'batch failed:\n' + '\n'.join(logs[-10:]) assert any('===' in line for line in logs), 'per-file headers missing' assert any('2/2' in line for line in logs), 'batch summary missing' assert (batch_dir / 'batchA_aligned.fasta').exists(), 'batchA not aligned' diff --git a/tests/e2e/test_cpu_monitor.py b/tests/e2e/test_cpu_monitor.py index c343e06..8399295 100644 --- a/tests/e2e/test_cpu_monitor.py +++ b/tests/e2e/test_cpu_monitor.py @@ -5,7 +5,8 @@ import pytest from datasets import phyml_config, synthetic # noqa: F401 (fixture) -from phyml_core.pipeline import PipelineWorker +from phyml_core.runner import PipelineRunner +from phyml_ui.worker import RunWorker pytestmark = [pytest.mark.needs_phyml, pytest.mark.needs_mpi, pytest.mark.needs_muscle] @@ -18,11 +19,12 @@ def test_cpu_samples_during_mpi_run(window, qapp, tmp_path, monkeypatch, binarie monkeypatch.chdir(run_dir) window._cpu_samples = [] - worker = PipelineWorker(phyml_config(binaries), str(fasta)) - window.worker = worker done, wlogs = [], [] - worker.log_signal.connect(wlogs.append) - worker.finished_signal.connect(lambda ok, tree: done.append((ok, tree))) + runner = PipelineRunner(phyml_config(binaries), str(fasta), window.paths, + on_log=lambda level, text: wlogs.append(text)) + worker = RunWorker(runner) # the window's CPU monitor asks a RunWorker for active_pid + window.worker = worker + worker.done.connect(lambda result: done.append((result.success, result))) worker.start() ticks = 0 while not done and ticks < 300: diff --git a/tests/e2e/test_iqtree.py b/tests/e2e/test_iqtree.py index 60ff953..396faaa 100644 --- a/tests/e2e/test_iqtree.py +++ b/tests/e2e/test_iqtree.py @@ -10,7 +10,8 @@ from datasets import synthetic # noqa: F401 (fixture) from phyml_core.engines import ENGINES -from phyml_core.pipeline import PipelineWorker +from phyml_core.paths import AppPaths +from phyml_core.runner import PipelineRunner pytestmark = pytest.mark.needs_iqtree @@ -37,26 +38,27 @@ def iqtree_config(binaries, **overrides): def run_worker(config, fasta): - """Run the worker synchronously; returns (worker, logs, finished results).""" - logs, done = [], [] - worker = PipelineWorker(config, str(fasta)) - worker.log_signal.connect(logs.append) - worker.finished_signal.connect(lambda ok, tree: done.append((ok, tree))) - worker.run() - return worker, logs, done + """Run the pipeline synchronously; returns (RunResult, logs, [(success, tree)]).""" + logs = [] + folder = Path(fasta).parent + paths = AppPaths(app_dir=str(folder / 'app'), data_dir=str(folder / 'data')) + runner = PipelineRunner(config, str(fasta), paths, + on_log=lambda level, text: logs.append(text)) + result = runner.run() + return result, logs, [(result.success, result.files[0].tree)] def test_mfp_ufboot_run(qapp, monkeypatch, binaries, run_dir): """Worker-level: ModelFinder + ultrafast bootstrap on pre-aligned synthetic data.""" monkeypatch.chdir(run_dir) - worker, logs, done = run_worker(iqtree_config(binaries), run_dir / 'test_prealigned.fasta') + result, logs, done = run_worker(iqtree_config(binaries), run_dir / 'test_prealigned.fasta') assert done[0][0], 'iqtree pipeline failed:\n' + '\n'.join(logs[-8:]) tree_path = done[0][1] assert Path(tree_path).exists(), tree_path tree = Phylo.read(tree_path, 'newick') assert len(tree.get_terminals()) == 8 - row = worker.results[0] + row = result.files[0].stats assert row['model'] and row['model'] != 'n/a', row iq_report = (run_dir / 'test_prealigned_iqtree.iqtree').read_text(encoding='utf-8') lnl_file = re.search(r'Log-likelihood of the tree:\s*(-?[\d.]+)', iq_report).group(1) @@ -88,7 +90,7 @@ def test_gui_validation_and_run_with_xlsx(window, warned, qtbot, binaries, run_d window.start_pipeline() qtbot.waitUntil(lambda: window.btn_run.isEnabled(), timeout=180_000) window.worker.wait() - assert window.worker.results[0]['status'] == 'OK', \ + assert window.last_result.files[0].status == 'ok', \ 'GUI iqtree run failed:\n' + window.log_console.toPlainText()[-800:] xlsx = run_dir / 'phymlgui_summary.xlsx' assert xlsx.exists(), 'xlsx summary missing' diff --git a/tests/e2e/test_phyml_pipeline.py b/tests/e2e/test_phyml_pipeline.py index 4f11bfa..e3914c0 100644 --- a/tests/e2e/test_phyml_pipeline.py +++ b/tests/e2e/test_phyml_pipeline.py @@ -1,4 +1,4 @@ -"""End-to-end tests for PipelineWorker with real MUSCLE + phyml-mpi. +"""End-to-end tests for PipelineRunner with real MUSCLE + phyml-mpi. Runs the production pipeline on synthetic data with a known topology: two clades A=(A1..A4), B=(B1..B4). After each run we check the output files and @@ -18,7 +18,8 @@ from Bio import Phylo from datasets import phyml_config, synthetic # noqa: F401 (fixture) -from phyml_core.pipeline import PipelineWorker +from phyml_core.paths import AppPaths +from phyml_core.runner import PipelineRunner pytestmark = [pytest.mark.needs_phyml, pytest.mark.needs_muscle] @@ -31,10 +32,12 @@ def run_pipeline(tmp_path, monkeypatch, source, folder, config): shutil.copy(source, fasta) monkeypatch.chdir(wd) logs = [] - worker = PipelineWorker(config, str(fasta)) - worker.log_signal.connect(logs.append) - worker.finished_signal.connect(lambda ok, tree: logs.append(f'__FINISHED__{ok}|{tree}')) - worker.run() # run synchronously in this thread + paths = AppPaths(app_dir=str(tmp_path / 'app'), data_dir=str(tmp_path / 'data')) + runner = PipelineRunner(config, str(fasta), paths, + on_log=lambda level, text: logs.append(text)) + result = runner.run() # synchronous, in this thread + tree = result.files[0].tree if result.files else '' + logs.append(f'__FINISHED__{result.success}|{tree}') return wd, logs diff --git a/tests/gui/test_log_console.py b/tests/gui/test_log_console.py index e88115e..68c105b 100644 --- a/tests/gui/test_log_console.py +++ b/tests/gui/test_log_console.py @@ -1,13 +1,30 @@ from phyml_ui.main_window import LogConsole -def test_roles_and_plain_text(qtbot): +def test_plain_text_keeps_every_entry(qtbot): log = LogConsole() qtbot.addWidget(log) - assert log._classify('[ERROR] bad') == 'error' - assert log._classify('[+] done') == 'success' - assert log._classify('[*] system line') == 'system' - assert log._classify('. PhyML raw output') == 'out' - log.append('[ERROR] bad') - log.append('raw') - assert '[ERROR] bad' in log.toPlainText() and 'raw' in log.toPlainText() + log.append('error', 'bad') + log.append('out', 'raw') + assert log.toPlainText() == 'bad\nraw' + assert not hasattr(log, '_classify'), 'the level comes from the caller, not the text' + + +def test_levels_drive_colors_and_filter(qtbot): + log = LogConsole() + qtbot.addWidget(log) + log.set_role_colors({'error': '#ff0000', 'success': '#00ff00', + 'system': '#0000ff', 'out': '#123456'}) + log.append('error', 'x') + log.append('out', 'y') + log.append('system', '[ERROR] looks like an error but is system') + html = log.view.toHtml().lower() + assert '#ff0000' in html and '#123456' in html + idx = log.filter_box.findData('error') + log.filter_box.setCurrentIndex(idx) + shown = log.view.toPlainText() + assert 'x' in shown.splitlines() + assert 'y' not in shown.splitlines() + assert '[ERROR] looks like' not in shown + log.filter_box.setCurrentIndex(log.filter_box.findData('system')) + assert log.view.toPlainText() == '[ERROR] looks like an error but is system' diff --git a/tests/gui/test_main_window.py b/tests/gui/test_main_window.py index e5ed61b..a20141a 100644 --- a/tests/gui/test_main_window.py +++ b/tests/gui/test_main_window.py @@ -2,10 +2,18 @@ import pytest from PySide6.QtWidgets import QSizePolicy +import phyml_ui.main_window as main_window +from phyml_core.engines.base import Issue, format_issue +from phyml_core.history import HistoryStore from phyml_core.i18n import _t +from phyml_core.report import CSV_NAME from phyml_core.utils import detect_cores from phyml_ui.main_window import SettingsDialog from phyml_ui.themes import build_qss, resolve +from unit.fakes import FakeEngine + +ALIGNED = '>A\nACGTACGTAC\n>B\nACGTACGAAC\n>C\nACGAACGTAC\n' +FAIL_MARK, OK_MARK = '✗', '✓' @pytest.fixture @@ -34,8 +42,11 @@ def test_mpi_binary_autodetect(window, tmp_path): def test_serial_bootstrap_on_mpi_build_blocked(window, warned, tmp_path): - """MPI binary + MPI off + bootstrap -> blocked with a warning.""" - window.phyml_path_input.setText(str(tmp_path / 'phyml-mpi.exe')) + """MPI binary + MPI off + bootstrap -> blocked with a warning (from PhyML.validate, + which reaches this check only for a binary that exists).""" + exe = tmp_path / 'phyml-mpi.exe' + exe.write_bytes(b'') + window.phyml_path_input.setText(str(exe)) window.chk_use_mpi.setChecked(False) # after the path: the path edit auto-enables MPI window.set_combo_by_data(window.cb_support_method, 'bootstrap') fasta = tmp_path / 'test_unaligned.fasta' @@ -43,7 +54,8 @@ def test_serial_bootstrap_on_mpi_build_blocked(window, warned, tmp_path): window.fasta_file = str(fasta) window.start_pipeline() assert warned, 'warning box must be shown' - assert warned[0][1] == _t('mpi_boot_title'), 'blocked for the wrong reason' + assert warned[0][1] == _t('err_box_title') + assert _t('mpi_boot_msg') in warned[0][2], 'blocked for the wrong reason' assert window.btn_run.isEnabled(), 'Run must stay enabled after blocked start' assert window.worker is None or not window.worker.isRunning() @@ -93,8 +105,8 @@ def test_theme_tabs_presets_log_filter_recent(window, qapp, tmp_path, restore_st # structured log: filter by role window.log_console.filter_box.setCurrentIndex(0) window.log_console.clear() - window.log_console.append('[ERROR] test failure line') - window.log_console.append('plain phyml output line') + window.log_console.append('error', '[ERROR] test failure line') + window.log_console.append('out', 'plain phyml output line') assert '[ERROR] test failure line' in window.log_console.toPlainText() assert 'plain phyml output line' in window.log_console.toPlainText() window.set_combo_by_data(window.log_console.filter_box, 'error') @@ -160,3 +172,159 @@ def test_fasta_info_label_is_blank_for_an_empty_file_and_counts_a_folder(window, assert window.fasta_info_label.text() == ' ' window.fasta_path_input.setText(str(tmp_path)) assert window.fasta_info_label.text() == _t('fasta_info').format(1, '—', '—', '—') + + +# ---------- runs through PipelineRunner (FakeEngine, no binaries) ---------- + +def use_fake_engine(window, monkeypatch, **params): + """Give the window a registry holding only FakeEngine; params go into every run config.""" + window.engines = {'fake': FakeEngine()} + window.engine_setting = 'fake' + window.update_models() + original = window.get_current_config + monkeypatch.setattr(window, 'get_current_config', lambda: {**original(), **params}) + + +def write_fasta(path): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(ALIGNED, encoding='utf-8') + return str(path) + + +def run_to_end(window, qtbot, files=None): + window.start_pipeline(files) + assert window.worker is not None, 'the run did not start' + qtbot.waitUntil(lambda: window.btn_run.isEnabled(), timeout=15000) + window.worker.wait() + + +def queue_markers(window): + return [window.queue_list.item(i).text().split(' ')[-1] + for i in range(window.queue_list.count())] + + +def test_window_recovers_after_engine_exception(window, qtbot, data_dir, tmp_path, + monkeypatch, warned): + use_fake_engine(window, monkeypatch, fake_mode='raise') + fasta = write_fasta(tmp_path / 'in' / 'a.fasta') + window.fasta_path_input.setText(fasta) + run_to_end(window, qtbot) + assert not warned + assert window.btn_run.isEnabled() and not window.btn_cancel.isEnabled() + assert 'boom' in window.log_console.toPlainText() + assert any(level == 'error' and 'boom' in text for level, text in window.log_console._entries) + assert queue_markers(window) == [FAIL_MARK] + assert window._failed_files == [fasta] + assert not window.retry_btn.isHidden() + rec = HistoryStore(window.paths.history_file).load()[0] + assert (rec['ok'], rec['n']) == (0, 1) + assert rec['files'][0]['status'] == 'fail' and 'boom' in rec['files'][0]['error'] + + +def test_successful_run_fills_results_history_and_tree(window, qtbot, tmp_path, monkeypatch, + warned): + use_fake_engine(window, monkeypatch, fake_mode='ok') + fasta = write_fasta(tmp_path / 'in' / 'a.fasta') + window.fasta_path_input.setText(fasta) + loaded = [] + monkeypatch.setattr(window.tree_view, 'load_tree', loaded.append) + run_to_end(window, qtbot) + result = window.last_result + assert result.success, window.log_console.toPlainText() + tree = result.files[0].tree + assert loaded == [tree] + assert window.stats_file_label.text().endswith('a.fasta') + assert window.stats_table.item(0, 1).text() == 'FAKE' + assert window.summary_table.rowCount() == 1 + assert window.summary_table.item(0, 0).text() == 'a.fasta' + assert window.summary_table.item(0, 1).text() == 'OK' + csv_path = tmp_path / 'in' / CSV_NAME + assert csv_path.is_file() + assert _t('summary_saved').format(str(csv_path)) in window.log_console.toPlainText() + assert window.retry_btn.isHidden() and window._failed_files == [] + assert window.open_dir_btn.isEnabled() + rec = HistoryStore(window.paths.history_file).load()[0] + assert rec['engine'] == 'fake' and rec['inputs'] == [fasta] + assert rec['files'][0]['file'] == fasta and rec['files'][0]['tree'] == tree + assert rec['params']['fake_mode'] == 'ok' + for key in ('presets', 'recent', 'window'): + assert key not in rec['params'] + + +def test_tree_not_loaded_when_show_tree_is_off(window, qtbot, tmp_path, monkeypatch): + use_fake_engine(window, monkeypatch, fake_mode='ok') + window.fasta_path_input.setText(write_fasta(tmp_path / 'a.fasta')) + window.chk_show_tree.setChecked(False) + loaded = [] + monkeypatch.setattr(window.tree_view, 'load_tree', loaded.append) + run_to_end(window, qtbot) + assert window.last_result.success and loaded == [] + + +def test_batch_with_a_failure_offers_retry_of_the_failed_file(window, qtbot, tmp_path, + monkeypatch): + folder = tmp_path / 'batch' + good = write_fasta(folder / 'good.fasta') + bad = write_fasta(folder / 'bad.fasta') + use_fake_engine(window, monkeypatch, fake_mode={'bad': 'exit3'}) + window.fasta_path_input.setText(str(folder)) + loaded = [] + monkeypatch.setattr(window.tree_view, 'load_tree', loaded.append) + run_to_end(window, qtbot) + assert [f.file for f in window.last_result.files] == [bad, good] + assert queue_markers(window) == [FAIL_MARK, OK_MARK] + assert window._failed_files == [bad] + assert not window.retry_btn.isHidden() + assert loaded == [], 'a batch with a failure shows no tree' + rec = HistoryStore(window.paths.history_file).load()[0] + assert rec['inputs'] == [str(folder)] and (rec['ok'], rec['n']) == (1, 2) + assert window.summary_table.rowCount() == 2 + + window._retry_failed() + qtbot.waitUntil(lambda: window.btn_run.isEnabled(), timeout=15000) + window.worker.wait() + assert [f.file for f in window.last_result.files] == [bad] + + +def test_cancel_from_the_window_marks_the_file_cancelled(window, qtbot, tmp_path, monkeypatch): + use_fake_engine(window, monkeypatch, fake_mode='hang') + window.fasta_path_input.setText(write_fasta(tmp_path / 'a.fasta')) + window.start_pipeline() + qtbot.waitUntil(lambda: window.worker.active_pid is not None, timeout=10000) + window.cancel_pipeline() + assert not window.btn_cancel.isEnabled() + qtbot.waitUntil(lambda: window.btn_run.isEnabled(), timeout=15000) + window.worker.wait() + assert window.last_result.cancelled + assert window.last_result.files[0].status == 'cancelled' + assert window._failed_files == [] and window.retry_btn.isHidden() + assert _t('sys_interrupt').strip() in window.log_console.toPlainText() + + +def test_summary_write_error_is_logged(window, qtbot, tmp_path, monkeypatch): + use_fake_engine(window, monkeypatch, fake_mode='ok') + window.fasta_path_input.setText(write_fasta(tmp_path / 'a.fasta')) + window.chk_show_tree.setChecked(False) + + def broken(rows, out_dir): + raise ValueError('xlsx exploded') + monkeypatch.setattr(main_window.report, 'write_summary', broken) + run_to_end(window, qtbot) + assert _t('err_summary_save').format('xlsx exploded') in window.log_console.toPlainText() + assert window.btn_run.isEnabled() + + +def test_prerun_validation_lists_all_errors(window, data_dir, tmp_path, warned): + window.engine_setting = 'phyml' + window.update_models() + window.fasta_path_input.setText(write_fasta(tmp_path / 'a.fasta')) + window.le_tstv.setText('abc') + window.le_classes.setText('x') + window.start_pipeline() + assert len(warned) == 1 + _, title, text = warned[0][:3] + assert title == _t('err_box_title') + lines = text.split('\n') + assert format_issue(Issue('error', 'val_not_number', ('tstv', 'abc'))) in lines + assert format_issue(Issue('error', 'val_not_number', ('classes', 'x'))) in lines + assert window.worker is None and window.btn_run.isEnabled() diff --git a/tests/gui/test_main_window_storage.py b/tests/gui/test_main_window_storage.py index ca082b5..ed60bc5 100644 --- a/tests/gui/test_main_window_storage.py +++ b/tests/gui/test_main_window_storage.py @@ -2,7 +2,6 @@ import json import os import shutil -import types from pathlib import Path import pytest @@ -13,6 +12,7 @@ from phyml_core.history import HistoryStore, display_input, new_record from phyml_core.i18n import _t from phyml_core.paths import AppPaths +from phyml_core.runner import STAT_KEYS, FileResult, RunResult from phyml_core.utils import detect_cores @@ -25,18 +25,18 @@ def log_text(window): return window.log_console.toPlainText() -def stub_worker(files, rows, **extra): - """What _append_history reads from a finished PipelineWorker. isRunning and active_process - are there for closeEvent and the CPU timer, which keep asking the worker.""" - return types.SimpleNamespace(fasta_files=list(files), results=list(rows), - isRunning=lambda: False, active_process=None, **extra) +def file_result(path, status='ok', tree='', **stats): + """A FileResult as PipelineRunner makes it: every stats key present.""" + full = {key: '' for key in STAT_KEYS} + full.update(stats) + return FileResult(path, status, tree=tree, stats=full) -def worker_row(path, status='OK', tree='', **stats): - row = {'file': os.path.basename(path), 'status': status, 'tree': tree, - 'model': '', 'taxa': '', 'lnL': '', 'gamma': '', 'tstv': '', 'time': ''} - row.update(stats) - return row +def finish_run(window, inputs, *files, config=None): + """What start_pipeline records about a run, then the history entry for its result.""" + window._run_inputs = list(inputs) + window._run_config = config if config is not None else window.get_current_config() + window._append_history(RunResult(list(files), cancelled=False)) def raising(exc): @@ -62,8 +62,7 @@ def test_window_uses_detected_paths_by_default(window, data_dir): def test_history_written_to_data_dir(window, data_dir): - window.worker = stub_worker(['x.fasta'], [worker_row('x.fasta', tree='x.txt')]) - window._append_history(1, 1, 'x.txt') + finish_run(window, ['x.fasta'], file_result('x.fasta', tree='x.txt')) records = HistoryStore(window.paths.history_file).load() assert records[0]['ok'] == 1 assert (data_dir / 'phyml_history.json').is_file() @@ -259,13 +258,11 @@ def test_history_record_holds_params_and_full_file_paths(window, tmp_path): folder = tmp_path / 'run folder' files = [str(folder / 'a.fasta'), str(folder / 'б.fasta')] trees = [str(folder / 'a.phy_phyml_tree.txt'), ''] - window.fasta_path_input.setText(str(folder)) window.set_combo_by_data(window.cb_model, 'GTR') - window.worker = stub_worker(files, [ - worker_row(files[0], 'OK', trees[0], model='GTR', taxa='8', lnL='-123.4', - gamma='0.5', tstv='2.1', time='3s'), - worker_row(files[1], 'FAIL')]) - window._append_history(1, 2, trees[0]) + finish_run(window, [str(folder)], + file_result(files[0], 'ok', trees[0], model='GTR', taxa='8', lnL='-123.4', + gamma='0.5', tstv='2.1', time='3s'), + file_result(files[1], 'fail')) rec = HistoryStore(window.paths.history_file).load()[0] assert rec['inputs'] == [str(folder)] @@ -290,16 +287,13 @@ def test_history_params_come_from_the_run_config(window): window.set_combo_by_data(window.cb_model, 'GTR') config = window.get_current_config() window.set_combo_by_data(window.cb_model, 'JC69') - window.worker = stub_worker(['x.fasta'], [worker_row('x.fasta')], config=config) - window._append_history(1, 1, '') + finish_run(window, ['x.fasta'], file_result('x.fasta'), config=config) assert HistoryStore(window.paths.history_file).load()[0]['params']['model'] == 'GTR' def test_single_file_run_is_listed_by_its_name(window, tmp_path): fasta = str(tmp_path / 'one.fasta') - window.fasta_path_input.setText(fasta) - window.worker = stub_worker([fasta], [worker_row(fasta, tree='t.txt')]) - window._append_history(1, 1, 't.txt') + finish_run(window, [fasta], file_result(fasta, tree='t.txt')) rec = HistoryStore(window.paths.history_file).load()[0] assert display_input(rec) == 'one.fasta' assert window.history_list.item(0).text() == _t('hist_entry').format(rec['ts'], 'one.fasta', 1, 1) @@ -334,8 +328,7 @@ def test_history_list_shows_old_and_new_records_and_opens_the_best_tree(window, def test_history_write_error_is_logged_without_crash(window, monkeypatch): monkeypatch.setattr(HistoryStore, 'append', raising(OSError('disk full'))) - window.worker = stub_worker(['x.fasta'], [worker_row('x.fasta')]) - window._append_history(1, 1, '') + finish_run(window, ['x.fasta'], file_result('x.fasta')) assert 'disk full' in log_text(window) assert _t('err_settings_save').split('{}')[0] in log_text(window) @@ -345,8 +338,7 @@ def test_unreadable_history_is_logged_without_crash(data_dir, make_window, monke w = make_window() assert 'history locked' in log_text(w) assert w.history_list.count() == 0 - w.worker = stub_worker(['x.fasta'], [worker_row('x.fasta')]) - w._append_history(1, 1, '') # must not raise either + finish_run(w, ['x.fasta'], file_result('x.fasta')) # must not raise either assert not (data_dir / 'phyml_history.json').exists() diff --git a/tests/gui/test_worker.py b/tests/gui/test_worker.py new file mode 100644 index 0000000..9c3c988 --- /dev/null +++ b/tests/gui/test_worker.py @@ -0,0 +1,65 @@ +"""RunWorker: PipelineRunner on a QThread, its callbacks turned into Qt signals.""" +import time + +import pytest + +from phyml_core.paths import AppPaths +from phyml_core.runner import LEVELS, PipelineRunner, RunResult +from phyml_ui.worker import RunWorker +from unit.fakes import FakeEngine + +ALIGNED = '>A\nACGTACGTAC\n>B\nACGTACGAAC\n>C\nACGAACGTAC\n' + + +@pytest.fixture +def paths(tmp_path): + return AppPaths(app_dir=str(tmp_path / 'app'), data_dir=str(tmp_path / 'data')) + + +def make_worker(tmp_path, paths, **params): + fasta = tmp_path / 'a.fasta' + fasta.write_text(ALIGNED, encoding='utf-8') + params.setdefault('engine', 'fake') + return RunWorker.for_run(params, [str(fasta)], paths, engines={'fake': FakeEngine()}) + + +def test_worker_emits_log_status_done(qtbot, tmp_path, paths): + w = make_worker(tmp_path, paths, fake_mode='ok') + logs, statuses = [], [] + w.log.connect(lambda level, text: logs.append((level, text))) + w.status.connect(lambda name, status: statuses.append((name, status))) + with qtbot.waitSignal(w.done, timeout=10000) as blocker: + w.start() + w.wait() + result = blocker.args[0] + assert isinstance(result, RunResult) + assert result.success + assert logs and all(level in LEVELS for level, _ in logs) + assert any('fake engine: done' in text for level, text in logs if level == 'out') + assert statuses == [('a.fasta', 'pending'), ('a.fasta', 'running'), ('a.fasta', 'ok')] + assert w.active_pid is None + + +def test_worker_wraps_a_given_runner(qtbot, tmp_path, paths): + fasta = tmp_path / 'a.fasta' + fasta.write_text(ALIGNED, encoding='utf-8') + runner = PipelineRunner({'engine': 'fake'}, [str(fasta)], paths, + engines={'fake': FakeEngine()}) + w = RunWorker(runner) + with qtbot.waitSignal(w.done, timeout=10000) as blocker: + w.start() + w.wait() + assert blocker.args[0].success + + +def test_cancel_returns_at_once_and_ends_the_run(qtbot, tmp_path, paths): + w = make_worker(tmp_path, paths, fake_mode='hang') + with qtbot.waitSignal(w.done, timeout=15000) as blocker: + w.start() + qtbot.waitUntil(lambda: w.active_pid is not None, timeout=10000) + started = time.monotonic() + w.cancel() + assert time.monotonic() - started < 0.5, 'cancel() blocked the calling thread' + w.wait() + result = blocker.args[0] + assert result.cancelled and result.files[0].status == 'cancelled' From a25dccca8061142e76e61b7429750c02b3e39e82 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:17:29 +0300 Subject: [PATCH 30/36] Allow one run at a time and kill the run's process tree synchronously when the window closes Co-Authored-By: Claude Opus 5.5 --- phyml_ui/main_window.py | 9 ++++- phyml_ui/worker.py | 10 ++++-- tests/gui/test_main_window.py | 66 +++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 4 deletions(-) diff --git a/phyml_ui/main_window.py b/phyml_ui/main_window.py index 7efbb0a..35426ae 100644 --- a/phyml_ui/main_window.py +++ b/phyml_ui/main_window.py @@ -530,7 +530,9 @@ def _open_recent(self, item): def closeEvent(self, event): if self.worker and self.worker.isRunning(): - self.worker.cancel() + # synchronous: after close the process may exit and cut off a daemon thread, + # leaving phyml-mpi / smpd children alive + self.worker.cancel(block=True) self.worker.wait() self.auto_save() # persist window geometry and splitter ratios event.accept() @@ -728,6 +730,8 @@ def set_combo_by_data(self, combo, data_val): if idx >= 0: combo.setCurrentIndex(idx) def start_pipeline(self, files_override=None): + if self.worker is not None and self.worker.isRunning(): + return # one run at a time: replacing a running worker would destroy its thread if isinstance(files_override, bool): # defensive: Qt clicked(checked) files_override = None if files_override is None: @@ -767,6 +771,7 @@ def start_pipeline(self, files_override=None): self._remember_recent(target) if not self._is_loading: self.auto_save() self.btn_run.setEnabled(False); self.btn_cancel.setEnabled(True); self.log_console.clear() + self.retry_btn.setVisible(False) self.result_tabs.setCurrentWidget(self.log_console) self._populate_queue(files) if self._batch_mode: @@ -791,6 +796,8 @@ def cancel_pipeline(self): def pipeline_finished(self, result): """Show a finished run (a RunResult): tables, summary files, history, retry, tree.""" + if self.worker is not None: + self.worker.wait() # done is emitted at the end of run(): this returns at once self.last_result = result self.btn_run.setEnabled(True); self.btn_cancel.setEnabled(False) avg = sum(self._cpu_samples) / len(self._cpu_samples) if self._cpu_samples else 0.0 diff --git a/phyml_ui/worker.py b/phyml_ui/worker.py index 74f3223..405b045 100644 --- a/phyml_ui/worker.py +++ b/phyml_ui/worker.py @@ -49,7 +49,11 @@ def active_pid(self) -> Optional[int]: def run(self) -> None: self.done.emit(self._runner.run()) # run() never raises - def cancel(self) -> None: - """Ask the runner to stop and return at once: killing the process tree can take - seconds, which must not be spent on the GUI thread.""" + def cancel(self, block: bool = False) -> None: + """Ask the runner to stop. By default this returns at once: killing the process + tree can take seconds, which must not be spent on the GUI thread. block=True kills + on the calling thread (at exit, where a daemon thread would be cut off).""" + if block: + self._runner.cancel() + return threading.Thread(target=self._runner.cancel, name='run-cancel', daemon=True).start() diff --git a/tests/gui/test_main_window.py b/tests/gui/test_main_window.py index a20141a..f02584c 100644 --- a/tests/gui/test_main_window.py +++ b/tests/gui/test_main_window.py @@ -328,3 +328,69 @@ def test_prerun_validation_lists_all_errors(window, data_dir, tmp_path, warned): assert format_issue(Issue('error', 'val_not_number', ('tstv', 'abc'))) in lines assert format_issue(Issue('error', 'val_not_number', ('classes', 'x'))) in lines assert window.worker is None and window.btn_run.isEnabled() + + +def wait_for_sleeper(window, qtbot): + """The process tree of the running step once fake_tool's sleeping child is up.""" + import psutil + from unit.fake_tool import SLEEPER + found = [] + + def up(): + pid = window.worker.active_pid + if pid is None: + return False + try: + root = psutil.Process(pid) + kids = root.children(recursive=True) + except psutil.Error: + return False + if any(SLEEPER in ' '.join(k.cmdline()) for k in kids): + found[:] = [root, *kids] + return True + return False + qtbot.waitUntil(up, timeout=10000) + return found + + +def test_retry_during_a_running_run_is_ignored(window, qtbot, tmp_path, monkeypatch, warned): + use_fake_engine(window, monkeypatch, fake_mode='hang') + fasta = write_fasta(tmp_path / 'a.fasta') + window.fasta_path_input.setText(fasta) + window._failed_files = [fasta] # as left by an earlier failed run + window.retry_btn.setVisible(True) + window.start_pipeline() + worker = window.worker + assert window.retry_btn.isHidden(), 'retry must not be offered while a run is going' + wait_for_sleeper(window, qtbot) + window._retry_failed() + window.start_pipeline() + window.btn_run.click() + assert window.worker is worker and worker.isRunning() + assert not warned + window.cancel_pipeline() + qtbot.waitUntil(lambda: window.btn_run.isEnabled(), timeout=15000) + assert worker.isFinished(), 'Run is enabled again only after the thread has ended' + assert window.last_result.cancelled + + +def test_closing_the_window_kills_the_running_process_tree(window, qtbot, tmp_path, monkeypatch): + import threading + use_fake_engine(window, monkeypatch, fake_mode='hang') + window.fasta_path_input.setText(write_fasta(tmp_path / 'a.fasta')) + window.start_pipeline() + procs = wait_for_sleeper(window, qtbot) + assert len(procs) >= 2 + runner = window.worker._runner + callers = [] + real_cancel = runner.cancel + + def cancel(): + callers.append(threading.current_thread()) + real_cancel() + monkeypatch.setattr(runner, 'cancel', cancel) + window.close() + # at exit a daemon thread would be cut off: the kill must finish before close() returns + assert threading.main_thread() in callers, 'close() left the kill to a daemon thread' + alive = [p.pid for p in procs if p.is_running()] + assert alive == [], f'processes left after close: {alive}' From fb1651932a201736eeee5769be71d4834b73f71a Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:20:23 +0300 Subject: [PATCH 31/36] Run the CLI on PipelineRunner without importing PySide6 Co-Authored-By: Claude Opus 5.5 --- phyml_core/cli.py | 123 +++++++++++++++++++++------- phymlgui.py | 16 ++-- tests/unit/test_cli.py | 130 ++++++++++++++++++++++++++++++ tests/unit/test_core_has_no_qt.py | 21 +++++ 4 files changed, 252 insertions(+), 38 deletions(-) create mode 100644 tests/unit/test_cli.py create mode 100644 tests/unit/test_core_has_no_qt.py diff --git a/phyml_core/cli.py b/phyml_core/cli.py index 6e28834..771574a 100644 --- a/phyml_core/cli.py +++ b/phyml_core/cli.py @@ -1,52 +1,117 @@ -"""Headless CLI: run the same pipeline without the GUI. +"""Headless CLI: run the same pipeline without the GUI (and without Qt). python phymlgui.py --file input.fasta python phymlgui.py --batch folder/ --config preset.json + +Settings come from the user's config file (as in the window); --config is a JSON object +laid over them. Log lines go to stdout, except level 'error' lines, which go to stderr. +Exit codes: 0 - every file succeeded; 1 - a file failed or the run was cancelled; +2 - usage or input problem (no --file/--batch, missing path, empty folder, unreadable +or invalid settings/--config). """ import argparse import json import os import sys +from . import settings +from .i18n import _t +from .paths import AppPaths +from .runner import PipelineRunner -def main(argv=None): - parser = argparse.ArgumentParser(prog='phymlgui', description='phymlGUI headless pipeline') - parser.add_argument('--file', help='single FASTA file') - parser.add_argument('--batch', help='folder with FASTA files') - parser.add_argument('--config', default='phyml_config.json', - help='config JSON (defaults to phyml_config.json)') - args = parser.parse_args(argv) +FASTA_EXTENSIONS = ('.fasta', '.fas', '.fa') + + +class _Usage(Exception): + """Ends the run with exit code 2; the argument is the message for stderr.""" + + +def _emit(stream, text): + """Print a line without ever failing on a console code page that lacks a character.""" + try: + print(text, file=stream, flush=True) + except UnicodeEncodeError: + encoding = getattr(stream, 'encoding', None) or 'ascii' + print(text.encode(encoding, 'replace').decode(encoding), file=stream, flush=True) + except (OSError, ValueError): # closed or broken pipe: nothing useful left to do + pass + + +def _make_tolerant(stream): + reconfigure = getattr(stream, 'reconfigure', None) + if reconfigure is not None: + try: + reconfigure(errors='replace') + except (OSError, ValueError): + pass - if not args.file and not args.batch: - parser.error('specify --file or --batch') - from phyml_core.i18n import _t - from phyml_core.paths import AppPaths - from phyml_core.runner import PipelineRunner +class _Parser(argparse.ArgumentParser): + def error(self, message): + raise _Usage(self.format_usage().rstrip() + '\n' + message) - config = {} - if os.path.exists(args.config): - with open(args.config, encoding='utf-8') as f: - config = json.load(f) +def _collect_files(args): if args.batch: + if not os.path.isdir(args.batch): + raise _Usage(_t('err_box_msg')) + # the same rule as the window: *_aligned.fasta are this app's own MUSCLE outputs files = sorted(os.path.join(args.batch, f) for f in os.listdir(args.batch) - if f.lower().endswith(('.fasta', '.fas', '.fa')) + if f.lower().endswith(FASTA_EXTENSIONS) and not f.lower().endswith('_aligned.fasta')) if not files: - print(_t("batch_empty")) - return 2 - else: - files = [args.file] - - runner = PipelineRunner(config, files, AppPaths.detect(), - on_log=lambda level, text: print(text)) - result = runner.run() # synchronous, never raises - trees = [f.tree for f in result.files if f.status == 'ok' and f.tree] + raise _Usage(_t('batch_empty')) + return files + if not os.path.isfile(args.file): + raise _Usage(_t('err_box_msg')) + return [args.file] + + +def _load_params(paths, config_arg): + try: + params = settings.from_disk(settings.load(paths.config_file), paths) + except OSError as e: + raise _Usage(f'{paths.config_file}: {e}') from e + if config_arg: + try: + with open(config_arg, encoding='utf-8-sig') as f: + overlay = json.load(f) + except (OSError, ValueError) as e: # includes JSONDecodeError and UnicodeDecodeError + raise _Usage(f'{config_arg}: {e}') from e + if not isinstance(overlay, dict): + raise _Usage(f'{config_arg}: not a JSON object') + params.update(overlay) + return params - print('OK' if result.success else 'FAILED') + +def main(argv=None) -> int: + _make_tolerant(sys.stdout) + _make_tolerant(sys.stderr) + parser = _Parser(prog='phymlgui', description='phymlGUI headless pipeline') + parser.add_argument('--file', help='single FASTA file') + parser.add_argument('--batch', help='folder with FASTA files') + parser.add_argument('--config', help='JSON object laid over the saved settings') + try: + args = parser.parse_args(argv) + if not args.file and not args.batch: + parser.error('specify --file or --batch') + files = _collect_files(args) + paths = AppPaths.detect() + params = _load_params(paths, args.config) + except _Usage as e: + _emit(sys.stderr, str(e)) + return 2 + except SystemExit as e: # --help (0) + return e.code if isinstance(e.code, int) else 0 + + def on_log(level, text): + _emit(sys.stderr if level == 'error' else sys.stdout, text.rstrip('\r\n')) + + result = PipelineRunner(params, files, paths, on_log=on_log).run() # never raises + trees = [f.tree for f in result.files if f.status == 'ok' and f.tree] + _emit(sys.stdout, 'OK' if result.success else 'FAILED') if trees: - print('tree:', trees[-1]) + _emit(sys.stdout, 'tree: ' + trees[-1]) return 0 if result.success else 1 diff --git a/phymlgui.py b/phymlgui.py index 20eb128..35a6603 100644 --- a/phymlgui.py +++ b/phymlgui.py @@ -1,19 +1,17 @@ -"""phymlGUI entry point. Application code lives in phyml_core/ and phyml_ui/.""" -import sys - -from PySide6.QtWidgets import QApplication, QMessageBox # noqa: F401 (re-exported for compatibility) +"""phymlGUI entry point. Application code lives in phyml_core/ and phyml_ui/. -from phyml_core import __version__ # noqa: F401 -from phyml_core.i18n import _t, LANG, STRINGS # noqa: F401 -from phyml_core.utils import psutil, detect_cores, is_mpi_binary, resource_path # noqa: F401 -from phyml_ui.main_window import PhyMLApp # noqa: F401 -from phyml_ui.tree_viewer import TreeViewerDialog # noqa: F401 +With arguments it runs the headless CLI and never imports PySide6; the GUI is imported +only when started without arguments. +""" +import sys def main(): if len(sys.argv) > 1: from phyml_core.cli import main as cli_main sys.exit(cli_main()) + from PySide6.QtWidgets import QApplication + from phyml_ui.main_window import PhyMLApp app = QApplication(sys.argv) window = PhyMLApp() window.show() diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py new file mode 100644 index 0000000..01e8a35 --- /dev/null +++ b/tests/unit/test_cli.py @@ -0,0 +1,130 @@ +import io +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from phyml_core import cli +from phyml_core.runner import FileResult, RunResult + +ROOT = Path(__file__).resolve().parents[2] +BLOCK_QT = "import sys; sys.modules['PySide6'] = None; " + + +def _env(tmp_path): + env = dict(os.environ) + env['PHYMLGUI_DATA_DIR'] = str(tmp_path / 'data') + return env + + +@pytest.fixture(autouse=True) +def data_dir(tmp_path, monkeypatch): + monkeypatch.setenv('PHYMLGUI_DATA_DIR', str(tmp_path / 'data')) + + +def test_cli_without_qt_fails_cleanly_on_empty_fasta(tmp_path): + empty = tmp_path / 'empty.fasta' + empty.write_text('') + code = BLOCK_QT + f"import sys; from phyml_core.cli import main; sys.exit(main(['--file', {str(empty)!r}]))" + r = subprocess.run([sys.executable, '-c', code], cwd=ROOT, env=_env(tmp_path), + capture_output=True, text=True) + assert r.returncode == 1, r.stderr + assert 'ImportError' not in r.stderr + assert 'FAILED' in r.stdout + + +def test_entry_point_cli_without_qt(tmp_path): + empty = tmp_path / 'empty.fasta' + empty.write_text('') + code = BLOCK_QT + (f"import sys; sys.argv = ['phymlgui.py', '--file', {str(empty)!r}]; " + "import phymlgui; phymlgui.main()") + r = subprocess.run([sys.executable, '-c', code], cwd=ROOT, env=_env(tmp_path), + capture_output=True, text=True) + assert r.returncode == 1, r.stderr + assert 'ImportError' not in r.stderr + + +def test_cli_usage_error_exit_2(capsys): + assert cli.main([]) == 2 + assert capsys.readouterr().err + + +def test_cli_empty_batch_dir_exit_2(tmp_path, capsys): + folder = tmp_path / 'b' + folder.mkdir() + (folder / 'x_aligned.fasta').write_text('>a\nA\n') + (folder / 'notes.txt').write_text('x') + assert cli.main(['--batch', str(folder)]) == 2 + assert capsys.readouterr().err + + +def test_cli_missing_paths_exit_2(tmp_path): + assert cli.main(['--file', str(tmp_path / 'nope.fasta')]) == 2 + assert cli.main(['--batch', str(tmp_path / 'nope')]) == 2 + f = tmp_path / 'a.fasta' + f.write_text('') + assert cli.main(['--file', str(f), '--config', str(tmp_path / 'nope.json')]) == 2 + bad = tmp_path / 'bad.json' + bad.write_text('{not json') + assert cli.main(['--file', str(f), '--config', str(bad)]) == 2 + + +def test_cli_unreadable_settings_exit_2(tmp_path, monkeypatch, capsys): + f = tmp_path / 'a.fasta' + f.write_text('') + + def boom(path, on_error=None): + raise PermissionError('locked') + monkeypatch.setattr(cli.settings, 'load', boom) + assert cli.main(['--file', str(f)]) == 2 + assert 'locked' in capsys.readouterr().err + + +class _Fake: + instances = [] + + def __init__(self, params, files, paths, on_log=None, on_status=None, engines=None): + self.params, self.files = params, files + _Fake.instances.append(self) + self.on_log = on_log + + def run(self): + self.on_log('system', 'a → b\n') + self.on_log('error', 'bad\n') + return RunResult([FileResult(self.files[0], 'ok', tree='t.nwk')], False) + + +def test_cli_logs_arrow_through_cp1251_stdout(tmp_path, monkeypatch): + monkeypatch.setattr(cli, 'PipelineRunner', _Fake) + f = tmp_path / 'a.fasta' + f.write_text('>a\nA\n') + raw = io.BytesIO() + out = io.TextIOWrapper(raw, encoding='cp1251', newline='') + err = io.TextIOWrapper(io.BytesIO(), encoding='cp1251', newline='') + monkeypatch.setattr(sys, 'stdout', out) + monkeypatch.setattr(sys, 'stderr', err) + assert cli.main(['--file', str(f)]) == 0 + out.flush() + text = raw.getvalue().decode('cp1251') + assert 'a ? b' in text and 'OK' in text and 'tree: t.nwk' in text + err.flush() + assert 'bad' in err.buffer.getvalue().decode('cp1251') + + +def test_cli_config_overlay_and_batch_selection(tmp_path, monkeypatch): + _Fake.instances.clear() + monkeypatch.setattr(cli, 'PipelineRunner', _Fake) + folder = tmp_path / 'b' + folder.mkdir() + for n in ('b.FA', 'a.fasta', 'c.fas', 'a_aligned.fasta', 'z.txt'): + (folder / n).write_text('>a\nA\n') + cfg = tmp_path / 'c.json' + cfg.write_text(json.dumps({'model': 'GTR'})) + cli.main(['--batch', str(folder), '--config', str(cfg)]) + inst = _Fake.instances[0] + assert [os.path.basename(p) for p in inst.files] == ['a.fasta', 'b.FA', 'c.fas'] + assert inst.params['model'] == 'GTR' + assert inst.params['engine'] == 'phyml' diff --git a/tests/unit/test_core_has_no_qt.py b/tests/unit/test_core_has_no_qt.py new file mode 100644 index 0000000..fbd8e6a --- /dev/null +++ b/tests/unit/test_core_has_no_qt.py @@ -0,0 +1,21 @@ +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +BLOCK_QT = "import sys; sys.modules['PySide6'] = None; " + + +def test_core_has_no_qt(): + code = BLOCK_QT + ("import pkgutil, importlib, phyml_core; " + "[importlib.import_module(m.name) for m in " + "pkgutil.walk_packages(phyml_core.__path__, 'phyml_core.')]") + r = subprocess.run([sys.executable, '-c', code], cwd=ROOT, capture_output=True, text=True) + assert r.returncode == 0, r.stderr + + +def test_entry_point_has_no_compat_reexports(): + code = BLOCK_QT + "import phymlgui; print(hasattr(phymlgui, 'PhyMLApp'), hasattr(phymlgui, 'QApplication'))" + r = subprocess.run([sys.executable, '-c', code], cwd=ROOT, capture_output=True, text=True) + assert r.returncode == 0, r.stderr + assert r.stdout.split() == ['False', 'False'] From 947c6fd9953bb383977e4a43275ba59f59aec50c Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:23:33 +0300 Subject: [PATCH 32/36] Split LogConsole, SettingsDialog and the status bar out of main_window Co-Authored-By: Claude Opus 5.5 --- phyml_ui/log_console.py | 84 +++++++++ phyml_ui/main_window.py | 270 +++------------------------- phyml_ui/settings_dialog.py | 97 ++++++++++ phyml_ui/status_bar.py | 98 ++++++++++ tests/e2e/test_batch.py | 2 +- tests/e2e/test_cpu_monitor.py | 8 +- tests/gui/test_log_console.py | 2 +- tests/gui/test_main_window.py | 10 +- tests/gui/test_main_window_smoke.py | 2 +- tests/gui/test_status_bar.py | 38 ++++ 10 files changed, 352 insertions(+), 259 deletions(-) create mode 100644 phyml_ui/log_console.py create mode 100644 phyml_ui/settings_dialog.py create mode 100644 phyml_ui/status_bar.py create mode 100644 tests/gui/test_status_bar.py diff --git a/phyml_ui/log_console.py b/phyml_ui/log_console.py new file mode 100644 index 0000000..4ecdd26 --- /dev/null +++ b/phyml_ui/log_console.py @@ -0,0 +1,84 @@ +from html import escape + +from PySide6.QtWidgets import (QWidget, QVBoxLayout, QHBoxLayout, QComboBox, + QLineEdit, QTextEdit) +from PySide6.QtGui import QFont + +from phyml_core.i18n import _t + + +class LogConsole(QWidget): + """Structured log: colored levels, severity filter, substring search. + + Every entry carries a level from phyml_core.runner.LEVELS ('error', 'success', + 'system', 'out'), given by whoever writes it. Entries are always stored in full; + filtering only affects what is displayed. + """ + + def __init__(self, parent=None): + super().__init__(parent) + self._entries = [] # (role, text) + self._role_colors = {'error': '#E53935', 'success': '#43A047', + 'system': '#1E9E6A', 'out': '#212121'} + layout = QVBoxLayout(self) + layout.setContentsMargins(0, 0, 0, 0) + layout.setSpacing(4) + + bar = QHBoxLayout() + self.filter_box = QComboBox() + self.filter_box.addItem(_t("log_filter_all"), "all") + self.filter_box.addItem(_t("log_filter_error"), "error") + self.filter_box.addItem(_t("log_filter_success"), "success") + self.filter_box.addItem(_t("log_filter_system"), "system") + self.search_edit = QLineEdit() + self.search_edit.setPlaceholderText(_t("log_search")) + self.filter_box.currentIndexChanged.connect(self._render) + self.search_edit.textChanged.connect(self._render) + bar.addWidget(self.filter_box) + bar.addWidget(self.search_edit) + layout.addLayout(bar) + + self.view = QTextEdit() + self.view.setReadOnly(True) + self.view.setFont(QFont('Consolas', 9)) + layout.addWidget(self.view) + + def append(self, level: str, text: str) -> None: + """Add an entry; a level this console has no color for is shown as 'out'.""" + if level not in self._role_colors: + level = 'out' + self._entries.append((level, text)) + if self._passes(level, text): + self._append_html(text, level) + + def clear(self): + self._entries.clear() + self.view.clear() + + def toPlainText(self): + return '\n'.join(t for _, t in self._entries) + + def _passes(self, role, text): + flt = self.filter_box.currentData() + if flt != 'all' and role != flt: + return False + needle = self.search_edit.text().strip().lower() + return not needle or needle in text.lower() + + def set_role_colors(self, colors): + self._role_colors = dict(colors) + self._render() + + def _append_html(self, text, role): + prefix = self._role_colors[role] + weight = '600' if role == 'system' else '400' + self.view.append(f'' + f'{escape(text)}') + sb = self.view.verticalScrollBar() + sb.setValue(sb.maximum()) + + def _render(self): + self.view.clear() + for role, text in self._entries: + if self._passes(role, text): + self._append_html(text, role) diff --git a/phyml_ui/main_window.py b/phyml_ui/main_window.py index 35426ae..547a336 100644 --- a/phyml_ui/main_window.py +++ b/phyml_ui/main_window.py @@ -3,19 +3,16 @@ import shutil from dataclasses import asdict from datetime import datetime -from html import escape from PySide6.QtWidgets import (QWidget, QVBoxLayout, QHBoxLayout, QFormLayout, QComboBox, QLineEdit, QPushButton, QFileDialog, QMessageBox, QTabWidget, - QCheckBox, QGroupBox, QSizePolicy, QLabel, - QStatusBar, QProgressBar, QSplitter, QToolBox, + QCheckBox, QGroupBox, QLabel, QSplitter, QToolBox, QToolButton, QListWidget, QListWidgetItem, - QTextEdit, QTableWidget, QTableWidgetItem, - QHeaderView, QScrollArea, QDialog, - QDialogButtonBox, QMenu, QInputDialog) -from PySide6.QtCore import QTimer, Qt -from PySide6.QtGui import QIcon, QFont, QShortcut, QKeySequence + QTableWidget, QTableWidgetItem, + QHeaderView, QScrollArea, QMenu, QInputDialog) +from PySide6.QtCore import Qt +from PySide6.QtGui import QIcon, QShortcut, QKeySequence from phyml_core import __version__, alignment, presets, report, settings from phyml_core.engines import ENGINES @@ -23,9 +20,12 @@ from phyml_core.history import HistoryStore, best_tree, display_input, new_record from phyml_core.i18n import _t from phyml_core.paths import AppPaths -from phyml_core.utils import psutil, detect_cores, is_mpi_binary, resource_path +from phyml_core.utils import detect_cores, is_mpi_binary, resource_path from phyml_ui.tree_viewer import TreeViewerWidget -from phyml_ui.themes import resolve, build_qss, role_colors, ACCENTS, PALETTES +from phyml_ui.themes import resolve, build_qss, role_colors, PALETTES +from phyml_ui.log_console import LogConsole +from phyml_ui.settings_dialog import SettingsDialog +from phyml_ui.status_bar import AppStatusBar from phyml_ui.worker import RunWorker # queue marker per file status of the runner @@ -35,172 +35,6 @@ _NOT_RUN_PARAMS = ('presets', 'recent', 'window') -class LogConsole(QWidget): - """Structured log: colored levels, severity filter, substring search. - - Every entry carries a level from phyml_core.runner.LEVELS ('error', 'success', - 'system', 'out'), given by whoever writes it. Entries are always stored in full; - filtering only affects what is displayed. - """ - - def __init__(self, parent=None): - super().__init__(parent) - self._entries = [] # (role, text) - self._role_colors = {'error': '#E53935', 'success': '#43A047', - 'system': '#1E9E6A', 'out': '#212121'} - layout = QVBoxLayout(self) - layout.setContentsMargins(0, 0, 0, 0) - layout.setSpacing(4) - - bar = QHBoxLayout() - self.filter_box = QComboBox() - self.filter_box.addItem(_t("log_filter_all"), "all") - self.filter_box.addItem(_t("log_filter_error"), "error") - self.filter_box.addItem(_t("log_filter_success"), "success") - self.filter_box.addItem(_t("log_filter_system"), "system") - self.search_edit = QLineEdit() - self.search_edit.setPlaceholderText(_t("log_search")) - self.filter_box.currentIndexChanged.connect(self._render) - self.search_edit.textChanged.connect(self._render) - bar.addWidget(self.filter_box) - bar.addWidget(self.search_edit) - layout.addLayout(bar) - - self.view = QTextEdit() - self.view.setReadOnly(True) - self.view.setFont(QFont('Consolas', 9)) - layout.addWidget(self.view) - - def append(self, level: str, text: str) -> None: - """Add an entry; a level this console has no color for is shown as 'out'.""" - if level not in self._role_colors: - level = 'out' - self._entries.append((level, text)) - if self._passes(level, text): - self._append_html(text, level) - - def clear(self): - self._entries.clear() - self.view.clear() - - def toPlainText(self): - return '\n'.join(t for _, t in self._entries) - - def _passes(self, role, text): - flt = self.filter_box.currentData() - if flt != 'all' and role != flt: - return False - needle = self.search_edit.text().strip().lower() - return not needle or needle in text.lower() - - def set_role_colors(self, colors): - self._role_colors = dict(colors) - self._render() - - def _append_html(self, text, role): - prefix = self._role_colors[role] - weight = '600' if role == 'system' else '400' - self.view.append(f'' - f'{escape(text)}') - sb = self.view.verticalScrollBar() - sb.setValue(sb.maximum()) - - def _render(self): - self.view.clear() - for role, text in self._entries: - if self._passes(role, text): - self._append_html(text, role) - - -class SettingsDialog(QDialog): - """Rarely changed settings: executable paths and appearance.""" - - FIELDS = ('path_phyml', 'path_mpirun', 'path_muscle') - - def __init__(self, app_window, parent=None): - super().__init__(parent) - self.app_window = app_window - self.setWindowTitle(_t("settings_title")) - self.setWindowIcon(QIcon(resource_path('icon.png'))) - self.resize(640, 340) - root = QVBoxLayout(self) - engine_box = QGroupBox(_t("settings_engine_box")) - eform = QFormLayout(engine_box) - self.engine_combo = QComboBox() - self.engine_combo.addItem(_t("engine_phyml"), "phyml") - self.engine_combo.addItem(_t("engine_iqtree"), "iqtree") - idx = self.engine_combo.findData(app_window.engine_setting) - self.engine_combo.setCurrentIndex(max(idx, 0)) - eform.addRow(_t("settings_engine"), self.engine_combo) - self.iqtree_edit = QLineEdit(app_window.iqtree_path_input.text()) - iq_btn = QPushButton(_t("btn_browse")) - iq_btn.clicked.connect(lambda: self._browse(self.iqtree_edit)) - irow = QHBoxLayout(); irow.addWidget(self.iqtree_edit); irow.addWidget(iq_btn) - iwrap = QWidget(); iwrap.setLayout(irow) - eform.addRow(_t("path_iqtree"), iwrap) - root.addWidget(engine_box) - - paths_box = QGroupBox(_t("settings_paths")) - form = QFormLayout(paths_box) - self.edits = [] - inputs = [app_window.phyml_path_input, app_window.mpirun_path_input, - app_window.muscle_path_input] - for key, source in zip(self.FIELDS, inputs): - edit = QLineEdit(source.text()) - btn = QPushButton(_t("btn_browse")) - btn.clicked.connect(lambda _=False, e=edit: self._browse(e)) - row = QHBoxLayout(); row.addWidget(edit); row.addWidget(btn) - wrap = QWidget(); wrap.setLayout(row) - form.addRow(_t(key), wrap) - self.edits.append(edit) - root.addWidget(paths_box) - - look_box = QGroupBox(_t("settings_appearance")) - look = QFormLayout(look_box) - self.theme_combo = QComboBox() - self.theme_combo.addItem(_t("theme_system"), "system") - self.theme_combo.addItem(_t("theme_light"), "light") - self.theme_combo.addItem(_t("theme_dark"), "dark") - idx = self.theme_combo.findData(app_window._theme_setting) - self.theme_combo.setCurrentIndex(max(idx, 0)) - self.accent_combo = QComboBox() - self.accent_combo.addItem(_t("accent_system"), "system") - for key, hexval in ACCENTS.items(): - self.accent_combo.addItem(_t("accent_" + key), key) - idx = self.accent_combo.findData(app_window._accent_setting) - self.accent_combo.setCurrentIndex(max(idx, 0)) - look.addRow(_t("settings_theme"), self.theme_combo) - look.addRow(_t("settings_accent"), self.accent_combo) - root.addWidget(look_box) - - buttons = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) - buttons.accepted.connect(self.accept) - buttons.rejected.connect(self.reject) - root.addWidget(buttons) - - def _browse(self, edit): - path, _ = QFileDialog.getOpenFileName(self, _t("file_exe")) - if path: - edit.setText(path) - - def accept(self): - inputs = [self.app_window.phyml_path_input, - self.app_window.mpirun_path_input, - self.app_window.muscle_path_input] - for edit, source in zip(self.edits, inputs): - source.setText(edit.text()) # programmatic: triggers autosave & MPI detect - self.app_window.iqtree_path_input.setText(self.iqtree_edit.text()) - engine = self.engine_combo.currentData() - if engine != self.app_window.engine_setting: - self.app_window.engine_setting = engine - self.app_window.update_models() - self.app_window.apply_theme(self.theme_combo.currentData(), - self.accent_combo.currentData()) - if not self.app_window._is_loading: - self.app_window.auto_save() - super().accept() - - # --- Main Application Window --- class PhyMLApp(QWidget): def __init__(self, paths=None, engines=None): @@ -216,8 +50,6 @@ def __init__(self, paths=None, engines=None): self.worker = None self.fasta_file = "" self._mpi_auto_path = "" - self._cpu_samples = [] - self._mon_procs = {} self._batch_mode = False self._theme_setting = 'system' self._accent_setting = 'green' @@ -249,7 +81,10 @@ def initUI(self): root_layout.setContentsMargins(6, 6, 6, 6) self.splitter = QSplitter(Qt.Horizontal) root_layout.addWidget(self.splitter, 1) - root_layout.addWidget(self._build_status_bar()) + self.status = AppStatusBar(self.phys_cores, self.logi_cores) + self.status.open_dir_clicked.connect(self._open_output_dir) + self.status.settings_clicked.connect(self.open_settings) + root_layout.addWidget(self.status) self.splitter.addWidget(self._build_middle_pane()) self.splitter.addWidget(self._build_right_pane()) @@ -444,36 +279,6 @@ def _build_right_pane(self): v.addWidget(split, 1) return right - def _build_status_bar(self): - self.status_bar = QStatusBar() - self.status_bar.setSizeGripEnabled(False) - self.cores_label = QLabel(_t("cores_info").format(self.phys_cores, self.logi_cores)) - self.cpu_sys_label = QLabel(_t("cpu_sys").format(0)) - self.cpu_sys_bar = QProgressBar() - self.cpu_sys_bar.setRange(0, 100) - self.cpu_sys_bar.setTextVisible(True) - self.cpu_sys_bar.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Preferred) - self.cpu_run_label = QLabel(_t("cpu_analysis_idle")) - self.open_dir_btn = QToolButton() - self.open_dir_btn.setText(_t("btn_open_dir")) - self.open_dir_btn.setEnabled(False) - self.open_dir_btn.clicked.connect(self._open_output_dir) - self.settings_btn = QPushButton(_t("btn_settings")) - self.settings_btn.clicked.connect(self.open_settings) - self.status_bar.addWidget(self.cores_label) - self.status_bar.addWidget(self.cpu_sys_label) - self.status_bar.addWidget(self.cpu_sys_bar, 1) # stretch: fills all free space - self.status_bar.addPermanentWidget(self.cpu_run_label) - self.status_bar.addPermanentWidget(self.open_dir_btn) - self.status_bar.addPermanentWidget(self.settings_btn) # far bottom-right corner - - self.cpu_timer = QTimer(self) - self.cpu_timer.setInterval(1000) - self.cpu_timer.timeout.connect(self.update_cpu_status) - if psutil is not None: - self.cpu_timer.start() - return self.status_bar - # ---------- drag & drop ---------- def dragEnterEvent(self, event): @@ -776,8 +581,7 @@ def start_pipeline(self, files_override=None): self._populate_queue(files) if self._batch_mode: self.result_tabs.setCurrentWidget(self.queue_list.parentWidget()) - self._cpu_samples = [] - self._mon_procs = {} + self.status.track(self._active_pid) self._run_started = datetime.now() self._run_config = config self._run_inputs = inputs @@ -788,6 +592,12 @@ def start_pipeline(self, files_override=None): self.worker.done.connect(self.pipeline_finished) self.worker.start() + def _active_pid(self): + """The root pid of the running analysis, for the CPU monitor.""" + if self.worker is not None and self.worker.isRunning(): + return self.worker.active_pid + return None + def cancel_pipeline(self): if self.worker and self.worker.isRunning(): self.worker.cancel() # returns at once; the kill runs on its own thread @@ -800,8 +610,7 @@ def pipeline_finished(self, result): self.worker.wait() # done is emitted at the end of run(): this returns at once self.last_result = result self.btn_run.setEnabled(True); self.btn_cancel.setEnabled(False) - avg = sum(self._cpu_samples) / len(self._cpu_samples) if self._cpu_samples else 0.0 - self.cpu_run_label.setText(_t("cpu_analysis_done").format(int(avg), round(avg / 100, 1))) + self.status.stop_tracking() if result.files: rows = report.summary_rows(result.files) self._populate_stats(result) @@ -811,7 +620,7 @@ def pipeline_finished(self, result): self._failed_files = [f.file for f in result.files if f.status == 'fail'] self.retry_btn.setVisible(bool(self._failed_files)) self._last_out_dir = self._run_out_dir - self.open_dir_btn.setEnabled(self._last_out_dir is not None) + self.status.set_open_dir_enabled(self._last_out_dir is not None) if result.success and self.chk_show_tree.isChecked(): tree = result.files[-1].tree if tree and os.path.exists(tree): @@ -986,36 +795,3 @@ def on_phyml_path_changed(self, path): self._log('system', _t("mpi_detected")) else: self._mpi_auto_path = "" - - def update_cpu_status(self): - if psutil is None: - return - sys_pct = psutil.cpu_percent(interval=None) - self.cpu_sys_bar.setValue(int(sys_pct)) - self.cpu_sys_label.setText(_t("cpu_sys").format(int(sys_pct))) - pid = self.worker.active_pid if self.worker is not None else None - if pid is not None and self.worker.isRunning(): - try: - root = psutil.Process(pid) - pids = {pid} | {c.pid for c in root.children(recursive=True)} - except psutil.Error: - return - # cpu_percent() keeps per-call state inside each Process object, - # so objects must be reused across ticks to get non-zero values - for k in list(self._mon_procs): - if k not in pids: - del self._mon_procs[k] - total = 0.0 - for p in pids: - obj = self._mon_procs.get(p) - if obj is None: - try: - obj = self._mon_procs[p] = psutil.Process(p) - except psutil.Error: - continue - try: - total += obj.cpu_percent(interval=None) - except psutil.Error: - pass - self._cpu_samples.append(total) - self.cpu_run_label.setText(_t("cpu_analysis").format(int(total), round(total / 100, 1))) diff --git a/phyml_ui/settings_dialog.py b/phyml_ui/settings_dialog.py new file mode 100644 index 0000000..985c6d5 --- /dev/null +++ b/phyml_ui/settings_dialog.py @@ -0,0 +1,97 @@ +from PySide6.QtWidgets import (QWidget, QVBoxLayout, QHBoxLayout, QFormLayout, + QComboBox, QLineEdit, QPushButton, QFileDialog, + QGroupBox, QDialog, QDialogButtonBox) +from PySide6.QtGui import QIcon + +from phyml_core.i18n import _t +from phyml_core.utils import resource_path +from phyml_ui.themes import ACCENTS + + +class SettingsDialog(QDialog): + """Rarely changed settings: executable paths and appearance.""" + + FIELDS = ('path_phyml', 'path_mpirun', 'path_muscle') + + def __init__(self, app_window, parent=None): + super().__init__(parent) + self.app_window = app_window + self.setWindowTitle(_t("settings_title")) + self.setWindowIcon(QIcon(resource_path('icon.png'))) + self.resize(640, 340) + root = QVBoxLayout(self) + engine_box = QGroupBox(_t("settings_engine_box")) + eform = QFormLayout(engine_box) + self.engine_combo = QComboBox() + self.engine_combo.addItem(_t("engine_phyml"), "phyml") + self.engine_combo.addItem(_t("engine_iqtree"), "iqtree") + idx = self.engine_combo.findData(app_window.engine_setting) + self.engine_combo.setCurrentIndex(max(idx, 0)) + eform.addRow(_t("settings_engine"), self.engine_combo) + self.iqtree_edit = QLineEdit(app_window.iqtree_path_input.text()) + iq_btn = QPushButton(_t("btn_browse")) + iq_btn.clicked.connect(lambda: self._browse(self.iqtree_edit)) + irow = QHBoxLayout(); irow.addWidget(self.iqtree_edit); irow.addWidget(iq_btn) + iwrap = QWidget(); iwrap.setLayout(irow) + eform.addRow(_t("path_iqtree"), iwrap) + root.addWidget(engine_box) + + paths_box = QGroupBox(_t("settings_paths")) + form = QFormLayout(paths_box) + self.edits = [] + inputs = [app_window.phyml_path_input, app_window.mpirun_path_input, + app_window.muscle_path_input] + for key, source in zip(self.FIELDS, inputs): + edit = QLineEdit(source.text()) + btn = QPushButton(_t("btn_browse")) + btn.clicked.connect(lambda _=False, e=edit: self._browse(e)) + row = QHBoxLayout(); row.addWidget(edit); row.addWidget(btn) + wrap = QWidget(); wrap.setLayout(row) + form.addRow(_t(key), wrap) + self.edits.append(edit) + root.addWidget(paths_box) + + look_box = QGroupBox(_t("settings_appearance")) + look = QFormLayout(look_box) + self.theme_combo = QComboBox() + self.theme_combo.addItem(_t("theme_system"), "system") + self.theme_combo.addItem(_t("theme_light"), "light") + self.theme_combo.addItem(_t("theme_dark"), "dark") + idx = self.theme_combo.findData(app_window._theme_setting) + self.theme_combo.setCurrentIndex(max(idx, 0)) + self.accent_combo = QComboBox() + self.accent_combo.addItem(_t("accent_system"), "system") + for key, hexval in ACCENTS.items(): + self.accent_combo.addItem(_t("accent_" + key), key) + idx = self.accent_combo.findData(app_window._accent_setting) + self.accent_combo.setCurrentIndex(max(idx, 0)) + look.addRow(_t("settings_theme"), self.theme_combo) + look.addRow(_t("settings_accent"), self.accent_combo) + root.addWidget(look_box) + + buttons = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) + buttons.accepted.connect(self.accept) + buttons.rejected.connect(self.reject) + root.addWidget(buttons) + + def _browse(self, edit): + path, _ = QFileDialog.getOpenFileName(self, _t("file_exe")) + if path: + edit.setText(path) + + def accept(self): + inputs = [self.app_window.phyml_path_input, + self.app_window.mpirun_path_input, + self.app_window.muscle_path_input] + for edit, source in zip(self.edits, inputs): + source.setText(edit.text()) # programmatic: triggers autosave & MPI detect + self.app_window.iqtree_path_input.setText(self.iqtree_edit.text()) + engine = self.engine_combo.currentData() + if engine != self.app_window.engine_setting: + self.app_window.engine_setting = engine + self.app_window.update_models() + self.app_window.apply_theme(self.theme_combo.currentData(), + self.accent_combo.currentData()) + if not self.app_window._is_loading: + self.app_window.auto_save() + super().accept() diff --git a/phyml_ui/status_bar.py b/phyml_ui/status_bar.py new file mode 100644 index 0000000..cc76187 --- /dev/null +++ b/phyml_ui/status_bar.py @@ -0,0 +1,98 @@ +from typing import Callable, Optional + +from PySide6.QtWidgets import (QStatusBar, QLabel, QProgressBar, QSizePolicy, + QToolButton, QPushButton) +from PySide6.QtCore import QTimer, Signal + +from phyml_core.i18n import _t +from phyml_core.utils import psutil + + +class AppStatusBar(QStatusBar): + """Core counts, system CPU load, the CPU use of the running analysis, and the + 'open folder' / 'settings' buttons.""" + + open_dir_clicked = Signal() + settings_clicked = Signal() + + def __init__(self, phys_cores, logi_cores, parent=None): + super().__init__(parent) + self.setSizeGripEnabled(False) + self.cores_label = QLabel(_t("cores_info").format(phys_cores, logi_cores)) + self.cpu_sys_label = QLabel(_t("cpu_sys").format(0)) + self.cpu_sys_bar = QProgressBar() + self.cpu_sys_bar.setRange(0, 100) + self.cpu_sys_bar.setTextVisible(True) + self.cpu_sys_bar.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Preferred) + self.cpu_run_label = QLabel(_t("cpu_analysis_idle")) + self.open_dir_btn = QToolButton() + self.open_dir_btn.setText(_t("btn_open_dir")) + self.open_dir_btn.setEnabled(False) + self.open_dir_btn.clicked.connect(self.open_dir_clicked) + self.settings_btn = QPushButton(_t("btn_settings")) + self.settings_btn.clicked.connect(self.settings_clicked) + self.addWidget(self.cores_label) + self.addWidget(self.cpu_sys_label) + self.addWidget(self.cpu_sys_bar, 1) # stretch: fills all free space + self.addPermanentWidget(self.cpu_run_label) + self.addPermanentWidget(self.open_dir_btn) + self.addPermanentWidget(self.settings_btn) # far bottom-right corner + + self.samples = [] # CPU use (percent of one core) of the run, one per tick + self._mon_procs = {} + self._pid_provider: Optional[Callable[[], Optional[int]]] = None + self.cpu_timer = QTimer(self) + self.cpu_timer.setInterval(1000) + self.cpu_timer.timeout.connect(self.update_cpu_status) + if psutil is not None: + self.cpu_timer.start() + + def set_open_dir_enabled(self, on: bool) -> None: + self.open_dir_btn.setEnabled(on) + + def track(self, pid_provider: Callable[[], Optional[int]]) -> None: + """Start collecting CPU samples of the process tree whose root pid the provider + returns (None while nothing runs).""" + self.samples = [] + self._mon_procs = {} + self._pid_provider = pid_provider + + def stop_tracking(self) -> float: + """Stop collecting, show the average of the samples and return it.""" + self._pid_provider = None + avg = sum(self.samples) / len(self.samples) if self.samples else 0.0 + self.cpu_run_label.setText(_t("cpu_analysis_done").format(int(avg), round(avg / 100, 1))) + return avg + + def update_cpu_status(self): + if psutil is None: + return + sys_pct = psutil.cpu_percent(interval=None) + self.cpu_sys_bar.setValue(int(sys_pct)) + self.cpu_sys_label.setText(_t("cpu_sys").format(int(sys_pct))) + pid = self._pid_provider() if self._pid_provider is not None else None + if pid is not None: + try: + root = psutil.Process(pid) + pids = {pid} | {c.pid for c in root.children(recursive=True)} + except psutil.Error: + return + # cpu_percent() keeps per-call state inside each Process object, + # so objects must be reused across ticks to get non-zero values + for k in list(self._mon_procs): + if k not in pids: + del self._mon_procs[k] + total = 0.0 + for p in pids: + obj = self._mon_procs.get(p) + if obj is None: + try: + obj = self._mon_procs[p] = psutil.Process(p) + except psutil.Error: + continue + try: + total += obj.cpu_percent(interval=None) + except psutil.Error: + pass + self.samples.append(total) + self.cpu_run_label.setText(_t("cpu_analysis").format(int(total), round(total / 100, 1))) diff --git a/tests/e2e/test_batch.py b/tests/e2e/test_batch.py index c00114e..f02341e 100644 --- a/tests/e2e/test_batch.py +++ b/tests/e2e/test_batch.py @@ -83,5 +83,5 @@ def test_queue_history_open_folder(gui_batch): records = HistoryStore(window.paths.history_file).load() assert records and records[0]['n'] == 2 and records[0]['ok'] == 2 assert window.history_list.count() >= 1 - assert window.open_dir_btn.isEnabled() + assert window.status.open_dir_btn.isEnabled() assert not window.retry_btn.isVisible() or not window._failed_files diff --git a/tests/e2e/test_cpu_monitor.py b/tests/e2e/test_cpu_monitor.py index 8399295..09c7e9a 100644 --- a/tests/e2e/test_cpu_monitor.py +++ b/tests/e2e/test_cpu_monitor.py @@ -18,24 +18,24 @@ def test_cpu_samples_during_mpi_run(window, qapp, tmp_path, monkeypatch, binarie shutil.copy(synthetic['unaligned'], fasta) monkeypatch.chdir(run_dir) - window._cpu_samples = [] done, wlogs = [], [] runner = PipelineRunner(phyml_config(binaries), str(fasta), window.paths, on_log=lambda level, text: wlogs.append(text)) worker = RunWorker(runner) # the window's CPU monitor asks a RunWorker for active_pid window.worker = worker + window.status.track(window._active_pid) worker.done.connect(lambda result: done.append((result.success, result))) worker.start() ticks = 0 while not done and ticks < 300: qapp.processEvents() time.sleep(0.3) - window.update_cpu_status() + window.status.update_cpu_status() ticks += 1 if done: worker.wait() # the signal is emitted just before the thread exits assert done, 'worker did not finish' assert done[0][0], 'pipeline failed:\n' + '\n'.join(wlogs[-10:]) - assert window._cpu_samples, 'no CPU samples collected' - peak = max(window._cpu_samples) + assert window.status.samples, 'no CPU samples collected' + peak = max(window.status.samples) assert peak > 50, f'analysis CPU unexpectedly low: peak {peak}' diff --git a/tests/gui/test_log_console.py b/tests/gui/test_log_console.py index 68c105b..334f587 100644 --- a/tests/gui/test_log_console.py +++ b/tests/gui/test_log_console.py @@ -1,4 +1,4 @@ -from phyml_ui.main_window import LogConsole +from phyml_ui.log_console import LogConsole def test_plain_text_keeps_every_entry(qtbot): diff --git a/tests/gui/test_main_window.py b/tests/gui/test_main_window.py index f02584c..00473e6 100644 --- a/tests/gui/test_main_window.py +++ b/tests/gui/test_main_window.py @@ -8,7 +8,7 @@ from phyml_core.i18n import _t from phyml_core.report import CSV_NAME from phyml_core.utils import detect_cores -from phyml_ui.main_window import SettingsDialog +from phyml_ui.settings_dialog import SettingsDialog from phyml_ui.themes import build_qss, resolve from unit.fakes import FakeEngine @@ -28,7 +28,7 @@ def test_defaults_match_detected_cores(window): phys, logi = detect_cores() assert window.le_mpi_np.text() == str(window.phys_cores), window.le_mpi_np.text() assert (window.phys_cores, window.logi_cores) == (phys, logi) - assert str(phys) in window.cores_label.text() and str(logi) in window.cores_label.text() + assert str(phys) in window.status.cores_label.text() and str(logi) in window.status.cores_label.text() def test_mpi_binary_autodetect(window, tmp_path): @@ -68,7 +68,7 @@ def test_theme_tabs_presets_log_filter_recent(window, qapp, tmp_path, restore_st assert qapp.styleSheet() == build_qss(theme, accent) assert window.log_console._role_colors['out'] == '#E6E8EB', 'phyml output must be themed' assert window.log_console._role_colors['system'] == '#2D7DD2', 'system lines use accent' - assert window.cpu_timer.isActive(), 'CPU monitor timer must be running' + assert window.status.cpu_timer.isActive(), 'CPU monitor timer must be running' window.apply_theme('system', 'system') assert window.result_tabs.count() == 4, 'expected 4 result tabs (log/stats/summary/queue)' @@ -124,7 +124,7 @@ def test_theme_tabs_presets_log_filter_recent(window, qapp, tmp_path, restore_st def test_cpu_bar_expands(window): """The CPU bar fills the free status bar space.""" - assert window.cpu_sys_bar.sizePolicy().horizontalPolicy() == QSizePolicy.Policy.Expanding + assert window.status.cpu_sys_bar.sizePolicy().horizontalPolicy() == QSizePolicy.Policy.Expanding def test_empty_folder_blocked(window, warned, tmp_path): @@ -242,7 +242,7 @@ def test_successful_run_fills_results_history_and_tree(window, qtbot, tmp_path, assert csv_path.is_file() assert _t('summary_saved').format(str(csv_path)) in window.log_console.toPlainText() assert window.retry_btn.isHidden() and window._failed_files == [] - assert window.open_dir_btn.isEnabled() + assert window.status.open_dir_btn.isEnabled() rec = HistoryStore(window.paths.history_file).load()[0] assert rec['engine'] == 'fake' and rec['inputs'] == [fasta] assert rec['files'][0]['file'] == fasta and rec['files'][0]['tree'] == tree diff --git a/tests/gui/test_main_window_smoke.py b/tests/gui/test_main_window_smoke.py index 76d19dd..50d9f4f 100644 --- a/tests/gui/test_main_window_smoke.py +++ b/tests/gui/test_main_window_smoke.py @@ -1,4 +1,4 @@ def test_window_builds(window): assert window.result_tabs.count() == 4 assert window.presets_list.count() >= 3 - assert window.cpu_timer.isActive() + assert window.status.cpu_timer.isActive() diff --git a/tests/gui/test_status_bar.py b/tests/gui/test_status_bar.py new file mode 100644 index 0000000..7589517 --- /dev/null +++ b/tests/gui/test_status_bar.py @@ -0,0 +1,38 @@ +import os + +import pytest + +from phyml_core.utils import psutil +from phyml_ui.status_bar import AppStatusBar + +pytestmark = pytest.mark.skipif(psutil is None, reason='psutil not installed') + + +def test_track_samples_the_process_and_stop_tracking_averages(qtbot): + bar = AppStatusBar(2, 4) + qtbot.addWidget(bar) + bar.track(lambda: os.getpid()) + bar.update_cpu_status() + bar.update_cpu_status() + assert len(bar.samples) == 2 + bar.samples = [100.0, 50.0] + assert bar.stop_tracking() == 75.0 + bar.update_cpu_status() # tracking stopped: no more samples + assert bar.samples == [100.0, 50.0] + + +def test_stop_tracking_without_samples_is_zero(qtbot): + bar = AppStatusBar(2, 4) + qtbot.addWidget(bar) + assert bar.stop_tracking() == 0.0 + + +def test_buttons_emit_signals(qtbot): + bar = AppStatusBar(2, 4) + qtbot.addWidget(bar) + assert not bar.open_dir_btn.isEnabled() + bar.set_open_dir_enabled(True) + with qtbot.waitSignal(bar.open_dir_clicked): + bar.open_dir_btn.click() + with qtbot.waitSignal(bar.settings_clicked): + bar.settings_btn.click() From 5788dee8f14e95f0a8251577fc823685750d1e93 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sat, 31 Oct 2026 05:42:57 +0300 Subject: [PATCH 33/36] Split parameter and navigation panels; main_window only composes the window ParamsPanel holds the model/evolution/search groups (params() / set_params() with one changed signal, set_engine() refills the models) and NavPanel the recent/presets/history lists with signals. The results tabs (ResultsPanel), the FASTA input row (FastaInput) and the settings file handling (settings.SettingsStore, same migration/blocked-save rules) move out too. Auto-save follows params.changed; the layout is pixel-identical. Co-Authored-By: Claude Opus 5.5 --- phyml_core/settings.py | 63 +++ phyml_ui/fasta_input.py | 67 +++ phyml_ui/main_window.py | 589 ++++++-------------------- phyml_ui/nav_panel.py | 87 ++++ phyml_ui/params_panel.py | 176 ++++++++ phyml_ui/results_panel.py | 96 +++++ tests/e2e/test_batch.py | 16 +- tests/e2e/test_iqtree.py | 6 +- tests/gui/test_main_window.py | 74 ++-- tests/gui/test_main_window_smoke.py | 2 +- tests/gui/test_main_window_storage.py | 94 ++-- tests/gui/test_nav_panel.py | 75 ++++ tests/gui/test_params_panel.py | 48 +++ tests/unit/test_settings.py | 74 ++++ 14 files changed, 919 insertions(+), 548 deletions(-) create mode 100644 phyml_ui/fasta_input.py create mode 100644 phyml_ui/nav_panel.py create mode 100644 phyml_ui/params_panel.py create mode 100644 phyml_ui/results_panel.py create mode 100644 tests/gui/test_nav_panel.py create mode 100644 tests/gui/test_params_panel.py diff --git a/phyml_core/settings.py b/phyml_core/settings.py index 465f5dc..cdb9ccb 100644 --- a/phyml_core/settings.py +++ b/phyml_core/settings.py @@ -177,3 +177,66 @@ def migrate_legacy(paths: AppPaths, cwd: str) -> List[str]: for name in pending: shutil.copy2(os.path.join(cwd, name), os.path.join(paths.data_dir, name)) return pending + + +class SettingsStore: + """The settings file as an application keeps it: one legacy migration, a load that + survives a damaged file, saves that keep the keys this version does not know. + + A file that exists but cannot be read (or a failed migration with no config yet) + blocks saving for the session, so defaults never replace the user's real settings + and the migration gate stays open. Messages go to log(level, text). + """ + + def __init__(self, paths: AppPaths, log: Callable[[str, str], None]): + self.paths = paths + self._log = log + self.blocked = False # never save: the file could not be read + self._save_failed = False # the last save failed (the error is logged once) + self._extra = {} # keys of the file this version does not know + + def load(self, cwd: Optional[str] = None) -> tuple: + """Migrate from cwd (default: the working folder) once, then read the file. + Returns (settings as on disk, filled with defaults; whether they read cleanly).""" + self._migrate(cwd) + problems = [] + try: + raw = load(self.paths.config_file, on_error=problems.append) + except OSError as e: + self.blocked = True + problems.append(_t('err_load').format(e)) + raw = copy.deepcopy(DEFAULTS) + for message in problems: + self._log('error', message) + self._extra = {k: v for k, v in raw.items() if k not in DEFAULTS} + return raw, not problems + + def _migrate(self, cwd: Optional[str]) -> None: + try: + cwd = os.getcwd() if cwd is None else cwd # getcwd fails for a deleted folder + copied = migrate_legacy(self.paths, cwd) + except OSError as e: + self._log('error', _t('err_load').format(e)) + if not os.path.exists(self.paths.config_file): + # Saving the defaults now would create the config that closes the migration + # gate, and the legacy settings would never be copied. + self.blocked = True + return + for name in copied: + self._log('system', _t('settings_migrated').format( + os.path.join(cwd, name), os.path.join(self.paths.data_dir, name))) + + def save(self, cfg: dict) -> None: + """Save cfg (the form of from_disk) with the unknown keys; failures are logged once.""" + if self.blocked: + return + data = dict(self._extra) + data.update(to_disk(cfg, self.paths)) + data['schema_version'] = SCHEMA_VERSION + try: + save(self.paths.config_file, data) + self._save_failed = False + except OSError as e: + if not self._save_failed: # every field change saves: report a failure once + self._log('error', _t('err_settings_save').format(e)) + self._save_failed = True diff --git a/phyml_ui/fasta_input.py b/phyml_ui/fasta_input.py new file mode 100644 index 0000000..2bb8dd7 --- /dev/null +++ b/phyml_ui/fasta_input.py @@ -0,0 +1,67 @@ +import os + +from PySide6.QtWidgets import (QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, + QPushButton, QFileDialog) +from PySide6.QtCore import Signal + +from phyml_core import alignment +from phyml_core.i18n import _t + +FASTA_SUFFIXES = ('.fasta', '.fas', '.fa') + + +def batch_files(folder: str) -> list: + """The FASTA files of a batch folder, sorted, full paths. *_aligned.fasta are left out: + those are this app's own MUSCLE outputs.""" + return [os.path.join(folder, f) for f in sorted(os.listdir(folder)) + if f.lower().endswith(FASTA_SUFFIXES) and not f.lower().endswith('_aligned.fasta')] + + +class FastaInput(QWidget): + """The input field (a FASTA file or a batch folder), its two pickers and a one-line + summary of what the field points at.""" + + path_chosen = Signal(str) # picked in a file or folder dialog + + def __init__(self, parent=None): + super().__init__(parent) + v = QVBoxLayout(self) + v.setContentsMargins(0, 0, 0, 0) + v.setSpacing(8) + v.addWidget(QLabel(_t("input_fasta"))) + self.path_edit = QLineEdit() + btn_fasta = QPushButton(_t("btn_select_fasta")) + btn_fasta.clicked.connect(self._browse_file) + btn_fasta_dir = QPushButton(_t("btn_select_dir")) + btn_fasta_dir.clicked.connect(self._browse_dir) + row = QHBoxLayout(); row.addWidget(self.path_edit, 1) + row.addWidget(btn_fasta); row.addWidget(btn_fasta_dir) + v.addLayout(row) + self.info_label = QLabel(' ') + self.info_label.setStyleSheet('color: #9AA0A6;') + self.path_edit.textChanged.connect(self._update_info) + v.addWidget(self.info_label) + + def _browse_file(self): + path, _ = QFileDialog.getOpenFileName(self, _t("file_fasta"), "", "FASTA Files (*.fasta *.fas *.fa)") + if path: + self.path_chosen.emit(path) + + def _browse_dir(self): + path = QFileDialog.getExistingDirectory(self, _t("file_dir")) + if path: + self.path_chosen.emit(path) + + def _update_info(self, path): + """Quick alignment summary under the field.""" + try: + if not path or not os.path.exists(path): + self.info_label.setText(' ') + elif os.path.isdir(path): + self.info_label.setText(_t('fasta_info').format(len(batch_files(path)), '—', '—', '—')) + else: + info = alignment.summarize(path) # an empty or broken file raises: no label + self.info_label.setText(_t('fasta_info').format( + info['n'], info['min_len'], info['max_len'], info['gap_pct'])) + except Exception: + self.info_label.setText(' ') diff --git a/phyml_ui/main_window.py b/phyml_ui/main_window.py index 547a336..be83155 100644 --- a/phyml_ui/main_window.py +++ b/phyml_ui/main_window.py @@ -1,58 +1,51 @@ -import copy import os import shutil from dataclasses import asdict from datetime import datetime -from PySide6.QtWidgets import (QWidget, QVBoxLayout, QHBoxLayout, - QFormLayout, QComboBox, QLineEdit, QPushButton, - QFileDialog, QMessageBox, QTabWidget, - QCheckBox, QGroupBox, QLabel, QSplitter, QToolBox, - QToolButton, QListWidget, QListWidgetItem, - QTableWidget, QTableWidgetItem, - QHeaderView, QScrollArea, QMenu, QInputDialog) +from PySide6.QtWidgets import (QWidget, QVBoxLayout, QHBoxLayout, QComboBox, QLineEdit, + QPushButton, QMessageBox, QCheckBox, QSplitter, QScrollArea, + QInputDialog, QApplication) from PySide6.QtCore import Qt -from PySide6.QtGui import QIcon, QShortcut, QKeySequence +from PySide6.QtGui import QIcon -from phyml_core import __version__, alignment, presets, report, settings +from phyml_core import __version__, presets, report, settings from phyml_core.engines import ENGINES from phyml_core.engines.base import format_issue -from phyml_core.history import HistoryStore, best_tree, display_input, new_record +from phyml_core.history import HistoryStore, best_tree, new_record from phyml_core.i18n import _t from phyml_core.paths import AppPaths from phyml_core.utils import detect_cores, is_mpi_binary, resource_path -from phyml_ui.tree_viewer import TreeViewerWidget -from phyml_ui.themes import resolve, build_qss, role_colors, PALETTES -from phyml_ui.log_console import LogConsole +from phyml_ui.fasta_input import FASTA_SUFFIXES, FastaInput, batch_files +from phyml_ui.nav_panel import NavPanel +from phyml_ui.params_panel import ParamsPanel, set_combo_by_data +from phyml_ui.results_panel import ResultsPanel from phyml_ui.settings_dialog import SettingsDialog from phyml_ui.status_bar import AppStatusBar +from phyml_ui.themes import resolve, build_qss, role_colors, PALETTES +from phyml_ui.tree_viewer import TreeViewerWidget from phyml_ui.worker import RunWorker -# queue marker per file status of the runner -_STATUS_MARKERS = {'pending': '…', 'running': '▶', 'ok': '✓', 'fail': '✗', - 'cancelled': '⊘'} # settings that say nothing about how a run was made: left out of a history record _NOT_RUN_PARAMS = ('presets', 'recent', 'window') -# --- Main Application Window --- class PhyMLApp(QWidget): + """The main window: builds the panels, keeps the settings file and the history in step + with them and starts / cancels / shows runs.""" + def __init__(self, paths=None, engines=None): super().__init__() self.paths = paths if paths is not None else AppPaths.detect() self.engines = ENGINES if engines is None else engines # tests pass fakes self.history_store = HistoryStore(self.paths.history_file) + self.settings_store = settings.SettingsStore(self.paths, self._log) self._is_loading = True - self._autosave_blocked = False # set when the settings file could not be read - self._save_failed = False # the last auto_save failed (the error is logged once) - self._extra_settings = {} # keys of the settings file this version does not know - self._history_records = [] # the records shown in history_list, same order + self._history_records = [] # the records shown in the history list, same order self.worker = None self.fasta_file = "" self._mpi_auto_path = "" - self._batch_mode = False - self._theme_setting = 'system' - self._accent_setting = 'green' + self._theme_setting, self._accent_setting = 'system', 'green' self.engine_setting = 'phyml' self._recent = [] self._user_presets = {} @@ -93,26 +86,8 @@ def initUI(self): self.bind_autosave(self) # after the whole tree is parented - def _build_nav_panel(self): - self.panel = QToolBox() - - self.recent_list = QListWidget() - self.recent_list.itemDoubleClicked.connect(self._open_recent) - self.panel.addItem(self.recent_list, _t("panel_recent")) - - self.presets_list = QListWidget() - self.presets_list.itemDoubleClicked.connect(self._preset_activated) - self.presets_list.setContextMenuPolicy(Qt.CustomContextMenu) - self.presets_list.customContextMenuRequested.connect(self._preset_menu) - self.panel.addItem(self.presets_list, _t("panel_presets")) - - self.history_list = QListWidget() - self.history_list.itemDoubleClicked.connect(self._open_history_entry) - self.panel.addItem(self.history_list, _t("panel_history")) - return self.panel - def _build_middle_pane(self): - """FASTA input + parameter groups (each on its own plate) + controls.""" + """FASTA input + parameter groups (each on its own plate) + navigation + controls.""" # hidden holders for engine paths; edited via the Settings dialog self.phyml_path_input = QLineEdit() self.phyml_path_input.textChanged.connect(self.on_phyml_path_changed) @@ -124,90 +99,27 @@ def _build_middle_pane(self): v = QVBoxLayout(middle) v.setContentsMargins(0, 0, 8, 0) # breathing room from the right splitter v.setSpacing(8) - - v.addWidget(QLabel(_t("input_fasta"))) - self.fasta_path_input = QLineEdit() - btn_fasta = QPushButton(_t("btn_select_fasta")) - btn_fasta.clicked.connect(self.browse_fasta) - btn_fasta_dir = QPushButton(_t("btn_select_dir")) - btn_fasta_dir.clicked.connect(self.browse_fasta_dir) - row = QHBoxLayout(); row.addWidget(self.fasta_path_input, 1) - row.addWidget(btn_fasta); row.addWidget(btn_fasta_dir) - v.addLayout(row) - self.fasta_info_label = QLabel(' ') - self.fasta_info_label.setStyleSheet('color: #9AA0A6;') - self.fasta_path_input.textChanged.connect(self._update_fasta_info) - v.addWidget(self.fasta_info_label) + self.fasta = FastaInput() + self.fasta.path_chosen.connect(self._set_input) + self.fasta_path_input = self.fasta.path_edit + v.addWidget(self.fasta) scroll = QScrollArea() scroll.setWidgetResizable(True) - plates = QWidget() - pv = QVBoxLayout(plates) - pv.setContentsMargins(0, 0, 4, 0) - pv.setSpacing(10) - - gb_model = QGroupBox(_t("tab_model")) - form_basic = QFormLayout(gb_model) - self.chk_use_mpi = QCheckBox(_t("lbl_use_mpi")) - self.le_mpi_np = QLineEdit(str(self.phys_cores)) - self.le_mpi_np.setFixedWidth(50) - mpi_hbox = QHBoxLayout(); mpi_hbox.addWidget(self.chk_use_mpi) - mpi_hbox.addWidget(QLabel(_t("lbl_mpi_np"))); mpi_hbox.addWidget(self.le_mpi_np) - mpi_hbox.addStretch() - form_basic.addRow(_t("mode_mpi"), mpi_hbox) - - self.cb_datatype = QComboBox() - self.cb_datatype.addItem(_t("dt_nt"), "nt") - self.cb_datatype.addItem(_t("dt_aa"), "aa") - self.cb_datatype.currentIndexChanged.connect(self.update_models) - form_basic.addRow(_t("lbl_dt"), self.cb_datatype) - - self.cb_model = QComboBox() - form_basic.addRow(_t("lbl_model"), self.cb_model) - - self.cb_support_method = QComboBox() - self.cb_support_method.addItem(_t("supp_boot"), "bootstrap") - self.cb_support_method.addItem(_t("supp_alrt1"), "-1"); self.cb_support_method.addItem(_t("supp_alrt2"), "-2") - self.cb_support_method.addItem(_t("supp_alrt4"), "-4"); self.cb_support_method.addItem(_t("supp_none"), "0") - self.cb_support_method.currentIndexChanged.connect(self.toggle_bootstrap_input) - form_basic.addRow(_t("lbl_supp"), self.cb_support_method) - - self.le_bootstrap_replicates = QLineEdit("100") - form_basic.addRow(_t("lbl_boot"), self.le_bootstrap_replicates) - pv.addWidget(gb_model) - - gb_evo = QGroupBox(_t("tab_evo")) - form_model = QFormLayout(gb_evo) - self.cb_freq = QComboBox() - self.cb_freq.addItem(_t("freq_def"), "default"); self.cb_freq.addItem(_t("freq_emp"), "e"); self.cb_freq.addItem(_t("freq_ml"), "m") - form_model.addRow(_t("lbl_freq"), self.cb_freq) - self.le_tstv = QLineEdit("e"); form_model.addRow(_t("lbl_tstv"), self.le_tstv) - self.le_pinv = QLineEdit("0.0"); form_model.addRow(_t("lbl_pinv"), self.le_pinv) - self.le_classes = QLineEdit("4"); form_model.addRow(_t("lbl_classes"), self.le_classes) - self.le_gamma = QLineEdit("e"); form_model.addRow(_t("lbl_gamma"), self.le_gamma) - pv.addWidget(gb_evo) - - gb_search = QGroupBox(_t("tab_search")) - form_tree = QFormLayout(gb_search) - self.cb_search = QComboBox() - self.cb_search.addItem(_t("search_nni"), "NNI"); self.cb_search.addItem(_t("search_spr"), "SPR"); self.cb_search.addItem(_t("search_best"), "BEST") - form_tree.addRow(_t("lbl_search"), self.cb_search) - self.cb_optimize = QComboBox() - self.cb_optimize.addItem(_t("opt_tlr"), "tlr"); self.cb_optimize.addItem(_t("opt_tl"), "tl"); self.cb_optimize.addItem(_t("opt_lr"), "lr") - self.cb_optimize.addItem(_t("opt_l"), "l"); self.cb_optimize.addItem(_t("opt_r"), "r"); self.cb_optimize.addItem(_t("opt_n"), "n") - form_tree.addRow(_t("lbl_opt"), self.cb_optimize) - self.chk_parsimony = QCheckBox(_t("chk_pars")) - form_tree.addRow(self.chk_parsimony) - pv.addWidget(gb_search) - pv.addStretch(1) - - scroll.setWidget(plates) + self.params = ParamsPanel(self.phys_cores) + self.params.changed.connect(self.trigger_autosave) + scroll.setWidget(self.params) v.addWidget(scroll, 1) # navigation (recent/presets/history) lives at the bottom of the column - self._build_nav_panel() - self.panel.setMaximumHeight(230) - v.addWidget(self.panel) + self.nav = NavPanel() + self.nav.setMaximumHeight(230) + self.nav.recent_activated.connect(self._open_recent) + self.nav.preset_activated.connect(self._preset_activated) + self.nav.preset_save_requested.connect(self._save_preset) + self.nav.preset_delete_requested.connect(self._delete_preset) + self.nav.history_activated.connect(self._open_history_entry) + v.addWidget(self.nav) control_row = QHBoxLayout() self.btn_run = QPushButton(_t("btn_run")) @@ -227,81 +139,54 @@ def _build_middle_pane(self): return middle def _build_right_pane(self): - """Big tree viewer on top, Log/Stats/Summary tabs underneath.""" + """Big tree viewer on top, Log/Stats/Summary/Queue tabs underneath.""" right = QWidget() v = QVBoxLayout(right) v.setContentsMargins(8, 0, 0, 0) # breathing room from the left splitter self.tree_view = TreeViewerWidget() - - self.result_tabs = QTabWidget() - self.log_console = LogConsole() - self.result_tabs.addTab(self.log_console, _t("tab_log")) - - stats_tab = QWidget() - stats_lay = QVBoxLayout(stats_tab) - self.stats_file_label = QLabel("—") - self.stats_table = QTableWidget(0, 2) - self.stats_table.horizontalHeader().setVisible(False) - self.stats_table.verticalHeader().setVisible(False) - self.stats_table.setAlternatingRowColors(True) - self.stats_table.horizontalHeader().setSectionResizeMode(0, QHeaderView.ResizeToContents) - self.stats_table.horizontalHeader().setSectionResizeMode(1, QHeaderView.Stretch) - stats_lay.addWidget(self.stats_file_label) - stats_lay.addWidget(self.stats_table) - self.result_tabs.addTab(stats_tab, _t("tab_stats")) - - self.summary_table = QTableWidget(0, 8) - self.summary_table.verticalHeader().setVisible(False) - self.summary_table.setAlternatingRowColors(True) - self.summary_table.horizontalHeader().setSectionResizeMode(0, QHeaderView.Stretch) - for col in range(1, 8): - self.summary_table.horizontalHeader().setSectionResizeMode(col, QHeaderView.ResizeToContents) - self.result_tabs.addTab(self.summary_table, _t("tab_summary")) - - queue_tab = QWidget() - queue_lay = QVBoxLayout(queue_tab) - self.queue_list = QListWidget() - self.queue_list.setAlternatingRowColors(True) - self.retry_btn = QToolButton() - self.retry_btn.setText(_t("btn_retry_failed")) - self.retry_btn.setVisible(False) - self.retry_btn.clicked.connect(self._retry_failed) - queue_lay.addWidget(self.queue_list, 1) - queue_lay.addWidget(self.retry_btn) - self.result_tabs.addTab(queue_tab, _t("tab_queue")) - - split = QSplitter(Qt.Vertical) - split.addWidget(self.tree_view) - split.addWidget(self.result_tabs) - split.setSizes([470, 260]) - split.setChildrenCollapsible(False) - self.right_split = split - v.addWidget(split, 1) + self.result_tabs = ResultsPanel() + self.result_tabs.retry_clicked.connect(self._retry_failed) + self.log_console = self.result_tabs.log_console + self.right_split = QSplitter(Qt.Vertical) + self.right_split.addWidget(self.tree_view) + self.right_split.addWidget(self.result_tabs) + self.right_split.setSizes([470, 260]) + self.right_split.setChildrenCollapsible(False) + v.addWidget(self.right_split, 1) return right - # ---------- drag & drop ---------- + # ---------- input, drag & drop, recent ---------- + + def _set_input(self, path): + self.fasta_path_input.setText(path) + self.fasta_file = path def dragEnterEvent(self, event): urls = event.mimeData().urls() if urls: p = urls[0].toLocalFile() - if os.path.isdir(p) or p.lower().endswith(('.fasta', '.fas', '.fa')): + if os.path.isdir(p) or p.lower().endswith(FASTA_SUFFIXES): event.acceptProposedAction() def dropEvent(self, event): - p = event.mimeData().urls()[0].toLocalFile() - self.fasta_path_input.setText(p) - self.fasta_file = p + self._set_input(event.mimeData().urls()[0].toLocalFile()) event.acceptProposedAction() - # ---------- panel / theme / settings helpers ---------- + def _remember_recent(self, path): + self._recent = ([path] + [p for p in self._recent if p != path])[:10] + self.nav.set_recent(self._recent) + + def _open_recent(self, path): + if os.path.exists(path): + self._set_input(path) + + # ---------- theme, settings, models ---------- def open_settings(self): SettingsDialog(self, self).exec() def apply_theme(self, theme_setting=None, accent_setting=None): """Resolve settings (with 'system' support), style the app and the log.""" - from PySide6.QtWidgets import QApplication if theme_setting is not None: self._theme_setting = theme_setting if accent_setting is not None: @@ -312,82 +197,27 @@ def apply_theme(self, theme_setting=None, accent_setting=None): pal = PALETTES[theme] self.tree_view.set_colors(pal['text'], accent, pal['base']) - def _refresh_recent(self): - self.recent_list.clear() - for p in self._recent: - self.recent_list.addItem(os.path.basename(p) + ('/' if os.path.isdir(p) else '')) - - def _remember_recent(self, path): - if path in self._recent: - self._recent.remove(path) - self._recent.insert(0, path) - self._recent = self._recent[:10] - self._refresh_recent() - - def _open_recent(self, item): - row = self.recent_list.row(item) - if row < len(self._recent) and os.path.exists(self._recent[row]): - p = self._recent[row] - self.fasta_path_input.setText(p) - self.fasta_file = p - - # ---------- pipeline plumbing ---------- - - def closeEvent(self, event): - if self.worker and self.worker.isRunning(): - # synchronous: after close the process may exit and cut off a daemon thread, - # leaving phyml-mpi / smpd children alive - self.worker.cancel(block=True) - self.worker.wait() - self.auto_save() # persist window geometry and splitter ratios - event.accept() - - def toggle_bootstrap_input(self): - is_bootstrap = self.cb_support_method.currentData() == "bootstrap" - self.le_bootstrap_replicates.setEnabled(is_bootstrap) - if not self._is_loading: self.auto_save() - def update_models(self): - self.cb_model.clear() # an unknown engine in a hand-edited config gets PhyML's list, as a run uses PhyML for it - engine = self.engines.get(self.engine_setting) or ENGINES['phyml'] - for m in engine.models(self.cb_datatype.currentData()): - self.cb_model.addItem(m, m) - if not self._is_loading: self.auto_save() + self.params.set_engine(self.engines.get(self.engine_setting) or ENGINES['phyml']) - def _update_fasta_info(self, path): - """Quick alignment summary under the FASTA field.""" - try: - if not path or not os.path.exists(path): - self.fasta_info_label.setText(' ') - return - if os.path.isdir(path): - n = sum(1 for f in os.listdir(path) - if f.lower().endswith(('.fasta', '.fas', '.fa')) - and not f.lower().endswith('_aligned.fasta')) - self.fasta_info_label.setText(_t('fasta_info').format(n, '—', '—', '—')) - return - info = alignment.summarize(path) # an empty or broken file raises: no label - self.fasta_info_label.setText(_t('fasta_info').format( - info['n'], info['min_len'], info['max_len'], info['gap_pct'])) - except Exception: - self.fasta_info_label.setText(' ') - - def browse_fasta(self): - path, _ = QFileDialog.getOpenFileName(self, _t("file_fasta"), "", "FASTA Files (*.fasta *.fas *.fa)") - if path: - self.fasta_path_input.setText(path) - self.fasta_file = path - - def browse_fasta_dir(self): - path = QFileDialog.getExistingDirectory(self, _t("file_dir")) - if path: - self.fasta_path_input.setText(path) - self.fasta_file = path + set_combo_by_data = staticmethod(set_combo_by_data) + + def on_phyml_path_changed(self, path): + if not is_mpi_binary(path): + self._mpi_auto_path = "" + elif not self.params.chk_use_mpi.isChecked() and self._mpi_auto_path != path: + self._mpi_auto_path = path + self.params.chk_use_mpi.setChecked(True) + if not self._is_loading: + self._log('system', _t("mpi_detected")) + + # ---------- settings file ---------- def bind_autosave(self, parent_widget): for child in parent_widget.children(): - if isinstance(child, QLineEdit): child.textChanged.connect(self.trigger_autosave) + if child is self.params: continue # it saves through params.changed + elif isinstance(child, QLineEdit): child.textChanged.connect(self.trigger_autosave) elif isinstance(child, QComboBox): child.currentIndexChanged.connect(self.trigger_autosave) elif isinstance(child, QCheckBox): child.toggled.connect(self.trigger_autosave) else: self.bind_autosave(child) @@ -395,6 +225,9 @@ def bind_autosave(self, parent_widget): def trigger_autosave(self, *args): if not self._is_loading: self.auto_save() + def auto_save(self): + self.settings_store.save(self.get_current_config()) + def get_current_config(self): return { "engine": self.engine_setting, @@ -402,84 +235,22 @@ def get_current_config(self): "phyml_path": self.phyml_path_input.text(), "mpirun_path": self.mpirun_path_input.text(), "muscle_path": self.muscle_path_input.text(), - "datatype": self.cb_datatype.currentData(), - "model": self.cb_model.currentData(), - "support_method": self.cb_support_method.currentData(), - "bootstrap_replicates": self.le_bootstrap_replicates.text(), - "use_mpi": self.chk_use_mpi.isChecked(), - "mpi_np": self.le_mpi_np.text(), - "freq": self.cb_freq.currentData(), - "tstv": self.le_tstv.text(), - "pinv": self.le_pinv.text(), - "classes": self.le_classes.text(), - "gamma": self.le_gamma.text(), - "search": self.cb_search.currentData(), - "optimize": self.cb_optimize.currentData(), - "parsimony": self.chk_parsimony.isChecked(), + **self.params.params(), "show_tree": self.chk_show_tree.isChecked(), "theme": self._theme_setting, "accent": self._accent_setting, "recent": self._recent, "presets": self._user_presets, - "window": { - "w": self.width(), "h": self.height(), - "splitter": self.splitter.sizes(), - "vsplit": self.right_split.sizes(), - } + "window": {"w": self.width(), "h": self.height(), + "splitter": self.splitter.sizes(), "vsplit": self.right_split.sizes()}, } - def _log(self, level, text): - """A log line at the given level; messages from the core modules may end with a newline.""" - self.log_console.append(level, text.rstrip('\n')) - - def auto_save(self): - if self._autosave_blocked: - return - data = dict(self._extra_settings) - data.update(settings.to_disk(self.get_current_config(), self.paths)) - data['schema_version'] = settings.SCHEMA_VERSION - try: - settings.save(self.paths.config_file, data) - self._save_failed = False - except OSError as e: - if not self._save_failed: # every field change saves: report a failure once - self._log('error', _t('err_settings_save').format(e)) - self._save_failed = True - - def _migrate_legacy_files(self): - """Copy config/history from the working folder of older versions, once.""" - try: - cwd = os.getcwd() - copied = settings.migrate_legacy(self.paths, cwd) - except OSError as e: - self._log('error', _t('err_load').format(e)) - if not os.path.exists(self.paths.config_file): - # Saving the defaults now would create the config that closes the migration - # gate, and the legacy settings would never be copied. - self._autosave_blocked = True - return - for name in copied: - self._log('system', _t('settings_migrated').format( - os.path.join(cwd, name), os.path.join(self.paths.data_dir, name))) - def load_config(self): self.update_models() - self._migrate_legacy_files() - problems = [] - try: - raw = settings.load(self.paths.config_file, on_error=problems.append) - except OSError as e: - # The file is there but unreadable: work with defaults, but never save them - # over the user's real settings. - self._autosave_blocked = True - problems.append(_t('err_load').format(e)) - raw = copy.deepcopy(settings.DEFAULTS) - for message in problems: - self._log('error', message) - self._extra_settings = {k: v for k, v in raw.items() if k not in settings.DEFAULTS} + raw, clean = self.settings_store.load() try: self._apply_config(settings.from_disk(raw, self.paths)) - if not problems and os.path.isfile(self.paths.config_file): + if clean and os.path.isfile(self.paths.config_file): self._log('system', _t("sys_load")) except Exception as e: self._log('error', _t("err_load").format(e)) @@ -497,42 +268,26 @@ def _apply_config(self, config): mpirun = shutil.which("mpiexec") or shutil.which("mpirun") or mpirun self.mpirun_path_input.setText(mpirun) self.muscle_path_input.setText(config["muscle_path"]) - self._apply_model_params(config) - self.chk_use_mpi.setChecked(config["use_mpi"]) - self.le_mpi_np.setText(config["mpi_np"] or str(self.phys_cores)) # '' = physical cores + self.update_models() + self.params.set_params(dict(presets.extract(config), use_mpi=config["use_mpi"], + mpi_np=config["mpi_np"] or str(self.phys_cores))) # '' = cores self.chk_show_tree.setChecked(config["show_tree"]) self._recent = [p for p in config["recent"] if isinstance(p, str) and os.path.exists(p)] - self._refresh_recent() + self.nav.set_recent(self._recent) self._user_presets = dict(config["presets"]) self.apply_theme(config["theme"], config["accent"]) win = config["window"] if win.get("w") and win.get("h"): self.resize(min(win["w"], 4096), min(win["h"], 4096)) - if isinstance(win.get("splitter"), list) and len(win["splitter"]) == 2: - self.splitter.setSizes(win["splitter"]) - if isinstance(win.get("vsplit"), list) and len(win["vsplit"]) == 2: - self.right_split.setSizes(win["vsplit"]) - self.toggle_bootstrap_input() - - def _apply_model_params(self, params): - """Show the model and search parameters (the part presets also carry).""" - self.set_combo_by_data(self.cb_datatype, params["datatype"]) - self.update_models() - self.set_combo_by_data(self.cb_model, params["model"]) - self.set_combo_by_data(self.cb_freq, params["freq"]) - self.set_combo_by_data(self.cb_support_method, params["support_method"]) - self.set_combo_by_data(self.cb_search, params["search"]) - self.set_combo_by_data(self.cb_optimize, params["optimize"]) - self.le_bootstrap_replicates.setText(params["bootstrap_replicates"]) - self.le_tstv.setText(params["tstv"]) - self.le_pinv.setText(params["pinv"]) - self.le_classes.setText(params["classes"]) - self.le_gamma.setText(params["gamma"]) - self.chk_parsimony.setChecked(params["parsimony"]) - - def set_combo_by_data(self, combo, data_val): - idx = combo.findData(data_val) - if idx >= 0: combo.setCurrentIndex(idx) + for key, split in (("splitter", self.splitter), ("vsplit", self.right_split)): + if isinstance(win.get(key), list) and len(win[key]) == 2: + split.setSizes(win[key]) + + # ---------- runs ---------- + + def _log(self, level, text): + """A log line at the given level; messages from the core modules may end with a newline.""" + self.log_console.append(level, text.rstrip('\n')) def start_pipeline(self, files_override=None): if self.worker is not None and self.worker.isRunning(): @@ -543,18 +298,10 @@ def start_pipeline(self, files_override=None): target = self.fasta_path_input.text() or self.fasta_file if not target or not os.path.exists(target): QMessageBox.warning(self, _t("err_box_title"), _t("err_box_msg")); return - if os.path.isdir(target): - # skip *_aligned.fasta: those are this app's own MUSCLE outputs - files = sorted(f for f in os.listdir(target) - if f.lower().endswith(('.fasta', '.fas', '.fa')) - and not f.lower().endswith('_aligned.fasta')) - if not files: - QMessageBox.warning(self, _t("err_box_title"), _t("batch_empty")); return - files = [os.path.join(target, f) for f in files] - out_dir = target - else: - files = [target] - out_dir = os.path.dirname(os.path.abspath(target)) + files = batch_files(target) if os.path.isdir(target) else [target] + if not files: + QMessageBox.warning(self, _t("err_box_title"), _t("batch_empty")); return + out_dir = target if os.path.isdir(target) else os.path.dirname(os.path.abspath(target)) inputs = [target] else: files = list(files_override) @@ -572,23 +319,16 @@ def start_pipeline(self, files_override=None): QMessageBox.warning(self, _t('err_box_title'), '\n'.join(format_issue(issue) for issue in errors)) return - self._batch_mode = len(files) > 1 self._remember_recent(target) if not self._is_loading: self.auto_save() - self.btn_run.setEnabled(False); self.btn_cancel.setEnabled(True); self.log_console.clear() - self.retry_btn.setVisible(False) - self.result_tabs.setCurrentWidget(self.log_console) - self._populate_queue(files) - if self._batch_mode: - self.result_tabs.setCurrentWidget(self.queue_list.parentWidget()) + self.btn_run.setEnabled(False); self.btn_cancel.setEnabled(True) + self.result_tabs.begin_run(files) self.status.track(self._active_pid) self._run_started = datetime.now() - self._run_config = config - self._run_inputs = inputs - self._run_out_dir = out_dir + self._run_config, self._run_inputs, self._run_out_dir = config, inputs, out_dir self.worker = RunWorker.for_run(config, files, self.paths, engines=self.engines) self.worker.log.connect(self._log) - self.worker.status.connect(self._on_file_status) + self.worker.status.connect(self.result_tabs.set_file_status) self.worker.done.connect(self.pipeline_finished) self.worker.start() @@ -604,6 +344,15 @@ def cancel_pipeline(self): self.btn_cancel.setEnabled(False) self._log('error', _t("sys_interrupt")) + def closeEvent(self, event): + if self.worker and self.worker.isRunning(): + # synchronous: after close the process may exit and cut off a daemon thread, + # leaving phyml-mpi / smpd children alive + self.worker.cancel(block=True) + self.worker.wait() + self.auto_save() # persist window geometry and splitter ratios + event.accept() + def pipeline_finished(self, result): """Show a finished run (a RunResult): tables, summary files, history, retry, tree.""" if self.worker is not None: @@ -613,12 +362,11 @@ def pipeline_finished(self, result): self.status.stop_tracking() if result.files: rows = report.summary_rows(result.files) - self._populate_stats(result) - self._populate_summary(rows) + self.result_tabs.show_result(result, rows) self._write_summary(rows) self._append_history(result) self._failed_files = [f.file for f in result.files if f.status == 'fail'] - self.retry_btn.setVisible(bool(self._failed_files)) + self.result_tabs.retry_btn.setVisible(bool(self._failed_files)) self._last_out_dir = self._run_out_dir self.status.set_open_dir_enabled(self._last_out_dir is not None) if result.success and self.chk_show_tree.isChecked(): @@ -637,21 +385,6 @@ def _write_summary(self, rows): key = 'xlsx_saved' if path.lower().endswith('.xlsx') else 'summary_saved' self._log('success', _t(key).format(path)) - # ---------- queue ---------- - - def _populate_queue(self, files): - self.queue_list.clear() - for f in files: - self.queue_list.addItem(os.path.basename(f) + ' …') - - def _on_file_status(self, name, status): - marker = _STATUS_MARKERS.get(status, '?') - for i in range(self.queue_list.count()): - item = self.queue_list.item(i) - if item.text().split(' ')[0] == name: - item.setText(f'{name} {marker}') - return - def _retry_failed(self): if self._failed_files: self.start_pipeline(files_override=self._failed_files) @@ -679,19 +412,15 @@ def _append_history(self, result): self._refresh_history() def _refresh_history(self): - self.history_list.clear() try: self._history_records = self.history_store.load( on_error=lambda message: self._log('error', message))[:50] except OSError as e: self._history_records = [] self._log('error', _t('err_load').format(e)) - for rec in self._history_records: - self.history_list.addItem(_t('hist_entry').format( - rec.get('ts', ''), display_input(rec), rec.get('ok', 0), rec.get('n', 0))) + self.nav.set_history(self._history_records) - def _open_history_entry(self, item): - row = self.history_list.row(item) + def _open_history_entry(self, row): if row >= len(self._history_records): return tree = best_tree(self._history_records[row]) @@ -703,39 +432,19 @@ def _open_history_entry(self, item): # ---------- presets ---------- def _refresh_presets(self): - self.presets_list.clear() - save_item = QListWidgetItem(_t('preset_save_current')) - save_item.setData(Qt.UserRole, '__save__') - self.presets_list.addItem(save_item) - for label_key in presets.BUILTIN: - it = QListWidgetItem(_t(label_key)) - it.setData(Qt.UserRole, ('builtin', label_key)) - self.presets_list.addItem(it) - for name in self._user_presets: - it = QListWidgetItem(name) - it.setData(Qt.UserRole, ('user', name)) - self.presets_list.addItem(it) - - def _preset_activated(self, item): - data = item.data(Qt.UserRole) - if data == '__save__': - self._save_preset() - return - kind, key = data - preset = presets.BUILTIN[key] if kind == 'builtin' else self._user_presets.get(key, {}) - self._apply_preset(preset) - self._log('success', _t('preset_applied').format(item.text())) + self.nav.set_presets(presets.BUILTIN, self._user_presets) - def _apply_preset(self, preset): + def _preset_activated(self, preset): params = presets.apply(preset, self.get_current_config()) - self._is_loading = True + self._is_loading = True # one save for the whole preset, after it is shown try: self.engine_setting = params['engine'] - self._apply_model_params(params) + self.update_models() + self.params.set_params(presets.extract(params)) finally: self._is_loading = False - self.toggle_bootstrap_input() self.auto_save() + self._log('success', _t('preset_applied').format(self.nav.activated_preset_name)) def _save_preset(self): name, ok = QInputDialog.getText(self, _t('preset_save_current'), _t('preset_prompt')) @@ -747,51 +456,7 @@ def _save_preset(self): self._refresh_presets() self._log('success', _t('preset_saved').format(name)) - def _preset_menu(self, pos): - item = self.presets_list.itemAt(pos) - if item is None: - return - data = item.data(Qt.UserRole) - if not (isinstance(data, tuple) and data[0] == 'user'): - return - menu = QMenu(self) - act = menu.addAction(_t('preset_delete')) - if menu.exec(self.presets_list.mapToGlobal(pos)) == act: - self._user_presets.pop(data[1], None) - self.auto_save() - self._refresh_presets() - - def _populate_stats(self, result): - """The statistics of the last file that finished 'ok'.""" - done = [f for f in result.files if f.status == 'ok'] - if not done: - return - last = done[-1] - self.stats_file_label.setText(_t("stats_field_file") + ': ' + os.path.basename(last.file)) - fields = [('stats_field_model', 'model'), ('stats_field_taxa', 'taxa'), - ('stats_field_lnl', 'lnL'), ('stats_field_gamma', 'gamma'), - ('stats_field_tstv', 'tstv'), ('stats_field_time', 'time')] - self.stats_table.setRowCount(len(fields)) - for i, (key, col) in enumerate(fields): - self.stats_table.setItem(i, 0, QTableWidgetItem(_t(key))) - self.stats_table.setItem(i, 1, QTableWidgetItem((last.stats or {}).get(col) or 'n/a')) - - def _populate_summary(self, rows): - cols = ['file', 'status', 'model', 'taxa', 'lnL', 'gamma', 'tstv', 'time'] - heads = [_t("sum_col_" + c) for c in cols] - self.summary_table.setColumnCount(len(cols)) - self.summary_table.setHorizontalHeaderLabels(heads) - self.summary_table.setRowCount(len(rows)) - for i, r in enumerate(rows): - for j, c in enumerate(cols): - self.summary_table.setItem(i, j, QTableWidgetItem(r.get(c) or '')) - - def on_phyml_path_changed(self, path): - if is_mpi_binary(path): - if not self.chk_use_mpi.isChecked() and self._mpi_auto_path != path: - self._mpi_auto_path = path - self.chk_use_mpi.setChecked(True) - if not self._is_loading: - self._log('system', _t("mpi_detected")) - else: - self._mpi_auto_path = "" + def _delete_preset(self, name): + self._user_presets.pop(name, None) + self.auto_save() + self._refresh_presets() diff --git a/phyml_ui/nav_panel.py b/phyml_ui/nav_panel.py new file mode 100644 index 0000000..34ad353 --- /dev/null +++ b/phyml_ui/nav_panel.py @@ -0,0 +1,87 @@ +import os + +from PySide6.QtWidgets import QToolBox, QListWidget, QListWidgetItem, QMenu +from PySide6.QtCore import Qt, Signal + +from phyml_core.history import display_input +from phyml_core.i18n import _t + +_SAVE = '__save__' # data of the "save current settings" row of the presets list + + +class NavPanel(QToolBox): + """Recent inputs, presets and run history; a double click asks the window to act.""" + + recent_activated = Signal(str) # the full path of the input + preset_activated = Signal(dict) # a copy of the preset's parameters + preset_save_requested = Signal() + preset_delete_requested = Signal(str) # the name of a user preset + history_activated = Signal(int) # the row, = the index in set_history's list + + def __init__(self, parent=None): + super().__init__(parent) + self._presets = {} # (kind, key) -> preset dict + self.activated_preset_name = '' # the shown name of the last activated preset + + self.recent_list = QListWidget() + self.recent_list.itemDoubleClicked.connect( + lambda item: self.recent_activated.emit(item.data(Qt.UserRole))) + self.addItem(self.recent_list, _t("panel_recent")) + + self.presets_list = QListWidget() + self.presets_list.itemDoubleClicked.connect(self._preset_double_clicked) + self.presets_list.setContextMenuPolicy(Qt.CustomContextMenu) + self.presets_list.customContextMenuRequested.connect(self._preset_menu) + self.addItem(self.presets_list, _t("panel_presets")) + + self.history_list = QListWidget() + self.history_list.itemDoubleClicked.connect( + lambda item: self.history_activated.emit(self.history_list.row(item))) + self.addItem(self.history_list, _t("panel_history")) + + def set_recent(self, paths: list) -> None: + self.recent_list.clear() + for p in paths: + item = QListWidgetItem(os.path.basename(p) + ('/' if os.path.isdir(p) else '')) + item.setData(Qt.UserRole, p) + self.recent_list.addItem(item) + + def set_presets(self, builtin: dict, user: dict) -> None: + """The save row, the built-in presets (keyed by their label key), the user ones.""" + self.presets_list.clear() + self._presets = {} + save_item = QListWidgetItem(_t('preset_save_current')) + save_item.setData(Qt.UserRole, _SAVE) + self.presets_list.addItem(save_item) + for kind, presets in (('builtin', builtin), ('user', user)): + for key, preset in presets.items(): + it = QListWidgetItem(_t(key) if kind == 'builtin' else key) + it.setData(Qt.UserRole, (kind, key)) + self.presets_list.addItem(it) + self._presets[(kind, key)] = preset + + def set_history(self, records: list) -> None: + self.history_list.clear() + for rec in records: + self.history_list.addItem(_t('hist_entry').format( + rec.get('ts', ''), display_input(rec), rec.get('ok', 0), rec.get('n', 0))) + + def _preset_double_clicked(self, item): + data = item.data(Qt.UserRole) + if data == _SAVE: + self.preset_save_requested.emit() + return + self.activated_preset_name = item.text() + self.preset_activated.emit(dict(self._presets.get(tuple(data), {}))) + + def _preset_menu(self, pos): + item = self.presets_list.itemAt(pos) + if item is None: + return + data = item.data(Qt.UserRole) + if not (isinstance(data, tuple) and data[0] == 'user'): + return + menu = QMenu(self) + act = menu.addAction(_t('preset_delete')) + if menu.exec(self.presets_list.mapToGlobal(pos)) == act: + self.preset_delete_requested.emit(data[1]) diff --git a/phyml_ui/params_panel.py b/phyml_ui/params_panel.py new file mode 100644 index 0000000..dc824f6 --- /dev/null +++ b/phyml_ui/params_panel.py @@ -0,0 +1,176 @@ +from typing import Optional + +from PySide6.QtWidgets import (QWidget, QVBoxLayout, QHBoxLayout, QFormLayout, + QComboBox, QLineEdit, QCheckBox, QGroupBox, QLabel) +from PySide6.QtCore import Signal + +from phyml_core.engines.base import Engine +from phyml_core.i18n import _t + +# params() keys, in the order the settings file has always listed them +_COMBOS = ('datatype', 'model', 'support_method') +_TEXTS = ('tstv', 'pinv', 'classes', 'gamma') + + +def set_combo_by_data(combo, data_val): + """Select the item whose data is data_val; an unknown value leaves the combo alone.""" + idx = combo.findData(data_val) + if idx >= 0: combo.setCurrentIndex(idx) + + +class ParamsPanel(QWidget): + """The model, evolution and tree-search groups, shown as and read back from a params + dict: the PRESET_KEYS without 'engine', plus 'use_mpi' and 'mpi_np'. + + changed is emitted once for every edit, and once for a whole set_params / set_engine. + """ + + changed = Signal() + + def __init__(self, default_np: int = 1, parent=None): + super().__init__(parent) + self._engine: Optional[Engine] = None + self._quiet = 0 # > 0 while fields are filled programmatically: no changed per field + pv = QVBoxLayout(self) + pv.setContentsMargins(0, 0, 4, 0) + pv.setSpacing(10) + + gb_model = QGroupBox(_t("tab_model")) + form_basic = QFormLayout(gb_model) + self.chk_use_mpi = QCheckBox(_t("lbl_use_mpi")) + self.le_mpi_np = QLineEdit(str(default_np)) + self.le_mpi_np.setFixedWidth(50) + mpi_hbox = QHBoxLayout(); mpi_hbox.addWidget(self.chk_use_mpi) + mpi_hbox.addWidget(QLabel(_t("lbl_mpi_np"))); mpi_hbox.addWidget(self.le_mpi_np) + mpi_hbox.addStretch() + form_basic.addRow(_t("mode_mpi"), mpi_hbox) + + self.cb_datatype = QComboBox() + self.cb_datatype.addItem(_t("dt_nt"), "nt") + self.cb_datatype.addItem(_t("dt_aa"), "aa") + form_basic.addRow(_t("lbl_dt"), self.cb_datatype) + + self.cb_model = QComboBox() + form_basic.addRow(_t("lbl_model"), self.cb_model) + + self.cb_support_method = QComboBox() + self.cb_support_method.addItem(_t("supp_boot"), "bootstrap") + self.cb_support_method.addItem(_t("supp_alrt1"), "-1"); self.cb_support_method.addItem(_t("supp_alrt2"), "-2") + self.cb_support_method.addItem(_t("supp_alrt4"), "-4"); self.cb_support_method.addItem(_t("supp_none"), "0") + form_basic.addRow(_t("lbl_supp"), self.cb_support_method) + + self.le_bootstrap_replicates = QLineEdit("100") + form_basic.addRow(_t("lbl_boot"), self.le_bootstrap_replicates) + pv.addWidget(gb_model) + + gb_evo = QGroupBox(_t("tab_evo")) + form_model = QFormLayout(gb_evo) + self.cb_freq = QComboBox() + self.cb_freq.addItem(_t("freq_def"), "default"); self.cb_freq.addItem(_t("freq_emp"), "e"); self.cb_freq.addItem(_t("freq_ml"), "m") + form_model.addRow(_t("lbl_freq"), self.cb_freq) + self.le_tstv = QLineEdit("e"); form_model.addRow(_t("lbl_tstv"), self.le_tstv) + self.le_pinv = QLineEdit("0.0"); form_model.addRow(_t("lbl_pinv"), self.le_pinv) + self.le_classes = QLineEdit("4"); form_model.addRow(_t("lbl_classes"), self.le_classes) + self.le_gamma = QLineEdit("e"); form_model.addRow(_t("lbl_gamma"), self.le_gamma) + pv.addWidget(gb_evo) + + gb_search = QGroupBox(_t("tab_search")) + form_tree = QFormLayout(gb_search) + self.cb_search = QComboBox() + self.cb_search.addItem(_t("search_nni"), "NNI"); self.cb_search.addItem(_t("search_spr"), "SPR"); self.cb_search.addItem(_t("search_best"), "BEST") + form_tree.addRow(_t("lbl_search"), self.cb_search) + self.cb_optimize = QComboBox() + self.cb_optimize.addItem(_t("opt_tlr"), "tlr"); self.cb_optimize.addItem(_t("opt_tl"), "tl"); self.cb_optimize.addItem(_t("opt_lr"), "lr") + self.cb_optimize.addItem(_t("opt_l"), "l"); self.cb_optimize.addItem(_t("opt_r"), "r"); self.cb_optimize.addItem(_t("opt_n"), "n") + form_tree.addRow(_t("lbl_opt"), self.cb_optimize) + self.chk_parsimony = QCheckBox(_t("chk_pars")) + form_tree.addRow(self.chk_parsimony) + pv.addWidget(gb_search) + pv.addStretch(1) + + self.cb_datatype.currentIndexChanged.connect(self._datatype_changed) + self.cb_support_method.currentIndexChanged.connect(self._toggle_bootstrap_input) + for combo in (self.cb_model, self.cb_support_method, self.cb_freq, + self.cb_search, self.cb_optimize): + combo.currentIndexChanged.connect(self._field_changed) + for edit in (self.le_mpi_np, self.le_bootstrap_replicates, self.le_tstv, + self.le_pinv, self.le_classes, self.le_gamma): + edit.textChanged.connect(self._field_changed) + for check in (self.chk_use_mpi, self.chk_parsimony): + check.toggled.connect(self._field_changed) + + # ---------- public ---------- + + def set_engine(self, engine: Engine) -> None: + """Show the model list of engine for the current data type.""" + self._engine = engine + self._quiet += 1 + try: + self._refill_models() + finally: + self._quiet -= 1 + self.changed.emit() + + def params(self) -> dict: + p = {key: self._combo(key).currentData() for key in _COMBOS} + p['bootstrap_replicates'] = self.le_bootstrap_replicates.text() + p['use_mpi'] = self.chk_use_mpi.isChecked() + p['mpi_np'] = self.le_mpi_np.text() + p['freq'] = self.cb_freq.currentData() + p.update({key: self._edit(key).text() for key in _TEXTS}) + p['search'] = self.cb_search.currentData() + p['optimize'] = self.cb_optimize.currentData() + p['parsimony'] = self.chk_parsimony.isChecked() + return p + + def set_params(self, params: dict) -> None: + """Show the keys of params() that params has; changed is emitted once, at the end.""" + self._quiet += 1 + try: + if 'datatype' in params: + set_combo_by_data(self.cb_datatype, params['datatype']) + self._refill_models() # the old type's list would not hold the model + for key in ('model', 'freq', 'support_method', 'search', 'optimize'): + if key in params: + set_combo_by_data(self._combo(key), params[key]) + for key in ('bootstrap_replicates', 'mpi_np', *_TEXTS): + if key in params: + self._edit(key).setText(params[key]) + if 'use_mpi' in params: + self.chk_use_mpi.setChecked(params['use_mpi']) + if 'parsimony' in params: + self.chk_parsimony.setChecked(params['parsimony']) + self._toggle_bootstrap_input() + finally: + self._quiet -= 1 + self.changed.emit() + + # ---------- internals ---------- + + def _combo(self, key): + return getattr(self, 'cb_' + key) + + def _edit(self, key): + return getattr(self, 'le_' + key) + + def _refill_models(self): + self.cb_model.clear() + if self._engine is not None: + for m in self._engine.models(self.cb_datatype.currentData()): + self.cb_model.addItem(m, m) + + def _datatype_changed(self): + self._quiet += 1 + try: + self._refill_models() + finally: + self._quiet -= 1 + self._field_changed() + + def _toggle_bootstrap_input(self): + is_bootstrap = self.cb_support_method.currentData() == "bootstrap" + self.le_bootstrap_replicates.setEnabled(is_bootstrap) + + def _field_changed(self, *args): + if not self._quiet: + self.changed.emit() diff --git a/phyml_ui/results_panel.py b/phyml_ui/results_panel.py new file mode 100644 index 0000000..7507145 --- /dev/null +++ b/phyml_ui/results_panel.py @@ -0,0 +1,96 @@ +import os + +from PySide6.QtWidgets import (QWidget, QVBoxLayout, QTabWidget, QLabel, QListWidget, + QTableWidget, QTableWidgetItem, QHeaderView, QToolButton) +from PySide6.QtCore import Signal + +from phyml_core.i18n import _t +from phyml_ui.log_console import LogConsole + +# queue marker per file status of the runner +_STATUS_MARKERS = {'pending': '…', 'running': '▶', 'ok': '✓', 'fail': '✗', + 'cancelled': '⊘'} +_STATS_FIELDS = [('stats_field_model', 'model'), ('stats_field_taxa', 'taxa'), + ('stats_field_lnl', 'lnL'), ('stats_field_gamma', 'gamma'), + ('stats_field_tstv', 'tstv'), ('stats_field_time', 'time')] +_SUMMARY_COLS = ['file', 'status', 'model', 'taxa', 'lnL', 'gamma', 'tstv', 'time'] + + +class ResultsPanel(QTabWidget): + """The Log / Statistics / Summary / Queue tabs under the tree viewer.""" + + retry_clicked = Signal() # "retry the failed files" in the queue tab + + def __init__(self, parent=None): + super().__init__(parent) + self.log_console = LogConsole() + self.addTab(self.log_console, _t("tab_log")) + + stats_tab = QWidget() + stats_lay = QVBoxLayout(stats_tab) + self.stats_file_label = QLabel("—") + self.stats_table = QTableWidget(0, 2) + self.stats_table.horizontalHeader().setVisible(False) + self.stats_table.verticalHeader().setVisible(False) + self.stats_table.setAlternatingRowColors(True) + self.stats_table.horizontalHeader().setSectionResizeMode(0, QHeaderView.ResizeToContents) + self.stats_table.horizontalHeader().setSectionResizeMode(1, QHeaderView.Stretch) + stats_lay.addWidget(self.stats_file_label) + stats_lay.addWidget(self.stats_table) + self.addTab(stats_tab, _t("tab_stats")) + + self.summary_table = QTableWidget(0, 8) + self.summary_table.verticalHeader().setVisible(False) + self.summary_table.setAlternatingRowColors(True) + self.summary_table.horizontalHeader().setSectionResizeMode(0, QHeaderView.Stretch) + for col in range(1, 8): + self.summary_table.horizontalHeader().setSectionResizeMode(col, QHeaderView.ResizeToContents) + self.addTab(self.summary_table, _t("tab_summary")) + + self.queue_tab = QWidget() + queue_lay = QVBoxLayout(self.queue_tab) + self.queue_list = QListWidget() + self.queue_list.setAlternatingRowColors(True) + self.retry_btn = QToolButton() + self.retry_btn.setText(_t("btn_retry_failed")) + self.retry_btn.setVisible(False) + self.retry_btn.clicked.connect(self.retry_clicked) + queue_lay.addWidget(self.queue_list, 1) + queue_lay.addWidget(self.retry_btn) + self.addTab(self.queue_tab, _t("tab_queue")) + + def begin_run(self, files: list) -> None: + """A run of files starts: empty log, a pending queue; a batch shows the queue.""" + self.log_console.clear() + self.retry_btn.setVisible(False) + self.setCurrentWidget(self.log_console) + self.queue_list.clear() + for f in files: + self.queue_list.addItem(os.path.basename(f) + ' …') + if len(files) > 1: + self.setCurrentWidget(self.queue_tab) + + def set_file_status(self, name: str, status: str) -> None: + marker = _STATUS_MARKERS.get(status, '?') + for i in range(self.queue_list.count()): + item = self.queue_list.item(i) + if item.text().split(' ')[0] == name: + item.setText(f'{name} {marker}') + return + + def show_result(self, result, rows: list) -> None: + """The statistics of the last file that finished 'ok', and the summary rows.""" + done = [f for f in result.files if f.status == 'ok'] + if done: + last = done[-1] + self.stats_file_label.setText(_t("stats_field_file") + ': ' + os.path.basename(last.file)) + self.stats_table.setRowCount(len(_STATS_FIELDS)) + for i, (key, col) in enumerate(_STATS_FIELDS): + self.stats_table.setItem(i, 0, QTableWidgetItem(_t(key))) + self.stats_table.setItem(i, 1, QTableWidgetItem((last.stats or {}).get(col) or 'n/a')) + self.summary_table.setColumnCount(len(_SUMMARY_COLS)) + self.summary_table.setHorizontalHeaderLabels([_t("sum_col_" + c) for c in _SUMMARY_COLS]) + self.summary_table.setRowCount(len(rows)) + for i, r in enumerate(rows): + for j, c in enumerate(_SUMMARY_COLS): + self.summary_table.setItem(i, j, QTableWidgetItem(r.get(c) or '')) diff --git a/tests/e2e/test_batch.py b/tests/e2e/test_batch.py index f02341e..df43cf7 100644 --- a/tests/e2e/test_batch.py +++ b/tests/e2e/test_batch.py @@ -30,10 +30,10 @@ def gui_batch(window, warned, qtbot, binaries, batch_dir, synthetic): window.phyml_path_input.setText(binaries['phyml']) window.mpirun_path_input.setText(binaries['mpiexec']) window.muscle_path_input.setText(binaries['muscle']) - window.chk_use_mpi.setChecked(True) - window.set_combo_by_data(window.cb_support_method, 'bootstrap') - window.le_bootstrap_replicates.setText('20') - window.le_mpi_np.setText('2') + window.params.chk_use_mpi.setChecked(True) + window.set_combo_by_data(window.params.cb_support_method, 'bootstrap') + window.params.le_bootstrap_replicates.setText('20') + window.params.le_mpi_np.setText('2') window.fasta_path_input.setText(str(batch_dir)) window.btn_run.click() # real signal path: clicked(bool) must not break start_pipeline assert not warned, f'run was blocked by a warning: {warned}' @@ -77,11 +77,11 @@ def test_gui_batch_writes_summary_csv(gui_batch): def test_queue_history_open_folder(gui_batch): """Queue statuses, history record and the open-folder button after a batch.""" window, _ = gui_batch - assert window.queue_list.count() == 2 - statuses = [window.queue_list.item(i).text().split(' ')[-1] for i in range(2)] + assert window.result_tabs.queue_list.count() == 2 + statuses = [window.result_tabs.queue_list.item(i).text().split(' ')[-1] for i in range(2)] assert statuses == ['✓', '✓'], statuses records = HistoryStore(window.paths.history_file).load() assert records and records[0]['n'] == 2 and records[0]['ok'] == 2 - assert window.history_list.count() >= 1 + assert window.nav.history_list.count() >= 1 assert window.status.open_dir_btn.isEnabled() - assert not window.retry_btn.isVisible() or not window._failed_files + assert not window.result_tabs.retry_btn.isVisible() or not window._failed_files diff --git a/tests/e2e/test_iqtree.py b/tests/e2e/test_iqtree.py index 396faaa..afdcbb3 100644 --- a/tests/e2e/test_iqtree.py +++ b/tests/e2e/test_iqtree.py @@ -78,15 +78,15 @@ def test_gui_validation_and_run_with_xlsx(window, warned, qtbot, binaries, run_d """GUI: engine validation, model list, full run + xlsx summary.""" window.engine_setting = 'iqtree' window.update_models() - assert window.cb_model.currentData() == 'MFP' + assert window.params.cb_model.currentData() == 'MFP' window.iqtree_path_input.setText('') # no binary -> blocked with warning window.fasta_path_input.setText(str(run_dir / 'test_prealigned.fasta')) window.start_pipeline() assert warned and (window.worker is None or not window.worker.isRunning()) window.iqtree_path_input.setText(binaries['iqtree']) - window.set_combo_by_data(window.cb_support_method, 'bootstrap') - window.le_bootstrap_replicates.setText('100') + window.set_combo_by_data(window.params.cb_support_method, 'bootstrap') + window.params.le_bootstrap_replicates.setText('100') window.start_pipeline() qtbot.waitUntil(lambda: window.btn_run.isEnabled(), timeout=180_000) window.worker.wait() diff --git a/tests/gui/test_main_window.py b/tests/gui/test_main_window.py index 00473e6..917f65c 100644 --- a/tests/gui/test_main_window.py +++ b/tests/gui/test_main_window.py @@ -26,17 +26,17 @@ def restore_stylesheet(qapp): def test_defaults_match_detected_cores(window): """np defaults to the physical core count; the status bar shows the core info.""" phys, logi = detect_cores() - assert window.le_mpi_np.text() == str(window.phys_cores), window.le_mpi_np.text() + assert window.params.le_mpi_np.text() == str(window.phys_cores), window.params.le_mpi_np.text() assert (window.phys_cores, window.logi_cores) == (phys, logi) assert str(phys) in window.status.cores_label.text() and str(logi) in window.status.cores_label.text() def test_mpi_binary_autodetect(window, tmp_path): - window.chk_use_mpi.setChecked(False) + window.params.chk_use_mpi.setChecked(False) window.phyml_path_input.setText(str(tmp_path / 'phyml.exe')) - assert not window.chk_use_mpi.isChecked(), 'plain phyml.exe must not enable MPI' + assert not window.params.chk_use_mpi.isChecked(), 'plain phyml.exe must not enable MPI' window.phyml_path_input.setText(str(tmp_path / 'phyml-mpi.exe')) - assert window.chk_use_mpi.isChecked(), 'phyml-mpi.exe must auto-enable MPI' + assert window.params.chk_use_mpi.isChecked(), 'phyml-mpi.exe must auto-enable MPI' log_text = window.log_console.toPlainText() assert 'MPI' in log_text, 'detection message missing from log' @@ -47,8 +47,8 @@ def test_serial_bootstrap_on_mpi_build_blocked(window, warned, tmp_path): exe = tmp_path / 'phyml-mpi.exe' exe.write_bytes(b'') window.phyml_path_input.setText(str(exe)) - window.chk_use_mpi.setChecked(False) # after the path: the path edit auto-enables MPI - window.set_combo_by_data(window.cb_support_method, 'bootstrap') + window.params.chk_use_mpi.setChecked(False) # after the path: the path edit auto-enables MPI + window.set_combo_by_data(window.params.cb_support_method, 'bootstrap') fasta = tmp_path / 'test_unaligned.fasta' fasta.write_text('>a\nACGT\n>b\nACGA\n') window.fasta_file = str(fasta) @@ -73,16 +73,16 @@ def test_theme_tabs_presets_log_filter_recent(window, qapp, tmp_path, restore_st assert window.result_tabs.count() == 4, 'expected 4 result tabs (log/stats/summary/queue)' # presets: built-ins listed, apply changes support method, user preset saves - assert window.presets_list.count() >= 3 - texts = [window.presets_list.item(i).text() for i in range(window.presets_list.count())] + assert window.nav.presets_list.count() >= 3 + texts = [window.nav.presets_list.item(i).text() for i in range(window.nav.presets_list.count())] screen_row = next(i for i, t in enumerate(texts) if 'скрининг' in t.lower() or 'screening' in t.lower()) - window._preset_activated(window.presets_list.item(screen_row)) - assert window.cb_support_method.currentData() == '-4', 'screening preset must select aLRT' + window.nav.presets_list.itemDoubleClicked.emit(window.nav.presets_list.item(screen_row)) + assert window.params.cb_support_method.currentData() == '-4', 'screening preset must select aLRT' window._user_presets['my-test'] = {k: window.get_current_config()[k] for k in ('model',)} window._refresh_presets() - assert any(window.presets_list.item(i).text() == 'my-test' - for i in range(window.presets_list.count())) + assert any(window.nav.presets_list.item(i).text() == 'my-test' + for i in range(window.nav.presets_list.count())) # tree viewer zoom / pan mode buttons window.tree_view.btn_zoom.click() @@ -100,7 +100,7 @@ def test_theme_tabs_presets_log_filter_recent(window, qapp, tmp_path, restore_st d.edits[0].setText(mpi_path) d.accept() assert window.phyml_path_input.text() == mpi_path - assert window.chk_use_mpi.isChecked(), 'settings edit must re-trigger MPI autodetect' + assert window.params.chk_use_mpi.isChecked(), 'settings edit must re-trigger MPI autodetect' # structured log: filter by role window.log_console.filter_box.setCurrentIndex(0) @@ -119,7 +119,7 @@ def test_theme_tabs_presets_log_filter_recent(window, qapp, tmp_path, restore_st window._remember_recent('X:/a.fasta') window._remember_recent('X:/b.fasta') window._remember_recent('X:/a.fasta') - assert window.recent_list.count() == 2 and window._recent[0] == 'X:/a.fasta' + assert window.nav.recent_list.count() == 2 and window._recent[0] == 'X:/a.fasta' def test_cpu_bar_expands(window): @@ -143,35 +143,35 @@ def test_model_combo_lists_the_models_of_the_engine(window, engine, datatype): """The model list comes from the engine registry (data and text both).""" from phyml_core.engines import ENGINES window.engine_setting = engine - window.set_combo_by_data(window.cb_datatype, datatype) + window.set_combo_by_data(window.params.cb_datatype, datatype) window.update_models() - shown = [(window.cb_model.itemText(i), window.cb_model.itemData(i)) for i in range(window.cb_model.count())] + shown = [(window.params.cb_model.itemText(i), window.params.cb_model.itemData(i)) for i in range(window.params.cb_model.count())] expected = ENGINES[engine].models(datatype) assert shown == [(m, m) for m in expected] - assert window.cb_model.currentData() == expected[0] + assert window.params.cb_model.currentData() == expected[0] def test_unknown_engine_in_the_settings_shows_the_phyml_models(window): from phyml_core.engines import ENGINES window.engine_setting = 'no-such-engine' window.update_models() - assert [window.cb_model.itemData(i) for i in range(window.cb_model.count())] == ENGINES['phyml'].models('nt') + assert [window.params.cb_model.itemData(i) for i in range(window.params.cb_model.count())] == ENGINES['phyml'].models('nt') def test_fasta_info_label_summarizes_the_alignment(window, tmp_path): fasta = tmp_path / 'выравнивание 1.fasta' fasta.write_text('>A\nACGT--\n>B\nAC\n>C\nACGTA\n', encoding='utf-8') window.fasta_path_input.setText(str(fasta)) - assert window.fasta_info_label.text() == _t('fasta_info').format(3, 2, 6, 15.4) + assert window.fasta.info_label.text() == _t('fasta_info').format(3, 2, 6, 15.4) def test_fasta_info_label_is_blank_for_an_empty_file_and_counts_a_folder(window, tmp_path): empty = tmp_path / 'empty.fasta' empty.write_text('') window.fasta_path_input.setText(str(empty)) - assert window.fasta_info_label.text() == ' ' + assert window.fasta.info_label.text() == ' ' window.fasta_path_input.setText(str(tmp_path)) - assert window.fasta_info_label.text() == _t('fasta_info').format(1, '—', '—', '—') + assert window.fasta.info_label.text() == _t('fasta_info').format(1, '—', '—', '—') # ---------- runs through PipelineRunner (FakeEngine, no binaries) ---------- @@ -199,8 +199,8 @@ def run_to_end(window, qtbot, files=None): def queue_markers(window): - return [window.queue_list.item(i).text().split(' ')[-1] - for i in range(window.queue_list.count())] + return [window.result_tabs.queue_list.item(i).text().split(' ')[-1] + for i in range(window.result_tabs.queue_list.count())] def test_window_recovers_after_engine_exception(window, qtbot, data_dir, tmp_path, @@ -215,7 +215,7 @@ def test_window_recovers_after_engine_exception(window, qtbot, data_dir, tmp_pat assert any(level == 'error' and 'boom' in text for level, text in window.log_console._entries) assert queue_markers(window) == [FAIL_MARK] assert window._failed_files == [fasta] - assert not window.retry_btn.isHidden() + assert not window.result_tabs.retry_btn.isHidden() rec = HistoryStore(window.paths.history_file).load()[0] assert (rec['ok'], rec['n']) == (0, 1) assert rec['files'][0]['status'] == 'fail' and 'boom' in rec['files'][0]['error'] @@ -233,15 +233,15 @@ def test_successful_run_fills_results_history_and_tree(window, qtbot, tmp_path, assert result.success, window.log_console.toPlainText() tree = result.files[0].tree assert loaded == [tree] - assert window.stats_file_label.text().endswith('a.fasta') - assert window.stats_table.item(0, 1).text() == 'FAKE' - assert window.summary_table.rowCount() == 1 - assert window.summary_table.item(0, 0).text() == 'a.fasta' - assert window.summary_table.item(0, 1).text() == 'OK' + assert window.result_tabs.stats_file_label.text().endswith('a.fasta') + assert window.result_tabs.stats_table.item(0, 1).text() == 'FAKE' + assert window.result_tabs.summary_table.rowCount() == 1 + assert window.result_tabs.summary_table.item(0, 0).text() == 'a.fasta' + assert window.result_tabs.summary_table.item(0, 1).text() == 'OK' csv_path = tmp_path / 'in' / CSV_NAME assert csv_path.is_file() assert _t('summary_saved').format(str(csv_path)) in window.log_console.toPlainText() - assert window.retry_btn.isHidden() and window._failed_files == [] + assert window.result_tabs.retry_btn.isHidden() and window._failed_files == [] assert window.status.open_dir_btn.isEnabled() rec = HistoryStore(window.paths.history_file).load()[0] assert rec['engine'] == 'fake' and rec['inputs'] == [fasta] @@ -274,11 +274,11 @@ def test_batch_with_a_failure_offers_retry_of_the_failed_file(window, qtbot, tmp assert [f.file for f in window.last_result.files] == [bad, good] assert queue_markers(window) == [FAIL_MARK, OK_MARK] assert window._failed_files == [bad] - assert not window.retry_btn.isHidden() + assert not window.result_tabs.retry_btn.isHidden() assert loaded == [], 'a batch with a failure shows no tree' rec = HistoryStore(window.paths.history_file).load()[0] assert rec['inputs'] == [str(folder)] and (rec['ok'], rec['n']) == (1, 2) - assert window.summary_table.rowCount() == 2 + assert window.result_tabs.summary_table.rowCount() == 2 window._retry_failed() qtbot.waitUntil(lambda: window.btn_run.isEnabled(), timeout=15000) @@ -297,7 +297,7 @@ def test_cancel_from_the_window_marks_the_file_cancelled(window, qtbot, tmp_path window.worker.wait() assert window.last_result.cancelled assert window.last_result.files[0].status == 'cancelled' - assert window._failed_files == [] and window.retry_btn.isHidden() + assert window._failed_files == [] and window.result_tabs.retry_btn.isHidden() assert _t('sys_interrupt').strip() in window.log_console.toPlainText() @@ -318,8 +318,8 @@ def test_prerun_validation_lists_all_errors(window, data_dir, tmp_path, warned): window.engine_setting = 'phyml' window.update_models() window.fasta_path_input.setText(write_fasta(tmp_path / 'a.fasta')) - window.le_tstv.setText('abc') - window.le_classes.setText('x') + window.params.le_tstv.setText('abc') + window.params.le_classes.setText('x') window.start_pipeline() assert len(warned) == 1 _, title, text = warned[0][:3] @@ -358,10 +358,10 @@ def test_retry_during_a_running_run_is_ignored(window, qtbot, tmp_path, monkeypa fasta = write_fasta(tmp_path / 'a.fasta') window.fasta_path_input.setText(fasta) window._failed_files = [fasta] # as left by an earlier failed run - window.retry_btn.setVisible(True) + window.result_tabs.retry_btn.setVisible(True) window.start_pipeline() worker = window.worker - assert window.retry_btn.isHidden(), 'retry must not be offered while a run is going' + assert window.result_tabs.retry_btn.isHidden(), 'retry must not be offered while a run is going' wait_for_sleeper(window, qtbot) window._retry_failed() window.start_pipeline() diff --git a/tests/gui/test_main_window_smoke.py b/tests/gui/test_main_window_smoke.py index 50d9f4f..e4c16eb 100644 --- a/tests/gui/test_main_window_smoke.py +++ b/tests/gui/test_main_window_smoke.py @@ -1,4 +1,4 @@ def test_window_builds(window): assert window.result_tabs.count() == 4 - assert window.presets_list.count() >= 3 + assert window.nav.presets_list.count() >= 3 assert window.status.cpu_timer.isActive() diff --git a/tests/gui/test_main_window_storage.py b/tests/gui/test_main_window_storage.py index ed60bc5..24c3fc9 100644 --- a/tests/gui/test_main_window_storage.py +++ b/tests/gui/test_main_window_storage.py @@ -48,7 +48,7 @@ def fail(*args, **kwargs): # ---------- where the files go ---------- def test_config_saved_in_data_dir(window, data_dir): - window.set_combo_by_data(window.cb_model, 'GTR') + window.set_combo_by_data(window.params.cb_model, 'GTR') saved = read_json(data_dir / 'phyml_config.json') assert saved['model'] == 'GTR' assert saved['schema_version'] == 2 @@ -67,7 +67,7 @@ def test_history_written_to_data_dir(window, data_dir): assert records[0]['ok'] == 1 assert (data_dir / 'phyml_history.json').is_file() assert not (Path.cwd() / 'phyml_history.json').exists() - assert window.history_list.count() == 1 + assert window.nav.history_list.count() == 1 def test_tool_path_inside_app_dir_saved_relative(data_dir, tmp_path, make_window): @@ -106,7 +106,7 @@ def test_legacy_config_migrated_on_first_start(data_dir, make_window): legacy = Path.cwd() / 'phyml_config.json' legacy.write_text(json.dumps({'model': 'GTR'}), encoding='utf-8') w = make_window() - assert w.cb_model.currentData() == 'GTR' + assert w.params.cb_model.currentData() == 'GTR' expected = _t('settings_migrated').format(str(legacy), str(data_dir / 'phyml_config.json')) assert expected in log_text(w) assert legacy.is_file(), 'the original stays in place' @@ -118,7 +118,7 @@ def test_legacy_history_migrated_and_listed(data_dir, make_window): files=[{'file': 'a.fasta', 'status': 'ok', 'tree': '', 'stats': {}, 'error': ''}]) (Path.cwd() / 'phyml_history.json').write_text(json.dumps([record]), encoding='utf-8') w = make_window() - assert w.history_list.count() == 1 + assert w.nav.history_list.count() == 1 assert log_text(w).count(_t('settings_migrated').split('{}')[0]) == 1 # no config to copy @@ -136,7 +136,7 @@ def test_failed_migration_is_logged_and_start_continues(data_dir, make_window, m w = make_window() assert 'copy denied' in log_text(w) assert _t('err_load').split('{}')[0].strip() in log_text(w) - assert w.cb_model.currentData() == 'HKY85' + assert w.params.cb_model.currentData() == 'HKY85' def test_failed_migration_blocks_autosave_and_retries_next_start(data_dir, make_window, monkeypatch): @@ -149,14 +149,14 @@ def test_failed_migration_blocks_autosave_and_retries_next_start(data_dir, make_ broken.setattr(main_window.settings, 'migrate_legacy', raising(PermissionError(13, 'copy denied'))) w = make_window() - assert w.cb_model.currentData() == 'HKY85' - w.set_combo_by_data(w.cb_model, 'JC69') + assert w.params.cb_model.currentData() == 'HKY85' + w.set_combo_by_data(w.params.cb_model, 'JC69') w.auto_save() w.close() assert not target.exists(), 'defaults were written over the config that was to be migrated' second = make_window() - assert second.cb_model.currentData() == 'GTR' + assert second.params.cb_model.currentData() == 'GTR' assert _t('settings_migrated').format(str(legacy), str(target)) in log_text(second) assert read_json(target)['model'] == 'GTR' @@ -167,14 +167,14 @@ def test_failed_migration_keeps_autosave_when_the_config_exists(data_dir, make_w monkeypatch.setattr(main_window.settings, 'migrate_legacy', raising(PermissionError(13, 'copy denied'))) w = make_window() - w.set_combo_by_data(w.cb_model, 'JC69') + w.set_combo_by_data(w.params.cb_model, 'JC69') assert read_json(data_dir / 'phyml_config.json')['model'] == 'JC69' # ---------- loading ---------- def test_missing_config_gives_defaults_without_load_message(window): - assert window.cb_model.currentData() == 'HKY85' + assert window.params.cb_model.currentData() == 'HKY85' assert _t('sys_load').strip() not in log_text(window) @@ -182,7 +182,7 @@ def test_existing_config_logs_load_message(data_dir, make_window): (data_dir / 'phyml_config.json').write_text( json.dumps({'model': 'GTR', 'bootstrap_replicates': '333'}), encoding='utf-8') w = make_window() - assert w.cb_model.currentData() == 'GTR' and w.le_bootstrap_replicates.text() == '333' + assert w.params.cb_model.currentData() == 'GTR' and w.params.le_bootstrap_replicates.text() == '333' assert _t('sys_load').strip() in log_text(w) assert not any(text.endswith('\n') for _, text in w.log_console._entries) @@ -191,7 +191,7 @@ def test_existing_config_logs_load_message(data_dir, make_window): def test_empty_mpi_np_means_physical_cores(data_dir, make_window, stored, shown): (data_dir / 'phyml_config.json').write_text(json.dumps({'mpi_np': stored}), encoding='utf-8') w = make_window() - assert w.le_mpi_np.text() == (shown or str(detect_cores()[0])) + assert w.params.le_mpi_np.text() == (shown or str(detect_cores()[0])) def test_bare_mpiexec_is_resolved_through_path(data_dir, make_window, monkeypatch): @@ -214,8 +214,8 @@ def test_damaged_config_reported_and_replaced_by_defaults(data_dir, make_window) broken = str(config) + '.broken' assert _t('err_settings_broken').format(broken) in log_text(w) assert Path(broken).read_bytes() == b'{ not json' - assert w.cb_model.currentData() == 'HKY85' - w.set_combo_by_data(w.cb_model, 'GTR') + assert w.params.cb_model.currentData() == 'HKY85' + w.set_combo_by_data(w.params.cb_model, 'GTR') assert read_json(config)['model'] == 'GTR', 'a damaged file does not stop saving' @@ -228,7 +228,7 @@ def test_unreadable_config_keeps_the_file_and_disables_autosave(data_dir, make_w assert 'locked' in log_text(w) assert _t('err_load').split('{}')[0].strip() in log_text(w) assert not any(text.endswith('\n') for _, text in w.log_console._entries) - w.set_combo_by_data(w.cb_model, 'JC69') + w.set_combo_by_data(w.params.cb_model, 'JC69') w.auto_save() w.close() assert config.read_bytes() == before, 'defaults must never replace the unreadable file' @@ -238,7 +238,7 @@ def test_unknown_settings_survive_a_save(data_dir, make_window): config = data_dir / 'phyml_config.json' config.write_text(json.dumps({'future_key': {'a': 1}}), encoding='utf-8') w = make_window() - w.set_combo_by_data(w.cb_model, 'GTR') + w.set_combo_by_data(w.params.cb_model, 'GTR') saved = read_json(config) assert saved['future_key'] == {'a': 1} and saved['model'] == 'GTR' @@ -246,7 +246,7 @@ def test_unknown_settings_survive_a_save(data_dir, make_window): def test_save_error_is_logged_once_per_failure(window, monkeypatch): monkeypatch.setattr(main_window.settings, 'save', raising(OSError('read-only disk'))) for model in ('GTR', 'JC69', 'K80'): - window.set_combo_by_data(window.cb_model, model) + window.set_combo_by_data(window.params.cb_model, model) prefix = _t('err_settings_save').split('{}')[0] assert log_text(window).count('read-only disk') == 1 assert prefix in log_text(window) @@ -258,7 +258,7 @@ def test_history_record_holds_params_and_full_file_paths(window, tmp_path): folder = tmp_path / 'run folder' files = [str(folder / 'a.fasta'), str(folder / 'б.fasta')] trees = [str(folder / 'a.phy_phyml_tree.txt'), ''] - window.set_combo_by_data(window.cb_model, 'GTR') + window.set_combo_by_data(window.params.cb_model, 'GTR') finish_run(window, [str(folder)], file_result(files[0], 'ok', trees[0], model='GTR', taxa='8', lnL='-123.4', gamma='0.5', tstv='2.1', time='3s'), @@ -278,15 +278,15 @@ def test_history_record_holds_params_and_full_file_paths(window, tmp_path): 'gamma': '0.5', 'tstv': '2.1', 'time': '3s'} assert 'error' in rec['files'][0] assert isinstance(rec['duration_s'], float) - assert window.history_list.item(0).text() == _t('hist_entry').format( + assert window.nav.history_list.item(0).text() == _t('hist_entry').format( rec['ts'], 'run folder/', 1, 2) def test_history_params_come_from_the_run_config(window): """A change made to the form while the run was going must not rewrite what was run.""" - window.set_combo_by_data(window.cb_model, 'GTR') + window.set_combo_by_data(window.params.cb_model, 'GTR') config = window.get_current_config() - window.set_combo_by_data(window.cb_model, 'JC69') + window.set_combo_by_data(window.params.cb_model, 'JC69') finish_run(window, ['x.fasta'], file_result('x.fasta'), config=config) assert HistoryStore(window.paths.history_file).load()[0]['params']['model'] == 'GTR' @@ -296,7 +296,7 @@ def test_single_file_run_is_listed_by_its_name(window, tmp_path): finish_run(window, [fasta], file_result(fasta, tree='t.txt')) rec = HistoryStore(window.paths.history_file).load()[0] assert display_input(rec) == 'one.fasta' - assert window.history_list.item(0).text() == _t('hist_entry').format(rec['ts'], 'one.fasta', 1, 1) + assert window.nav.history_list.item(0).text() == _t('hist_entry').format(rec['ts'], 'one.fasta', 1, 1) def test_history_list_shows_old_and_new_records_and_opens_the_best_tree(window, tmp_path, monkeypatch): @@ -312,16 +312,16 @@ def test_history_list_shows_old_and_new_records_and_opens_the_best_tree(window, {'file': str(tmp_path / 'batch' / 'b.fasta'), 'status': 'ok', 'tree': str(tree), 'stats': {}, 'error': ''}])) window._refresh_history() - assert window.history_list.count() == 2 - new_item, old_item = window.history_list.item(0), window.history_list.item(1) + assert window.nav.history_list.count() == 2 + new_item, old_item = window.nav.history_list.item(0), window.nav.history_list.item(1) assert new_item.text().split(' | ')[1] == 'batch/' assert old_item.text() == _t('hist_entry').format('2020-01-01 10:00', 'old.fasta', 2, 3) shown = [] monkeypatch.setattr(window.tree_view, 'load_tree', shown.append) - window._open_history_entry(new_item) + window.nav.history_list.itemDoubleClicked.emit(new_item) assert shown == [str(tree)] - window._open_history_entry(old_item) + window.nav.history_list.itemDoubleClicked.emit(old_item) assert shown == [str(tree)], 'a vanished tree is not loaded' assert _t('hist_no_tree') in log_text(window) @@ -337,7 +337,7 @@ def test_unreadable_history_is_logged_without_crash(data_dir, make_window, monke monkeypatch.setattr(HistoryStore, 'load', raising(PermissionError(13, 'history locked'))) w = make_window() assert 'history locked' in log_text(w) - assert w.history_list.count() == 0 + assert w.nav.history_list.count() == 0 finish_run(w, ['x.fasta'], file_result('x.fasta')) # must not raise either assert not (data_dir / 'phyml_history.json').exists() @@ -345,8 +345,8 @@ def test_unreadable_history_is_logged_without_crash(data_dir, make_window, monke # ---------- presets ---------- def preset_item(window, label_key): - for i in range(window.presets_list.count()): - item = window.presets_list.item(i) + for i in range(window.nav.presets_list.count()): + item = window.nav.presets_list.item(i) if item.data(main_window.Qt.UserRole) == ('builtin', label_key): return item raise AssertionError(label_key) @@ -355,10 +355,10 @@ def preset_item(window, label_key): def test_builtin_preset_switches_the_engine(window): window.engine_setting = 'iqtree' window.update_models() - window._preset_activated(preset_item(window, 'preset_builtin_pub')) + window.nav.presets_list.itemDoubleClicked.emit(preset_item(window, 'preset_builtin_pub')) assert window.engine_setting == 'phyml' - assert window.cb_model.currentData() == 'GTR' - assert window.cb_search.currentData() == 'BEST' + assert window.params.cb_model.currentData() == 'GTR' + assert window.params.cb_search.currentData() == 'BEST' assert read_json(window.paths.config_file)['engine'] == 'phyml' @@ -367,18 +367,38 @@ def test_user_preset_without_engine_keeps_the_current_one(window): window.update_models() window._user_presets['old'] = {'datatype': 'nt', 'model': 'HKY+F+G4', 'search': 'SPR'} window._refresh_presets() - item = next(window.presets_list.item(i) for i in range(window.presets_list.count()) - if window.presets_list.item(i).text() == 'old') - window._preset_activated(item) + item = next(window.nav.presets_list.item(i) for i in range(window.nav.presets_list.count()) + if window.nav.presets_list.item(i).text() == 'old') + window.nav.presets_list.itemDoubleClicked.emit(item) assert window.engine_setting == 'iqtree' - assert window.cb_model.currentData() == 'HKY+F+G4' and window.cb_search.currentData() == 'SPR' + assert window.params.cb_model.currentData() == 'HKY+F+G4' and window.params.cb_search.currentData() == 'SPR' def test_saved_user_preset_holds_the_engine_and_is_persisted(window, monkeypatch): monkeypatch.setattr(QInputDialog, 'getText', staticmethod(lambda *a, **k: ('mine', True))) - window.set_combo_by_data(window.cb_model, 'GTR') + window.set_combo_by_data(window.params.cb_model, 'GTR') window._save_preset() expected = presets.extract(window.get_current_config()) assert window._user_presets['mine'] == expected assert expected['engine'] == 'phyml' assert read_json(window.paths.config_file)['presets']['mine'] == expected + + +def test_loading_a_config_does_not_save_it(data_dir, make_window): + config = data_dir / 'phyml_config.json' + config.write_text(json.dumps({'model': 'GTR', 'datatype': 'nt'}), encoding='utf-8') + before = config.read_bytes() + w = make_window() + assert w.params.cb_model.currentData() == 'GTR' + assert config.read_bytes() == before, 'the window saved while it was loading' + + +def test_a_preset_is_saved_once_and_a_field_edit_saves(window, monkeypatch): + saves = [] + real_save = main_window.settings.save + monkeypatch.setattr(main_window.settings, 'save', lambda *a: saves.append(a) or real_save(*a)) + window.nav.presets_list.itemDoubleClicked.emit(preset_item(window, 'preset_builtin_pub')) + assert len(saves) == 1 + assert read_json(window.paths.config_file)['bootstrap_replicates'] == '1000' + window.params.le_tstv.setText('2.0') + assert len(saves) == 2 and read_json(window.paths.config_file)['tstv'] == '2.0' diff --git a/tests/gui/test_nav_panel.py b/tests/gui/test_nav_panel.py new file mode 100644 index 0000000..d29d79d --- /dev/null +++ b/tests/gui/test_nav_panel.py @@ -0,0 +1,75 @@ +"""NavPanel: recent files, presets and history lists and their signals.""" +from PySide6.QtCore import Qt + +from phyml_core import presets +from phyml_core.history import display_input +from phyml_core.i18n import _t +from phyml_ui.nav_panel import NavPanel + + +def double_click(qtbot, lst, row): + item = lst.item(row) + lst.scrollToItem(item) + pos = lst.visualItemRect(item).center() + qtbot.mouseClick(lst.viewport(), Qt.LeftButton, pos=pos) # a view acts on the pressed row + qtbot.mouseDClick(lst.viewport(), Qt.LeftButton, pos=pos) + + +def make_nav(qtbot): + nav = NavPanel() + qtbot.addWidget(nav) + nav.resize(300, 600) + nav.show() + qtbot.waitExposed(nav) + return nav + + +def test_nav_signals(qtbot): + nav = make_nav(qtbot) + user = {'mine': {'model': 'GTR', 'engine': 'iqtree'}} + nav.set_presets(presets.BUILTIN, user) + texts = [nav.presets_list.item(i).text() for i in range(nav.presets_list.count())] + assert texts == [_t('preset_save_current'), *(_t(k) for k in presets.BUILTIN), 'mine'] + + nav.setCurrentWidget(nav.presets_list) + with qtbot.waitSignal(nav.preset_activated, timeout=1000) as sig: + double_click(qtbot, nav.presets_list, 1) + assert sig.args == [presets.BUILTIN[next(iter(presets.BUILTIN))]] + assert nav.activated_preset_name == texts[1] + with qtbot.waitSignal(nav.preset_activated, timeout=1000) as sig: + double_click(qtbot, nav.presets_list, len(texts) - 1) + assert sig.args == [user['mine']] + with qtbot.waitSignal(nav.preset_save_requested, timeout=1000): + double_click(qtbot, nav.presets_list, 0) + + records = [{'ts': '2020-01-01 10:00', 'input': 'a.fasta', 'ok': 1, 'n': 1}, + {'ts': '2020-01-02 10:00', 'input': 'b.fasta', 'ok': 0, 'n': 2}] + nav.set_history(records) + assert nav.history_list.item(1).text() == _t('hist_entry').format( + '2020-01-02 10:00', display_input(records[1]), 0, 2) + nav.setCurrentWidget(nav.history_list) + with qtbot.waitSignal(nav.history_activated, timeout=1000) as sig: + double_click(qtbot, nav.history_list, 0) + assert sig.args == [0] + + +def test_recent_shows_names_and_emits_the_full_path(qtbot, tmp_path): + nav = make_nav(qtbot) + folder = tmp_path / 'batch' + folder.mkdir() + nav.set_recent([str(tmp_path / 'x.fasta'), str(folder)]) + assert [nav.recent_list.item(i).text() for i in range(2)] == ['x.fasta', 'batch/'] + nav.setCurrentWidget(nav.recent_list) + with qtbot.waitSignal(nav.recent_activated, timeout=1000) as sig: + double_click(qtbot, nav.recent_list, 1) + assert sig.args == [str(folder)] + + +def test_emitted_preset_is_a_copy(qtbot): + nav = make_nav(qtbot) + nav.set_presets(presets.BUILTIN, {}) + got = [] + nav.preset_activated.connect(got.append) + nav.presets_list.itemDoubleClicked.emit(nav.presets_list.item(1)) + got[0]['model'] = 'changed' + assert presets.BUILTIN[next(iter(presets.BUILTIN))]['model'] != 'changed' diff --git a/tests/gui/test_params_panel.py b/tests/gui/test_params_panel.py new file mode 100644 index 0000000..02bceeb --- /dev/null +++ b/tests/gui/test_params_panel.py @@ -0,0 +1,48 @@ +"""ParamsPanel: the parameter groups <-> a params dict.""" +from phyml_core.engines import ENGINES +from phyml_ui.params_panel import ParamsPanel + +FULL = {'datatype': 'aa', 'model': 'WAG', 'support_method': '-2', + 'bootstrap_replicates': '250', 'use_mpi': True, 'mpi_np': '3', + 'freq': 'm', 'tstv': '2.5', 'pinv': 'e', 'classes': '6', 'gamma': '0.7', + 'search': 'SPR', 'optimize': 'tl', 'parsimony': True} + + +def make_panel(qtbot, engine='phyml'): + panel = ParamsPanel() + qtbot.addWidget(panel) + panel.set_engine(ENGINES[engine]) + return panel + + +def test_params_roundtrip(qtbot): + panel = make_panel(qtbot) + panel.set_params(FULL) + assert panel.params() == FULL + assert not panel.le_bootstrap_replicates.isEnabled(), 'aLRT: no replicates field' + + +def test_set_params_emits_changed_once(qtbot): + panel = make_panel(qtbot) + count = [] + panel.changed.connect(lambda: count.append(1)) + with qtbot.waitSignal(panel.changed, timeout=1000): + panel.set_params(FULL) + assert len(count) == 1 + + +def test_set_engine_refills_models(qtbot): + panel = make_panel(qtbot) + assert panel.cb_model.itemData(0) == ENGINES['phyml'].models('nt')[0] + panel.set_engine(ENGINES['iqtree']) + models = [panel.cb_model.itemData(i) for i in range(panel.cb_model.count())] + assert models == ENGINES['iqtree'].models('nt') + assert panel.cb_model.currentData() == 'MFP' + + +def test_editing_a_field_emits_changed(qtbot): + panel = make_panel(qtbot) + with qtbot.waitSignal(panel.changed, timeout=1000): + panel.le_tstv.setText('3') + with qtbot.waitSignal(panel.changed, timeout=1000): + panel.chk_use_mpi.toggle() diff --git a/tests/unit/test_settings.py b/tests/unit/test_settings.py index c6526a5..7a0029d 100644 --- a/tests/unit/test_settings.py +++ b/tests/unit/test_settings.py @@ -322,3 +322,77 @@ def test_migrate_legacy_noop_when_data_dir_has_config(tmp_path): p = AppPaths(app_dir=str(tmp_path), data_dir=str(data)) assert settings.migrate_legacy(p, str(cwd)) == [] assert (data / CONFIG_NAME).read_text('utf-8') == '{"model": "JC69"}' # never overwritten + + +# ---------- SettingsStore ---------- + +def make_store(tmp_path): + data = tmp_path / 'data' + lines = [] + store = settings.SettingsStore(AppPaths(app_dir=str(tmp_path), data_dir=str(data)), + lambda level, text: lines.append((level, text))) + return store, data, lines + + +def test_store_save_keeps_unknown_keys_and_writes_the_schema(tmp_path): + store, data, lines = make_store(tmp_path) + data.mkdir() + (data / CONFIG_NAME).write_text(json.dumps({'future': [1], 'model': 'GTR'}), encoding='utf-8') + raw, clean = store.load(cwd=str(tmp_path / 'elsewhere')) + assert clean and raw['model'] == 'GTR' and lines == [] + store.save(dict(settings.DEFAULTS, model='JC69')) + saved = json.loads((data / CONFIG_NAME).read_text(encoding='utf-8')) + assert saved['future'] == [1] and saved['model'] == 'JC69' + assert saved['schema_version'] == settings.SCHEMA_VERSION + + +def test_store_reports_a_failed_save_once_until_one_succeeds(tmp_path, monkeypatch): + store, data, lines = make_store(tmp_path) + data.mkdir() + store.load(cwd=str(tmp_path)) + real_save = settings.save + monkeypatch.setattr(settings, 'save', lambda *a: (_ for _ in ()).throw(OSError('disk full'))) + store.save(dict(settings.DEFAULTS)) + store.save(dict(settings.DEFAULTS)) + assert [level for level, _ in lines] == ['error'] + monkeypatch.setattr(settings, 'save', real_save) + store.save(dict(settings.DEFAULTS)) + monkeypatch.setattr(settings, 'save', lambda *a: (_ for _ in ()).throw(OSError('disk full'))) + store.save(dict(settings.DEFAULTS)) + assert [level for level, _ in lines] == ['error', 'error'] + + +def test_store_unreadable_file_blocks_saving(tmp_path, monkeypatch): + store, data, lines = make_store(tmp_path) + data.mkdir() + (data / CONFIG_NAME).write_text('{"model": "GTR"}', encoding='utf-8') + monkeypatch.setattr(settings, 'load', lambda *a, **k: (_ for _ in ()).throw(PermissionError(13, 'locked'))) + raw, clean = store.load(cwd=str(tmp_path)) + assert not clean and store.blocked and raw == settings.DEFAULTS + assert 'locked' in lines[0][1] and lines[0][0] == 'error' + store.save(dict(settings.DEFAULTS, model='JC69')) + assert json.loads((data / CONFIG_NAME).read_text(encoding='utf-8')) == {'model': 'GTR'} + + +@pytest.mark.parametrize('config_exists', [False, True]) +def test_store_failed_migration_blocks_saving_only_without_a_config(tmp_path, monkeypatch, config_exists): + store, data, lines = make_store(tmp_path) + if config_exists: + data.mkdir() + (data / CONFIG_NAME).write_text('{}', encoding='utf-8') + monkeypatch.setattr(settings, 'migrate_legacy', + lambda *a: (_ for _ in ()).throw(PermissionError(13, 'copy denied'))) + store.load(cwd=str(tmp_path)) + assert 'copy denied' in lines[0][1] + assert store.blocked is not config_exists + + +def test_store_logs_each_migrated_file(tmp_path): + store, data, lines = make_store(tmp_path) + legacy = tmp_path / 'old' + legacy.mkdir() + (legacy / CONFIG_NAME).write_text('{"model": "GTR"}', encoding='utf-8') + raw, clean = store.load(cwd=str(legacy)) + assert raw['model'] == 'GTR' and clean + assert lines == [('system', _t('settings_migrated').format( + str(legacy / CONFIG_NAME), str(data / CONFIG_NAME)))] From d32e2acb79fe050857aa4fc7029da47069f9e9f5 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sat, 31 Oct 2026 05:51:18 +0300 Subject: [PATCH 34/36] Run the full pytest suite in CI with pinned IQ-TREE and MUSCLE Add packaging/third_party.json and scripts/fetch_third_party.py (SHA256-verified downloads into bin/), a Python 3.11/3.13 CI matrix that caches bin/, README sections on settings location and tests. The IQ-TREE GUI test now isolates the app folder, since an empty path field falls back to bin/iqtree2.exe. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/tests.yml | 23 ++++++-- .gitignore | 3 +- README.md | 22 +++++++- packaging/third_party.json | 17 ++++++ scripts/fetch_third_party.py | 81 ++++++++++++++++++++++++++++ tests/e2e/test_iqtree.py | 5 +- tests/unit/test_fetch_third_party.py | 52 ++++++++++++++++++ 7 files changed, 195 insertions(+), 8 deletions(-) create mode 100644 packaging/third_party.json create mode 100644 scripts/fetch_third_party.py create mode 100644 tests/unit/test_fetch_third_party.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 93b54ad..f4c592d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,14 +6,29 @@ on: pull_request: jobs: - smoke: + test: runs-on: windows-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.11', '3.13'] + env: + QT_QPA_PLATFORM: offscreen steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: ${{ matrix.python-version }} - name: Install dependencies run: pip install -r requirements.txt -r requirements-dev.txt - - name: Tests (no analysis binaries) - run: python -m pytest tests/unit tests/gui + - name: Cache pinned IQ-TREE and MUSCLE + id: third-party + uses: actions/cache@v4 + with: + path: bin + key: third-party-${{ hashFiles('packaging/third_party.json') }} + - name: Fetch pinned IQ-TREE and MUSCLE + if: steps.third-party.outputs.cache-hit != 'true' + run: python scripts/fetch_third_party.py --dest bin + - name: Tests + run: python -m pytest -q -rs diff --git a/.gitignore b/.gitignore index 2a35062..a73ddc5 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ phyml_history.json release_body.md release.json release_resp.json -asset_resp.json \ No newline at end of file +asset_resp.json +/bin/ diff --git a/README.md b/README.md index bd40f1e..97b815c 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,17 @@ ### ⚙️ Requirements * **Executables:** `phyml.exe` (or MPI version) and `muscle.exe`. -* **Python packages:** Python 3.9+ with `PySide6`, `matplotlib`, `biopython` and `psutil` (`pip install -r requirements.txt`). +* **Python packages:** Python 3.11+ with `PySide6`, `matplotlib`, `biopython` and `psutil` (`pip install -r requirements.txt`). * **MPI Environment:** For parallel mode, **Microsoft MPI** is required (install `msmpisetup.exe` from the official Microsoft GitHub). +* **Settings and history:** stored next to the exe when that folder is writable, otherwise in `%APPDATA%\phymlGUI`. Set `PHYMLGUI_DATA_DIR` to use another folder. + +### 🧪 Tests +``` +pip install -r requirements.txt -r requirements-dev.txt +python scripts/fetch_third_party.py --dest bin # pinned IQ-TREE and MUSCLE (SHA256-verified) +python -m pytest +``` +Analysis binaries are looked up in `bin/` and in the variables `PHYMLGUI_PHYML`, `PHYMLGUI_MPIEXEC`, `PHYMLGUI_IQTREE`, `PHYMLGUI_MUSCLE`, `PHYMLGUI_RAXML` (full path to the executable). Tests that need a missing binary are skipped. Markers: `needs_phyml`, `needs_mpi`, `needs_iqtree`, `needs_muscle`, `slow`; e.g. `python -m pytest -m "not needs_phyml and not needs_mpi"` is what CI runs without PhyML/MPI. Tests never touch your real settings: they use an isolated data folder. --- @@ -47,8 +56,17 @@ ### ⚙️ Требования * **Программные файлы:** `phyml.exe` (или MPI-версия) и `muscle.exe`. -* **Python-зависимости:** Python 3.9+ с пакетами `PySide6`, `matplotlib`, `biopython` и `psutil` (`pip install -r requirements.txt`). +* **Python-зависимости:** Python 3.11+ с пакетами `PySide6`, `matplotlib`, `biopython` и `psutil` (`pip install -r requirements.txt`). * **Среда MPI:** Для работы в параллельном режиме требуется установленный **Microsoft MPI** (инсталлятор `msmpisetup.exe` можно найти на официальном GitHub Microsoft). +* **Настройки и история:** хранятся рядом с exe, если папка доступна для записи, иначе в `%APPDATA%\phymlGUI`. Другую папку можно задать переменной `PHYMLGUI_DATA_DIR`. + +### 🧪 Тесты +``` +pip install -r requirements.txt -r requirements-dev.txt +python scripts/fetch_third_party.py --dest bin # закреплённые IQ-TREE и MUSCLE (проверка SHA256) +python -m pytest +``` +Бинарники анализа ищутся в `bin/` и в переменных `PHYMLGUI_PHYML`, `PHYMLGUI_MPIEXEC`, `PHYMLGUI_IQTREE`, `PHYMLGUI_MUSCLE`, `PHYMLGUI_RAXML` (полный путь к exe). Тесты, которым нужен отсутствующий бинарник, пропускаются. Маркеры: `needs_phyml`, `needs_mpi`, `needs_iqtree`, `needs_muscle`, `slow`; например `python -m pytest -m "not needs_phyml and not needs_mpi"` — запуск CI без PhyML/MPI. Тесты не трогают ваши настройки: они работают в изолированной папке данных. --- diff --git a/packaging/third_party.json b/packaging/third_party.json new file mode 100644 index 0000000..b45d1d3 --- /dev/null +++ b/packaging/third_party.json @@ -0,0 +1,17 @@ +{ + "iqtree": { + "version": "2.4.0", + "url": "https://github.com/iqtree/iqtree2/releases/download/v2.4.0/iqtree-2.4.0-Windows.zip", + "sha256": "93f488a6cfef96ef664b23fefe5be97c2c80ebcae9cd2031c0cac92b467278be", + "extract": { + "iqtree-2.4.0-Windows/bin/iqtree2.exe": "iqtree2.exe", + "iqtree-2.4.0-Windows/bin/libiomp5md.dll": "libiomp5md.dll" + } + }, + "muscle": { + "version": "5.3", + "url": "https://github.com/rcedgar/muscle/releases/download/v5.3/muscle-win64.v5.3.exe", + "sha256": "a86637db99deb3efa1cb6762c4e3160ca33644608270eb167e4214f1f9409711", + "save_as": "muscle.exe" + } +} diff --git a/scripts/fetch_third_party.py b/scripts/fetch_third_party.py new file mode 100644 index 0000000..b32438c --- /dev/null +++ b/scripts/fetch_third_party.py @@ -0,0 +1,81 @@ +"""Download the pinned third-party binaries (IQ-TREE, MUSCLE) into a folder. + +Every download is verified against the SHA256 in packaging/third_party.json +before anything is written to the destination. +""" +import argparse +import hashlib +import json +import os +import shutil +import sys +import tempfile +import urllib.request +import zipfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +MANIFEST = os.path.join(ROOT, 'packaging', 'third_party.json') + + +def _download(url, target): + sha = hashlib.sha256() + with urllib.request.urlopen(url) as resp, open(target, 'wb') as out: + while True: + chunk = resp.read(1024 * 1024) + if not chunk: + break + sha.update(chunk) + out.write(chunk) + return sha.hexdigest() + + +def fetch(name, spec, dest): + """Download spec['url'], check SHA256, put the files into dest; return their paths.""" + with tempfile.TemporaryDirectory(prefix='phymlgui_fetch_') as tmp: + archive = os.path.join(tmp, 'download') + actual = _download(spec['url'], archive) + expected = spec['sha256'].lower() + if actual != expected: + raise ValueError(f'{name}: SHA256 mismatch: expected {expected}, got {actual}') + os.makedirs(dest, exist_ok=True) + written = [] + if 'extract' in spec: + with zipfile.ZipFile(archive) as z: + for member, out_name in spec['extract'].items(): + target = os.path.join(dest, out_name) + with z.open(member) as src, open(target, 'wb') as out: + shutil.copyfileobj(src, out) + written.append(target) + else: + target = os.path.join(dest, spec['save_as']) + shutil.copyfile(archive, target) + written.append(target) + return written + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--dest', default='bin') + parser.add_argument('--only', action='append', choices=None, + help='fetch only this tool (repeatable)') + args = parser.parse_args(argv) + with open(MANIFEST, encoding='utf-8') as f: + manifest = json.load(f) + names = args.only or list(manifest) + unknown = [n for n in names if n not in manifest] + if unknown: + print(f'unknown tool(s): {", ".join(unknown)}', file=sys.stderr) + return 2 + for name in names: + try: + paths = fetch(name, manifest[name], args.dest) + except (ValueError, OSError, KeyError, zipfile.BadZipFile) as exc: + print(f'{name}: {exc}', file=sys.stderr) + return 1 + for p in paths: + print(f'{name}: {p}') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/e2e/test_iqtree.py b/tests/e2e/test_iqtree.py index afdcbb3..a5e0be6 100644 --- a/tests/e2e/test_iqtree.py +++ b/tests/e2e/test_iqtree.py @@ -1,4 +1,5 @@ """IQ-TREE engine, ModelFinder and the XLSX summary.""" +import dataclasses import random import re import shutil @@ -74,11 +75,13 @@ def test_100_replicates_raised_to_1000(qapp, monkeypatch, binaries, run_dir): assert any(' -B 1000' in line for line in logs), 'UFBoot replicates were not raised to 1000' -def test_gui_validation_and_run_with_xlsx(window, warned, qtbot, binaries, run_dir): +def test_gui_validation_and_run_with_xlsx(window, warned, qtbot, binaries, run_dir, monkeypatch): """GUI: engine validation, model list, full run + xlsx summary.""" window.engine_setting = 'iqtree' window.update_models() assert window.params.cb_model.currentData() == 'MFP' + # an app folder without bin/: an empty field would otherwise find bin/iqtree2.exe + monkeypatch.setattr(window, 'paths', dataclasses.replace(window.paths, app_dir=str(run_dir))) window.iqtree_path_input.setText('') # no binary -> blocked with warning window.fasta_path_input.setText(str(run_dir / 'test_prealigned.fasta')) window.start_pipeline() diff --git a/tests/unit/test_fetch_third_party.py b/tests/unit/test_fetch_third_party.py new file mode 100644 index 0000000..4383cb9 --- /dev/null +++ b/tests/unit/test_fetch_third_party.py @@ -0,0 +1,52 @@ +import hashlib +import importlib.util +import zipfile +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +_spec = importlib.util.spec_from_file_location( + 'fetch_third_party', ROOT / 'scripts' / 'fetch_third_party.py') +fetch_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(fetch_mod) + + +def _sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def test_fetch_verifies_sha256(tmp_path): + src = tmp_path / 'tool.exe' + src.write_bytes(b'payload') + dest = tmp_path / 'bin' + spec = {'url': src.as_uri(), 'sha256': '0' * 64, 'save_as': 'tool.exe'} + with pytest.raises(ValueError) as err: + fetch_mod.fetch('tool', spec, str(dest)) + assert '0' * 64 in str(err.value) + assert _sha(src) in str(err.value) + assert not dest.exists() or list(dest.iterdir()) == [] + + +def test_fetch_save_as(tmp_path): + src = tmp_path / 'muscle-win64.v5.3.exe' + src.write_bytes(b'payload') + dest = tmp_path / 'bin' + spec = {'url': src.as_uri(), 'sha256': _sha(src), 'save_as': 'muscle.exe'} + paths = fetch_mod.fetch('muscle', spec, str(dest)) + assert [Path(p) for p in paths] == [dest / 'muscle.exe'] + assert (dest / 'muscle.exe').read_bytes() == b'payload' + + +def test_fetch_extracts_listed_members(tmp_path): + archive = tmp_path / 'pkg.zip' + with zipfile.ZipFile(archive, 'w') as z: + z.writestr('pkg/bin/tool.exe', b'exe') + z.writestr('pkg/readme.txt', b'text') + dest = tmp_path / 'bin' + spec = {'url': archive.as_uri(), 'sha256': _sha(archive), + 'extract': {'pkg/bin/tool.exe': 'tool.exe'}} + paths = fetch_mod.fetch('pkg', spec, str(dest)) + assert [Path(p) for p in paths] == [dest / 'tool.exe'] + assert sorted(p.name for p in dest.iterdir()) == ['tool.exe'] + assert (dest / 'tool.exe').read_bytes() == b'exe' From af62efe33f4ee21d426521bb0d49a0904ba5aace Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sat, 31 Oct 2026 06:02:31 +0300 Subject: [PATCH 35/36] Fix findings of the final foundation review Resolve an empty mpi_np to the physical core count in the core so the CLI works on a fresh machine, report a damaged config and an unusable data folder from the CLI, show a message box when the window cannot start, fall back to mpiexec for a blank mpirun_path, use a dedicated message for history write errors, add network and job timeouts, drop dead locale keys and correct the README. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/tests.yml | 1 + README.md | 8 ++-- locales.json | 8 +--- phyml_core/__init__.py | 2 +- phyml_core/cli.py | 18 ++++--- phyml_core/engines/phyml.py | 4 +- phyml_core/settings.py | 10 ++++ phyml_ui/main_window.py | 6 +-- phymlgui.py | 7 ++- scripts/fetch_third_party.py | 2 +- tests/gui/test_main_window_storage.py | 7 ++- tests/unit/test_cli.py | 68 +++++++++++++++++++++++++++ tests/unit/test_engine_phyml.py | 10 ++++ 13 files changed, 126 insertions(+), 25 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f4c592d..2794238 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,6 +8,7 @@ on: jobs: test: runs-on: windows-latest + timeout-minutes: 30 strategy: fail-fast: false matrix: diff --git a/README.md b/README.md index 97b815c..65b3a6a 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ * **Hardware Awareness:** Physical/logical core detection (the `np` field is pre-filled), plus a live CPU status bar showing both system load and the utilization of the analysis process tree. * **Tree Viewer:** Rectangular / slanted / circular layouts, support-value threshold highlighting, clade collapsing, outgroup rerooting, majority-rule consensus from bootstrap trees, Robinson–Foulds topology comparison and PNG/SVG/PDF export. * **Run History and Presets:** every run is recorded (double-click reloads its tree); built-in and user parameter presets. -* **Headless CLI:** `python phymlgui.py --file data.fasta` / `--batch folder/` runs the same pipeline without the GUI. +* **Headless CLI:** `python phymlgui.py --file data.fasta` / `--batch folder/` runs the same pipeline without the GUI. Settings come from the data folder (not from a `phyml_config.json` in the current directory); `--config preset.json` is a JSON object laid over them. Exit codes: 0 success, 1 a file failed or the run was cancelled, 2 usage or input problem. * **Data Sanitization / Conversion:** cleans sequence IDs, converts `FASTA` to `PHYLIP-relaxed`. * **Themes:** light/dark (can follow the Windows system theme) with a selectable accent color. @@ -33,7 +33,7 @@ pip install -r requirements.txt -r requirements-dev.txt python scripts/fetch_third_party.py --dest bin # pinned IQ-TREE and MUSCLE (SHA256-verified) python -m pytest ``` -Analysis binaries are looked up in `bin/` and in the variables `PHYMLGUI_PHYML`, `PHYMLGUI_MPIEXEC`, `PHYMLGUI_IQTREE`, `PHYMLGUI_MUSCLE`, `PHYMLGUI_RAXML` (full path to the executable). Tests that need a missing binary are skipped. Markers: `needs_phyml`, `needs_mpi`, `needs_iqtree`, `needs_muscle`, `slow`; e.g. `python -m pytest -m "not needs_phyml and not needs_mpi"` is what CI runs without PhyML/MPI. Tests never touch your real settings: they use an isolated data folder. +Analysis binaries are looked up in `bin/` and in the variables `PHYMLGUI_PHYML`, `PHYMLGUI_MPIEXEC`, `PHYMLGUI_IQTREE`, `PHYMLGUI_MUSCLE`, `PHYMLGUI_RAXML` (full path to the executable). Tests that need a missing binary are skipped. Markers: `needs_phyml`, `needs_mpi`, `needs_iqtree`, `needs_muscle`, `slow`; CI runs plain `python -m pytest -q -rs`; the tests whose binaries are not installed there are skipped. Tests never touch your real settings: they use an isolated data folder. --- @@ -50,7 +50,7 @@ Analysis binaries are looked up in `bin/` and in the variables `PHYMLGUI_PHYML`, * **Аппаратная осведомлённость:** Определение физических/логических ядер и живая строка состояния с загрузкой CPU — системы и дерева процессов анализа. * **Просмотрщик деревьев:** Прямоугольная/сланцевая/круговая раскладки, подсветка поддержек по порогу, сворачивание клад, рерутинг по outgroup, консенсус большинства из бутстреп-деревьев, сравнение топологий по Робинсону–Фаулдсу, экспорт PNG/SVG/PDF. * **История и пресеты:** каждый запуск записывается (двойной клик загружает его дерево); встроенные и пользовательские пресеты параметров. -* **Консольный режим:** `python phymlgui.py --file data.fasta` / `--batch папка/` — тот же пайплайн без GUI. +* **Консольный режим:** `python phymlgui.py --file data.fasta` / `--batch папка/` — тот же пайплайн без GUI. Настройки берутся из папки данных (а не из `phyml_config.json` в текущей папке); `--config preset.json` — JSON-объект, накладываемый поверх них. Коды выхода: 0 — успех, 1 — файл не обработан или запуск отменён, 2 — ошибка аргументов или входных данных. * **Очистка и конвертация:** санитарная очистка ID, перевод `FASTA` → `PHYLIP-relaxed`. * **Темы:** светлая/тёмная (могут следовать системной Windows) с выбором акцентного цвета. @@ -66,7 +66,7 @@ pip install -r requirements.txt -r requirements-dev.txt python scripts/fetch_third_party.py --dest bin # закреплённые IQ-TREE и MUSCLE (проверка SHA256) python -m pytest ``` -Бинарники анализа ищутся в `bin/` и в переменных `PHYMLGUI_PHYML`, `PHYMLGUI_MPIEXEC`, `PHYMLGUI_IQTREE`, `PHYMLGUI_MUSCLE`, `PHYMLGUI_RAXML` (полный путь к exe). Тесты, которым нужен отсутствующий бинарник, пропускаются. Маркеры: `needs_phyml`, `needs_mpi`, `needs_iqtree`, `needs_muscle`, `slow`; например `python -m pytest -m "not needs_phyml and not needs_mpi"` — запуск CI без PhyML/MPI. Тесты не трогают ваши настройки: они работают в изолированной папке данных. +Бинарники анализа ищутся в `bin/` и в переменных `PHYMLGUI_PHYML`, `PHYMLGUI_MPIEXEC`, `PHYMLGUI_IQTREE`, `PHYMLGUI_MUSCLE`, `PHYMLGUI_RAXML` (полный путь к exe). Тесты, которым нужен отсутствующий бинарник, пропускаются. Маркеры: `needs_phyml`, `needs_mpi`, `needs_iqtree`, `needs_muscle`, `slow`; CI запускает просто `python -m pytest -q -rs`; тесты, для которых на CI нет бинарников, пропускаются. Тесты не трогают ваши настройки: они работают в изолированной папке данных. --- diff --git a/locales.json b/locales.json index bc518ca..e47e02a 100644 --- a/locales.json +++ b/locales.json @@ -16,10 +16,8 @@ "conv_success": "[+] Conversion complete: {}", "err_conv": "[ERROR] Conversion failed: {}", "err_id_collision": "[ERROR] Sequence IDs collide after cleaning: {}", - "err_phyml": "[ERROR] Invalid path to PhyML.", "run_phyml": "[*] Running Command: {}", "cancel_analysis": "\n[-] Analysis cancelled by user.", - "err_crit_phyml": "\n[CRITICAL ERROR] PhyML exited with code {}.", "err_engine_exit": "[CRITICAL ERROR] {} exited with code {}.", "err_no_tree": "[ERROR] {} finished but the tree file is missing: {}", "success_pipe": "\n[+] PIPELINE COMPLETED SUCCESSFULLY.", @@ -81,6 +79,7 @@ "err_load": "[ERROR] Failed to read config: {}\n", "settings_migrated": "[System] Settings copied from {} to {}.", "err_settings_save": "[ERROR] Cannot save settings: {}", + "err_history_save": "[ERROR] Cannot save run history: {}", "err_settings_broken": "[ERROR] Settings file is damaged; defaults loaded, a copy is saved as {}.", "err_history_broken": "[ERROR] Run history file is damaged and was ignored: {}", "err_box_title": "Error", @@ -168,7 +167,6 @@ "engine_phyml": "PhyML (MPI-capable)", "engine_iqtree": "IQ-TREE (ModelFinder, UFBoot)", "path_iqtree": "Path to IQ-TREE (iqtree2.exe):", - "err_iqtree": "[ERROR] Invalid path to IQ-TREE.", "iqtree_ufboot_min": "[*] IQ-TREE ultrafast bootstrap needs at least 1000 replicates: using {} instead of {}.", "fasta_info": "{} taxa | {}-{} sites | gaps {}%", "xlsx_saved": "[+] Excel workbook saved: {}", @@ -213,10 +211,8 @@ "conv_success": "[+] Конвертация завершена: {}", "err_conv": "[ОШИБКА] Сбой при конвертации: {}", "err_id_collision": "[ОШИБКА] После очистки совпали ID последовательностей: {}", - "err_phyml": "[ОШИБКА] Некорректный путь к PhyML.", "run_phyml": "[*] Выполняемая команда: {}", "cancel_analysis": "\n[-] Анализ прерван пользователем.", - "err_crit_phyml": "\n[КРИТИЧЕСКАЯ ОШИБКА] PhyML завершил работу с кодом {}.", "err_engine_exit": "[КРИТИЧЕСКАЯ ОШИБКА] {} завершил работу с кодом {}.", "err_no_tree": "[ОШИБКА] {} завершился, но файл дерева не найден: {}", "success_pipe": "\n[+] ПАЙПЛАЙН УСПЕШНО ЗАВЕРШЕН.", @@ -278,6 +274,7 @@ "err_load": "[ОШИБКА] Сбой чтения конфига: {}\n", "settings_migrated": "[Система] Настройки скопированы из {} в {}.", "err_settings_save": "[ОШИБКА] Не удалось сохранить настройки: {}", + "err_history_save": "[ОШИБКА] Не удалось сохранить историю запусков: {}", "err_settings_broken": "[ОШИБКА] Файл настроек повреждён: загружены настройки по умолчанию, копия сохранена как {}.", "err_history_broken": "[ОШИБКА] Файл истории запусков повреждён и пропущен: {}", "err_box_title": "Ошибка", @@ -365,7 +362,6 @@ "engine_phyml": "PhyML (с поддержкой MPI)", "engine_iqtree": "IQ-TREE (ModelFinder, UFBoot)", "path_iqtree": "Путь к IQ-TREE (iqtree2.exe):", - "err_iqtree": "[ОШИБКА] Некорректный путь к IQ-TREE.", "iqtree_ufboot_min": "[*] Ультрабыстрому бутстрепу IQ-TREE нужно не меньше 1000 реплик: используется {} вместо {}.", "fasta_info": "{} такс | {}-{} сайтов | гэпы {}%", "xlsx_saved": "[+] Excel-файл сохранён: {}", diff --git a/phyml_core/__init__.py b/phyml_core/__init__.py index 55d5524..f9d5722 100644 --- a/phyml_core/__init__.py +++ b/phyml_core/__init__.py @@ -1,3 +1,3 @@ -"""phymlGUI core: pipeline, i18n and helpers (no GUI code here).""" +"""phymlGUI core: settings, runner, engines, i18n and helpers (no GUI code here).""" __version__ = "1.2" diff --git a/phyml_core/cli.py b/phyml_core/cli.py index 771574a..159e2b0 100644 --- a/phyml_core/cli.py +++ b/phyml_core/cli.py @@ -3,11 +3,12 @@ python phymlgui.py --file input.fasta python phymlgui.py --batch folder/ --config preset.json -Settings come from the user's config file (as in the window); --config is a JSON object -laid over them. Log lines go to stdout, except level 'error' lines, which go to stderr. +Settings come from the config file in the data folder (as in the window; the working +folder is not consulted); --config is a JSON object laid over them. A damaged config file +is reported on stderr and the defaults are used, as in the window. Log lines go to stdout, except level 'error' lines, which go to stderr. Exit codes: 0 - every file succeeded; 1 - a file failed or the run was cancelled; -2 - usage or input problem (no --file/--batch, missing path, empty folder, unreadable -or invalid settings/--config). +2 - usage or input problem (no --file/--batch, missing path, empty folder, no usable +data folder, unreadable settings, unreadable or invalid --config). """ import argparse import json @@ -69,7 +70,9 @@ def _collect_files(args): def _load_params(paths, config_arg): try: - params = settings.from_disk(settings.load(paths.config_file), paths) + raw = settings.load(paths.config_file, + on_error=lambda message: _emit(sys.stderr, message)) + params = settings.resolve_cores(settings.from_disk(raw, paths)) except OSError as e: raise _Usage(f'{paths.config_file}: {e}') from e if config_arg: @@ -96,7 +99,10 @@ def main(argv=None) -> int: if not args.file and not args.batch: parser.error('specify --file or --batch') files = _collect_files(args) - paths = AppPaths.detect() + try: + paths = AppPaths.detect() + except OSError as e: + raise _Usage(str(e)) from e params = _load_params(paths, args.config) except _Usage as e: _emit(sys.stderr, str(e)) diff --git a/phyml_core/engines/phyml.py b/phyml_core/engines/phyml.py index 4711a3a..67dcae1 100644 --- a/phyml_core/engines/phyml.py +++ b/phyml_core/engines/phyml.py @@ -73,7 +73,7 @@ def validate(self, params: dict, paths: AppPaths) -> List[Issue]: issues.append(Issue('error', 'mpi_boot_msg')) if use_mpi: - mpiexec = param_text(params, 'mpirun_path', 'mpiexec') + mpiexec = param_text(params, 'mpirun_path', 'mpiexec') or 'mpiexec' if not _mpiexec_found(mpiexec): issues.append(Issue('error', 'val_no_mpiexec', (mpiexec,))) @@ -135,5 +135,5 @@ def launcher(self, params: dict) -> Union[Native, Mpi]: """Call validate() first: a bad mpi_np raises ValueError here.""" if not params.get('use_mpi', False): return Native() - mpiexec = param_text(params, 'mpirun_path', 'mpiexec') + mpiexec = param_text(params, 'mpirun_path', 'mpiexec') or 'mpiexec' return Mpi((mpiexec and shutil.which(mpiexec)) or mpiexec, int(param_text(params, 'mpi_np', '2'))) diff --git a/phyml_core/settings.py b/phyml_core/settings.py index cdb9ccb..88289a2 100644 --- a/phyml_core/settings.py +++ b/phyml_core/settings.py @@ -8,6 +8,7 @@ from .i18n import _t from .paths import CONFIG_NAME, HISTORY_NAME, AppPaths +from .utils import detect_cores SCHEMA_VERSION = 2 @@ -155,6 +156,15 @@ def from_disk(cfg: dict, paths: AppPaths) -> dict: return out +def resolve_cores(cfg: dict) -> dict: + """A copy of cfg where an empty mpi_np ('' = "not chosen yet") becomes the number of + physical cores. The window and the CLI both call this before using the settings.""" + out = dict(cfg) + if not str(out.get('mpi_np') or '').strip(): + out['mpi_np'] = str(detect_cores()[0]) + return out + + def migrate_legacy(paths: AppPaths, cwd: str) -> List[str]: """Copy config/history from the old location (the working folder) into data_dir. diff --git a/phyml_ui/main_window.py b/phyml_ui/main_window.py index be83155..5f62231 100644 --- a/phyml_ui/main_window.py +++ b/phyml_ui/main_window.py @@ -249,7 +249,7 @@ def load_config(self): self.update_models() raw, clean = self.settings_store.load() try: - self._apply_config(settings.from_disk(raw, self.paths)) + self._apply_config(settings.resolve_cores(settings.from_disk(raw, self.paths))) if clean and os.path.isfile(self.paths.config_file): self._log('system', _t("sys_load")) except Exception as e: @@ -270,7 +270,7 @@ def _apply_config(self, config): self.muscle_path_input.setText(config["muscle_path"]) self.update_models() self.params.set_params(dict(presets.extract(config), use_mpi=config["use_mpi"], - mpi_np=config["mpi_np"] or str(self.phys_cores))) # '' = cores + mpi_np=config["mpi_np"])) self.chk_show_tree.setChecked(config["show_tree"]) self._recent = [p for p in config["recent"] if isinstance(p, str) and os.path.exists(p)] self.nav.set_recent(self._recent) @@ -408,7 +408,7 @@ def _append_history(self, result): try: self.history_store.append(rec) except OSError as e: - self._log('error', _t('err_settings_save').format(e)) + self._log('error', _t('err_history_save').format(e)) self._refresh_history() def _refresh_history(self): diff --git a/phymlgui.py b/phymlgui.py index 35a6603..52cc3c0 100644 --- a/phymlgui.py +++ b/phymlgui.py @@ -12,8 +12,13 @@ def main(): sys.exit(cli_main()) from PySide6.QtWidgets import QApplication from phyml_ui.main_window import PhyMLApp + from PySide6.QtWidgets import QMessageBox app = QApplication(sys.argv) - window = PhyMLApp() + try: + window = PhyMLApp() + except OSError as e: # e.g. the data folder cannot be created + QMessageBox.critical(None, 'phymlGUI', str(e)) + sys.exit(1) window.show() sys.exit(app.exec()) diff --git a/scripts/fetch_third_party.py b/scripts/fetch_third_party.py index b32438c..907e2df 100644 --- a/scripts/fetch_third_party.py +++ b/scripts/fetch_third_party.py @@ -19,7 +19,7 @@ def _download(url, target): sha = hashlib.sha256() - with urllib.request.urlopen(url) as resp, open(target, 'wb') as out: + with urllib.request.urlopen(url, timeout=120) as resp, open(target, 'wb') as out: while True: chunk = resp.read(1024 * 1024) if not chunk: diff --git a/tests/gui/test_main_window_storage.py b/tests/gui/test_main_window_storage.py index 24c3fc9..823e204 100644 --- a/tests/gui/test_main_window_storage.py +++ b/tests/gui/test_main_window_storage.py @@ -194,6 +194,10 @@ def test_empty_mpi_np_means_physical_cores(data_dir, make_window, stored, shown) assert w.params.le_mpi_np.text() == (shown or str(detect_cores()[0])) +def test_no_config_shows_physical_cores_for_mpi_np(window): + assert window.params.le_mpi_np.text() == str(detect_cores()[0]) + + def test_bare_mpiexec_is_resolved_through_path(data_dir, make_window, monkeypatch): found = str(data_dir / 'bin' / 'mpiexec.exe') monkeypatch.setattr(shutil, 'which', lambda name: found if name == 'mpiexec' else None) @@ -330,7 +334,8 @@ def test_history_write_error_is_logged_without_crash(window, monkeypatch): monkeypatch.setattr(HistoryStore, 'append', raising(OSError('disk full'))) finish_run(window, ['x.fasta'], file_result('x.fasta')) assert 'disk full' in log_text(window) - assert _t('err_settings_save').split('{}')[0] in log_text(window) + assert _t('err_history_save').split('{}')[0] in log_text(window) + assert _t('err_settings_save').split('{}')[0] not in log_text(window) def test_unreadable_history_is_logged_without_crash(data_dir, make_window, monkeypatch): diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 01e8a35..8769adb 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -128,3 +128,71 @@ def test_cli_config_overlay_and_batch_selection(tmp_path, monkeypatch): assert [os.path.basename(p) for p in inst.files] == ['a.fasta', 'b.FA', 'c.fas'] assert inst.params['model'] == 'GTR' assert inst.params['engine'] == 'phyml' + + +def test_cli_empty_mpi_np_becomes_physical_cores(tmp_path, monkeypatch): + """A fresh machine has no config: the runner must get a real np, not ''.""" + from phyml_core.utils import detect_cores + _Fake.instances.clear() + monkeypatch.setattr(cli, 'PipelineRunner', _Fake) + f = tmp_path / 'a.fasta' + f.write_text('>a\nA\n') + assert cli.main(['--file', str(f), '--config', _write(tmp_path, {'engine': 'iqtree'})]) == 0 + params = _Fake.instances[0].params + assert params['engine'] == 'iqtree' + assert params['mpi_np'] == str(detect_cores()[0]) + + +def _write(tmp_path, obj): + path = tmp_path / 'overlay.json' + path.write_text(json.dumps(obj)) + return str(path) + + +def test_cli_damaged_config_is_reported_on_stderr_and_defaults_are_used(tmp_path, monkeypatch, capsys): + from phyml_core.i18n import _t + _Fake.instances.clear() + monkeypatch.setattr(cli, 'PipelineRunner', _Fake) + data = tmp_path / 'data' + data.mkdir() + (data / 'phyml_config.json').write_text('{not json') + f = tmp_path / 'a.fasta' + f.write_text('>a\nA\n') + assert cli.main(['--file', str(f)]) == 0 + assert _t('err_settings_broken').split('{}')[0] in capsys.readouterr().err + assert _Fake.instances[0].params['model'] == 'HKY85' + + +def test_cli_unusable_data_dir_exit_2_without_traceback(tmp_path, monkeypatch, capsys): + blocker = tmp_path / 'file' + blocker.write_text('x') + monkeypatch.setenv('PHYMLGUI_DATA_DIR', str(blocker / 'sub')) + f = tmp_path / 'a.fasta' + f.write_text('>a\nA\n') + assert cli.main(['--file', str(f)]) == 2 + err = capsys.readouterr().err + assert err and 'Traceback' not in err + + +def test_gui_start_shows_a_message_box_when_the_data_folder_fails(tmp_path): + """Run in a subprocess: it creates its own QApplication.""" + code = '\n'.join([ + "import sys", + "from PySide6.QtWidgets import QMessageBox", + "from phyml_core.paths import AppPaths", + "shown = []", + "def boom(cls, *a, **k): raise PermissionError(13, 'no data folder')", + "AppPaths.detect = classmethod(boom)", + "QMessageBox.critical = lambda *a, **k: shown.append(a[-1])", + "sys.argv = ['phymlgui.py']", + "import phymlgui", + "try:", + " phymlgui.main()", + "except SystemExit as e:", + " print('EXIT', e.code, shown)", + ]) + env = _env(tmp_path) + env['QT_QPA_PLATFORM'] = 'offscreen' + r = subprocess.run([sys.executable, '-c', code], cwd=ROOT, env=env, + capture_output=True, text=True, timeout=60) + assert 'EXIT 1' in r.stdout and 'no data folder' in r.stdout, r.stdout + r.stderr diff --git a/tests/unit/test_engine_phyml.py b/tests/unit/test_engine_phyml.py index f6ec498..0cd1f30 100644 --- a/tests/unit/test_engine_phyml.py +++ b/tests/unit/test_engine_phyml.py @@ -401,6 +401,16 @@ def test_validate_mpiexec_present_as_a_file_or_a_command_name(paths, exe, tmp_pa assert PhyML().validate(dict(params, mpirun_path='mpiexec'), paths) == [] +@pytest.mark.parametrize('blank', ['', ' ']) +def test_blank_mpirun_path_falls_back_to_mpiexec(paths, exe, monkeypatch, blank): + monkeypatch.setattr(shutil, 'which', lambda name, *a, **k: 'C:/MS/mpiexec.exe' if name == 'mpiexec' else None) + params = dict(P, phyml_path=exe, use_mpi=True, mpirun_path=blank, mpi_np='3') + assert PhyML().validate(params, paths) == [] + assert PhyML().launcher(params) == Mpi('C:/MS/mpiexec.exe', 3) + monkeypatch.setattr(shutil, 'which', lambda *a, **k: None) + assert PhyML().validate(params, paths) == [Issue('error', 'val_no_mpiexec', ('mpiexec',))] + + def test_validate_mpiexec_is_not_checked_when_mpi_is_off(paths, exe, monkeypatch): monkeypatch.setattr(shutil, 'which', lambda *a, **k: None) assert PhyML().validate(dict(P, phyml_path=exe, use_mpi=False, mpirun_path='nowhere'), paths) == [] From eab0e34e0c7c038c851449a52faeb5a693516648 Mon Sep 17 00:00:00 2001 From: Aleksey Drobin <43707931+adrobi@users.noreply.github.com> Date: Sat, 31 Oct 2026 12:12:06 +0300 Subject: [PATCH 36/36] Make the fake tool write bytes so Cyrillic paths survive a cp1252 pipe English Windows runners pipe child output as cp1252; the fake crashed printing a Cyrillic path. The Cyrillic-path test now forces cp1252 everywhere. Co-Authored-By: Claude Opus 5.5 --- tests/unit/fake_tool.py | 3 ++- tests/unit/test_runner.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/fake_tool.py b/tests/unit/fake_tool.py index 69f705e..d6fcf21 100644 --- a/tests/unit/fake_tool.py +++ b/tests/unit/fake_tool.py @@ -27,7 +27,8 @@ def say(text): - sys.stdout.write(text + '\n') + # bytes, like real programs: a cp1252 pipe cannot encode a Cyrillic path + sys.stdout.buffer.write((text + '\n').encode('utf-8')) sys.stdout.flush() diff --git a/tests/unit/test_runner.py b/tests/unit/test_runner.py index b04ff38..bbd65d7 100644 --- a/tests/unit/test_runner.py +++ b/tests/unit/test_runner.py @@ -307,7 +307,9 @@ def test_undecodable_engine_output_is_replaced(tmp_path, paths): assert any(text.startswith('progress ') for text in rec.texts('out')) -def test_run_in_cyrillic_dir_with_spaces(tmp_path, paths): +def test_run_in_cyrillic_dir_with_spaces(tmp_path, paths, monkeypatch): + # an English Windows (CI) pipes child output as cp1252, which has no Cyrillic + monkeypatch.setenv('PYTHONIOENCODING', 'cp1252') folder = tmp_path / 'Мои данные' / 'проект 1' aligned = write_fasta(folder / 'выборка 1.fasta', ALIGNED) unaligned = write_fasta(folder / 'выборка 2.fasta', UNALIGNED)