diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000000..b13ccf0cfb --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,30 @@ +#!/bin/bash + +# MFC pre-commit hook +# Runs ./mfc.sh precheck before allowing commits +# Bypass with: git commit --no-verify + +# Only run if we're in the MFC repo root (where mfc.sh exists) +if [ ! -f "$(git rev-parse --show-toplevel)/mfc.sh" ]; then + exit 0 +fi + +cd "$(git rev-parse --show-toplevel)" + +# Auto-detect CPU count (capped at 12 to avoid hogging resources) +JOBS=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) +[ "$JOBS" -gt 12 ] && JOBS=12 + +echo "" +echo "mfc: Running precheck before commit (-j $JOBS)..." +echo "" + +if ./mfc.sh precheck -j "$JOBS"; then + echo "" + exit 0 +else + echo "" + echo "mfc: Commit blocked. Fix issues above or use 'git commit --no-verify' to bypass." + echo "" + exit 1 +fi diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 1427f9d693..f75631b2dd 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -6,11 +6,15 @@ on: types: [submitted] workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: file-changes: name: Detect File Changes runs-on: 'ubuntu-latest' - outputs: + outputs: checkall: ${{ steps.changes.outputs.checkall }} steps: - name: Clone @@ -19,13 +23,65 @@ jobs: - name: Detect Changes uses: dorny/paths-filter@v3 id: changes - with: + with: filters: ".github/file-filter.yml" + wait-for-tests: + name: Wait for Test Suite + runs-on: ubuntu-latest + steps: + - name: Wait for Test Suite to Pass + env: + GH_TOKEN: ${{ github.token }} + run: | + echo "Waiting for Test Suite workflow to complete..." + SHA="${{ github.event.pull_request.head.sha || github.sha }}" + + # Poll every 60 seconds for up to 3 hours + for i in $(seq 1 180); do + # Get the Test Suite workflow runs for this commit + STATUS=$(gh api repos/${{ github.repository }}/commits/$SHA/check-runs \ + --jq '.check_runs[] | select(.name == "Lint Gate") | .conclusion' | head -1) + + if [ "$STATUS" = "success" ]; then + echo "Lint Gate passed. Checking test jobs..." + + # Check if any Github test jobs failed + FAILED=$(gh api repos/${{ github.repository }}/commits/$SHA/check-runs \ + --jq '[.check_runs[] | select(.name | startswith("Github")) | select(.conclusion == "failure")] | length') + + if [ "$FAILED" != "0" ]; then + echo "::error::Test Suite has failing jobs. Benchmarks will not run." + exit 1 + fi + + # Check if Github tests are still running + PENDING=$(gh api repos/${{ github.repository }}/commits/$SHA/check-runs \ + --jq '[.check_runs[] | select(.name | startswith("Github")) | select(.conclusion == null)] | length') + + if [ "$PENDING" = "0" ]; then + echo "All Test Suite jobs completed successfully!" + exit 0 + fi + + echo "Tests still running ($PENDING pending)..." + elif [ "$STATUS" = "failure" ]; then + echo "::error::Lint Gate failed. Benchmarks will not run." + exit 1 + else + echo "Lint Gate status: ${STATUS:-pending}..." + fi + + sleep 60 + done + + echo "::error::Timeout waiting for Test Suite to complete." + exit 1 + self: name: "${{ matrix.name }} (${{ matrix.device }}${{ matrix.interface != 'none' && format('-{0}', matrix.interface) || '' }})" if: ${{ github.repository=='MFlowCode/MFC' && needs.file-changes.outputs.checkall=='true' && ((github.event_name=='pull_request_review' && github.event.review.state=='approved') || (github.event_name=='pull_request' && (github.event.pull_request.user.login=='sbryngelson' || github.event.pull_request.user.login=='wilfonba'))) }} - needs: file-changes + needs: [file-changes, wait-for-tests] strategy: fail-fast: false matrix: diff --git a/.github/workflows/cleanliness.yml b/.github/workflows/cleanliness.yml index b02df12898..bcbe35caaa 100644 --- a/.github/workflows/cleanliness.yml +++ b/.github/workflows/cleanliness.yml @@ -2,6 +2,10 @@ name: Cleanliness on: [push, pull_request, workflow_dispatch] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: file-changes: name: Detect File Changes diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ad0ea7a220..469048f8a6 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -2,6 +2,10 @@ name: Coverage Check on: [push, pull_request, workflow_dispatch] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: file-changes: name: Detect File Changes diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f0dc72783d..fb16384045 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,8 +1,50 @@ name: 'Test Suite' on: [push, pull_request, workflow_dispatch] - + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: + lint-gate: + name: Lint Gate + runs-on: ubuntu-latest + steps: + - name: Clone + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Initialize MFC + run: ./mfc.sh init + + - name: Check Formatting + run: | + ./mfc.sh format -j $(nproc) + git diff --exit-code || (echo "::error::Code is not formatted. Run './mfc.sh format' locally." && exit 1) + + - name: Spell Check + run: ./mfc.sh spelling + + - name: Lint Toolchain + run: ./mfc.sh lint + + - name: Lint Source - No Raw Directives + run: | + ! grep -iR '!\$acc\|!\$omp' --exclude="parallel_macros.fpp" --exclude="acc_macros.fpp" --exclude="omp_macros.fpp" --exclude="shared_parallel_macros.fpp" --exclude="syscheck.fpp" ./src/* + + - name: Lint Source - No Double Precision Intrinsics + run: | + ! grep -iR 'double_precision\|dsqrt\|dexp\|dlog\|dble\|dabs\|double\ precision\|real(8)\|real(4)\|dprod\|dmin\|dmax\|dfloat\|dreal\|dcos\|dsin\|dtan\|dsign\|dtanh\|dsinh\|dcosh\|d0' --exclude-dir=syscheck --exclude="*nvtx*" --exclude="*precision_select*" ./src/* + + - name: Lint Source - No Junk Code + run: | + ! grep -iR -e '\.\.\.' -e '\-\-\-' -e '===' ./src/* + file-changes: name: Detect File Changes runs-on: 'ubuntu-latest' @@ -21,7 +63,7 @@ jobs: github: name: Github if: needs.file-changes.outputs.checkall == 'true' - needs: file-changes + needs: [lint-gate, file-changes] strategy: matrix: os: ['ubuntu', 'macos'] @@ -95,7 +137,7 @@ jobs: self: name: "${{ matrix.cluster_name }} (${{ matrix.device }}${{ matrix.interface != 'none' && format('-{0}', matrix.interface) || '' }})" if: github.repository == 'MFlowCode/MFC' && needs.file-changes.outputs.checkall == 'true' - needs: file-changes + needs: [lint-gate, file-changes] continue-on-error: false timeout-minutes: 480 strategy: diff --git a/mfc.sh b/mfc.sh index 843ea4dc19..fa3d420df9 100755 --- a/mfc.sh +++ b/mfc.sh @@ -10,6 +10,39 @@ fi # Load utility script . "$(pwd)/toolchain/util.sh" +# Auto-install git pre-commit hook (once, silently) +if [ -d "$(pwd)/.git" ] && [ ! -e "$(pwd)/.git/hooks/pre-commit" ] && [ -f "$(pwd)/.githooks/pre-commit" ]; then + ln -sf "$(pwd)/.githooks/pre-commit" "$(pwd)/.git/hooks/pre-commit" + log "Installed git pre-commit hook (runs$MAGENTA ./mfc.sh precheck$COLOR_RESET before commits)." +fi + +# Auto-install shell completions (once) +COMPLETION_DIR="$HOME/.local/share/mfc/completions" +if [ ! -d "$COMPLETION_DIR" ]; then + mkdir -p "$COMPLETION_DIR" + cp "$(pwd)/toolchain/completions/mfc.bash" "$COMPLETION_DIR/" + cp "$(pwd)/toolchain/completions/_mfc" "$COMPLETION_DIR/" + + # Add to shell rc file based on current shell + if [[ "$SHELL" == *"zsh"* ]]; then + RC_FILE="$HOME/.zshrc" + RC_LINE="fpath=(\"$COMPLETION_DIR\" \$fpath)" + SOURCE_CMD="source $COMPLETION_DIR/_mfc" + else + RC_FILE="$HOME/.bashrc" + RC_LINE="[ -f \"$COMPLETION_DIR/mfc.bash\" ] && source \"$COMPLETION_DIR/mfc.bash\"" + SOURCE_CMD="source $COMPLETION_DIR/mfc.bash" + fi + + if [ -f "$RC_FILE" ] && ! grep -q "$COMPLETION_DIR" "$RC_FILE" 2>/dev/null; then + echo "" >> "$RC_FILE" + echo "# MFC shell completion" >> "$RC_FILE" + echo "$RC_LINE" >> "$RC_FILE" + fi + + log "Installed tab completions. Restart shell or run:$MAGENTA $SOURCE_CMD$COLOR_RESET" +fi + # Print startup message immediately for user feedback log "Starting..." @@ -56,6 +89,10 @@ elif [ "$1" '==' "spelling" ] && [ "$2" != "--help" ] && [ "$2" != "-h" ]; then . "$(pwd)/toolchain/bootstrap/python.sh" shift; . "$(pwd)/toolchain/bootstrap/spelling.sh" $@; exit 0 +elif [ "$1" '==' "precheck" ]; then + . "$(pwd)/toolchain/bootstrap/python.sh" + + shift; . "$(pwd)/toolchain/bootstrap/precheck.sh" $@; exit 0 fi mkdir -p "$(pwd)/build" diff --git a/toolchain/bootstrap/precheck.sh b/toolchain/bootstrap/precheck.sh new file mode 100755 index 0000000000..e215b5945e --- /dev/null +++ b/toolchain/bootstrap/precheck.sh @@ -0,0 +1,138 @@ +#!/bin/bash +set -e +set -o pipefail + +# Function to display help message +show_help() { + echo "Usage: ./mfc.sh precheck [OPTIONS]" + echo "Run the same fast checks that CI runs before expensive tests start." + echo "Use this locally before pushing to catch issues early." + echo "" + echo "Options:" + echo " -h, --help Display this help message and exit." + echo " -j, --jobs JOBS Runs JOBS number of parallel jobs for formatting." + echo "" + exit 0 +} + +# Cross-platform hash function (macOS uses md5, Linux uses md5sum) +compute_hash() { + if command -v md5sum > /dev/null 2>&1; then + md5sum | cut -d' ' -f1 + elif command -v md5 > /dev/null 2>&1; then + md5 -q + else + # Fallback: use cksum if neither available + cksum | cut -d' ' -f1 + fi +} + +JOBS=1 + +while [[ $# -gt 0 ]]; do + case "$1" in + -j|--jobs) + if [[ -z "$2" || "$2" == -* ]]; then + echo "Precheck: -j/--jobs requires a value." + exit 1 + fi + if ! [[ "$2" =~ ^[0-9]+$ ]]; then + echo "Precheck: jobs value '$2' is not a valid number." + exit 1 + fi + JOBS="$2" + shift + ;; + -h | --help) + show_help + ;; + *) + echo "Precheck: unknown argument: $1." + exit 1 + ;; + esac + + shift +done + +FAILED=0 + +log "Running$MAGENTA precheck$COLOR_RESET (same checks as CI lint-gate)..." +echo "" + +# 1. Check formatting +log "[$CYAN 1/4$COLOR_RESET] Checking$MAGENTA formatting$COLOR_RESET..." +# Capture state before formatting +BEFORE_HASH=$(git diff -- '*.f90' '*.fpp' '*.py' 2>/dev/null | compute_hash) +if ! ./mfc.sh format -j "$JOBS" > /dev/null 2>&1; then + error "Formatting check failed to run." + FAILED=1 +else + # Check if formatting changed any Fortran/Python files + AFTER_HASH=$(git diff -- '*.f90' '*.fpp' '*.py' 2>/dev/null | compute_hash) + if [ "$BEFORE_HASH" != "$AFTER_HASH" ]; then + error "Code was not formatted. Files have been auto-formatted; review and stage the changes." + echo "" + git diff --stat -- '*.f90' '*.fpp' '*.py' 2>/dev/null || true + echo "" + FAILED=1 + else + ok "Formatting check passed." + fi +fi + +# 2. Spell check +log "[$CYAN 2/4$COLOR_RESET] Running$MAGENTA spell check$COLOR_RESET..." +if ./mfc.sh spelling > /dev/null 2>&1; then + ok "Spell check passed." +else + error "Spell check failed. Run$MAGENTA ./mfc.sh spelling$COLOR_RESET for details." + FAILED=1 +fi + +# 3. Lint toolchain (Python) +log "[$CYAN 3/4$COLOR_RESET] Running$MAGENTA toolchain lint$COLOR_RESET..." +if ./mfc.sh lint > /dev/null 2>&1; then + ok "Toolchain lint passed." +else + error "Toolchain lint failed. Run$MAGENTA ./mfc.sh lint$COLOR_RESET for details." + FAILED=1 +fi + +# 4. Source code lint checks +log "[$CYAN 4/4$COLOR_RESET] Running$MAGENTA source lint$COLOR_RESET checks..." +SOURCE_FAILED=0 + +# Check for raw OpenACC/OpenMP directives +if grep -qiR '!\$acc\|!\$omp' --exclude="parallel_macros.fpp" --exclude="acc_macros.fpp" --exclude="omp_macros.fpp" --exclude="shared_parallel_macros.fpp" --exclude="syscheck.fpp" ./src/* 2>/dev/null; then + error "Found raw OpenACC/OpenMP directives. Use macros instead." + SOURCE_FAILED=1 +fi + +# Check for double precision intrinsics +if grep -qiR 'double_precision\|dsqrt\|dexp\|dlog\|dble\|dabs\|double\ precision\|real(8)\|real(4)\|dprod\|dmin\|dmax\|dfloat\|dreal\|dcos\|dsin\|dtan\|dsign\|dtanh\|dsinh\|dcosh\|d0' --exclude-dir=syscheck --exclude="*nvtx*" --exclude="*precision_select*" ./src/* 2>/dev/null; then + error "Found double precision intrinsics. Use generic intrinsics." + SOURCE_FAILED=1 +fi + +# Check for junk code patterns +if grep -qiR -e '\.\.\.' -e '\-\-\-' -e '===' ./src/* 2>/dev/null; then + error "Found junk code patterns (..., ---, ===) in source." + SOURCE_FAILED=1 +fi + +if [ $SOURCE_FAILED -eq 0 ]; then + ok "Source lint passed." +else + FAILED=1 +fi + +echo "" + +if [ $FAILED -eq 0 ]; then + ok "All precheck tests passed! Safe to push." + exit 0 +else + error "Some precheck tests failed. Fix issues before pushing." + exit 1 +fi diff --git a/toolchain/main.py b/toolchain/main.py index 10e6e44ab0..568db1db67 100644 --- a/toolchain/main.py +++ b/toolchain/main.py @@ -22,14 +22,39 @@ def __do_regenerate(toolchain: str): completions_dir = Path(toolchain) / "completions" completions_dir.mkdir(exist_ok=True) - # Generate completion files + # Generate and write completion files directly (completions_dir / "mfc.bash").write_text(generate_bash_completion(MFC_CLI_SCHEMA)) (completions_dir / "_mfc").write_text(generate_zsh_completion(MFC_CLI_SCHEMA)) # Generate JSON schema - schema = generate_json_schema(include_descriptions=True) with open(Path(toolchain) / "mfc-case-schema.json", 'w', encoding='utf-8') as f: - json.dump(schema, f, indent=2) + json.dump(generate_json_schema(include_descriptions=True), f, indent=2) + + +def __update_installed_completions(toolchain: str): + """Update installed shell completions if they're older than generated ones.""" + import shutil # pylint: disable=import-outside-toplevel + from pathlib import Path # pylint: disable=import-outside-toplevel + + src_dir = Path(toolchain) / "completions" + dst_dir = Path.home() / ".local" / "share" / "mfc" / "completions" + + # Only update if already installed (mfc.sh handles initial install) + if not dst_dir.exists(): + return + + # Update if installed but older than generated + updated = False + for name in ["mfc.bash", "_mfc"]: + if (src_dir / name).exists() and (dst_dir / name).exists(): + if os.path.getmtime(src_dir / name) > os.path.getmtime(dst_dir / name): + shutil.copy2(src_dir / name, dst_dir / name) + updated = True + + if updated: + is_zsh = "zsh" in os.environ.get("SHELL", "") + src_cmd = f"source {dst_dir}/_mfc" if is_zsh else f"source {dst_dir}/mfc.bash" + cons.print(f"[dim]Tab completions updated. Run: {src_cmd}[/dim]") def __ensure_generated_files(): @@ -64,6 +89,9 @@ def __ensure_generated_files(): if needs_regen: __do_regenerate(toolchain) + # Always check if completions need to be installed or updated + __update_installed_completions(toolchain) + def __print_greeting(): MFC_LOGO_LINES = MFC_LOGO.splitlines() max_logo_line_length = max(len(line) for line in MFC_LOGO_LINES) diff --git a/toolchain/mfc/cli/commands.py b/toolchain/mfc/cli/commands.py index 0f827a4b74..8ad8c4bd07 100644 --- a/toolchain/mfc/cli/commands.py +++ b/toolchain/mfc/cli/commands.py @@ -85,7 +85,7 @@ Argument( name="verbose", short="v", - help="Increase verbosity level. Use -v, -vv, or -vvv for more detail.", + help="Increase output verbosity (-v, -vv, -vvv for more detail).", action=ArgAction.COUNT, default=0, ), @@ -98,7 +98,7 @@ Argument( name="debug-log", short="d", - help="Enable debug logging for troubleshooting.", + help="Enable Python toolchain debug logging (not MFC code).", action=ArgAction.STORE_TRUE, dest="debug_log", ), @@ -519,10 +519,10 @@ ], examples=[ Example("./mfc.sh validate case.py", "Check syntax and constraints"), - Example("./mfc.sh validate case.py -d", "Validate with debug output"), + Example("./mfc.sh validate case.py -d", "Validate with toolchain debug output"), ], key_options=[ - ("-d, --debug-log", "Enable debug logging"), + ("-d, --debug-log", "Enable toolchain debug logging"), ], ) @@ -751,6 +751,27 @@ ], ) +PRECHECK_COMMAND = Command( + name="precheck", + help="Run CI lint checks locally before committing.", + description="Run the same fast checks that CI runs before expensive tests start. Use this locally before pushing to catch issues early.", + arguments=[ + Argument( + name="jobs", + short="j", + help="Number of parallel jobs for formatting.", + metavar="N", + ), + ], + examples=[ + Example("./mfc.sh precheck", "Run all lint checks"), + Example("./mfc.sh precheck -j 8", "Run with 8 parallel jobs"), + ], + key_options=[ + ("-j, --jobs N", "Number of parallel jobs for formatting"), + ], +) + INTERACTIVE_COMMAND = Command( name="interactive", help="Launch interactive menu-driven interface.", @@ -990,6 +1011,7 @@ LINT_COMMAND, FORMAT_COMMAND, SPELLING_COMMAND, + PRECHECK_COMMAND, INTERACTIVE_COMMAND, BENCH_COMMAND, BENCH_DIFF_COMMAND, diff --git a/toolchain/mfc/cli/completion_gen.py b/toolchain/mfc/cli/completion_gen.py index d34d7d9cdc..7a286bc981 100644 --- a/toolchain/mfc/cli/completion_gen.py +++ b/toolchain/mfc/cli/completion_gen.py @@ -245,9 +245,11 @@ def generate_bash_completion(schema: CLISchema) -> str: ' return 0', '}', '', - 'complete -o filenames -o bashdefault -F _mfc_completions ./mfc.sh', - 'complete -o filenames -o bashdefault -F _mfc_completions mfc.sh', - 'complete -o filenames -o bashdefault -F _mfc_completions mfc', + '# -o filenames: handle escaping/slashes for file completions', + '# Removed -o bashdefault to prevent unwanted directory fallback', + 'complete -o filenames -F _mfc_completions ./mfc.sh', + 'complete -o filenames -F _mfc_completions mfc.sh', + 'complete -o filenames -F _mfc_completions mfc', ]) return '\n'.join(lines) @@ -317,8 +319,8 @@ def _generate_zsh_command_args(cmd: Command, schema: CLISchema) -> List[str]: "'--no-mpi[Disable MPI]'", "'--gpu[Enable GPU]:mode:(acc mp)'", "'--no-gpu[Disable GPU]'", - "'--debug[Enable debug mode]'", - "'--no-debug[Disable debug mode]'", + "'--debug[Build with debug compiler flags (for MFC code)]'", + "'--no-debug[Build without debug flags]'", "'--gcov[Enable gcov coverage]'", "'--no-gcov[Disable gcov coverage]'", "'--unified[Enable unified memory]'", @@ -401,6 +403,9 @@ def generate_zsh_completion(schema: CLISchema) -> str: if arg_lines: lines.append(' _arguments \\') lines.append(' ' + ' \\\n '.join(arg_lines)) + else: + # Explicitly disable default completion for commands with no args + lines.append(' :') lines.append(' ;;') lines.extend([ diff --git a/toolchain/mfc/cli/docs_gen.py b/toolchain/mfc/cli/docs_gen.py index 0c4b8b69a7..cf120fc07f 100644 --- a/toolchain/mfc/cli/docs_gen.py +++ b/toolchain/mfc/cli/docs_gen.py @@ -83,7 +83,7 @@ def _generate_options_table(cmd: Command, schema: CLISchema) -> List[str]: if "mfc_config" in cmd.include_common: lines.append("| `--mpi`, `--no-mpi` | Enable/disable MPI | `true` |") lines.append("| `--gpu [acc/mp]`, `--no-gpu` | Enable GPU (OpenACC/OpenMP) | `no` |") - lines.append("| `--debug`, `--no-debug` | Enable debug mode | `false` |") + lines.append("| `--debug`, `--no-debug` | Build with debug compiler flags | `false` |") lines.append("") @@ -270,16 +270,23 @@ def generate_cli_reference(schema: CLISchema) -> str: "|------|-------------|", "| `--mpi` / `--no-mpi` | Enable/disable MPI support |", "| `--gpu [acc/mp]` / `--no-gpu` | Enable GPU with OpenACC or OpenMP |", - "| `--debug` / `--no-debug` | Enable debug build |", + "| `--debug` / `--no-debug` | Build with debug compiler flags |", "| `--gcov` / `--no-gcov` | Enable code coverage |", "| `--single` / `--no-single` | Single precision |", "| `--mixed` / `--no-mixed` | Mixed precision |", "", "### Verbosity (`-v, --verbose`)", "", + "Controls output verbosity level:", + "", "- `-v` - Basic verbose output", "- `-vv` - Show build commands", - "- `-vvv` - Full debug output including CMake debug", + "- `-vvv` - Full verbose output including CMake details", + "", + "### Debug Logging (`-d, --debug-log`)", + "", + "Enables debug logging for the Python toolchain (mfc.sh internals).", + "This is for troubleshooting the build system, not the MFC simulation code.", "", ])