diff --git a/.github/workflows/integration_tests_cuda.yml b/.github/workflows/integration_tests_cuda.yml new file mode 100644 index 00000000..7bb21519 --- /dev/null +++ b/.github/workflows/integration_tests_cuda.yml @@ -0,0 +1,79 @@ +name: Integration Tests - CUDA + +on: + push: + branches: + - main + pull_request: + types: [opened, reopened, synchronize, ready_for_review] + schedule: + - cron: "0 5 * * *" + +jobs: + tests: + runs-on: + group: dahlia + labels: RTX5060 + container: + image: daisytuner/docc-run-env-llvm19-ubuntu-24.04:latest-amd64 + options: >- + --cap-add=PERFMON + --gpus=all + + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} + submodules: recursive + lfs: true + + - name: Build + run: | + mkdir build + cd build + cmake -G Ninja \ + -DCMAKE_C_COMPILER=clang-19 \ + -DCMAKE_CXX_COMPILER=clang++-19 \ + -DCMAKE_BUILD_TYPE=Debug \ + -DLLVM_BUILD_FRONTEND=ON \ + -DLLVM_BUILD_TESTS=ON \ + -DSDFG_BUILD_TESTS=OFF \ + -DINSTALL_GTEST=OFF \ + -DBUILD_TESTS:BOOL=OFF \ + -DBUILD_BENCHMARKS:BOOL=OFF \ + -DBUILD_BENCHMARKS_GOOGLE:BOOL=OFF \ + .. + ninja -j$(nproc) + cpack -G DEB + + apt-get install -y ./docc*.deb + apt-get install -y libhdf5-dev + + - name: Integration tests - CUDA + run: | + export CPATH=/usr/local/include:$CPATH + export LIBRARY_PATH=/usr/local/lib:$LIBRARY_PATH + export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + export PATH=/usr/local/bin:$PATH + export LLVM_SYMBOLIZER_PATH=$(which llvm-symbolizer-19) + export CPLUS_INCLUDE_PATH=$(pwd)/rtl/include/daisy_rtl:$CPLUS_INCLUDE_PATH + + pip install pytest==7.1.3 --break-system-packages + pip install pytest-parallel --break-system-packages + + cd integration + + nvidia-smi + + # Basic tests + pytest -v -rx basic_test.py + pytest -v -rx rtl_test.py + + # PolyBench + pytest -v -rx polybench_cuda_test.py + + # Rodinia + pytest -v -rx rodinia_cuda_test.py + + # Apps + pytest -v -rx apps_cuda_test.py diff --git a/.github/workflows/integration_tests_openmp.yml b/.github/workflows/integration_tests_openmp.yml new file mode 100644 index 00000000..7443d853 --- /dev/null +++ b/.github/workflows/integration_tests_openmp.yml @@ -0,0 +1,81 @@ +name: Integration Tests - OpenMP + +on: + push: + branches: + - main + pull_request: + types: [opened, reopened, synchronize, ready_for_review] + schedule: + - cron: "0 5 * * *" + +jobs: + tests: + runs-on: + group: dahlia + labels: openmp + container: + image: daisytuner/docc-run-env-llvm19-ubuntu-24.04:latest-amd64 + + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} + submodules: recursive + lfs: true + + - name: Build + run: | + mkdir build + cd build + cmake -G Ninja \ + -DCMAKE_C_COMPILER=clang-19 \ + -DCMAKE_CXX_COMPILER=clang++-19 \ + -DCMAKE_BUILD_TYPE=Debug \ + -DLLVM_BUILD_FRONTEND=ON \ + -DLLVM_BUILD_TESTS=ON \ + -DSDFG_BUILD_TESTS=OFF \ + -DINSTALL_GTEST=OFF \ + -DBUILD_TESTS:BOOL=OFF \ + -DBUILD_BENCHMARKS:BOOL=OFF \ + -DBUILD_BENCHMARKS_GOOGLE:BOOL=OFF \ + .. + ninja -j$(nproc) + cpack -G DEB + + apt-get install -y ./docc*.deb + apt-get install -y libhdf5-dev + + # - name: Build Plugins # out-of-tree build currently broken, as needed deps are not installed as part of docc-llvm deb + # run: | + # cd integration/tests/plugins/simple + # mkdir build && cd build + # cmake -GNinja .. + # ninja -j$(nproc) + + - name: Integration tests - OpenMP + run: | + export LLVM_SYMBOLIZER_PATH=$(which llvm-symbolizer-19) + + pip install pytest==7.1.3 --break-system-packages + pip install pytest-parallel --break-system-packages + + cd integration + + # Plugin + # LD_LIBRARY_PATH=../../build/integration/tests/plugins/simple:$LD_LIBRARY_PATH pytest -v -s plugin_test.py + + # Polybench + pytest --workers auto -v -rx polybench_none_test.py + pytest --workers auto -v -rx polybench_sequential_test.py + pytest --workers auto -v -rx polybench_openmp_test.py + + # Rodinia + pytest -v -rx rodinia_none_test.py + pytest -v -rx rodinia_sequential_test.py + pytest -v -rx rodinia_openmp_test.py + + # Apps + pytest -v -rx apps_none_test.py + pytest -v -rx apps_sequential_test.py + pytest -v -rx apps_openmp_test.py diff --git a/.github/workflows/integration_tests_transfertuning.yml b/.github/workflows/integration_tests_transfertuning.yml new file mode 100644 index 00000000..66f13594 --- /dev/null +++ b/.github/workflows/integration_tests_transfertuning.yml @@ -0,0 +1,80 @@ +name: Integration Tests - Transfertuning + +on: + push: + branches: + - main + pull_request: + types: [opened, reopened, synchronize, ready_for_review] + schedule: + - cron: "0 5 * * *" + +jobs: + tests: + runs-on: + group: dahlia + labels: openmp + container: + image: daisytuner/docc-run-env-llvm19-ubuntu-24.04:latest-amd64 + + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} + submodules: recursive + lfs: true + + - name: Mark GitHub Actions workdir as safe + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Dependencies + run: | + pip install pytest==7.1.3 pytest-parallel tqdm + pip install pandas numpy scipy + + - name: Build + run: | + mkdir build + cd build + cmake -G Ninja \ + -DCMAKE_C_COMPILER=clang-19 \ + -DCMAKE_CXX_COMPILER=clang++-19 \ + -DCMAKE_BUILD_TYPE=Debug \ + -DLLVM_BUILD_FRONTEND=ON \ + -DLLVM_BUILD_TESTS=ON \ + -DSDFG_BUILD_TESTS=OFF \ + -DINSTALL_GTEST=OFF \ + -DBUILD_TESTS:BOOL=OFF \ + -DBUILD_BENCHMARKS:BOOL=OFF \ + -DBUILD_BENCHMARKS_GOOGLE:BOOL=OFF \ + .. + ninja -j$(nproc) + cpack -G DEB + + apt-get install -y ./docc*.deb + + - name: Integration tests - Transfertuning + env: + DOCC_ACCESS_TOKEN: ${{ secrets.DOCC_CI_TOKEN }} + run: | + export CPATH=/usr/local/include:$CPATH + export LIBRARY_PATH=/usr/local/lib:$LIBRARY_PATH + export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + export PATH=$PATH:/usr/local/bin + export LLVM_SYMBOLIZER_PATH=$(which llvm-symbolizer-19) + + export DOCC_OFFLOAD_REPORT=1 + + # End-to-end tests for transfer tuning + cd integration/tests/matmul/ + docc -g -O3 -docc-tune=sequential -docc-transfer-tune -docc-save-temps matmul.c -o matmul.out + ./matmul.out + cd ../../../ + + cd integration + pytest -v -rx -s polybench_transfertuning_test.py diff --git a/.github/workflows/llvm_tests_san.yml b/.github/workflows/llvm_tests_san.yml index e4328f19..0c6dc96c 100644 --- a/.github/workflows/llvm_tests_san.yml +++ b/.github/workflows/llvm_tests_san.yml @@ -23,6 +23,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} submodules: recursive - name: Mark GitHub Actions workdir as safe diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7a74ae85..f9bf636c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,6 +27,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} submodules: recursive - uses: pypa/cibuildwheel@v3.3.1 @@ -64,6 +65,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} submodules: recursive # Pin docc-compiler version to match release @@ -156,6 +158,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} submodules: recursive - name: Define Version diff --git a/.github/workflows/sanitizer_tests_asan.yml b/.github/workflows/sanitizer_tests_asan.yml index 5ebca361..94efb177 100644 --- a/.github/workflows/sanitizer_tests_asan.yml +++ b/.github/workflows/sanitizer_tests_asan.yml @@ -18,6 +18,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} submodules: recursive - name: Formatting @@ -55,6 +56,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} submodules: recursive - name: Install dependencies diff --git a/.github/workflows/sanitizer_tests_lsan.yml b/.github/workflows/sanitizer_tests_lsan.yml index 69d5c5a3..f7ea4728 100644 --- a/.github/workflows/sanitizer_tests_lsan.yml +++ b/.github/workflows/sanitizer_tests_lsan.yml @@ -18,6 +18,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} submodules: recursive - name: Formatting diff --git a/.github/workflows/sanitizer_tests_ubsan.yml b/.github/workflows/sanitizer_tests_ubsan.yml index f8d95b67..9ea843ef 100644 --- a/.github/workflows/sanitizer_tests_ubsan.yml +++ b/.github/workflows/sanitizer_tests_ubsan.yml @@ -18,6 +18,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} submodules: recursive - name: Formatting diff --git a/.github/workflows/unit_tests_linux.yml b/.github/workflows/unit_tests_linux.yml index 9d9b4e15..9ede436d 100644 --- a/.github/workflows/unit_tests_linux.yml +++ b/.github/workflows/unit_tests_linux.yml @@ -25,6 +25,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} submodules: recursive - name: Mark GitHub Actions workdir as safe @@ -153,6 +154,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} submodules: recursive - name: Mark GitHub Actions workdir as safe @@ -237,6 +239,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} submodules: recursive - name: Set up Python ${{ matrix.python-version }} @@ -321,6 +324,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} submodules: recursive - name: Set up Python ${{ matrix.python-version }} @@ -403,6 +407,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} submodules: recursive - name: Set up Python ${{ matrix.python-version }} @@ -477,6 +482,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} submodules: recursive - name: Set up Python ${{ matrix.python-version }} @@ -549,6 +555,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} submodules: recursive - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/unit_tests_macos.yml b/.github/workflows/unit_tests_macos.yml index fe8db86f..83c130d7 100644 --- a/.github/workflows/unit_tests_macos.yml +++ b/.github/workflows/unit_tests_macos.yml @@ -21,6 +21,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} submodules: recursive - name: Install dependencies diff --git a/.github/workflows/unit_tests_release.yml b/.github/workflows/unit_tests_release.yml index e881251b..430be78a 100644 --- a/.github/workflows/unit_tests_release.yml +++ b/.github/workflows/unit_tests_release.yml @@ -23,6 +23,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} submodules: recursive - name: Mark GitHub Actions workdir as safe @@ -79,6 +80,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: ${{ secrets.BENCHMARK_TESTS_TOKEN }} submodules: recursive - name: Install dependencies diff --git a/.gitmodules b/.gitmodules index 745a3a84..00109c1f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "cmake"] path = cmake url = git@github.com:daisytuner/cmake-scripts.git +[submodule "integration/tests"] + path = integration/tests + url = git@github.com:daisytuner/benchmark-tests.git diff --git a/integration/CFG_pruner.py b/integration/CFG_pruner.py new file mode 100644 index 00000000..cf71cbed --- /dev/null +++ b/integration/CFG_pruner.py @@ -0,0 +1,601 @@ +from typing import Dict, Any, Tuple +import sys +import os +import pydot +from pydot import Dot +from pyvis.network import Network +import networkx as nx +import argparse +import re +from collections import defaultdict +from tqdm import tqdm + +#!/usr/bin/env python3 +""" +CFG_pruner.py - small helper to read a Graphviz DOT file into a NetworkX graph. + +Usage: + python CFG_pruner.py path/to/graph_files perf_trace_file [--total_time_limit 0.01] [--top_n 5] [--output cfgs_annotated] [--entry-points] + +""" + +def remove_empty_elements(g: Dot): + for node in g.get_nodes(): + if node.get_name() == "" or node.get_label() == "" or node.get_label() is None: + g.del_node(node.get_name()) + for edge in g.get_edges(): + if edge.get_source() == "" or edge.get_destination() == "" or edge.get_label() == "" or edge.get_label() is None: + g.del_edge(edge.get_source(), edge.get_destination()) + +def read_dot_file(path: str): + try: + # Try using pygraphviz via networkx for faster parsing + graphs = [nx.drawing.nx_pydot.to_pydot(nx.nx_agraph.read_dot(path))] + except Exception: + graphs = pydot.graph_from_dot_file(path) + + # graphs = pydot.graph_from_dot_file(path) + graph = graphs[0] + remove_empty_elements(graph) + return graph + +def _print_summary(g: Dot): + print("Graph Summary:") + print("Number of nodes:", len(g.get_nodes())) + print("Number of edges:", len(g.get_edges())) + print("root node:", g.get_node_list()[0].get_label()) + call_nodes = [node for node in g.get_nodes() if "()" in node.get_label()] + print("Number of call nodes:", len(call_nodes)) + +def remove_node(g: Dot, node_name: str, in_edges, out_edges): + node_to_remove = g.get_node(node_name) + if node_to_remove: + g.del_node(node_name) + # Remove incoming edges + for edge in in_edges.get(node_name, []): + g.del_edge(edge.get_source(), edge.get_destination()) + # Remove outgoing edges + for edge in out_edges.get(node_name, []): + g.del_edge(edge.get_source(), edge.get_destination()) + +def bfs(g: Dot, in_edges, out_edges) -> set: + root = g.get_node_list()[0] + visited = set() + queue = [root.get_name()] + while queue: + current = queue.pop(0) + visited.add(current) + for edge in in_edges.get(current, []): + if edge.get_dir() != "both": + continue + src = edge.get_source() + if src not in visited and src not in queue: + queue.append(src) + for edge in out_edges.get(current, []): + dst = edge.get_destination() + if dst not in visited and dst not in queue: + queue.append(dst) + return visited + +def remove_orphaned_nodes(g: Dot): + in_edges, out_edges = build_adjacency_index(g) + if len(g.get_node_list()) < 1: + return + visited = bfs(g, in_edges, out_edges) + print("Number of reachable nodes after pruning:", len(visited)) + for node in g.get_nodes(): + if node.get_name() not in visited: + remove_node(g, node.get_name(), in_edges, out_edges) + +def filter_prunable_nodes(g: Dot, total_time_limit: float, duration: float, children: defaultdict[str, set]) -> list: + prunable_nodes = [] + call_nodes = [node for node in g.get_nodes() if "()" in node.get_label() and "main()" not in node.get_label()] + + root = g.get_node_list()[0] + + for call_node in call_nodes: + if call_node.get_name() == root.get_name(): + continue + total_time = totaltimes.get(call_node.get_name(), 0) + if total_time < total_time_limit * duration: + for child in children.get(call_node.get_name(), []): + child_time = totaltimes.get(child, 0) + if child_time >= total_time_limit * duration: + total_time = child_time + if total_time < total_time_limit * duration: + prunable_nodes.append(call_node) + + return prunable_nodes + +def build_adjacency_index(graph): + in_edges = defaultdict(list) + out_edges = defaultdict(list) + + for edge in graph.get_edge_list(): + source = edge.get_source().split(":")[0] + destination = edge.get_destination().split(":")[0] + + out_edges[source].append(edge) + in_edges[destination].append(edge) + + return in_edges, out_edges + +def replace_node(g: Dot, old_node_name: str, new_node_name: str, in_edges, out_edges): + node_to_remove = g.get_node(old_node_name) + if not node_to_remove: + return + node_to_remove = node_to_remove[0] + label = node_to_remove.get_label() + shape = node_to_remove.get_shape() + style = node_to_remove.get_style() + if style is None: + style = "filled" + fillcolor = node_to_remove.get_fillcolor() + if fillcolor is None: + fillcolor = "white" + g.add_node(pydot.Node(new_node_name, label=label, shape=shape, style=style, fillcolor=fillcolor)) + if node_to_remove: + g.del_node(old_node_name) + # Remove outgoing edges + for edge in out_edges.get(old_node_name, []): + g.del_edge(edge.get_source(), edge.get_destination()) + + # Replace incoming edges + for edge in in_edges.get(old_node_name, []): + g.add_edge(pydot.Edge(edge.get_source(), new_node_name, label=edge.get_label(), dir=edge.get_dir())) + g.del_edge(edge.get_source(), edge.get_destination()) + +def prune_leafs(g: Dot, total_time_limit: float, duration: float): + in_edges, out_edges = build_adjacency_index(g) + nodes_to_remove = [] + for node in g.get_nodes(): + if out_edges.get(node.get_name(), []) != []: + continue + if "()" in node.get_label() and "main()" not in node.get_label(): + total_time = totaltimes.get(node.get_name(), 0) + if total_time < total_time_limit * duration: + nodes_to_remove.append(node.get_name()) + + for node_name in nodes_to_remove: + remove_node(g, node_name, in_edges, out_edges) + +def subtree_pruning(g: Dot, total_time_limit: float, duration: float, children: defaultdict[str, set]): + prunable_nodes = filter_prunable_nodes(g, total_time_limit, duration, children) + + in_edges, out_edges = build_adjacency_index(g) + + print("number of prunable calls:", len(prunable_nodes)) + print("Total nodes before pruning:", len(g.get_nodes())) + + for call_node in prunable_nodes: + replace_node(g, call_node.get_name(), call_node.get_name() + "_pruned", in_edges, out_edges) + + remove_orphaned_nodes(g) + + prune_leafs(g, total_time_limit, duration) + + remove_orphaned_nodes(g) + + print("Total nodes after pruning:", len(g.get_nodes())) + + +def color_graph(g: Dot, total_time_limit: float, color: str = "purple", top_n: int = 5): + selftime_ordered_nodes = [] + for node in g.get_nodes(): + time = totaltimes.get(node.get_name(), 0) + if time > 0: + selftime_ordered_nodes.append((time, node)) + selftime_ordered_nodes.sort(reverse=True, key=lambda x: x[0]) + + for _, node in selftime_ordered_nodes[:top_n]: + g.get_node(node.get_name())[0].set_style("filled") + g.get_node(node.get_name())[0].set_fillcolor(color) + + return selftime_ordered_nodes[:top_n] + +def remove_subnodes(g: Dot): + subnode_edges = [] + for edge in g.get_edges(): + if ":" in edge.get_source() or ":" in edge.get_destination(): + subnode_edges.append(edge) + for edge in subnode_edges: + g.del_edge(edge.get_source(), edge.get_destination()) + g.add_edge(pydot.Edge(edge.get_source().split(":")[0], edge.get_destination().split(":")[0], label=edge.get_label(), dir=edge.get_dir())) + +def clear_redundant_edges(g: Dot): + unique_edges = set() + for edge in g.get_edges(): + edge_tuple = (edge.get_source(), edge.get_destination(), edge.get_label(), edge.get_dir()) + if edge_tuple in unique_edges: + continue + else: + unique_edges.add(edge_tuple) + for edge in g.get_edges(): + g.del_edge(edge.get_source(), edge.get_destination()) + for edge_tuple in unique_edges: + if edge_tuple[3] is None: + g.add_edge(pydot.Edge(edge_tuple[0], edge_tuple[1], label=edge_tuple[2])) + else: + g.add_edge(pydot.Edge(edge_tuple[0], edge_tuple[1], label=edge_tuple[2], dir=edge_tuple[3])) + +def remove_empty_records(g: Dot): + for node in g.get_nodes(): + label = node.get_label() + max_entries = label.count("|") + if max_entries == 0: + continue + for i in range(max_entries): + label = label.replace(f"| \ ", "") + node.set_label(label) + +def visualize_graph(g: Dot, output_dir: str, filename: str): + remove_subnodes(g) + clear_redundant_edges(g) + remove_empty_records(g) + os.makedirs(output_dir, exist_ok=True) + g.write_raw(f"{output_dir}/{filename}") + +def calculate_selftimes(input_stream: str) -> Tuple[defaultdict[str, float], float]: + """ + Parses perf script output, auto-detects frequency, + and calculates self-time per function. + """ + + # Dictionary to store sample counts: { "function_name": count } + selftime_counts = defaultdict(int) + + # 1. Regex for the Header Line to capture Timestamp + # Looks for a floating point number followed immediately by a colon + # Matches: " ... 513845.025989: ..." + header_pattern = re.compile(r'\s+([0-9]+\.[0-9]+):') + + # 2. Regex for the Stack Frame (Function Name) + # Matches: " 7ffff... function_name+0xoffset (...)" + stack_pattern = re.compile(r'^\s*[0-9a-f]+\s+(.+?)(?:\+0x[0-9a-f]+)?\s+\(.+\)') + + # State tracking + looking_for_stack_tip = False + + # Frequency detection variables + first_timestamp = None + last_timestamp = None + total_samples_detected = 0 + + for line in input_stream: + line = line.rstrip() + if not line: + continue + + # --- Check if line is a Stack Frame (indented) --- + if line[0].isspace(): + if looking_for_stack_tip: + match = stack_pattern.match(line) + if match: + func_name = match.group(1).strip() + selftime_counts[func_name] += 1 + + # We found the tip; stop looking until next header + looking_for_stack_tip = False + + # --- Check if line is a Header (not indented) --- + else: + # It is a header line. Try to find the timestamp. + ts_match = header_pattern.search(line) + if ts_match: + timestamp = float(ts_match.group(1)) + + if first_timestamp is None: + first_timestamp = timestamp + last_timestamp = timestamp + + total_samples_detected += 1 + + # A header implies a new sample follows; enable flag + looking_for_stack_tip = True + + # --- Calculate Frequency --- + if total_samples_detected < 2 or first_timestamp == last_timestamp: + # Fallback if we don't have enough data to calculate duration + calculated_frequency = 99.0 + duration = 0 + print("Warning: Not enough samples to calculate frequency. Defaulting to 99 Hz.") + else: + duration = last_timestamp - first_timestamp + # Frequency = Total Samples / Duration in Seconds + calculated_frequency = total_samples_detected / duration + + sorted_stats = sorted(selftime_counts.items(), key=lambda x: x[1], reverse=True) + + selftimes = defaultdict(float) + + for func, count in sorted_stats: + # Time = Count / Frequency + time_seconds = count / calculated_frequency + selftimes[func] = time_seconds + + return selftimes, duration + +def calculate_totaltimes(input_stream: str) -> Tuple[defaultdict[str, float], float]: + """ + Parses perf script output and calculates TOTAL (inclusive) time per function. + + Total Time = The duration a function was present anywhere in the call stack. + """ + + # Dictionary to store sample counts: { "function_name": count } + total_time_counts = defaultdict(int) + + # Set to track functions in the CURRENT sample/stack + current_sample_stack = set() + + # 1. Regex for the Header Line + header_pattern = re.compile(r'\s+([0-9]+\.[0-9]+):') + + # 2. Regex for the Stack Frame + stack_pattern = re.compile(r'^\s*[0-9a-f]+\s+(.+?)(?:\+0x[0-9a-f]+)?\s+\(.+\)') + + # Frequency detection variables + first_timestamp = None + last_timestamp = None + total_samples_detected = 0 + + # Helper to commit the current stack to the global totals + def commit_current_sample(): + for func_name in current_sample_stack: + total_time_counts[func_name] += 1 + current_sample_stack.clear() + + for line in input_stream: + line = line.rstrip() + if not line: + continue + + # --- Check if line is a Stack Frame (indented) --- + if line[0].isspace(): + # We are inside a stack trace; capture every function + match = stack_pattern.match(line) + if match: + func_name = match.group(1).strip() + # Use a set to handle recursion (count function once per sample) + current_sample_stack.add(func_name) + + # --- Check if line is a Header (not indented) --- + else: + # A new header means the previous sample is finished. + # Commit the functions found in the previous stack (if any). + if current_sample_stack: + commit_current_sample() + + # Process the timestamp + ts_match = header_pattern.search(line) + if ts_match: + timestamp = float(ts_match.group(1)) + + if first_timestamp is None: + first_timestamp = timestamp + last_timestamp = timestamp + + total_samples_detected += 1 + + # --- End of Stream: Commit the final sample --- + if current_sample_stack: + commit_current_sample() + + # --- Calculate Frequency --- + if total_samples_detected < 2 or first_timestamp == last_timestamp: + calculated_frequency = 99.0 + duration = 0 + print("Warning: Not enough samples to calculate frequency. Defaulting to 99 Hz.") + else: + duration = last_timestamp - first_timestamp + calculated_frequency = total_samples_detected / duration + + # --- Convert Counts to Seconds --- + sorted_stats = sorted(total_time_counts.items(), key=lambda x: x[1], reverse=True) + totaltimes = defaultdict(float) + + for func, count in sorted_stats: + time_seconds = count / calculated_frequency + totaltimes[func] = time_seconds + + return totaltimes, duration + +def calculate_children(input_stream: str) -> Dict[str, set[str]]: + descendant_map = defaultdict(set) + stack_pattern = re.compile(r'^\s*[0-9a-f]+\s+(.+?)(?:\+0x[0-9a-f]+)?\s+\(.+\)') + + current_stack = [] + + def commit_stack(): + # The stack is captured as [innermost, caller, ..., main] + # We need to reverse it to [main, ..., caller, innermost] to map parent -> children + reversed_stack = current_stack[::-1] + for i in range(len(reversed_stack)): + parent_func = reversed_stack[i] + for j in range(i + 1, len(reversed_stack)): + child_func = reversed_stack[j] + descendant_map[parent_func].add(child_func) + current_stack.clear() + + for line in input_stream: + line = line.rstrip() + if not line: + continue + + if line[0].isspace(): + match = stack_pattern.match(line) + if match: + func_name = match.group(1).strip() + current_stack.append(func_name) + else: + if current_stack: + commit_stack() + + # Commit the final stack if the file ends with a stack trace + if current_stack: + commit_stack() + + return descendant_map + +def parse_profiles(perf_trace: str) -> Tuple[defaultdict[str, float], defaultdict[str, float], float]: + cmd_mangled = f"perf script --no-demangle -i {perf_trace} -F +srcline --full-source-path > mangled.profile" + cmd_demangled = f"perf script -i {perf_trace} -F +srcline --full-source-path > demangled.profile" + os.system(cmd_mangled) + os.system(cmd_demangled) + + with open("mangled.profile", "r") as f: + mangled_content = f.readlines() + mangled_times_total, duration_mangled_total = calculate_totaltimes(mangled_content) + mangled_times_self, duration_mangled_self = calculate_selftimes(mangled_content) + mangled_children = calculate_children(mangled_content) + + with open("demangled.profile", "r") as f: + demangled_content = f.readlines() + demangled_times_total, duration_demangled_total = calculate_totaltimes(demangled_content) + demangled_times_self, duration_demangled_self = calculate_selftimes(demangled_content) + demangled_children = calculate_children(demangled_content) + + duration = max(duration_mangled_total, duration_demangled_total, duration_mangled_self, duration_demangled_self) + + combined_times_self = defaultdict(float) + for func in set(mangled_times_self.keys()).union(set(demangled_times_self.keys())): + combined_times_self[func] = max(mangled_times_self.get(func, 0.0), demangled_times_self.get(func, 0.0)) + + combined_times_total = defaultdict(float) + for func in set(mangled_times_total.keys()).union(set(demangled_times_total.keys())): + combined_times_total[func] = max(mangled_times_total.get(func, 0.0), demangled_times_total.get(func, 0.0)) + + children = defaultdict(set) + for func in set(mangled_children.keys()).union(set(demangled_children.keys())): + children[func] = mangled_children.get(func, set()).union(demangled_children.get(func, set())) + + os.remove("mangled.profile") + os.remove("demangled.profile") + + return combined_times_total, combined_times_self, duration, children + +def annotate_times(g: Dot, selftimes: defaultdict[str, float], totaltimes: defaultdict[str, float]): + for node in g.get_nodes(): + label = node.get_label() + + # Only annotate nodes whose label contains '()' + if "()" not in label: + continue + func_name = label.split("()")[0].split()[-1] + + if func_name in selftimes: + time_val_self = selftimes[func_name] + self_str = f"| Self-time: {time_val_self:.6f}s|" + else: + self_str = "" + if func_name in totaltimes: + time_val_total = totaltimes[func_name] + total_str = f"| Total-time: {time_val_total:.6f}s|" + else: + total_str = "" + + new_label = re.sub(rf"\b{re.escape(func_name)}\b", f"{self_str} {total_str} {func_name}", label, count=1) + node.set_label(new_label) + node.set('shape', 'record') + +def total_time(g: Dot) -> float: + total = 0.0 + root = g.get_node_list()[0] + label = root.get_label() + if "Total-time:" in label: + time_str = label.split("Total-time:")[1].split("s")[0] + try: + time_val = float(time_str) + if time_val > total: + total = time_val + except ValueError: + pass + return total + +def preprocess_graph(g: Dot, selftimes: defaultdict[str, float], totaltimes: defaultdict[str, float]): + print("Preprocessing graph...") + annotate_times(g, selftimes, totaltimes) + +def handle_single_cfg(g: Dot, total_time_limit: float, top_n:int, duration: float, children: defaultdict[str, set]): + subtree_pruning(g, total_time_limit, duration, children) + + color_graph(g, total_time_limit=total_time_limit, color="pink", top_n=top_n) + +def read_graphs(matches: list[str]) -> list[Tuple[str, Dot]]: + base_graphs = [] + for path in tqdm(matches, desc="Reading DOT files"): + g = read_dot_file(path) + base_graphs.append((path, g)) + + return base_graphs + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Prune and analyze a Graphviz DOT file representing a CFG.") + parser.add_argument("path", type=str, help="Path to .dot file or directory containing .main.cfg.dot files") + parser.add_argument("perf_trace", type=str, help="Path to perf trace file") + parser.add_argument("--total_time_limit", type=float, default=0.01, help="Fraction of total self-time to use as pruning threshold (default: 0.01)") + parser.add_argument("--top_n", type=int, default=5, help="Number of top nodes by self-time to highlight (default: 5)") + parser.add_argument("--output", type=str, default="cfgs_annotated", help="Output directory for annotated CFGs (default: cfgs_annotated)") + parser.add_argument("--entry-points", type=str, help="Dumps a list of relevant entry points to the specified file") + + args = parser.parse_args() + + path = args.path + total_time_limit = args.total_time_limit + top_n = args.top_n + perf_trace = args.perf_trace + output_dir = args.output + + global totaltimes + + totaltimes, selftimes, duration, children = parse_profiles(perf_trace) + + print("Total runtime: " + str(duration)) + + sorted_totaltimes = sorted(totaltimes.items(), key=lambda x: x[1], reverse=True) + print("Top total times:") + for func, time in sorted_totaltimes: + if time <= total_time_limit * duration: + break + print(f"{func}: {time:.6f}s") + if args.entry_points: + mode = "w" if func == sorted_totaltimes[0][0] else "a" + with open(args.entry_points, mode) as f: + f.write(f"{func}\n") + + if not os.path.exists(path): + print("File not found:", path) + sys.exit(2) + + if os.path.isdir(path): + matches = [] + for root, _, filenames in os.walk(path): + for fname in filenames: + if fname.endswith(".cfg.dot"): + matches.append(os.path.join(root, fname)) + + if not matches: + print("No .cfg.dot files found in", path) + sys.exit(3) + + # use the first match as the dot file to process + if len(matches) > 1: + print("Multiple .cfg.dot files found: ", len(matches)) + + if not os.path.exists(output_dir): + os.makedirs(output_dir, exist_ok=True) + + unique_graphs = read_graphs(matches) + + for path, g in unique_graphs: + filename = path.split("/")[-1] + preprocess_graph(g, selftimes, totaltimes) + if total_time(g) <= total_time_limit * duration: + print("Skipping ", path, " due to low total total-time.") + continue + + handle_single_cfg(g, total_time_limit, top_n, duration, children) + + visualize_graph(g, output_dir, filename) + \ No newline at end of file diff --git a/integration/apps_cuda_test.py b/integration/apps_cuda_test.py new file mode 100644 index 00000000..acb68ef1 --- /dev/null +++ b/integration/apps_cuda_test.py @@ -0,0 +1,471 @@ +import subprocess +import numpy as np +import pandas as pd +import pytest + +from pathlib import Path +from functools import partial + +from test_runner import TestRunner, SDFGVerification + + +def evaluate_hpccg(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final residual"): + header = data.pop(0).strip() + + ref_final_residual = np.array([float(header.split(":")[1].strip())]) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final residual"): + header = data.pop(0).strip() + + test_final_residual = np.array([float(header.split(":")[1].strip())]) + assert np.abs(test_final_residual - ref_final_residual) <= 1e-19 + +@pytest.mark.skip(reason="Flaky.") +def test_HPCCG(): + test_case = Path(__file__).parent / "tests" / "apps" / "HPCCG" / "main.cpp" + verifier = SDFGVerification( + verification={ + "CUDAOffloading": 26, + "FOR": 20, + "MAP": 6, + "CUDA": 4, + "WHILE": 6, + "SEQUENTIAL": 2, + "ExternalOffloading": 40, + }, + ) + runner = TestRunner( + "Apps", + test_case, + "docc-cpp", + "clang++-19", + ["-O3", "-g", "-lm"], + "cuda", + [ + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "compute_residual.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "ddot.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "generate_matrix.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "HPC_Sparse_Matrix.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "HPC_sparsemv.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "HPCCG.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "mytimer.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "waxpby.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "YAML_Doc.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "YAML_Element.cpp", + ], + partial( + evaluate_hpccg, + args=["50", "50", "50"], + ), + sdfg_verification=verifier, + docc_flags=["-docc-offload-unknown-sizes"], + ) + runner.run() + + +def evaluate_lulesh(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final Origin Energy"): + header = data.pop(0).strip() + + ref_origin_energy = float(header.split("=")[1].strip()) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final Origin Energy"): + header = data.pop(0).strip() + + test_origin_energy = float(header.split("=")[1].strip()) + + print(test_origin_energy, ref_origin_energy) + assert test_origin_energy == ref_origin_energy + + +@pytest.mark.xfail(reason="Verifier changed & Output incorrect") +def test_LULESH(): + test_case = Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh.cc" + + verifier = SDFGVerification( + verification={ + "sdfgs": 29, + "FOR": 39, + "WHILE": 21, + "CUDAOffloading": 46, + "MAP": 14, + "CUDA": 7, + "Free": 6, + "SEQUENTIAL": 7, + }, + ) + runner = TestRunner( + "Apps", + test_case, + "docc-cpp", + "clang++-19", + [ + "-O3", + "-fopenmp", + "-g", + "-DUSE_MPI=0", + "-DUSE_OMP=0", + "-lm", + ], + "cuda", + [ + Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh-comm.cc", + Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh-init.cc", + Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh-util.cc", + Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh-viz.cc", + ], + partial( + evaluate_lulesh, + args=["-s", "30", "-i", "10"], + ), + sdfg_verification=verifier, + ) + runner.run() + + +def evaluate_miniFE(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final Resid Norm"): + header = data.pop(0).strip() + + ref_final_residual = float(header.split(":")[1].strip()) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final Resid Norm"): + header = data.pop(0).strip() + + test_final_residual = float(header.split(":")[1].strip()) + + assert abs(test_final_residual - ref_final_residual) < 10e-8 + + +@pytest.mark.parametrize( + "data_layout, precision", + [ + pytest.param("MINIFE_CSR_MATRIX", "float"), + pytest.param("MINIFE_ELL_MATRIX", "float"), + ], +) +@pytest.mark.skip(reason="Times out because of 15 minute compile time.") +def test_miniFE(data_layout, precision): + test_case = Path(__file__).parent / "tests" / "apps" / "miniFE" / "src" / "main.cpp" + verifier_csr = SDFGVerification( + verification={ + "WHILE": 57, + "FOR": 183, + "CUDAOffloading": 40, + "MAP": 21, + "SEQUENTIAL": 9, + "CUDA": 12, + } + ) + verifier_ell = SDFGVerification( + verification={ + "WHILE": 45, + "FOR": 128, + "CUDA": 5, + "CUDAOffloading": 16, + "MAP": 11, + "SEQUENTIAL": 6, + } + ) + if data_layout == "MINIFE_CSR_MATRIX": + verifier = verifier_csr + else: + verifier = verifier_ell + + runner = TestRunner( + "Apps", + test_case, + "docc-cpp", + "clang++-19", + [ + "-O3", + "-g", + "-fopenmp", + "-I" + str((Path(__file__).parent / "tests" / "apps" / "miniFE" / "src").absolute()), + "-I" + + str((Path(__file__).parent / "tests" / "apps" / "miniFE" / "utils").absolute()), + "-I" + str((Path(__file__).parent / "tests" / "apps" / "miniFE" / "fem").absolute()), + "-DMINIFE_SCALAR=" + precision, + "-DMINIFE_LOCAL_ORDINAL=int", + "-DMINIFE_GLOBAL_ORDINAL=int", + "-D" + data_layout, + "-UHAVE_MPI", + "-lm", + ], + "cuda", + [ + Path(__file__).parent / "tests" / "apps" / "miniFE" / "utils" / "param_utils.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "utils" / "utils.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "utils" / "mytimer.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "src" / "YAML_Element.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "src" / "YAML_Doc.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "basic" / "BoxPartition.cpp", + ], + partial( + evaluate_miniFE, + args=[], + ), + sdfg_verification=verifier, + docc_flags=["-docc-lower-invoke"], + ) + runner.run(1500) + + +def evaluate_miniAMR2(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while header != "BEGIN DUMP OUTPUT": + header = data.pop(0) + assert header == "BEGIN DUMP OUTPUT" + + reference_arrays = {"output": []} + current_array = "output" + for line in data: + if line == "END DUMP OUTPUT": + break + + row = np.fromstring(line, dtype=np.float64, sep=" ") + reference_arrays[current_array].append(row) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while header != "BEGIN DUMP OUTPUT": + header = data.pop(0) + assert header == "BEGIN DUMP OUTPUT" + + test_arrays = {"output": []} + current_array = "output" + for line in data: + if line == "END DUMP OUTPUT": + break + + row = np.fromstring(line, dtype=np.float64, sep=" ") + test_arrays[current_array].append(row) + + if not reference_arrays["output"]: + return + + reference_arrays["output"] = np.concatenate(reference_arrays["output"]) + test_arrays["output"] = np.concatenate(test_arrays["output"]) + + for array in reference_arrays: + assert np.array_equal( + reference_arrays[array], test_arrays[array], equal_nan=True + ) + + +@pytest.mark.xfail(reason="Verifier changed") +def test_miniAMR2(): + test_case = Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "main.c" + + verifier = SDFGVerification( + verification={'sdfgs': 48, 'CUDAOffloading': 4, 'MAP': 3, 'CUDA': 2, 'FOR': 55, 'SEQUENTIAL': 1} + ) + runner = TestRunner( + "Apps", + test_case, + "docc", + "clang-19", + ["-O3", "-g", "-fopenmp", "-latomic", "-DMANTEVO_DUMP_ARRAYS"], + "cuda", + [ + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "block.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "calc.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "params.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "plot.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "profile.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "queue.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "refine.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "stencil.c", + ], + partial( + evaluate_miniAMR2, + args=[ + "--nx", + "16", + "--ny", + "16", + "--nz", + "16", + ], + ), + sdfg_verification=verifier, + ) + runner.run() + + +def evaluate_cloudsc(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + +@pytest.mark.skip(reason="Compile time") +def test_cloudsc(): + test_case = Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "dwarf_cloudsc.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 48, + "MAP": 3, + "FOR": 57, + "WHILE": 24, + "SEQUENTIAL": 3, + "Malloc": 13, + "Free": 7, + } + ) + runner = TestRunner( + "Apps", + test_case, + "docc", + "clang-19", + [ + "-O3", + "-g", + "-fopenmp", + "-DHAVE_HDF5", + "-I/usr/include/hdf5/serial", + "-lhdf5_serial", + ], + "cuda", + [ + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "cloudsc_c.c", + Path(__file__).parent + / "tests" + / "apps" + / "cloudsc_c" + / "cloudsc" + / "cloudsc_driver.c", + Path(__file__).parent + / "tests" + / "apps" + / "cloudsc_c" + / "cloudsc" + / "cloudsc_validate.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "load_state.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "mycpu.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "yoecldp_c.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "yoethf_c.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "yomcst_c.c", + ], + partial( + evaluate_cloudsc, + args=[ + "1", + "8192", + "128", + ], + ), + sdfg_verification=verifier, + ) + runner.run() diff --git a/integration/apps_none_test.py b/integration/apps_none_test.py new file mode 100644 index 00000000..4b28adea --- /dev/null +++ b/integration/apps_none_test.py @@ -0,0 +1,455 @@ +import subprocess +import numpy as np +import pandas as pd +import pytest + +from pathlib import Path +from functools import partial + +from test_runner import TestRunner, SDFGVerification + + +def evaluate_hpccg(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final residual"): + header = data.pop(0).strip() + + ref_final_residual = float(header.split(":")[1].strip()) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final residual"): + header = data.pop(0).strip() + + test_final_residual = float(header.split(":")[1].strip()) + + assert test_final_residual == ref_final_residual + + +def test_HPCCG(): + test_case = Path(__file__).parent / "tests" / "apps" / "HPCCG" / "main.cpp" + verifier = SDFGVerification( + verification={ + "FOR": 20, + "MAP": 6, + "SEQUENTIAL": 6, + "WHILE": 6, + }, + ) + runner = TestRunner( + "HPCCG", + test_case, + "docc-cpp", + "clang++-19", + ["-O3", "-g", "-lm"], + "none", + [ + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "compute_residual.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "ddot.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "generate_matrix.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "HPC_Sparse_Matrix.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "HPC_sparsemv.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "HPCCG.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "mytimer.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "waxpby.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "YAML_Doc.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "YAML_Element.cpp", + ], + partial( + evaluate_hpccg, + args=["50", "50", "50"], + ), + sdfg_verification=verifier, + ) + runner.run() + + +def evaluate_lulesh(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final Origin Energy"): + header = data.pop(0).strip() + + ref_origin_energy = float(header.split("=")[1].strip()) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + found = False + while not header.startswith("Final Origin Energy"): + header = data.pop(0).strip() + found = True + + assert found, "Final Origin Energy not found in output" + + test_origin_energy = float(header.split("=")[1].strip()) + + print(test_origin_energy, ref_origin_energy) + assert test_origin_energy == ref_origin_energy + + +def test_LULESH(): + test_case = Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh.cc" + + verifier = SDFGVerification( + verification={ + "sdfgs": 29, + "FOR": 36, + "SEQUENTIAL": 14, + "WHILE": 24, + "MAP": 14, + "Free": 6, + } + ) + runner = TestRunner( + "Apps", + test_case, + "docc-cpp", + "clang++-19", + [ + "-O3", + "-fopenmp", + "-g", + "-DUSE_MPI=0", + "-DUSE_OMP=0", + "-lm", + ], + "none", + [ + Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh-comm.cc", + Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh-init.cc", + Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh-util.cc", + Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh-viz.cc", + ], + partial( + evaluate_lulesh, + args=["-s", "30", "-i", "10"], + ), + sdfg_verification=verifier, + ) + runner.run() + + +def evaluate_miniFE(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final Resid Norm"): + header = data.pop(0).strip() + + ref_final_residual = float(header.split(":")[1].strip()) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final Resid Norm"): + header = data.pop(0).strip() + + test_final_residual = float(header.split(":")[1].strip()) + + assert abs(test_final_residual - ref_final_residual) < 10e-8 + + +@pytest.mark.parametrize( + "data_layout, precision", + [ + pytest.param("MINIFE_CSR_MATRIX", "float", marks=pytest.mark.xfail(reason="Compilation segfaults")), + pytest.param("MINIFE_ELL_MATRIX", "float", marks=pytest.mark.xfail(reason="Compilation segfaults")), + ], +) +def test_miniFE(data_layout, precision): + test_case = Path(__file__).parent / "tests" / "apps" / "miniFE" / "src" / "main.cpp" + verifier_csr = SDFGVerification( + verification={ + "sdfgs": 76, + "MAP": 25, + "FOR": 183, + "WHILE": 57, + "SEQUENTIAL": 25, + } + ) + verifier_ell = SDFGVerification( + verification={ + "sdfgs": 73, + "MAP": 15, + "FOR": 128, + "WHILE": 45, + "SEQUENTIAL": 15, + } + ) + if data_layout == "MINIFE_CSR_MATRIX": + verifier = verifier_csr + else: + verifier = verifier_ell + + runner = TestRunner( + "Apps", + test_case, + "docc-cpp", + "clang++-19", + [ + "-O3", + "-g", + "-fopenmp", + "-I" + + str((Path(__file__).parent / "tests" / "apps" / "miniFE" / "src").absolute()), + "-I" + + str((Path(__file__).parent / "tests" / "apps" / "miniFE" / "utils").absolute()), + "-I" + + str((Path(__file__).parent / "tests" / "apps" / "miniFE" / "fem").absolute()), + "-DMINIFE_SCALAR=" + precision, + "-DMINIFE_LOCAL_ORDINAL=int", + "-DMINIFE_GLOBAL_ORDINAL=int", + "-D" + data_layout, + "-UHAVE_MPI", + "-lm", + ], + "none", + [ + Path(__file__).parent / "tests" / "apps" / "miniFE" / "utils" / "param_utils.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "utils" / "utils.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "utils" / "mytimer.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "src" / "YAML_Element.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "src" / "YAML_Doc.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "basic" / "BoxPartition.cpp", + ], + partial( + evaluate_miniFE, + args=[], + ), + sdfg_verification=verifier, + docc_flags=["-docc-lower-invoke"], + ) + runner.run(timeout=1500) + +def evaluate_miniAMR2(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while header != "BEGIN DUMP OUTPUT": + header = data.pop(0) + assert header == "BEGIN DUMP OUTPUT" + + reference_arrays = {"output": []} + current_array = "output" + for line in data: + if line == "END DUMP OUTPUT": + break + + row = np.fromstring(line, dtype=np.float64, sep=" ") + reference_arrays[current_array].append(row) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while header != "BEGIN DUMP OUTPUT": + header = data.pop(0) + assert header == "BEGIN DUMP OUTPUT" + + test_arrays = {"output": []} + current_array = "output" + for line in data: + if line == "END DUMP OUTPUT": + break + + row = np.fromstring(line, dtype=np.float64, sep=" ") + test_arrays[current_array].append(row) + + if not reference_arrays["output"]: + return + + reference_arrays["output"] = np.concatenate(reference_arrays["output"]) + test_arrays["output"] = np.concatenate(test_arrays["output"]) + + for array in reference_arrays: + assert np.array_equal( + reference_arrays[array], test_arrays[array], equal_nan=True + ) + + +def test_miniAMR2(): + test_case = Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "main.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 48, + "MAP": 1, + "FOR": 22, + "WHILE": 59, + "SEQUENTIAL": 1, + "Malloc": 13, + "Free": 7, + } + ) + runner = TestRunner( + "Apps", + test_case, + "docc", + "clang-19", + ["-O3", "-g", "-fopenmp", "-latomic", "-DMANTEVO_DUMP_ARRAYS"], + "none", + [ + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "block.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "calc.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "params.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "plot.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "profile.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "queue.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "refine.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "stencil.c", + ], + partial( + evaluate_miniAMR2, + args=[ + "--nx", + "16", + "--ny", + "16", + "--nz", + "16", + ], + ), + sdfg_verification=verifier + ) + runner.run() + +def evaluate_cloudsc(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + +@pytest.mark.skip(reason="Compile time") +def test_cloudsc(): + test_case = Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "dwarf_cloudsc.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 48, + "MAP": 3, + "FOR": 57, + "WHILE": 24, + "SEQUENTIAL": 3, + "Malloc": 13, + "Free": 7, + } + ) + runner = TestRunner( + "Apps", + test_case, + "docc", + "clang-19", + ["-O3", "-g", "-fopenmp", "-DHAVE_HDF5", "-I/usr/include/hdf5/serial", "-lhdf5_serial"], + "none", + [ + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "cloudsc_c.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "cloudsc_driver.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "cloudsc_validate.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "load_state.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "mycpu.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "yoecldp_c.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "yoethf_c.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "yomcst_c.c", + ], + partial( + evaluate_cloudsc, + args=[ + "1", + "8192", + "128", + ], + ), + sdfg_verification=verifier + ) + runner.run() diff --git a/integration/apps_openmp_test.py b/integration/apps_openmp_test.py new file mode 100644 index 00000000..35af4046 --- /dev/null +++ b/integration/apps_openmp_test.py @@ -0,0 +1,452 @@ +import subprocess +import numpy as np +import pandas as pd +import pytest + +from pathlib import Path +from functools import partial + +from test_runner import TestRunner, SDFGVerification + + +def evaluate_hpccg(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final residual"): + header = data.pop(0).strip() + + ref_final_residual = float(header.split(":")[1].strip()) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final residual"): + header = data.pop(0).strip() + + test_final_residual = float(header.split(":")[1].strip()) + assert np.abs(test_final_residual - ref_final_residual) <= 1e-19 + +@pytest.mark.skip(reason="Flaky test") +def test_HPCCG(): + test_case = Path(__file__).parent / "tests" / "apps" / "HPCCG" / "main.cpp" + + verifier = SDFGVerification( + verification={ + "FOR": 19, + "MAP": 6, + "CPU_PARALLEL": 6, + "WHILE": 6, + "DOT": 1, + }, + ) + runner = TestRunner( + "Apps", + test_case, + "docc-cpp", + "clang++-19", + ["-O3", "-g", "-lm"], + "openmp", + [ + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "compute_residual.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "ddot.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "generate_matrix.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "HPC_Sparse_Matrix.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "HPC_sparsemv.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "HPCCG.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "mytimer.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "waxpby.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "YAML_Doc.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "YAML_Element.cpp", + ], + partial( + evaluate_hpccg, + args=["50", "50", "50"], + ), + sdfg_verification=verifier, + docc_flags=["-docc-einsum", "-lblas"], + ) + runner.run() + + +def evaluate_lulesh(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final Origin Energy"): + header = data.pop(0).strip() + + ref_origin_energy = float(header.split("=")[1].strip()) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final Origin Energy"): + header = data.pop(0).strip() + + test_origin_energy = float(header.split("=")[1].strip()) + + print(test_origin_energy, ref_origin_energy) + assert test_origin_energy == ref_origin_energy + + +@pytest.mark.xfail(reason="Verifier changed & Output incorrect") +def test_LULESH(): + test_case = Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh.cc" + + verifier = SDFGVerification( + verification={ + "sdfgs": 29, + "FOR": 39, + "SEQUENTIAL": 4, + "CPU_PARALLEL": 10, + "WHILE": 21, + "MAP": 14, + } + ) + runner = TestRunner( + "Apps", + test_case, + "docc-cpp", + "clang++-19", + [ + "-O3", + "-fopenmp", + "-g", + "-DUSE_MPI=0", + "-DUSE_OMP=0", + "-lm", + ], + "openmp", + [ + Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh-comm.cc", + Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh-init.cc", + Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh-util.cc", + Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh-viz.cc", + ], + partial( + evaluate_lulesh, + args=["-s", "30", "-i", "10"], + ), + sdfg_verification=verifier, + ) + runner.run() + + +def evaluate_miniFE(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final Resid Norm"): + header = data.pop(0).strip() + + ref_final_residual = float(header.split(":")[1].strip()) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final Resid Norm"): + header = data.pop(0).strip() + + test_final_residual = float(header.split(":")[1].strip()) + + assert abs(test_final_residual - ref_final_residual) < 10e-8 + + +@pytest.mark.parametrize( + "data_layout, precision", + [ + pytest.param("MINIFE_CSR_MATRIX", "float", marks=pytest.mark.xfail(reason="Compilation segfaults")), + pytest.param("MINIFE_ELL_MATRIX", "float", marks=pytest.mark.xfail(reason="Compilation segfaults")), + ], +) +def test_miniFE(data_layout, precision): + test_case = Path(__file__).parent / "tests" / "apps" / "miniFE" / "src" / "main.cpp" + verifier_csr = SDFGVerification( + verification={ + "MAP": 25, + "FOR": 183, + "WHILE": 57, + "SEQUENTIAL": 8, + "CPU_PARALLEL": 17, + } + ) + verifier_ell = SDFGVerification( + verification={ + "MAP": 15, + "FOR": 128, + "WHILE": 45, + "SEQUENTIAL": 5, + "CPU_PARALLEL": 10, + } + ) + if data_layout == "MINIFE_CSR_MATRIX": + verifier = verifier_csr + else: + verifier = verifier_ell + + runner = TestRunner( + "Apps", + test_case, + "docc-cpp", + "clang++-19", + [ + "-O3", + "-g", + "-fopenmp", + "-I" + + str((Path(__file__).parent / "tests" / "apps" / "miniFE" / "src").absolute()), + "-I" + + str((Path(__file__).parent / "tests" / "apps" / "miniFE" / "utils").absolute()), + "-I" + + str((Path(__file__).parent / "tests" / "apps" / "miniFE" / "fem").absolute()), + "-DMINIFE_SCALAR=" + precision, + "-DMINIFE_LOCAL_ORDINAL=int", + "-DMINIFE_GLOBAL_ORDINAL=int", + "-D" + data_layout, + "-UHAVE_MPI", + "-lm", + ], + "openmp", + [ + Path(__file__).parent / "tests" / "apps" / "miniFE" / "utils" / "param_utils.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "utils" / "utils.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "utils" / "mytimer.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "src" / "YAML_Element.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "src" / "YAML_Doc.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "basic" / "BoxPartition.cpp", + ], + partial( + evaluate_miniFE, + args=[], + ), + sdfg_verification=verifier, + docc_flags=["-docc-lower-invoke"], + ) + runner.run(timeout=1500) + +def evaluate_miniAMR2(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while header != "BEGIN DUMP OUTPUT": + header = data.pop(0) + assert header == "BEGIN DUMP OUTPUT" + + reference_arrays = {"output": []} + current_array = "output" + for line in data: + if line == "END DUMP OUTPUT": + break + + row = np.fromstring(line, dtype=np.float64, sep=" ") + reference_arrays[current_array].append(row) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while header != "BEGIN DUMP OUTPUT": + header = data.pop(0) + assert header == "BEGIN DUMP OUTPUT" + + test_arrays = {"output": []} + current_array = "output" + for line in data: + if line == "END DUMP OUTPUT": + break + + row = np.fromstring(line, dtype=np.float64, sep=" ") + test_arrays[current_array].append(row) + + if not reference_arrays["output"]: + return + + reference_arrays["output"] = np.concatenate(reference_arrays["output"]) + test_arrays["output"] = np.concatenate(test_arrays["output"]) + + for array in reference_arrays: + assert np.array_equal( + reference_arrays[array], test_arrays[array], equal_nan=True + ) + + +def test_miniAMR2(): + test_case = Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "main.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 48, + "MAP": 1, + "FOR": 22, + "WHILE": 59, + "CPU_PARALLEL": 1, + } + ) + runner = TestRunner( + "Apps", + test_case, + "docc", + "clang-19", + ["-O3", "-g", "-fopenmp", "-latomic", "-DMANTEVO_DUMP_ARRAYS"], + "openmp", + [ + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "block.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "calc.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "params.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "plot.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "profile.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "queue.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "refine.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "stencil.c", + ], + partial( + evaluate_miniAMR2, + args=[ + "--nx", + "16", + "--ny", + "16", + "--nz", + "16", + ], + ), + sdfg_verification=verifier + ) + runner.run() + +def evaluate_cloudsc(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + +@pytest.mark.skip(reason="Compile time") +def test_cloudsc(): + test_case = Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "dwarf_cloudsc.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 48, + "MAP": 3, + "FOR": 57, + "WHILE": 24, + "SEQUENTIAL": 3, + "Malloc": 13, + "Free": 7, + } + ) + runner = TestRunner( + "Apps", + test_case, + "docc", + "clang-19", + ["-O3", "-g", "-fopenmp", "-DHAVE_HDF5", "-I/usr/include/hdf5/serial", "-lhdf5_serial"], + "openmp", + [ + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "cloudsc_c.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "cloudsc_driver.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "cloudsc_validate.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "load_state.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "mycpu.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "yoecldp_c.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "yoethf_c.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "yomcst_c.c", + ], + partial( + evaluate_cloudsc, + args=[ + "1", + "8192", + "128", + ], + ), + sdfg_verification=verifier + ) + runner.run() diff --git a/integration/apps_sequential_test.py b/integration/apps_sequential_test.py new file mode 100644 index 00000000..9c04fabe --- /dev/null +++ b/integration/apps_sequential_test.py @@ -0,0 +1,453 @@ +import subprocess +import numpy as np +import pandas as pd +import pytest + +from pathlib import Path +from functools import partial + +from test_runner import TestRunner, SDFGVerification + + +def evaluate_hpccg(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final residual"): + header = data.pop(0).strip() + + ref_final_residual = float(header.split(":")[1].strip()) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final residual"): + header = data.pop(0).strip() + + test_final_residual = float(header.split(":")[1].strip()) + + assert test_final_residual == ref_final_residual + + +def test_HPCCG(): + test_case = Path(__file__).parent / "tests" / "apps" / "HPCCG" / "main.cpp" + + verifier = SDFGVerification( + verification={ + "FOR": 23, + "MAP": 9, + "SEQUENTIAL": 9, + "WHILE": 6, + }, + ) + runner = TestRunner( + "Apps", + test_case, + "docc-cpp", + "clang++-19", + ["-O3", "-g", "-lm"], + "sequential", + [ + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "compute_residual.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "ddot.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "generate_matrix.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "HPC_Sparse_Matrix.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "HPC_sparsemv.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "HPCCG.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "mytimer.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "waxpby.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "YAML_Doc.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "YAML_Element.cpp", + ], + partial( + evaluate_hpccg, + args=["50", "50", "50"], + ), + sdfg_verification=verifier, + ) + runner.run() + + +def evaluate_lulesh(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final Origin Energy"): + header = data.pop(0).strip() + + ref_origin_energy = float(header.split("=")[1].strip()) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final Origin Energy"): + header = data.pop(0).strip() + + test_origin_energy = float(header.split("=")[1].strip()) + + print(test_origin_energy, ref_origin_energy) + assert test_origin_energy == ref_origin_energy + + +@pytest.mark.xfail(reason="Verifier changed & Output incorrect") +def test_LULESH(): + test_case = Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh.cc" + + verifier = SDFGVerification( + verification={ + "sdfgs": 29, + "FOR": 39, + "SEQUENTIAL": 14, + "WHILE": 21, + "MAP": 14, + "Free": 6, + } + ) + runner = TestRunner( + "Apps", + test_case, + "docc-cpp", + "clang++-19", + [ + "-O3", + "-fopenmp", + "-g", + "-DUSE_MPI=0", + "-DUSE_OMP=0", + "-lm", + ], + "sequential", + [ + Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh-comm.cc", + Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh-init.cc", + Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh-util.cc", + Path(__file__).parent / "tests" / "apps" / "LULESH" / "lulesh-viz.cc", + ], + partial( + evaluate_lulesh, + args=["-s", "30", "-i", "10"], + ), + sdfg_verification=verifier, + ) + runner.run() + + +def evaluate_miniFE(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final Resid Norm"): + header = data.pop(0).strip() + + ref_final_residual = float(header.split(":")[1].strip()) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final Resid Norm"): + header = data.pop(0).strip() + + test_final_residual = float(header.split(":")[1].strip()) + + assert abs(test_final_residual - ref_final_residual) < 10e-8 + + +@pytest.mark.parametrize( + "data_layout, precision", + [ + pytest.param("MINIFE_CSR_MATRIX", "float", marks=pytest.mark.xfail(reason="Compilation segfaults")), + pytest.param("MINIFE_ELL_MATRIX", "float", marks=pytest.mark.xfail(reason="Compilation segfaults")), + ], +) +def test_miniFE(data_layout, precision): + test_case = Path(__file__).parent / "tests" / "apps" / "miniFE" / "src" / "main.cpp" + verifier_csr = SDFGVerification( + verification={ + # "sdfgs": 76, + "MAP": 25, + "FOR": 183, + "WHILE": 57, + "SEQUENTIAL": 25, + } + ) + verifier_ell = SDFGVerification( + verification={ + "sdfgs": 73, + "MAP": 15, + "FOR": 128, + "WHILE": 45, + "SEQUENTIAL": 15, + } + ) + if data_layout == "MINIFE_CSR_MATRIX": + verifier = verifier_csr + else: + verifier = verifier_ell + + runner = TestRunner( + "Apps", + test_case, + "docc-cpp", + "clang++-19", + [ + "-O3", + "-g", + "-fopenmp", + "-I" + + str((Path(__file__).parent / "tests" / "apps" / "miniFE" / "src").absolute()), + "-I" + + str((Path(__file__).parent / "tests" / "apps" / "miniFE" / "utils").absolute()), + "-I" + + str((Path(__file__).parent / "tests" / "apps" / "miniFE" / "fem").absolute()), + "-DMINIFE_SCALAR=" + precision, + "-DMINIFE_LOCAL_ORDINAL=int", + "-DMINIFE_GLOBAL_ORDINAL=int", + "-D" + data_layout, + "-UHAVE_MPI", + "-lm", + ], + "sequential", + [ + Path(__file__).parent / "tests" / "apps" / "miniFE" / "utils" / "param_utils.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "utils" / "utils.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "utils" / "mytimer.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "src" / "YAML_Element.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "src" / "YAML_Doc.cpp", + Path(__file__).parent / "tests" / "apps" / "miniFE" / "basic" / "BoxPartition.cpp", + ], + partial( + evaluate_miniFE, + args=[], + ), + sdfg_verification=verifier, + docc_flags=["-docc-lower-invoke"], + ) + runner.run(timeout=1500) + +def evaluate_miniAMR2(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while header != "BEGIN DUMP OUTPUT": + header = data.pop(0) + assert header == "BEGIN DUMP OUTPUT" + + reference_arrays = {"output": []} + current_array = "output" + for line in data: + if line == "END DUMP OUTPUT": + break + + row = np.fromstring(line, dtype=np.float64, sep=" ") + reference_arrays[current_array].append(row) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while header != "BEGIN DUMP OUTPUT": + header = data.pop(0) + assert header == "BEGIN DUMP OUTPUT" + + test_arrays = {"output": []} + current_array = "output" + for line in data: + if line == "END DUMP OUTPUT": + break + + row = np.fromstring(line, dtype=np.float64, sep=" ") + test_arrays[current_array].append(row) + + if not reference_arrays["output"]: + return + + reference_arrays["output"] = np.concatenate(reference_arrays["output"]) + test_arrays["output"] = np.concatenate(test_arrays["output"]) + + for array in reference_arrays: + assert np.array_equal( + reference_arrays[array], test_arrays[array], equal_nan=True + ) + + +def test_miniAMR2(): + test_case = Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "main.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 48, + "MAP": 1, + "FOR": 22, + "WHILE": 59, + "SEQUENTIAL": 1, + "Malloc": 13, + "Free": 7, + } + ) + runner = TestRunner( + "Apps", + test_case, + "docc", + "clang-19", + ["-O3", "-g", "-fopenmp", "-latomic", "-DMANTEVO_DUMP_ARRAYS"], + "sequential", + [ + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "block.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "calc.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "params.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "plot.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "profile.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "queue.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "refine.c", + Path(__file__).parent / "tests" / "apps" / "miniAMR2" / "stencil.c", + ], + partial( + evaluate_miniAMR2, + args=[ + "--nx", + "16", + "--ny", + "16", + "--nz", + "16", + ], + ), + sdfg_verification=verifier + ) + runner.run() + +def evaluate_cloudsc(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + +@pytest.mark.skip(reason="Compile time") +def test_cloudsc(): + test_case = Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "dwarf_cloudsc.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 48, + "MAP": 3, + "FOR": 57, + "WHILE": 24, + "SEQUENTIAL": 3, + "Malloc": 13, + "Free": 7, + } + ) + runner = TestRunner( + "Apps", + test_case, + "docc", + "clang-19", + ["-O3", "-g", "-fopenmp", "-DHAVE_HDF5", "-I/usr/include/hdf5/serial", "-lhdf5_serial"], + "sequential", + [ + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "cloudsc_c.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "cloudsc_driver.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "cloudsc_validate.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "load_state.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "mycpu.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "yoecldp_c.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "yoethf_c.c", + Path(__file__).parent / "tests" / "apps" / "cloudsc_c" / "cloudsc" / "yomcst_c.c", + ], + partial( + evaluate_cloudsc, + args=[ + "1", + "8192", + "128", + ], + ), + sdfg_verification=verifier + ) + runner.run() diff --git a/integration/apps_tenstorrent_test.py b/integration/apps_tenstorrent_test.py new file mode 100644 index 00000000..555a8562 --- /dev/null +++ b/integration/apps_tenstorrent_test.py @@ -0,0 +1,81 @@ +import subprocess +import numpy as np +import pandas as pd +import pytest + +from pathlib import Path +from functools import partial + +from test_runner import TestRunner, SDFGVerification + + +def evaluate_hpccg(reference_file: Path, test_file: Path, args) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final residual"): + header = data.pop(0).strip() + + ref_final_residual = np.array([float(header.split(":")[1].strip())]) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while not header.startswith("Final residual"): + header = data.pop(0).strip() + + test_final_residual = np.array([float(header.split(":")[1].strip())]) + print("###", test_final_residual, ref_final_residual) + assert np.abs(test_final_residual - ref_final_residual) <= 1e-17 + +@pytest.mark.skip("Dot expansion regression") +def test_HPCCG(): + test_case = Path(__file__).parent / "tests" / "apps" / "HPCCG" / "main.cpp" + verifier = SDFGVerification( + verification={"FOR": 22, "MAP": 9, "WHILE": 6, 'SEQUENTIAL': 3, "TTOffloading": 18, "DOT": 1}, + ) + runner = TestRunner( + "Apps", + test_case, + "docc-cpp", + "clang++-19", + ["-O3", "-g", "-lm"], + "tenstorrent", + [ + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "compute_residual.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "ddot.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "generate_matrix.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "HPC_Sparse_Matrix.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "HPC_sparsemv.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "HPCCG.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "mytimer.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "waxpby.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "YAML_Doc.cpp", + Path(__file__).parent / "tests" / "apps" / "HPCCG" / "YAML_Element.cpp", + ], + partial( + evaluate_hpccg, + args=["50", "50", "50"], + ), + sdfg_verification=verifier, + docc_flags=["-docc-einsum", "-docc-no-offloading-transfer-opt"], + ) + runner.run() diff --git a/integration/basic_test.py b/integration/basic_test.py new file mode 100644 index 00000000..53e45176 --- /dev/null +++ b/integration/basic_test.py @@ -0,0 +1,395 @@ +import os +import subprocess + +from pathlib import Path + +import pytest + +from test_runner import SDFGVerification + + +@pytest.mark.parametrize("opt_level", ["-O0", "-O1", "-O2", "-O3"]) +def test_static_inline(opt_level): + benchmark_path = Path(__file__).parent / "tests" / "basic" / "static_inline" + output_path = benchmark_path / f"static_inline_{opt_level.replace('-', '')}.out" + cmd = [ + "docc", + "-g", + opt_level, + str(benchmark_path / "main.c"), + str(benchmark_path / "square.c"), + str(benchmark_path / "cube.c"), + "-o", + str(output_path), + ] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + + # Run benchmark + process = subprocess.Popen( + [str(output_path), "2"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + square_res = stdout.splitlines()[0] + cube_res = stdout.splitlines()[1] + assert square_res == "Square: 4.000000" + assert cube_res == "Cube: 8.000000" + + +@pytest.mark.parametrize("opt_level", ["-O0", "-O1", "-O2", "-O3"]) +def test_static_global(opt_level): + benchmark_path = Path(__file__).parent / "tests" / "basic" / "static_global" + output_path = benchmark_path / f"static_global_{opt_level.replace('-', '')}.out" + cmd = [ + "docc-cpp", + "-g", + opt_level, + str(benchmark_path / "main.cpp"), + str(benchmark_path / "square.cpp"), + str(benchmark_path / "cube.cpp"), + "-o", + str(output_path), + ] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + + # Run benchmark + process = subprocess.Popen( + [str(output_path), "2"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + square_res = stdout.splitlines()[0] + cube_res = stdout.splitlines()[1] + assert square_res == "Square: 4" + assert cube_res == "Cube: 8" + +def test_memcpy(): + benchmark_path = Path(__file__).parent / "tests" / "basic" / "memory_lto" + output_path = benchmark_path / f"memcpy.out" + cmd = [ + "docc", + "-g", + "-O3", + str(benchmark_path / "memcpy.c"), + "-o", + str(output_path), + ] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + + # Run benchmark + process = subprocess.Popen( + [str(output_path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + success = stdout.splitlines()[0] + assert success == "Success: 1522756.000000 == 1522756.000000" + + +def test_device_transfers(): + benchmark_path = Path(__file__).parent / "tests" / "basic" / "memory_lto" + output_path = benchmark_path / f"device_transfers.out" + cmd = [ + "docc", + "-mllvm", + "-docc-tune=cuda", + "-g", + "-O3", + str(benchmark_path / "device_transfers.c"), + str(benchmark_path / "device_transfers_lib.c"), + "-o", + str(output_path), + ] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + + # Run benchmark + process = subprocess.Popen( + [str(output_path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + success = stdout.splitlines()[0] + assert success == "Success: ptr1[100] = 10000.000000" + + +def test_device_transfers_lib(): + benchmark_path = Path(__file__).parent / "tests" / "basic" / "memory_lto" + output_path_lib = benchmark_path / f"libdevice_transfers.so" + cmd_lib = [ + "docc", + "-mllvm", + "-docc-tune=cuda", + "-fPIC", + "-shared", + "-g", + "-O3", + str(benchmark_path / "device_transfers_lib.c"), + "-o", + str(output_path_lib), + ] + process = subprocess.Popen( + cmd_lib, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + + output_path = benchmark_path / f"device_transfers.out" + cmd = [ + "docc", + "-g", + "-O3", + str(benchmark_path / "device_transfers.c"), + "-o", + str(output_path), + "-L" + str(benchmark_path), + "-l" + "device_transfers" + ] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + + # Run benchmark + os.environ["LD_LIBRARY_PATH"] = str(benchmark_path) + ":" + os.environ.get("LD_LIBRARY_PATH", "") + process = subprocess.Popen( + [str(output_path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + + success = stdout.splitlines()[0] + assert success == "Success: ptr1[100] = 10000.000000" + + +def test_long_name(): + benchmark_path = Path(__file__).parent / "tests" / "basic" / "names" + output_path = benchmark_path / f"long_name.out" + cmd = [ + "docc", + "-g", + "-O3", + str(benchmark_path / "long_name.c"), + "-o", + str(output_path), + ] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + + # Run benchmark + process = subprocess.Popen( + [str(output_path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + success = stdout.splitlines()[0] + assert success == "Success: data[10] = 20.000000" + +@pytest.mark.xfail(reason="Compilation segfaults") +def test_transfer_minimization_external(): + # Compile lib + benchmark_path = Path(__file__).parent / "tests" / "basic" / "transfer_minimization" + output_path_lib = benchmark_path / "libvecadd.so" + cmd = [ + "clang-19", + "-fPIC", + "-shared", + "-g", + "-O3", + str(benchmark_path / "vecadd.c"), + "-o", + str(output_path_lib), + ] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + + # Compile test + os.environ["DOCC_OPT_REPORT"] = "1" + output_path = benchmark_path / "test_external.out" + cmd = [ + "docc", + "-g", + "-O3", + str(benchmark_path / "test_external.c"), + "-o", + str(output_path), + "-L" + str(benchmark_path), + "-l" + "vecadd" + ] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + + verification = SDFGVerification({ + "sdfgs": 1, + "ExternalOffloading": 11, + "Call": 6, + }) + verification.verify(stderr) + + # Run test + os.environ["LD_LIBRARY_PATH"] = str(benchmark_path) + ":" + os.environ.get("LD_LIBRARY_PATH", "") + process = subprocess.Popen( + [str(output_path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + assert "Correct" in stdout.splitlines() + +@pytest.mark.xfail(reason="Verifier changed") +def test_transfer_minimization_cuda(): + # Compile test + os.environ["DOCC_OPT_REPORT"] = "1" + benchmark_path = Path(__file__).parent / "tests" / "basic" / "transfer_minimization" + output_path = benchmark_path / "test_cuda.out" + cmd = [ + "docc", + "-docc-tune=cuda", + "-g", + "-O3", + str(benchmark_path / "test_cuda.c"), + "-o", + str(output_path) + ] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + + verification = SDFGVerification({ + "sdfgs": 1, + "CUDA": 3, + "CUDAOffloading": 11, + }) + verification.verify(stderr) + + # Run test + process = subprocess.Popen( + [str(output_path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + assert "Correct" in stdout.splitlines() diff --git a/integration/blas_tenstorrent_test.py b/integration/blas_tenstorrent_test.py new file mode 100644 index 00000000..2f7395cc --- /dev/null +++ b/integration/blas_tenstorrent_test.py @@ -0,0 +1,189 @@ +import subprocess +import pytest +import numpy as np + +from functools import partial +from pathlib import Path + +from test_runner import TestRunner, SDFGVerification + + +def verify( + reference_file: Path, + test_file: Path, + dtype, + rtol: float = 1e-05, + atol: float = 1e-08, +): + cmd = [reference_file] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + # Skip lines until we find the header + while data: + header = data.pop(0) + if header == "==BEGIN DUMP_ARRAYS==": + break + else: + raise AssertionError( + "Header '==BEGIN DUMP_ARRAYS==' not found in reference output" + ) + + dtype_ = dtype + reference_array = None + for line in data: + if line == "==END DUMP_ARRAYS==": + break + + if reference_array is None: + reference_array = [] + + row = np.fromstring(line, dtype=dtype_, sep=" ") + reference_array.append(row) + + cmd = [test_file] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("Crash: STDERR ==============") + print(stderr) + print("Crash: STDOUT ==============") + print(stdout) + assert process.returncode == 0 + + data = stdout.splitlines() + # Skip lines until we find the header + while data: + header = data.pop(0) + if header == "==BEGIN DUMP_ARRAYS==": + break + else: + raise AssertionError( + "Header '==BEGIN DUMP_ARRAYS==' not found in reference output" + ) + + dtype_ = dtype + test_array = None + for line in data: + if line == "==END DUMP_ARRAYS==": + break + + if test_array is None: + test_array = [] + + row = np.fromstring(line, dtype=dtype_, sep=" ") + test_array.append(row) + + assert np.allclose(reference_array, test_array, rtol=rtol, atol=atol) + + +@pytest.mark.skip(reason="docc-expand will be deprecated") +@pytest.mark.parametrize( + "precision, expand, rtol", + [ + pytest.param("s", "tenstorrent", 1e-03), + ], +) +def test_gemm(precision: str, expand: str, rtol: float): + benchmark_path = Path(__file__).parent / "tests" / "blas" + + verifier_no_expand = SDFGVerification( + verification={ + "sdfgs": 1, + "GEMM": 1, + "FOR": 5, + "SEQUENTIAL": 2, + "MAP": 2, + }, + ) + verifier_expand = SDFGVerification( + verification={ + "sdfgs": 1, + "FOR": 8, + "MAP": 5, + "SEQUENTIAL": 5, + }, + ) + test_case = benchmark_path / "gemm.c" + runner = TestRunner( + "BLAS", + test_case, + "docc", + "clang-19", + [ + "-g", + f"-DDATA_TYPE_IS_{precision}", + "-O3", + "-lblas", + ], + "none", + [], + partial( + verify, dtype=np.float64 if precision == "d" else np.float32, rtol=rtol + ), + sdfg_verification=verifier_expand if expand == "all" else verifier_no_expand, + docc_flags=["-mllvm", "-docc-expand=" + expand], + ) + return runner.run() + +@pytest.mark.skip(reason="docc-expand will be deprecated") +@pytest.mark.parametrize( + "precision, expand, rtol", + [ + pytest.param("s", "tenstorrent", 1e-03), + ], +) +def test_dot(precision: str, expand: str, rtol: float): + benchmark_path = Path(__file__).parent / "tests" / "blas" + + verifier_no_expand = SDFGVerification( + verification={ + "sdfgs": 1, + "DOT": 1, + "FOR": 5, + "SEQUENTIAL": 1, + "MAP": 1, + }, + ) + verifier_expand = SDFGVerification( + verification={ + "sdfgs": 1, + "FOR": 4, + "MAP": 3, + "TTOffloading": 4, + "SEQUENTIAL": 3, + }, + ) + test_case = benchmark_path / "dot.c" + runner = TestRunner( + "BLAS", + test_case, + "docc", + "clang-19", + [ + "-g", + f"-DDATA_TYPE_IS_{precision}", + "-O3", + "-lblas", + ], + "sequential", + [], + partial( + verify, dtype=np.float64 if precision == "d" else np.float32, rtol=rtol + ), + sdfg_verification=verifier_expand if expand == "tenstorrent" else verifier_no_expand, + docc_flags=["-mllvm", "-docc-expand=" + expand], + ) + return runner.run() \ No newline at end of file diff --git a/integration/plugin_test.py b/integration/plugin_test.py new file mode 100644 index 00000000..014c6d76 --- /dev/null +++ b/integration/plugin_test.py @@ -0,0 +1,79 @@ +import os +import subprocess + +from pathlib import Path + + +def test_simple_plugin(): + benchmark_path = Path(__file__).parent / "tests" / "plugins" / "simple" / "test" + output_path = benchmark_path / "simple.out" + + # Build without plugin library + cmd = [ + "docc-cpp", + "-g", + "-O0", + str(benchmark_path / "main.cpp"), + "-o", + str(output_path), + ] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + + # Run benchmark without plugin library + process = subprocess.Popen( + [str(output_path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 1 + res = stdout.splitlines() + assert "foo() was NOT replaced by plugin!" in res + + # Build with plugin library + cmd = [ + "docc-cpp", + "-g", + "-O0", + "-mllvm", + "-docc-plugins=libSimplePlugin.so", + str(benchmark_path / "main.cpp"), + "-o", + str(output_path), + ] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + res = stderr.splitlines() + assert "[DEBUG] [docc] Loaded plugin: simple (0.0.1)" in res + + # Run benchmark + process = subprocess.Popen( + [str(output_path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + res = stdout.splitlines() + assert "foo() was replaced by plugin!" in res diff --git a/integration/polybench_cuda_test.py b/integration/polybench_cuda_test.py new file mode 100644 index 00000000..57fe1a45 --- /dev/null +++ b/integration/polybench_cuda_test.py @@ -0,0 +1,1489 @@ +import subprocess +import os +import numpy as np +import pandas as pd + +import re +import pytest + +from pathlib import Path +from functools import partial + +from test_runner import TestRunner, SDFGVerification + + +def verify(reference_file: Path, test_file: Path, dtype, max_ulps): + cmd = [reference_file] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stderr.splitlines() + header = data.pop(0) + assert header == "==BEGIN DUMP_ARRAYS==" + + dtype_ = dtype + reference_arrays = {} + current_array = None + for line in data: + if line == "==END DUMP_ARRAYS==": + break + + if line.startswith("begin"): + assert current_array is None + current_array = line[line.find(":") + 1 :].strip() + reference_arrays[current_array] = [] + elif line.startswith("end"): + assert ( + current_array is not None + and current_array == line[line.find(":") + 1 :].strip() + ) + reference_arrays[current_array] = np.hstack(reference_arrays[current_array]) + current_array = None + else: + assert current_array is not None + row = np.fromstring(line, dtype=dtype_, sep=" ") + reference_arrays[current_array].append(row) + + cmd = [test_file] + omp_env = os.environ.copy() + omp_env["OMP_NUM_THREADS"] = "10" + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + env=omp_env, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stderr.splitlines() + header = data.pop(0) + assert header == "==BEGIN DUMP_ARRAYS==" + + test_arrays = {} + current_array = None + for line in data: + if line == "==END DUMP_ARRAYS==": + break + + if line.startswith("begin"): + assert current_array is None + current_array = line[line.find(":") + 1 :].strip() + test_arrays[current_array] = [] + elif line.startswith("end"): + assert ( + current_array is not None + and current_array == line[line.find(":") + 1 :].strip() + ) + test_arrays[current_array] = np.hstack(test_arrays[current_array]) + current_array = None + else: + assert current_array is not None + row = np.fromstring(line, dtype=dtype, sep=" ") + test_arrays[current_array].append(row) + + for array in reference_arrays: + tol = max_ulps * np.abs(np.spacing(reference_arrays[array])) + assert np.all(np.abs(test_arrays[array] - reference_arrays[array]) <= tol) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_correlation(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "datamining" / "correlation" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 25, + "MAP": 17, + "SEQUENTIAL": 5, + "CUDA": 12, + }, + ) + test_case = benchmark_path / "correlation.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_covariance(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "datamining" / "covariance" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 20, + "MAP": 13, + "SEQUENTIAL": 4, + "CUDA": 9, + }, + ) + test_case = benchmark_path / "covariance.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE", marks=pytest.mark.xfail(reason="Verifier changed")), + pytest.param("-DDATA_TYPE_IS_FLOAT", marks=pytest.mark.xfail(reason="Verifier changed")), + ], +) +def test_gemm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemm" + ) + + verifier = SDFGVerification( + verification={'sdfgs': 8, 'Calloc': 3, 'FOR': 17, 'Free': 3, 'barrier_local': 4, 'CUDAOffloading': 8, 'MAP': 10, 'CUDA': 10}, + ) + test_case = benchmark_path / "gemm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gemver(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemver" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 13, + "MAP": 7, + "SEQUENTIAL": 1, + "CUDA": 6, + }, + ) + test_case = benchmark_path / "gemver.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gesummv(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gesummv" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 10, + "MAP": 5, + "CUDA": 5, + }, + ) + test_case = benchmark_path / "gesummv.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_symm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "symm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 14, + "MAP": 7, + "SEQUENTIAL": 3, + "CUDA": 4, + }, + ) + test_case = benchmark_path / "symm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_syr2k(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "syr2k" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 14, + "MAP": 8, + "SEQUENTIAL": 2, + "CUDA": 6, + }, + ) + test_case = benchmark_path / "syr2k.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_syrk(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "syrk" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 14, + "MAP": 8, + "SEQUENTIAL": 2, + "CUDA": 6, + }, + ) + test_case = benchmark_path / "syrk.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE", marks=pytest.mark.xfail(reason="Verifier changed & Program crashes")), + pytest.param("-DDATA_TYPE_IS_FLOAT", marks=pytest.mark.xfail(reason="Verifier changed & Program crashes")), + ], +) +def test_trmm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "trmm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 14, + "MAP": 7, + "SEQUENTIAL": 2, + "CUDA": 5, + }, + ) + test_case = benchmark_path / "trmm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + # sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE", marks=pytest.mark.xfail(reason="Verifier changed")), + pytest.param("-DDATA_TYPE_IS_FLOAT", marks=pytest.mark.xfail(reason="Verifier changed")), + ], +) +def test_2mm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "2mm" + ) + + verifier = SDFGVerification( + verification={'sdfgs': 8, 'FOR': 25, 'barrier_local': 8, 'CUDAOffloading': 13, 'MAP': 16, 'CUDA': 16}, + ) + test_case = benchmark_path / "2mm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE", marks=pytest.mark.xfail(reason="Verifier changed")), + pytest.param("-DDATA_TYPE_IS_FLOAT", marks=pytest.mark.xfail(reason="Verifier changed")), + ], +) +def test_3mm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "3mm" + ) + + verifier = SDFGVerification( + verification={'sdfgs': 8, 'FOR': 31, 'barrier_local': 12, 'CUDAOffloading': 17, 'MAP': 20, 'CUDA': 20}, + ) + test_case = benchmark_path / "3mm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_atax(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "atax" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 13, + "MAP": 7, + "SEQUENTIAL": 1, + "CUDA": 6, + }, + ) + test_case = benchmark_path / "atax.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE", marks=pytest.mark.xfail(reason="Verifier changed & Output incorrect")), + pytest.param("-DDATA_TYPE_IS_FLOAT", marks=pytest.mark.xfail(reason="Verifier changed & Output incorrect")), + ], +) +def test_bicg(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "bicg" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 12, + "MAP": 5, + "CUDA": 5, + }, + ) + test_case = benchmark_path / "bicg.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_doitgen(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "doitgen" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 9, + "FOR": 22, + "MAP": 10, + "SEQUENTIAL": 6, + "CUDA": 4, + }, + ) + test_case = benchmark_path / "doitgen.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_mvt(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "mvt" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 11, + "MAP": 4, + "SEQUENTIAL": 1, + "CUDA": 3, + }, + ) + test_case = benchmark_path / "mvt.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_cholesky(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "cholesky" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 21, + "MAP": 11, + "SEQUENTIAL": 2, + "CUDA": 9, + }, + ) + test_case = benchmark_path / "cholesky.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + # pytest.param("-DDATA_TYPE_IS_DOUBLE"), + # pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_durbin(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "durbin" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 9, + "MAP": 3, + "SEQUENTIAL": 0, + "CUDA": 3, + }, + ) + test_case = benchmark_path / "durbin.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + +@pytest.mark.skip(reason="Wrong results, needs investigation") +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gramschmidt(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent + / "tests" + / "polybench" + / "linear-algebra" + / "solvers" + / "gramschmidt" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 19, + "MAP": 9, + "SEQUENTIAL": 3, + "CUDA": 6, + }, + ) + test_case = benchmark_path / "gramschmidt.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_lu(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "lu" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 22, + "MAP": 12, + "SEQUENTIAL": 3, + "CUDA": 9, + }, + ) + test_case = benchmark_path / "lu.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_ludcmp(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "ludcmp" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 26, + "MAP": 13, + "SEQUENTIAL": 3, + "CUDA": 10, + }, + ) + test_case = benchmark_path / "ludcmp.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE", marks=pytest.mark.xfail(reason="Verifier changed & Output incorrect")), + pytest.param("-DDATA_TYPE_IS_FLOAT", marks=pytest.mark.xfail(reason="Verifier changed & Output incorrect")), + ], +) +def test_trisolv(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "trisolv" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 8, + "MAP": 2, + "SEQUENTIAL": 1, + "CUDA": 1, + }, + ) + test_case = benchmark_path / "trisolv.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE", marks=pytest.mark.xfail(reason="Verifier changed")), + pytest.param("-DDATA_TYPE_IS_FLOAT", marks=pytest.mark.xfail(reason="Verifier changed")), + ], +) +def test_deriche(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "deriche" + + verifier = SDFGVerification( + verification={'sdfgs': 8, 'CUDAOffloading': 22, 'MAP': 10, 'FOR': 19, 'CUDA': 10}, + ) + test_case = benchmark_path / "deriche.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +def test_floyd_warshall(compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "floyd-warshall" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 10, + "MAP": 2, + "SEQUENTIAL": 0, + "CUDA": 2, + }, + ) + test_case = benchmark_path / "floyd-warshall.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + "-DDATA_TYPE_IS_INT", + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial(verify, dtype=np.int64, max_ulps=1e6), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +def test_nussinov(compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "nussinov" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 10, + "MAP": 3, + "SEQUENTIAL": 0, + "CUDA": 3, + }, + ) + test_case = benchmark_path / "nussinov.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + "-DDATA_TYPE_IS_INT", + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial(verify, dtype=np.int64, max_ulps=1e6), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE", marks=pytest.mark.xfail(reason="Verifier changed")), + pytest.param("-DDATA_TYPE_IS_FLOAT", marks=pytest.mark.xfail(reason="Verifier changed")), + ], +) +def test_adi(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "adi" + + verifier = SDFGVerification( + verification={'sdfgs': 8, 'FOR': 20, 'CUDAOffloading': 30, 'MAP': 10, 'CUDA': 10}, + ) + test_case = benchmark_path / "adi.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_fdtd_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "fdtd-2d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 20, + "MAP": 10, + "CUDA": 10, + }, + ) + test_case = benchmark_path / "fdtd-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_heat_3d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "heat-3d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 13, + "MAP": 6, + "CUDA": 6, + }, + ) + test_case = benchmark_path / "heat-3d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_jacobi_1d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "jacobi-1d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 8, + "MAP": 3, + "CUDA": 3, + }, + ) + test_case = benchmark_path / "jacobi-1d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_jacobi_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "jacobi-2d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 12, + "MAP": 6, + "CUDA": 6, + }, + ) + test_case = benchmark_path / "jacobi-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_seidel_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "seidel-2d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 10, + "MAP": 2, + "SEQUENTIAL": 0, + "CUDA": 2, + }, + ) + test_case = benchmark_path / "seidel-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "cuda", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) diff --git a/integration/polybench_none_test.py b/integration/polybench_none_test.py new file mode 100644 index 00000000..ec3a00b3 --- /dev/null +++ b/integration/polybench_none_test.py @@ -0,0 +1,1435 @@ +import subprocess +import numpy as np +import pandas as pd + +import re +import pytest + +from pathlib import Path +from functools import partial + +from test_runner import TestRunner, SDFGVerification + + +def verify(reference_file: Path, test_file: Path, dtype): + cmd = [reference_file] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stderr.splitlines() + header = data.pop(0) + assert header == "==BEGIN DUMP_ARRAYS==" + + dtype_ = dtype + reference_arrays = {} + current_array = None + for line in data: + if line == "==END DUMP_ARRAYS==": + break + + if line.startswith("begin"): + assert current_array is None + current_array = line[line.find(":") + 1 :].strip() + reference_arrays[current_array] = [] + elif line.startswith("end"): + assert ( + current_array is not None + and current_array == line[line.find(":") + 1 :].strip() + ) + reference_arrays[current_array] = np.hstack(reference_arrays[current_array]) + current_array = None + else: + assert current_array is not None + row = np.fromstring(line, dtype=dtype_, sep=" ") + reference_arrays[current_array].append(row) + + cmd = [test_file] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stderr.splitlines() + header = data.pop(0) + assert header == "==BEGIN DUMP_ARRAYS==" + + test_arrays = {} + current_array = None + for line in data: + if line == "==END DUMP_ARRAYS==": + break + + if line.startswith("begin"): + assert current_array is None + current_array = line[line.find(":") + 1 :].strip() + test_arrays[current_array] = [] + elif line.startswith("end"): + assert ( + current_array is not None + and current_array == line[line.find(":") + 1 :].strip() + ) + test_arrays[current_array] = np.hstack(test_arrays[current_array]) + current_array = None + else: + assert current_array is not None + row = np.fromstring(line, dtype=dtype, sep=" ") + test_arrays[current_array].append(row) + + for array in reference_arrays: + assert np.array_equal( + reference_arrays[array], test_arrays[array], equal_nan=True + ) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_correlation(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "datamining" / "correlation" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 16, + "MAP": 8, + "SEQUENTIAL": 8, + }, + ) + test_case = benchmark_path / "correlation.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_covariance(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "datamining" / "covariance" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 14, + "MAP": 7, + "SEQUENTIAL": 7, + }, + ) + test_case = benchmark_path / "covariance.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gemm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 15, + "MAP": 9, + "SEQUENTIAL": 9, + }, + ) + test_case = benchmark_path / "gemm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gemver(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemver" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 13, + "MAP": 7, + "SEQUENTIAL": 7, + }, + ) + test_case = benchmark_path / "gemver.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gesummv(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gesummv" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 8, + "MAP": 3, + "SEQUENTIAL": 3, + }, + ) + test_case = benchmark_path / "gesummv.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_symm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "symm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 13, + "MAP": 6, + "SEQUENTIAL": 6, + }, + ) + test_case = benchmark_path / "symm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_syr2k(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "syr2k" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 13, + "MAP": 7, + "SEQUENTIAL": 7, + }, + ) + test_case = benchmark_path / "syr2k.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_syrk(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "syrk" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 13, + "MAP": 7, + "SEQUENTIAL": 7, + }, + ) + test_case = benchmark_path / "syrk.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_trmm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "trmm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 11, + "MAP": 5, + "SEQUENTIAL": 5, + }, + ) + test_case = benchmark_path / "trmm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_2mm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "2mm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 19, + "MAP": 12, + "SEQUENTIAL": 12, + }, + ) + test_case = benchmark_path / "2mm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_3mm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "3mm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 22, + "MAP": 14, + "SEQUENTIAL": 14, + }, + ) + test_case = benchmark_path / "3mm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_atax(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "atax" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 11, + "MAP": 5, + "SEQUENTIAL": 5, + }, + ) + test_case = benchmark_path / "atax.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_bicg(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "bicg" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 11, + "MAP": 4, + "SEQUENTIAL": 4, + }, + ) + test_case = benchmark_path / "bicg.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_doitgen(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "doitgen" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 9, + "FOR": 21, + "MAP": 9, + "SEQUENTIAL": 9, + }, + ) + test_case = benchmark_path / "doitgen.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_mvt(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "mvt" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 11, + "MAP": 4, + "SEQUENTIAL": 4, + }, + ) + test_case = benchmark_path / "mvt.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_cholesky(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "cholesky" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 19, + "MAP": 9, + "SEQUENTIAL": 9, + }, + ) + test_case = benchmark_path / "cholesky.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_durbin(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "durbin" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 9, + "MAP": 3, + "SEQUENTIAL": 3, + }, + ) + test_case = benchmark_path / "durbin.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gramschmidt(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent + / "tests" + / "polybench" + / "linear-algebra" + / "solvers" + / "gramschmidt" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 17, + "MAP": 7, + "SEQUENTIAL": 7, + }, + ) + test_case = benchmark_path / "gramschmidt.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_lu(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "lu" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 20, + "MAP": 10, + "SEQUENTIAL": 10, + }, + ) + test_case = benchmark_path / "lu.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_ludcmp(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "ludcmp" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 24, + "MAP": 11, + "SEQUENTIAL": 11, + }, + ) + test_case = benchmark_path / "ludcmp.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_trisolv(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "trisolv" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 8, + "MAP": 2, + "SEQUENTIAL": 2, + }, + ) + test_case = benchmark_path / "trisolv.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_deriche(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "deriche" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 19, + "MAP": 10, + "SEQUENTIAL": 10, + }, + ) + test_case = benchmark_path / "deriche.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +def test_floyd_warshall(compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "floyd-warshall" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 10, + "MAP": 2, + "SEQUENTIAL": 2, + }, + ) + test_case = benchmark_path / "floyd-warshall.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + "-DDATA_TYPE_IS_INT", + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial(verify, dtype=np.int64), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +def test_nussinov(compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "nussinov" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 10, + "MAP": 3, + "SEQUENTIAL": 3, + "WHILE": 1, + }, + ) + test_case = benchmark_path / "nussinov.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + "-DDATA_TYPE_IS_INT", + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial(verify, dtype=np.int64), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_adi(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "adi" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 14, + "MAP": 4, + "SEQUENTIAL": 4, + }, + ) + test_case = benchmark_path / "adi.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_fdtd_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "fdtd-2d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 20, + "MAP": 10, + "SEQUENTIAL": 10, + }, + ) + test_case = benchmark_path / "fdtd-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_heat_3d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "heat-3d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 16, + "MAP": 9, + "SEQUENTIAL": 9, + }, + ) + test_case = benchmark_path / "heat-3d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_jacobi_1d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "jacobi-1d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 8, + "MAP": 3, + "SEQUENTIAL": 3, + }, + ) + test_case = benchmark_path / "jacobi-1d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_jacobi_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "jacobi-2d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 12, + "MAP": 6, + "SEQUENTIAL": 6, + }, + ) + test_case = benchmark_path / "jacobi-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_seidel_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "seidel-2d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 10, + "MAP": 2, + "SEQUENTIAL": 2, + }, + ) + test_case = benchmark_path / "seidel-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "none", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=120) diff --git a/integration/polybench_openmp_test.py b/integration/polybench_openmp_test.py new file mode 100644 index 00000000..318dcfa7 --- /dev/null +++ b/integration/polybench_openmp_test.py @@ -0,0 +1,1525 @@ +import subprocess +import numpy as np +import pandas as pd + +import re +import pytest + +from pathlib import Path +from functools import partial + +from test_runner import TestRunner, SDFGVerification + + +def verify(reference_file: Path, test_file: Path, dtype, max_ulps = None): + cmd = [reference_file] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stderr.splitlines() + header = data.pop(0) + assert header == "==BEGIN DUMP_ARRAYS==" + + dtype_ = dtype + reference_arrays = {} + current_array = None + for line in data: + if line == "==END DUMP_ARRAYS==": + break + + if line.startswith("begin"): + assert current_array is None + current_array = line[line.find(":") + 1 :].strip() + reference_arrays[current_array] = [] + elif line.startswith("end"): + assert ( + current_array is not None + and current_array == line[line.find(":") + 1 :].strip() + ) + reference_arrays[current_array] = np.hstack(reference_arrays[current_array]) + current_array = None + else: + assert current_array is not None + row = np.fromstring(line, dtype=dtype_, sep=" ") + reference_arrays[current_array].append(row) + + cmd = [test_file] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stderr.splitlines() + header = data.pop(0) + assert header == "==BEGIN DUMP_ARRAYS==" + + test_arrays = {} + current_array = None + for line in data: + if line == "==END DUMP_ARRAYS==": + break + + if line.startswith("begin"): + assert current_array is None + current_array = line[line.find(":") + 1 :].strip() + test_arrays[current_array] = [] + elif line.startswith("end"): + assert ( + current_array is not None + and current_array == line[line.find(":") + 1 :].strip() + ) + test_arrays[current_array] = np.hstack(test_arrays[current_array]) + current_array = None + else: + assert current_array is not None + row = np.fromstring(line, dtype=dtype, sep=" ") + test_arrays[current_array].append(row) + + for array in reference_arrays: + if max_ulps: + tol = max_ulps * np.abs(np.spacing(reference_arrays[array])) + assert np.all(np.abs(test_arrays[array] - reference_arrays[array]) <= tol) + else: + assert np.array_equal( + reference_arrays[array], test_arrays[array], equal_nan=True + ) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_correlation(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "datamining" / "correlation" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 23, + "MAP": 15, + "SEQUENTIAL": 5, + "CPU_PARALLEL": 10 + }, + ) + test_case = benchmark_path / "correlation.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_covariance(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "datamining" / "covariance" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 18, + "MAP": 11, + "SEQUENTIAL": 4, + "CPU_PARALLEL": 7, + }, + ) + test_case = benchmark_path / "covariance.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gemm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 9, + "MAP": 4, + "CPU_PARALLEL": 4, + "GEMM": 1, + }, + ) + test_case = benchmark_path / "gemm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e2, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gemver(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemver" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 12, + "MAP": 6, + "SEQUENTIAL": 2, + "CPU_PARALLEL": 4, + }, + ) + test_case = benchmark_path / "gemver.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e2, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gesummv(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gesummv" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 10, + "MAP": 5, + "SEQUENTIAL": 1, + "CPU_PARALLEL": 4, + }, + ) + test_case = benchmark_path / "gesummv.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_symm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "symm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 13, + "MAP": 6, + "SEQUENTIAL": 3, + "CPU_PARALLEL": 3, + }, + ) + test_case = benchmark_path / "symm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_syr2k(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "syr2k" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 12, + "MAP": 6, + "SEQUENTIAL": 2, + "CPU_PARALLEL": 4, + }, + ) + test_case = benchmark_path / "syr2k.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_syrk(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "syrk" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 12, + "MAP": 6, + "SEQUENTIAL": 2, + "CPU_PARALLEL": 4, + }, + ) + test_case = benchmark_path / "syrk.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE", marks=pytest.mark.xfail(reason="Verifier changed & Output incorrect")), + pytest.param("-DDATA_TYPE_IS_FLOAT", marks=pytest.mark.xfail(reason="Verifier changed & Output incorrect")), + ], +) +def test_trmm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "trmm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 13, + "MAP": 6, + "SEQUENTIAL": 2, + "CPU_PARALLEL": 4, + }, + ) + test_case = benchmark_path / "trmm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_2mm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "2mm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 11, + "MAP": 6, + "CPU_PARALLEL": 6, + "GEMM": 2, + }, + ) + test_case = benchmark_path / "2mm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e2, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_3mm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "3mm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 12, + "MAP": 7, + "CPU_PARALLEL": 7, + "GEMM": 3, + }, + ) + test_case = benchmark_path / "3mm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e2, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_atax(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "atax" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 12, + "MAP": 6, + "SEQUENTIAL": 1, + "CPU_PARALLEL": 5, + }, + ) + test_case = benchmark_path / "atax.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE", marks=pytest.mark.xfail(reason="Verifier changed & Output incorrect")), + pytest.param("-DDATA_TYPE_IS_FLOAT", marks=pytest.mark.xfail(reason="Verifier changed & Output incorrect")), + ], +) +def test_bicg(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "bicg" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 12, + "MAP": 5, + "SEQUENTIAL": 1, + "CPU_PARALLEL": 4, + }, + ) + test_case = benchmark_path / "bicg.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_doitgen(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "doitgen" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 9, + "FOR": 20, + "SEQUENTIAL": 6, + "CPU_PARALLEL": 2, + "MAP": 8, + }, + ) + test_case = benchmark_path / "doitgen.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_mvt(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "mvt" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 11, + "MAP": 4, + "SEQUENTIAL": 2, + "CPU_PARALLEL": 2, + }, + ) + test_case = benchmark_path / "mvt.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_cholesky(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "cholesky" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 18, + "MAP": 8, + "SEQUENTIAL": 2, + "CPU_PARALLEL": 6, + "CMath": 1, + }, + ) + test_case = benchmark_path / "cholesky.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_durbin(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "durbin" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 9, + "MAP": 3, + "CPU_PARALLEL": 3, + }, + ) + test_case = benchmark_path / "durbin.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gramschmidt(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent + / "tests" + / "polybench" + / "linear-algebra" + / "solvers" + / "gramschmidt" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 16, + "MAP": 6, + "SEQUENTIAL": 1, + "CPU_PARALLEL": 5, + "CMath": 1, + }, + ) + test_case = benchmark_path / "gramschmidt.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_lu(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "lu" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 19, + "MAP": 9, + "SEQUENTIAL": 3, + "CPU_PARALLEL": 6, + }, + ) + test_case = benchmark_path / "lu.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_ludcmp(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "ludcmp" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 23, + "MAP": 10, + "SEQUENTIAL": 3, + "CPU_PARALLEL": 7, + }, + ) + test_case = benchmark_path / "ludcmp.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE", marks=pytest.mark.xfail(reason="Verifier changed & Output incorrect")), + pytest.param("-DDATA_TYPE_IS_FLOAT", marks=pytest.mark.xfail(reason="Verifier changed & Output incorrect")), + ], +) +def test_trisolv(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "trisolv" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 8, + "MAP": 2, + "SEQUENTIAL": 1, + "CPU_PARALLEL": 1, + }, + ) + test_case = benchmark_path / "trisolv.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_deriche(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "deriche" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 16, + "MAP": 7, + "CPU_PARALLEL": 7, + }, + ) + test_case = benchmark_path / "deriche.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +def test_floyd_warshall(compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "floyd-warshall" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 9, + "MAP": 1, + "CPU_PARALLEL": 1, + }, + ) + test_case = benchmark_path / "floyd-warshall.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + "-DDATA_TYPE_IS_INT", + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial(verify, dtype=np.int64), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +def test_nussinov(compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "nussinov" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 9, + "MAP": 2, + "CPU_PARALLEL": 2, + "WHILE": 1, + }, + ) + test_case = benchmark_path / "nussinov.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + "-DDATA_TYPE_IS_INT", + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial(verify, dtype=np.int64), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_adi(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "adi" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 19, + "MAP": 9, + "CPU_PARALLEL": 9, + }, + ) + test_case = benchmark_path / "adi.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_fdtd_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "fdtd-2d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 16, + "MAP": 6, + "CPU_PARALLEL": 6, + }, + ) + test_case = benchmark_path / "fdtd-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_heat_3d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "heat-3d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 10, + "MAP": 3, + "CPU_PARALLEL": 3, + }, + ) + test_case = benchmark_path / "heat-3d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_jacobi_1d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "jacobi-1d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 8, + "MAP": 3, + "CPU_PARALLEL": 3, + }, + ) + test_case = benchmark_path / "jacobi-1d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_jacobi_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "jacobi-2d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 9, + "MAP": 3, + "CPU_PARALLEL": 3, + }, + ) + test_case = benchmark_path / "jacobi-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_seidel_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "seidel-2d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 9, + "MAP": 1, + "CPU_PARALLEL": 1, + }, + ) + test_case = benchmark_path / "seidel-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "openmp", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) diff --git a/integration/polybench_rocm_test.py b/integration/polybench_rocm_test.py new file mode 100644 index 00000000..0fb795be --- /dev/null +++ b/integration/polybench_rocm_test.py @@ -0,0 +1,1518 @@ +import subprocess +import os +import numpy as np +import pandas as pd + +import re +import pytest + +from pathlib import Path +from functools import partial + +from test_runner import TestRunner, SDFGVerification + + +def verify(reference_file: Path, test_file: Path, dtype, max_ulps): + cmd = [reference_file] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stderr.splitlines() + header = data.pop(0) + assert header == "==BEGIN DUMP_ARRAYS==" + + dtype_ = dtype + reference_arrays = {} + current_array = None + for line in data: + if line == "==END DUMP_ARRAYS==": + break + + if line.startswith("begin"): + assert current_array is None + current_array = line[line.find(":") + 1 :].strip() + reference_arrays[current_array] = [] + elif line.startswith("end"): + assert ( + current_array is not None + and current_array == line[line.find(":") + 1 :].strip() + ) + reference_arrays[current_array] = np.hstack(reference_arrays[current_array]) + current_array = None + else: + assert current_array is not None + row = np.fromstring(line, dtype=dtype_, sep=" ") + reference_arrays[current_array].append(row) + + cmd = [test_file] + omp_env = os.environ.copy() + omp_env["OMP_NUM_THREADS"] = "10" + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + env=omp_env, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stderr.splitlines() + header = data.pop(0) + assert header == "==BEGIN DUMP_ARRAYS==" + + test_arrays = {} + current_array = None + for line in data: + if line == "==END DUMP_ARRAYS==": + break + + if line.startswith("begin"): + assert current_array is None + current_array = line[line.find(":") + 1 :].strip() + test_arrays[current_array] = [] + elif line.startswith("end"): + assert ( + current_array is not None + and current_array == line[line.find(":") + 1 :].strip() + ) + test_arrays[current_array] = np.hstack(test_arrays[current_array]) + current_array = None + else: + assert current_array is not None + row = np.fromstring(line, dtype=dtype, sep=" ") + test_arrays[current_array].append(row) + + for array in reference_arrays: + tol = max_ulps * np.abs(np.spacing(reference_arrays[array])) + assert np.all(np.abs(test_arrays[array] - reference_arrays[array]) <= tol) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_correlation(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "datamining" / "correlation" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 25, + "MAP": 17, + "SEQUENTIAL": 5, + "ROCM": 12, + }, + ) + test_case = benchmark_path / "correlation.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_covariance(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "datamining" / "covariance" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 20, + "MAP": 13, + "SEQUENTIAL": 4, + "ROCM": 9, + }, + ) + test_case = benchmark_path / "covariance.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gemm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 17, + "MAP": 10, + "SEQUENTIAL": 0, + "ROCM": 10, + }, + ) + test_case = benchmark_path / "gemm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gemver(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemver" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 13, + "MAP": 7, + "SEQUENTIAL": 1, + "ROCM": 6, + }, + ) + test_case = benchmark_path / "gemver.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gesummv(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gesummv" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 10, + "MAP": 5, + "ROCM": 5, + }, + ) + test_case = benchmark_path / "gesummv.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_symm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "symm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 14, + "MAP": 7, + "SEQUENTIAL": 3, + "ROCM": 4, + }, + ) + test_case = benchmark_path / "symm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_syr2k(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "syr2k" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 14, + "MAP": 8, + "SEQUENTIAL": 2, + "ROCM": 6, + }, + ) + test_case = benchmark_path / "syr2k.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_syrk(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "syrk" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 14, + "MAP": 8, + "SEQUENTIAL": 2, + "ROCM": 6, + }, + ) + test_case = benchmark_path / "syrk.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_trmm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "trmm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 14, + "MAP": 7, + "SEQUENTIAL": 2, + "ROCM": 5, + }, + ) + test_case = benchmark_path / "trmm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_2mm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "2mm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 25, + "MAP": 16, + "SEQUENTIAL": 0, + "ROCM": 16, + }, + ) + test_case = benchmark_path / "2mm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_3mm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "3mm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 31, + "MAP": 20, + "SEQUENTIAL": 0, + "ROCM": 20, + }, + ) + test_case = benchmark_path / "3mm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_atax(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "atax" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 13, + "MAP": 7, + "SEQUENTIAL": 1, + "ROCM": 6, + }, + ) + test_case = benchmark_path / "atax.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_bicg(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "bicg" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 12, + "MAP": 5, + "ROCM": 5, + }, + ) + test_case = benchmark_path / "bicg.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_doitgen(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "doitgen" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 9, + "FOR": 23, + "MAP": 11, + "SEQUENTIAL": 7, + "ROCM": 4, + }, + ) + test_case = benchmark_path / "doitgen.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_mvt(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "mvt" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 11, + "MAP": 4, + "SEQUENTIAL": 1, + "ROCM": 3, + }, + ) + test_case = benchmark_path / "mvt.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_cholesky(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "cholesky" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 21, + "MAP": 11, + "SEQUENTIAL": 2, + "ROCM": 9, + }, + ) + test_case = benchmark_path / "cholesky.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + # pytest.param("-DDATA_TYPE_IS_DOUBLE"), + # pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_durbin(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "durbin" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 9, + "MAP": 3, + "SEQUENTIAL": 0, + "ROCM": 3, + }, + ) + test_case = benchmark_path / "durbin.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + +@pytest.mark.skip(reason="Wrong results, needs investigation") +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gramschmidt(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent + / "tests" + / "polybench" + / "linear-algebra" + / "solvers" + / "gramschmidt" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 19, + "MAP": 9, + "SEQUENTIAL": 3, + "ROCM": 6, + }, + ) + test_case = benchmark_path / "gramschmidt.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_lu(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "lu" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 22, + "MAP": 12, + "SEQUENTIAL": 3, + "ROCM": 9, + }, + ) + test_case = benchmark_path / "lu.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_ludcmp(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "ludcmp" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 26, + "MAP": 13, + "SEQUENTIAL": 3, + "ROCM": 10, + }, + ) + test_case = benchmark_path / "ludcmp.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_trisolv(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "trisolv" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 8, + "MAP": 2, + "SEQUENTIAL": 1, + "ROCM": 1, + }, + ) + test_case = benchmark_path / "trisolv.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_deriche(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "deriche" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 19, + "MAP": 10, + "ROCM": 10, + }, + ) + test_case = benchmark_path / "deriche.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +def test_floyd_warshall(compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "floyd-warshall" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 10, + "MAP": 2, + "SEQUENTIAL": 0, + "ROCM": 2, + }, + ) + test_case = benchmark_path / "floyd-warshall.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + "-DDATA_TYPE_IS_INT", + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial(verify, dtype=np.int64, max_ulps=1e6), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +def test_nussinov(compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "nussinov" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 10, + "MAP": 3, + "SEQUENTIAL": 0, + "ROCM": 3, + }, + ) + test_case = benchmark_path / "nussinov.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + "-DDATA_TYPE_IS_INT", + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial(verify, dtype=np.int64, max_ulps=1e6), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_adi(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "adi" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 20, + "MAP": 10, + "ROCM": 10, + }, + ) + test_case = benchmark_path / "adi.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_fdtd_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "fdtd-2d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 20, + "MAP": 10, + "ROCM": 10, + }, + ) + test_case = benchmark_path / "fdtd-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_heat_3d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "heat-3d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 16, + "MAP": 9, + "SEQUENTIAL": 3, + "ROCM": 6, + }, + ) + test_case = benchmark_path / "heat-3d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_jacobi_1d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "jacobi-1d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 8, + "MAP": 3, + "ROCM": 3, + }, + ) + test_case = benchmark_path / "jacobi-1d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_jacobi_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "jacobi-2d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 12, + "MAP": 6, + "ROCM": 6, + }, + ) + test_case = benchmark_path / "jacobi-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_seidel_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "seidel-2d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 10, + "MAP": 2, + "SEQUENTIAL": 0, + "ROCM": 2, + }, + ) + test_case = benchmark_path / "seidel-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "rocm", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e6, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) diff --git a/integration/polybench_sequential_test.py b/integration/polybench_sequential_test.py new file mode 100644 index 00000000..f75c2273 --- /dev/null +++ b/integration/polybench_sequential_test.py @@ -0,0 +1,1461 @@ +import subprocess +import numpy as np +import pandas as pd + +import re +import pytest + +from pathlib import Path +from functools import partial + +from test_runner import TestRunner, SDFGVerification + + +def verify(reference_file: Path, test_file: Path, dtype, max_ulps = None): + cmd = [reference_file] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stderr.splitlines() + header = data.pop(0) + assert header == "==BEGIN DUMP_ARRAYS==" + + dtype_ = dtype + reference_arrays = {} + current_array = None + for line in data: + if line == "==END DUMP_ARRAYS==": + break + + if line.startswith("begin"): + assert current_array is None + current_array = line[line.find(":") + 1 :].strip() + reference_arrays[current_array] = [] + elif line.startswith("end"): + assert ( + current_array is not None + and current_array == line[line.find(":") + 1 :].strip() + ) + reference_arrays[current_array] = np.hstack(reference_arrays[current_array]) + current_array = None + else: + assert current_array is not None + row = np.fromstring(line, dtype=dtype_, sep=" ") + reference_arrays[current_array].append(row) + + cmd = [test_file] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stderr.splitlines() + header = data.pop(0) + assert header == "==BEGIN DUMP_ARRAYS==" + + test_arrays = {} + current_array = None + for line in data: + if line == "==END DUMP_ARRAYS==": + break + + if line.startswith("begin"): + assert current_array is None + current_array = line[line.find(":") + 1 :].strip() + test_arrays[current_array] = [] + elif line.startswith("end"): + assert ( + current_array is not None + and current_array == line[line.find(":") + 1 :].strip() + ) + test_arrays[current_array] = np.hstack(test_arrays[current_array]) + current_array = None + else: + assert current_array is not None + row = np.fromstring(line, dtype=dtype, sep=" ") + test_arrays[current_array].append(row) + + for array in reference_arrays: + if max_ulps: + tol = max_ulps * np.abs(np.spacing(reference_arrays[array])) + assert np.all(np.abs(test_arrays[array] - reference_arrays[array]) <= tol) + else: + assert np.array_equal( + reference_arrays[array], test_arrays[array], equal_nan=True + ) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_correlation(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "datamining" / "correlation" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 25, + "MAP": 17, + "SEQUENTIAL": 17, + }, + ) + test_case = benchmark_path / "correlation.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_covariance(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "datamining" / "covariance" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 20, + "MAP": 13, + "SEQUENTIAL": 13, + }, + ) + test_case = benchmark_path / "covariance.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gemm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemm" + ) + + verifier = SDFGVerification( + verification={'sdfgs': 8, 'FOR': 13, 'MAP': 8, 'SEQUENTIAL': 8, 'GEMM': 1}, + ) + test_case = benchmark_path / "gemm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e2, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gemver(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemver" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 13, + "MAP": 7, + "SEQUENTIAL": 7, + }, + ) + test_case = benchmark_path / "gemver.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e2, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gesummv(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gesummv" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 10, + "MAP": 5, + "SEQUENTIAL": 5, + }, + ) + test_case = benchmark_path / "gesummv.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_symm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "symm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 14, + "MAP": 7, + "SEQUENTIAL": 7, + }, + ) + test_case = benchmark_path / "symm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_syr2k(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "syr2k" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 14, + "MAP": 8, + "SEQUENTIAL": 8, + }, + ) + test_case = benchmark_path / "syr2k.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_syrk(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "syrk" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 14, + "MAP": 8, + "SEQUENTIAL": 8, + }, + ) + test_case = benchmark_path / "syrk.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_trmm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "trmm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 15, + "MAP": 9, + "SEQUENTIAL": 9, + }, + ) + test_case = benchmark_path / "trmm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_2mm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "2mm" + ) + + verifier = SDFGVerification( + verification={'sdfgs': 8, 'Call': 18, 'MAP': 12, 'FOR': 17, 'SEQUENTIAL': 12, 'GEMM': 2, 'Calloc': 3, 'Unreachable': 4, 'Free': 3}, + ) + test_case = benchmark_path / "2mm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e2, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_3mm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "3mm" + ) + + verifier = SDFGVerification( + verification={'sdfgs': 8, 'Call': 20, 'MAP': 14, 'FOR': 19, 'SEQUENTIAL': 14, 'GEMM': 3, 'Calloc': 3, 'Unreachable': 4, 'Free': 3}, + ) + test_case = benchmark_path / "3mm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e2, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_atax(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "atax" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 13, + "MAP": 7, + "SEQUENTIAL": 7, + }, + ) + test_case = benchmark_path / "atax.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_bicg(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "bicg" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 12, + "MAP": 6, + "SEQUENTIAL": 6, + }, + ) + test_case = benchmark_path / "bicg.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_doitgen(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "doitgen" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 9, + "FOR": 23, + "SEQUENTIAL": 11, + "MAP": 11, + }, + ) + test_case = benchmark_path / "doitgen.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_mvt(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "mvt" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 11, + "MAP": 4, + "SEQUENTIAL": 4, + }, + ) + test_case = benchmark_path / "mvt.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_cholesky(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "cholesky" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 21, + "MAP": 11, + "SEQUENTIAL": 11, + }, + ) + test_case = benchmark_path / "cholesky.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_durbin(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "durbin" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 9, + "MAP": 3, + "SEQUENTIAL": 3, + }, + ) + test_case = benchmark_path / "durbin.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gramschmidt(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent + / "tests" + / "polybench" + / "linear-algebra" + / "solvers" + / "gramschmidt" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 19, + "MAP": 9, + "SEQUENTIAL": 9, + }, + ) + test_case = benchmark_path / "gramschmidt.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_lu(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "lu" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 22, + "MAP": 12, + "SEQUENTIAL": 12, + }, + ) + test_case = benchmark_path / "lu.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_ludcmp(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "ludcmp" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 26, + "MAP": 13, + "SEQUENTIAL": 13, + }, + ) + test_case = benchmark_path / "ludcmp.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE", marks=pytest.mark.xfail(reason="Verfier changed")), + pytest.param("-DDATA_TYPE_IS_FLOAT", marks=pytest.mark.xfail(reason="Verfier changed")), + ], +) +def test_trisolv(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "trisolv" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 8, + "MAP": 2, + "SEQUENTIAL": 2, + }, + ) + test_case = benchmark_path / "trisolv.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_deriche(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "deriche" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 19, + "MAP": 10, + "SEQUENTIAL": 10, + }, + ) + test_case = benchmark_path / "deriche.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +def test_floyd_warshall(compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "floyd-warshall" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 10, + "MAP": 2, + "SEQUENTIAL": 2, + }, + ) + test_case = benchmark_path / "floyd-warshall.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + "-DDATA_TYPE_IS_INT", + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial(verify, dtype=np.int64), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +def test_nussinov(compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "nussinov" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 10, + "MAP": 3, + "SEQUENTIAL": 3, + "WHILE": 1, + }, + ) + test_case = benchmark_path / "nussinov.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + "-DDATA_TYPE_IS_INT", + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial(verify, dtype=np.int64), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_adi(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "adi" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 20, + "MAP": 10, + "SEQUENTIAL": 10, + }, + ) + test_case = benchmark_path / "adi.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_fdtd_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "fdtd-2d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 20, + "MAP": 10, + "SEQUENTIAL": 10, + }, + ) + test_case = benchmark_path / "fdtd-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_heat_3d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "heat-3d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 16, + "MAP": 9, + "SEQUENTIAL": 9, + }, + ) + test_case = benchmark_path / "heat-3d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_jacobi_1d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "jacobi-1d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 8, + "MAP": 3, + "SEQUENTIAL": 3, + }, + ) + test_case = benchmark_path / "jacobi-1d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_jacobi_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "jacobi-2d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 12, + "MAP": 6, + "SEQUENTIAL": 6, + }, + ) + test_case = benchmark_path / "jacobi-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_seidel_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "seidel-2d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 10, + "MAP": 2, + "SEQUENTIAL": 2, + }, + ) + test_case = benchmark_path / "seidel-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-mllvm", "-docc-einsum"], + ) + return runner.run(timeout=120) diff --git a/integration/polybench_tenstorrent_test.py b/integration/polybench_tenstorrent_test.py new file mode 100644 index 00000000..bec69c11 --- /dev/null +++ b/integration/polybench_tenstorrent_test.py @@ -0,0 +1,1504 @@ +import subprocess +import os +import numpy as np +import pandas as pd + +import re +import pytest + +from pathlib import Path +from functools import partial + +from test_runner import TestRunner, SDFGVerification + + +def verify( + reference_file: Path, + test_file: Path, + dtype, + rtol: float = 1e-03, + atol: float = 1e-05, +): + cmd = [reference_file] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stderr.splitlines() + header = data.pop(0) + assert header == "==BEGIN DUMP_ARRAYS==" + + dtype_ = dtype + reference_arrays = {} + current_array = None + for line in data: + if line == "==END DUMP_ARRAYS==": + break + + if line.startswith("begin"): + assert current_array is None + current_array = line[line.find(":") + 1 :].strip() + reference_arrays[current_array] = [] + elif line.startswith("end"): + assert ( + current_array is not None + and current_array == line[line.find(":") + 1 :].strip() + ) + reference_arrays[current_array] = np.hstack(reference_arrays[current_array]) + current_array = None + else: + assert current_array is not None + row = np.fromstring(line, dtype=dtype_, sep=" ") + reference_arrays[current_array].append(row) + + cmd = [test_file] + omp_env = os.environ.copy() + omp_env["OMP_NUM_THREADS"] = "10" + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + env=omp_env, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stderr.splitlines() + header = data.pop(0) + assert header == "==BEGIN DUMP_ARRAYS==" + + test_arrays = {} + current_array = None + for line in data: + if line == "==END DUMP_ARRAYS==": + break + + if line.startswith("begin"): + assert current_array is None + current_array = line[line.find(":") + 1 :].strip() + test_arrays[current_array] = [] + elif line.startswith("end"): + assert ( + current_array is not None + and current_array == line[line.find(":") + 1 :].strip() + ) + test_arrays[current_array] = np.hstack(test_arrays[current_array]) + current_array = None + else: + assert current_array is not None + row = np.fromstring(line, dtype=dtype, sep=" ") + test_arrays[current_array].append(row) + + for array in reference_arrays: + assert np.allclose(reference_arrays[array], test_arrays[array], rtol=rtol, atol=atol) + + +@pytest.mark.skip(reason="Validation fails") +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_correlation(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "datamining" / "correlation" + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 25, + # "MAP": 17, + # "SEQUENTIAL": 7, + # "CUDA": 10, + }, + ) + test_case = benchmark_path / "correlation.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_covariance(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "datamining" / "covariance" + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 20, + # "MAP": 13, + # "SEQUENTIAL": 4, + # "CUDA": 9, + }, + ) + test_case = benchmark_path / "covariance.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32 + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gemm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 13, + "MAP": 8, + "SEQUENTIAL": 8, + "GEMM": 1 + # "TTOffloading": 8, + }, + ) + test_case = benchmark_path / "gemm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + atol=1e0 + ), + sdfg_verification=verifier, + docc_flags=["-docc-einsum"], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gemver(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemver" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 14, + "MAP": 8, + "SEQUENTIAL": 6, + "TTOffloading": 4, + }, + ) + test_case = benchmark_path / "gemver.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gesummv(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gesummv" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 12, + "MAP": 7, + "SEQUENTIAL": 3, + "TTOffloading": 8, + }, + ) + test_case = benchmark_path / "gesummv.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_symm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "symm" + ) + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 14, + # "MAP": 7, + # "SEQUENTIAL": 3, + # "CUDA": 4, + }, + ) + test_case = benchmark_path / "symm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_syr2k(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "syr2k" + ) + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 14, + # "MAP": 8, + # "SEQUENTIAL": 2, + # "CUDA": 6, + }, + ) + test_case = benchmark_path / "syr2k.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_syrk(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "syrk" + ) + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 14, + # "MAP": 8, + # "SEQUENTIAL": 2, + # "CUDA": 6, + }, + ) + test_case = benchmark_path / "syrk.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + # pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_trmm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "trmm" + ) + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 14, + # "MAP": 7, + # "SEQUENTIAL": 3, + # "CUDA": 4, + }, + ) + test_case = benchmark_path / "trmm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_2mm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "2mm" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 23, + "MAP": 16, + "SEQUENTIAL": 16, + }, + ) + test_case = benchmark_path / "2mm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + atol=1e0 + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_3mm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "3mm" + ) + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 28, + # "MAP": 20, + # "SEQUENTIAL": 3, + # "CUDA": 17, + }, + ) + test_case = benchmark_path / "3mm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + atol=1e0 + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_atax(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "atax" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 16, + "MAP": 10, + "SEQUENTIAL": 4, + "TTOffloading": 6, + }, + ) + test_case = benchmark_path / "atax.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_bicg(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "bicg" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 15, + "MAP": 8, + "SEQUENTIAL": 2, + "TTOffloading": 6, + }, + ) + test_case = benchmark_path / "bicg.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_doitgen(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "doitgen" + ) + + verifier = SDFGVerification( + verification={ + # "sdfgs": 9, + # "FOR": 22, + # "MAP": 10, + # "SEQUENTIAL": 6, + # "CUDA": 4, + }, + ) + test_case = benchmark_path / "doitgen.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_mvt(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "mvt" + ) + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 11, + # "MAP": 4, + # "SEQUENTIAL": 1, + # "CUDA": 3, + }, + ) + test_case = benchmark_path / "mvt.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_cholesky(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "cholesky" + ) + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 21, + # "MAP": 11, + # "SEQUENTIAL": 2, + # "CUDA": 9, + }, + ) + test_case = benchmark_path / "cholesky.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_durbin(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "durbin" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 10, + "MAP": 4, + "SEQUENTIAL": 2, + "TTOffloading": 2, + }, + ) + test_case = benchmark_path / "durbin.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gramschmidt(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent + / "tests" + / "polybench" + / "linear-algebra" + / "solvers" + / "gramschmidt" + ) + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 19, + # "MAP": 9, + # "SEQUENTIAL": 5, + # "CUDA": 4, + }, + ) + test_case = benchmark_path / "gramschmidt.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_lu(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "lu" + ) + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 22, + # "MAP": 12, + # "SEQUENTIAL": 3, + # "CUDA": 9, + }, + ) + test_case = benchmark_path / "lu.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_ludcmp(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "ludcmp" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 27, + "MAP": 14, + "SEQUENTIAL": 12, + "TTOffloading": 6, + }, + ) + test_case = benchmark_path / "ludcmp.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_trisolv(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "trisolv" + ) + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 8, + # "MAP": 2, + # "SEQUENTIAL": 2 + }, + ) + test_case = benchmark_path / "trisolv.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + # pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_deriche(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "deriche" + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 19, + # "MAP": 10, + # "CUDA": 10, + }, + ) + test_case = benchmark_path / "deriche.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +def test_floyd_warshall(compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "floyd-warshall" + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 10, + # "MAP": 2, + # "SEQUENTIAL": 0, + # "CUDA": 2, + }, + ) + test_case = benchmark_path / "floyd-warshall.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + "-DDATA_TYPE_IS_INT", + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.int32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +def test_nussinov(compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "nussinov" + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 10, + # "MAP": 3, + # "SEQUENTIAL": 0, + # "CUDA": 3, + }, + ) + test_case = benchmark_path / "nussinov.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + "-DDATA_TYPE_IS_INT", + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.int32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_adi(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "adi" + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 20, + # "MAP": 10, + # "SEQUENTIAL": 8, + # "CUDA": 2, + }, + ) + test_case = benchmark_path / "adi.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_fdtd_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "fdtd-2d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 21, + "MAP": 11, + "SEQUENTIAL": 9, + "TTOffloading": 2, + }, + ) + test_case = benchmark_path / "fdtd-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_heat_3d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "heat-3d" + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 16, + # "MAP": 9, + # "SEQUENTIAL": 7, + # "CUDA": 2, + }, + ) + test_case = benchmark_path / "heat-3d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_jacobi_1d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "jacobi-1d" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "FOR": 9, + "MAP": 4, + "SEQUENTIAL": 2, + "TTOffloading": 4, + }, + ) + test_case = benchmark_path / "jacobi-1d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_jacobi_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "jacobi-2d" + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 12, + # "MAP": 6, + # "SEQUENTIAL": 4, + # "CUDA": 2, + }, + ) + test_case = benchmark_path / "jacobi-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_seidel_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "seidel-2d" + + verifier = SDFGVerification( + verification={ + # "sdfgs": 8, + # "FOR": 10, + # "MAP": 2, + # "SEQUENTIAL": 0, + # "CUDA": 2, + }, + ) + test_case = benchmark_path / "seidel-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + ], + "tenstorrent", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + sdfg_verification=verifier, + docc_flags=[], + ) + return runner.run() diff --git a/integration/polybench_transfertuning_test.py b/integration/polybench_transfertuning_test.py new file mode 100644 index 00000000..e6e7dcc8 --- /dev/null +++ b/integration/polybench_transfertuning_test.py @@ -0,0 +1,1317 @@ +import subprocess +import numpy as np +import pandas as pd + +import re +import pytest + +from pathlib import Path +from functools import partial + +from test_runner import TestRunner, TransformationVerification + + +def verify(reference_file: Path, test_file: Path, dtype, max_ulps = None): + cmd = [reference_file] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stderr.splitlines() + header = data.pop(0) + assert header == "==BEGIN DUMP_ARRAYS==" + + dtype_ = dtype + reference_arrays = {} + current_array = None + for line in data: + if line == "==END DUMP_ARRAYS==": + break + + if line.startswith("begin"): + assert current_array is None + current_array = line[line.find(":") + 1 :].strip() + reference_arrays[current_array] = [] + elif line.startswith("end"): + assert ( + current_array is not None + and current_array == line[line.find(":") + 1 :].strip() + ) + reference_arrays[current_array] = np.hstack(reference_arrays[current_array]) + current_array = None + else: + assert current_array is not None + row = np.fromstring(line, dtype=dtype_, sep=" ") + reference_arrays[current_array].append(row) + + cmd = [test_file] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stderr.splitlines() + header = data.pop(0) + assert header == "==BEGIN DUMP_ARRAYS==" + + test_arrays = {} + current_array = None + for line in data: + if line == "==END DUMP_ARRAYS==": + break + + if line.startswith("begin"): + assert current_array is None + current_array = line[line.find(":") + 1 :].strip() + test_arrays[current_array] = [] + elif line.startswith("end"): + assert ( + current_array is not None + and current_array == line[line.find(":") + 1 :].strip() + ) + test_arrays[current_array] = np.hstack(test_arrays[current_array]) + current_array = None + else: + assert current_array is not None + row = np.fromstring(line, dtype=dtype, sep=" ") + test_arrays[current_array].append(row) + + for array in reference_arrays: + if max_ulps: + tol = max_ulps * np.abs(np.spacing(reference_arrays[array])) + assert np.all(np.abs(test_arrays[array] - reference_arrays[array]) <= tol) + else: + assert np.array_equal( + reference_arrays[array], test_arrays[array], equal_nan=True + ) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_correlation(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "datamining" / "correlation" + + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {}, 'tuned_loops': 12}}) + + test_case = benchmark_path / "correlation.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_covariance(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "datamining" / "covariance" + + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {}, 'tuned_loops': 8}}) + + test_case = benchmark_path / "covariance.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gemm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemm" + ) + + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {}, 'tuned_loops': 5}}) + + test_case = benchmark_path / "gemm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gemver(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemver" + ) + + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {}, 'tuned_loops': 5}}) + + test_case = benchmark_path / "gemver.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e2, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gesummv(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gesummv" + ) + + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {}, 'tuned_loops': 4}}) + + test_case = benchmark_path / "gesummv.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_symm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "symm" + ) + + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {}, 'tuned_loops': 4}}) + + test_case = benchmark_path / "symm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_syr2k(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "syr2k" + ) + + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {}, 'tuned_loops': 4}}) + + test_case = benchmark_path / "syr2k.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_syrk(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "syrk" + ) + + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {}, 'tuned_loops': 4}}) + + test_case = benchmark_path / "syrk.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_trmm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "trmm" + ) + + transformation_verification = TransformationVerification({}) + + test_case = benchmark_path / "trmm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_2mm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "2mm" + ) + + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {}, 'tuned_loops': 8}}) + + test_case = benchmark_path / "2mm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + max_ulps=1e2, + ), + transformation_verification=transformation_verification, + docc_flags=["-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_3mm(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "3mm" + ) + + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {}, 'tuned_loops': 10}}) + + test_case = benchmark_path / "3mm.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_atax(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "atax" + ) + + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {}, 'tuned_loops': 6}}) + + test_case = benchmark_path / "atax.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_bicg(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "bicg" + ) + + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {}, 'tuned_loops': 5}}) + + test_case = benchmark_path / "bicg.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_doitgen(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "doitgen" + ) + + transformation_verification = TransformationVerification({}) + + test_case = benchmark_path / "doitgen.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_mvt(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "mvt" + ) + + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {}, 'tuned_loops': 3}}) + + test_case = benchmark_path / "mvt.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification = transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_cholesky(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "cholesky" + ) + + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {}, 'tuned_loops': 7}}) + + test_case = benchmark_path / "cholesky.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_durbin(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "durbin" + ) + + transformation_verification = TransformationVerification({}) + + test_case = benchmark_path / "durbin.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_gramschmidt(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent + / "tests" + / "polybench" + / "linear-algebra" + / "solvers" + / "gramschmidt" + ) + + transformation_verification = TransformationVerification({}) + + test_case = benchmark_path / "gramschmidt.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_lu(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "lu" + ) + + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {}, 'tuned_loops': 6}}) + + test_case = benchmark_path / "lu.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_ludcmp(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "ludcmp" + ) + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {}, 'tuned_loops': 9}}) + + test_case = benchmark_path / "ludcmp.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_trisolv(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = ( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "trisolv" + ) + + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {}, 'tuned_loops': 3}}) + + test_case = benchmark_path / "trisolv.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_deriche(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "deriche" + + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {}, 'tuned_loops': 7}}) + + test_case = benchmark_path / "deriche.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +def test_floyd_warshall(compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "floyd-warshall" + + transformation_verification = TransformationVerification({"RPCNodeTransform":{1}}) + + test_case = benchmark_path / "floyd-warshall.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + "-DDATA_TYPE_IS_INT", + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial(verify, dtype=np.int64), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +def test_nussinov(compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "medley" / "nussinov" + + transformation_verification = TransformationVerification({}) + + test_case = benchmark_path / "nussinov.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + "-DDATA_TYPE_IS_INT", + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial(verify, dtype=np.int64), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_adi(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "adi" + + transformation_verification = TransformationVerification({}) + test_case = benchmark_path / "adi.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_fdtd_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "fdtd-2d" + + transformation_verification = TransformationVerification({}) + + test_case = benchmark_path / "fdtd-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_heat_3d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "heat-3d" + + transformation_verification = TransformationVerification({}) + + test_case = benchmark_path / "heat-3d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_jacobi_1d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "jacobi-1d" + + transformation_verification = TransformationVerification({}) + + test_case = benchmark_path / "jacobi-1d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_jacobi_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "jacobi-2d" + + transformation_verification = TransformationVerification({}) + + test_case = benchmark_path / "jacobi-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) + + +@pytest.mark.parametrize( + "datatype", + [ + pytest.param("-DDATA_TYPE_IS_DOUBLE"), + pytest.param("-DDATA_TYPE_IS_FLOAT"), + ], +) +def test_seidel_2d(datatype, compiler="clang-19", size="MEDIUM_DATASET"): + benchmark_path = Path(__file__).parent / "tests" / "polybench" / "stencils" / "seidel-2d" + + transformation_verification = TransformationVerification({"RPCNodeTransform":{'loop_nests': {1}, 'tuned_loops': 2}}) + test_case = benchmark_path / "seidel-2d.c" + runner = TestRunner( + "Polybench", + test_case, + "docc", + compiler, + [ + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-D" + size, + datatype, + "-I" + str((Path(__file__).parent / "tests" / "polybench" / "utilities").absolute()), + "-lm", + "-lblas" + ], + "sequential", + [Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"], + partial( + verify, + dtype=np.float64 if datatype == "-DDATA_TYPE_IS_DOUBLE" else np.float32, + ), + transformation_verification=transformation_verification, + docc_flags=["-mllvm", "-docc-einsum", "-docc-transfer-tune", "-docc-tune=sequential", "-docc-save-temps"], + ) + return runner.run(timeout=120) diff --git a/integration/rodinia_cuda_test.py b/integration/rodinia_cuda_test.py new file mode 100644 index 00000000..2cf078f4 --- /dev/null +++ b/integration/rodinia_cuda_test.py @@ -0,0 +1,1108 @@ +import subprocess +import numpy as np +import pandas as pd +import pytest + +from pathlib import Path +from functools import partial + +from test_runner import TestRunner, SDFGVerification + + +def evaluate( + reference_file: Path, test_file: Path, args, max_ulps, dtype=np.float64 +) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while header != "BEGIN DUMP OUTPUT": + header = data.pop(0) + assert header == "BEGIN DUMP OUTPUT" + + reference_arrays = {"output": []} + current_array = "output" + for line in data: + if line == "END DUMP OUTPUT": + break + + row = np.fromstring(line, dtype=dtype, sep=" ") + reference_arrays[current_array].append(row) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while header != "BEGIN DUMP OUTPUT": + header = data.pop(0) + assert header == "BEGIN DUMP OUTPUT" + + test_arrays = {"output": []} + current_array = "output" + for line in data: + if line == "END DUMP OUTPUT": + break + + row = np.fromstring(line, dtype=dtype, sep=" ") + test_arrays[current_array].append(row) + + if not reference_arrays["output"]: + return + + reference_arrays["output"] = np.concatenate(reference_arrays["output"]) + test_arrays["output"] = np.concatenate(test_arrays["output"]) + + for array in reference_arrays: + tol = max_ulps * np.abs(np.spacing(reference_arrays[array])) + assert np.all(np.abs(test_arrays[array] - reference_arrays[array]) <= tol) + + +@pytest.mark.xfail(reason="Verifier changed") +def test_bplustree(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "b+tree" / "main.c" + verifier = SDFGVerification( + verification={ + "sdfgs": 80, + "FOR": 39, + "Malloc": 14, + "WHILE": 103, + "Free": 31, + "MAP": 4, + "SEQUENTIAL": 4, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "cuda", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "b+tree" + / "kernel" + / "kernel_cpu.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "b+tree" + / "kernel" + / "kernel_cpu_2.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "b+tree" + / "util" + / "timer" + / "timer.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "b+tree" + / "util" + / "num" + / "num.c", + ], + partial( + evaluate, + max_ulps=1e2, + args=[ + "file", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "b+tree" + / "mil.data" + ).absolute() + ), + "command", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "b+tree" + / "command.data" + ).absolute() + ), + ], + dtype=np.int64, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed") +def test_backprop(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "backprop" / "backprop.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 25, + "FOR": 56, + "Call": 60, + "Unreachable": 2, + "WHILE": 2, + "Free": 6, + "Malloc": 5, + "CUDA": 2, + "CUDAOffloading": 4, + "MAP": 15, + "SEQUENTIAL": 13, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "cuda", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "backprop" + / "backprop_kernel.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "backprop" / "facetrain.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "backprop" / "imagenet.c", + ], + partial(evaluate, max_ulps=1e2, args=["1024"], dtype=np.float32), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed") +def test_bfs(compiler="clang++-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "bfs" / "bfs.cpp" + + verifier = SDFGVerification( + verification={ + "sdfgs": 3, + "Malloc": 6, + "WHILE": 2, + "FOR": 6, + "CUDAOffloading": 2, + "MAP": 2, + "SEQUENTIAL": 1, + "CUDA": 1, + "Free": 6, + "Unreachable": 1, + "Call": 20, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "cuda", + [], + partial( + evaluate, + max_ulps=1e2, + args=[ + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "bfs" + / "graph1MW_6.data" + ).absolute() + ) + ], + dtype=np.int64, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed") +def test_cfd(compiler="clang++-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "cfd" / "euler3d_cpu.cpp" + + verifier = SDFGVerification( + verification={ + "sdfgs": 6, + "ExternalOffloading": 6, + "CUDAOffloading": 36, + "MAP": 5, + "SEQUENTIAL": 1, + "FOR": 19, + "CUDA": 4, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "cuda", + [], + partial( + evaluate, + max_ulps=1e8, + args=[ + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "cfd" + / "fvcorr.domn.097K.data" + ).absolute() + ) + ], + dtype=np.float32, + ), + sdfg_verification=verifier, + docc_flags=["-docc-offload-unknown-sizes"], + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed & Output incorrect") +def test_heartwall(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "heartwall" / "main.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 50, + "Call": 96, + "Unreachable": 5, + "CUDAOffloading": 2, + "MAP": 4, + "Malloc": 27, + "FOR": 51, + "SEQUENTIAL": 3, + "CUDA": 1, + "Free": 37, + "WHILE": 70, + "Memcpy": 2, + "Memset": 6, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-I" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "heartwall" / "AVI/" + ).absolute() + ), + "-lm", + ], + "cuda", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "heartwall" + / "AVI" + / "avilib.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "heartwall" + / "AVI" + / "avimod.c", + ], + partial( + evaluate, + max_ulps=1e2, + args=[ + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "heartwall" + / "test.avi.data" + ).absolute() + ), + "3", + "1", + ], + dtype=np.int64, + ), + sdfg_verification=verifier, + docc_flags=["-docc-offload-unknown-sizes"], + ) + return runner.run(timeout=350) + +@pytest.mark.skip(reason="Test is flaky, needs investigation") +def test_hotspot(compiler="clang++-19"): + test_case = ( + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "hotspot" / "hotspot_openmp.cpp" + ) + + verifier = SDFGVerification( + verification={"sdfgs": 8, "Free": 2, "FOR": 8, "Calloc": 3, "WHILE": 3}, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "cuda", + [], + partial( + evaluate, + max_ulps=1e2, + args=[ + "1024", + "1024", + "10", + "1", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "hotspot" + / "temp_1024.data" + ).absolute() + ), + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "hotspot" + / "power_1024.data" + ).absolute() + ), + "/dev/null", + ], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_hotspot3D(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "hotspot3D" / "3D.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "Malloc": 1, + "Free": 3, + "Calloc": 4, + "WHILE": 2, + "FOR": 17, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "cuda", + [], + partial( + evaluate, + max_ulps=1e2, + args=[ + "512", + "8", + "100", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "hotspot3D" + / "power_512x8.data" + ).absolute() + ), + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "hotspot3D" + / "temp_512x8.data" + ).absolute() + ), + "/dev/null", + ], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed & Output incorrect") +def test_kmeans(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "kmeans" / "kmeans.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 6, + "Free": 10, + "MAP": 4, + "CUDA": 1, + "CUDAOffloading": 2, + "Calloc": 2, + "WHILE": 11, + "SEQUENTIAL": 3, + "FOR": 16, + "Malloc": 6, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "cuda", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "kmeans" + / "kmeans_clustering.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "kmeans" / "cluster.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "kmeans" / "getopt.c", + ], + partial( + evaluate, + max_ulps=1e2, + args=[ + "-i", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "kmeans" + / "kdd_cup.data" + ).absolute() + ), + ], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed") +def test_lavaMD(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "lavaMD" / "main.c" + + verifier = SDFGVerification( + verification={"sdfgs": 4, "FOR": 21, "WHILE": 5, "Malloc": 8, "Free": 8}, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "cuda", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "lavaMD" + / "kernel" + / "kernel_cpu.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "lavaMD" + / "util" + / "num" + / "num.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "lavaMD" + / "util" + / "timer" + / "timer.c", + ], + partial(evaluate, max_ulps=1e2, args=["-boxes1d", "10"], dtype=np.float64), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + +@pytest.mark.skip(reason="Crashes on execution") +def test_lud(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "lud" / "lud.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 14, + "WHILE": 3, + "Free": 9, + "CUDAOffloading": 10, + "MAP": 11, + "CUDA": 4, + "Malloc": 2, + "SEQUENTIAL": 7, + "FOR": 40, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "cuda", + [ + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "lud" / "lud_omp.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "lud" / "common.c", + ], + partial(evaluate, max_ulps=1e2, args=["-s", "4096", "-v"], dtype=np.float32), + sdfg_verification=verifier, + docc_flags=["-docc-offload-unknown-sizes"], + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Output incorrect") +def test_nw(compiler="clang++-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "nw" / "nw.cpp" + + verifier = SDFGVerification( + verification={ + "sdfgs": 25, + "FOR": 54, + "Free": 6, + "MAP": 15, + "CPU_PARALLEL": 11, + "SEQUENTIAL": 4, + "Malloc": 7, + "WHILE": 4, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "cuda", + [], + partial(evaluate, max_ulps=1e2, args=["16384", "10", "1"], dtype=np.int64), + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed") +def test_particlefilter(compiler="clang-19"): + test_case = ( + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "particlefilter" + / "particlefilter.c" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 17, + "Free": 2, + "Malloc": 4, + "CUDA": 11, + "WHILE": 5, + "CUDAOffloading": 30, + "MAP": 18, + "SEQUENTIAL": 7, + "FOR": 56, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-w", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "cuda", + [], + partial( + evaluate, + max_ulps=1e2, + args=["-x", "2048", "-y", "2048", "-z", "100", "-np", "100"], + dtype=np.float64, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed & Output incorrect") +def test_pathfinder(compiler="clang++-19"): + test_case = ( + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "pathfinder" / "pathfinder.cpp" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 1, + "FOR": 7, + "MAP": 2, + "CUDA": 1, + "SEQUENTIAL": 1, + "CUDAOffloading": 4, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "cuda", + [], + partial(evaluate, max_ulps=1e2, args=["65536", "1000"], dtype=np.int64), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed") +def test_srad(compiler="clang++-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "srad" / "srad.cpp" + + verifier = SDFGVerification( + verification={ + "sdfgs": 3, + "Malloc": 11, + "CUDAOffloading": 12, + "MAP": 7, + "FOR": 16, + "SEQUENTIAL": 4, + "CUDA": 3, + "Free": 11, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "cuda", + [], + partial( + evaluate, + max_ulps=1e2, + args=["2048", "2048", "0", "127", "0", "127", "1", "0.5", "2"], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed & Output incorrect") +def test_streamcluster(compiler="clang++-19"): + test_case = ( + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "streamcluster" + / "streamcluster.cpp" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 25, + "MAP": 3, + "SEQUENTIAL": 2, + "CUDA": 1, + "CUDAOffloading": 4, + "WHILE": 28, + "FOR": 45, + "Calloc": 3, + "Malloc": 7, + "Free": 8, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "cuda", + [], + partial( + evaluate, + max_ulps=1e2, + args=["2", "3", "256", "65536", "65536", "1000", "none", "/dev/null", "1"], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) diff --git a/integration/rodinia_none_test.py b/integration/rodinia_none_test.py new file mode 100644 index 00000000..2f4c558b --- /dev/null +++ b/integration/rodinia_none_test.py @@ -0,0 +1,1037 @@ +import subprocess +import numpy as np +import pandas as pd +import pytest + +from pathlib import Path +from functools import partial + +from test_runner import TestRunner, SDFGVerification +import signal + + +def evaluate(reference_file: Path, test_file: Path, args, dtype=np.float64) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while header != "BEGIN DUMP OUTPUT": + header = data.pop(0) + assert header == "BEGIN DUMP OUTPUT" + + reference_arrays = {"output": []} + current_array = "output" + for line in data: + if line == "END DUMP OUTPUT": + break + + row = np.fromstring(line, dtype=dtype, sep=" ") + reference_arrays[current_array].append(row) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while header != "BEGIN DUMP OUTPUT": + header = data.pop(0) + assert header == "BEGIN DUMP OUTPUT" + + test_arrays = {"output": []} + current_array = "output" + for line in data: + if line == "END DUMP OUTPUT": + break + + row = np.fromstring(line, dtype=dtype, sep=" ") + test_arrays[current_array].append(row) + + if not reference_arrays["output"]: + return + + reference_arrays["output"] = np.concatenate(reference_arrays["output"]) + test_arrays["output"] = np.concatenate(test_arrays["output"]) + + for array in reference_arrays: + assert np.array_equal( + reference_arrays[array], test_arrays[array], equal_nan=True + ) + + +def test_bplustree(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "b+tree" / "main.c" + verifier = SDFGVerification( + verification={ + "sdfgs": 80, + "FOR": 28, + "Malloc": 14, + "WHILE": 113, + "Free": 31, + "MAP": 2, + "SEQUENTIAL": 2, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "none", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "b+tree" + / "kernel" + / "kernel_cpu.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "b+tree" + / "kernel" + / "kernel_cpu_2.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "b+tree" + / "util" + / "timer" + / "timer.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "b+tree" + / "util" + / "num" + / "num.c", + ], + partial( + evaluate, + args=[ + "file", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "b+tree" + / "mil.data" + ).absolute() + ), + "command", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "b+tree" + / "command.data" + ).absolute() + ), + ], + dtype=np.int64, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_backprop(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "backprop" / "backprop.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 25, + "Free": 6, + "Malloc": 5, + "MAP": 15, + "SEQUENTIAL": 15, + "FOR": 52, + "WHILE": 6, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "none", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "backprop" + / "backprop_kernel.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "backprop" / "facetrain.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "backprop" / "imagenet.c", + ], + partial(evaluate, args=["1024"], dtype=np.float32), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_bfs(compiler="clang++-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "bfs" / "bfs.cpp" + + verifier = SDFGVerification( + verification={'sdfgs': 3, 'Malloc': 6, 'WHILE': 7, 'FOR': 1, 'SEQUENTIAL': 1, 'MAP': 1, 'Free': 6}, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "none", + [], + partial( + evaluate, + args=[ + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "bfs" + / "graph1MW_6.data" + ).absolute() + ) + ], + dtype=np.int64, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_cfd(compiler="clang++-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "cfd" / "euler3d_cpu.cpp" + + verifier = SDFGVerification( + verification={ + "sdfgs": 6, + "MAP": 6, + "SEQUENTIAL": 6, + "FOR": 19, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "none", + [], + partial( + evaluate, + args=[ + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "cfd" + / "fvcorr.domn.097K.data" + ).absolute() + ) + ], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + + return runner.run(timeout=240) + + +def test_heartwall(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "heartwall" / "main.c" + + verifier = SDFGVerification( + verification={'sdfgs': 50, 'Malloc': 27, 'MAP': 7, 'FOR': 48, 'SEQUENTIAL': 7, 'Free': 37, 'WHILE': 73}, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-I" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "heartwall" / "AVI/" + ).absolute() + ), + "-lm", + ], + "none", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "heartwall" + / "AVI" + / "avilib.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "heartwall" + / "AVI" + / "avimod.c", + ], + partial( + evaluate, + args=[ + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "heartwall" + / "test.avi.data" + ).absolute() + ), + "3", + "1", + ], + dtype=np.int64, + ), + sdfg_verification=verifier, + ) + + return runner.run(timeout=350) + + +@pytest.mark.skip(reason="Test is flaky, needs investigation") +def test_hotspot(compiler="clang++-19"): + test_case = ( + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "hotspot" / "hotspot_openmp.cpp" + ) + verifier = SDFGVerification( + verification={"sdfgs": 8, "Free": 2, "FOR": 8, "WHILE": 3, "Calloc": 3}, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "none", + [], + partial( + evaluate, + args=[ + "1024", + "1024", + "10", + "1", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "hotspot" + / "temp_1024.data" + ).absolute() + ), + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "hotspot" + / "power_1024.data" + ).absolute() + ), + "/dev/null", + ], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_hotspot3D(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "hotspot3D" / "3D.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "Malloc": 1, + "Memcpy": 1, + "Free": 3, + "Calloc": 4, + "WHILE": 2, + "FOR": 17, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "none", + [], + partial( + evaluate, + args=[ + "512", + "8", + "100", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "hotspot3D" + / "power_512x8.data" + ).absolute() + ), + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "hotspot3D" + / "temp_512x8.data" + ).absolute() + ), + "/dev/null", + ], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.skip(reason="Reference executable segfaults (SIGSEGV) in CI environment") +def test_kmeans(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "kmeans" / "kmeans.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 6, + "Free": 10, + "FOR": 16, + "MAP": 4, + "Calloc": 2, + "Malloc": 6, + "WHILE": 11, + "SEQUENTIAL": 4, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "none", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "kmeans" + / "kmeans_clustering.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "kmeans" / "cluster.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "kmeans" / "getopt.c", + ], + partial( + evaluate, + args=[ + "-i", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "kmeans" + / "kdd_cup.data" + ).absolute() + ), + ], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_lavaMD(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "lavaMD" / "main.c" + + verifier = SDFGVerification( + verification={'sdfgs': 4, 'FOR': 20, 'WHILE': 6, 'Malloc': 8, 'Free': 8} + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "none", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "lavaMD" + / "kernel" + / "kernel_cpu.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "lavaMD" + / "util" + / "num" + / "num.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "lavaMD" + / "util" + / "timer" + / "timer.c", + ], + partial(evaluate, args=["-boxes1d", "10"], dtype=np.float64), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.skip(reason="Timeout") +def test_lud(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "lud" / "lud.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 14, + "Free": 9, + "Malloc": 2, + "FOR": 40, + "MAP": 11, + "WHILE": 3, + "SEQUENTIAL": 11, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "none", + [ + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "lud" / "lud_omp.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "lud" / "common.c", + ], + partial(evaluate, args=["-s", "4096", "-v"], dtype=np.float32), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_nw(compiler="clang++-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "nw" / "nw.cpp" + + verifier = SDFGVerification( + verification={ + "sdfgs": 7, + "Malloc": 3, + "WHILE": 5, + "Free": 15, + "FOR": 28, + "SEQUENTIAL": 13, + "MAP": 13, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "none", + [], + partial(evaluate, args=["16384", "10", "1"], dtype=np.int64), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_particlefilter(compiler="clang-19"): + test_case = ( + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "particlefilter" + / "particlefilter.c" + ) + + verifier = SDFGVerification( + verification={ + 'sdfgs': 17, + 'Free': 2, + 'Malloc': 4, + 'WHILE': 12, + 'MAP': 18, + 'SEQUENTIAL': 18, + 'FOR': 49 + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-w", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "none", + [], + partial( + evaluate, + args=["-x", "2048", "-y", "2048", "-z", "100", "-np", "100"], + dtype=np.float64, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_pathfinder(compiler="clang++-19"): + test_case = ( + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "pathfinder" / "pathfinder.cpp" + ) + + verifier = SDFGVerification( + verification={"sdfgs": 1, "FOR": 7}, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "none", + [], + partial(evaluate, args=["65536", "1000"], dtype=np.int64), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_srad(compiler="clang++-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "srad" / "srad.cpp" + + verifier = SDFGVerification( + verification={ + "sdfgs": 3, + "Malloc": 11, + "MAP": 7, + "FOR": 16, + "SEQUENTIAL": 7, + "Free": 11, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "none", + [], + partial( + evaluate, + args=["2048", "2048", "0", "127", "0", "127", "1", "0.5", "2"], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_streamcluster(compiler="clang++-19"): + test_case = ( + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "streamcluster" + / "streamcluster.cpp" + ) + verifier = SDFGVerification( + verification={ + "sdfgs": 25, + "MAP": 2, + "SEQUENTIAL": 2, + "FOR": 41, + "Malloc": 7, + "Free": 8, + "Calloc": 3, + "WHILE": 32, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "none", + [], + partial( + evaluate, + args=["2", "3", "256", "65536", "65536", "1000", "none", "/dev/null", "1"], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) diff --git a/integration/rodinia_openmp_test.py b/integration/rodinia_openmp_test.py new file mode 100644 index 00000000..45eefd5b --- /dev/null +++ b/integration/rodinia_openmp_test.py @@ -0,0 +1,1027 @@ +import subprocess +import numpy as np +import pandas as pd +import pytest + +from pathlib import Path +from functools import partial + +from test_runner import TestRunner, SDFGVerification + + +def evaluate(reference_file: Path, test_file: Path, args, dtype=np.float64) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while header != "BEGIN DUMP OUTPUT": + header = data.pop(0) + assert header == "BEGIN DUMP OUTPUT" + + reference_arrays = {"output": []} + current_array = "output" + for line in data: + if line == "END DUMP OUTPUT": + break + + row = np.fromstring(line, dtype=dtype, sep=" ") + reference_arrays[current_array].append(row) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while header != "BEGIN DUMP OUTPUT": + header = data.pop(0) + assert header == "BEGIN DUMP OUTPUT" + + test_arrays = {"output": []} + current_array = "output" + for line in data: + if line == "END DUMP OUTPUT": + break + + row = np.fromstring(line, dtype=dtype, sep=" ") + test_arrays[current_array].append(row) + + if not reference_arrays["output"]: + return + + reference_arrays["output"] = np.concatenate(reference_arrays["output"]) + test_arrays["output"] = np.concatenate(test_arrays["output"]) + + for array in reference_arrays: + assert np.array_equal( + reference_arrays[array], test_arrays[array], equal_nan=True + ) + + +def test_bplustree(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "b+tree" / "main.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 80, + "FOR": 28, + "WHILE": 113, + "MAP": 2, + "CPU_PARALLEL": 2, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "openmp", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "b+tree" + / "kernel" + / "kernel_cpu.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "b+tree" + / "kernel" + / "kernel_cpu_2.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "b+tree" + / "util" + / "timer" + / "timer.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "b+tree" + / "util" + / "num" + / "num.c", + ], + partial( + evaluate, + args=[ + "file", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "b+tree" + / "mil.data" + ).absolute() + ), + "command", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "b+tree" + / "command.data" + ).absolute() + ), + ], + dtype=np.int64, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_backprop(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "backprop" / "backprop.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 25, + "FOR": 57, + "MAP": 20, + "CPU_PARALLEL": 20, + "WHILE": 6, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "openmp", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "backprop" + / "backprop_kernel.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "backprop" / "facetrain.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "backprop" / "imagenet.c", + ], + partial(evaluate, args=["1024"], dtype=np.float32), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_bfs(compiler="clang++-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "bfs" / "bfs.cpp" + + verifier = SDFGVerification( + verification={'sdfgs': 3, 'WHILE': 7, 'FOR': 1, 'MAP': 1, 'CPU_PARALLEL': 1}, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "openmp", + [], + partial( + evaluate, + args=[ + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "bfs" + / "graph1MW_6.data" + ).absolute() + ) + ], + dtype=np.int64, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_cfd(compiler="clang++-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "cfd" / "euler3d_cpu.cpp" + + verifier = SDFGVerification( + verification={ + "sdfgs": 6, + "MAP": 14, + "SEQUENTIAL": 9, + "CPU_PARALLEL": 5, + "FOR": 27, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "openmp", + [], + partial( + evaluate, + args=[ + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "cfd" + / "fvcorr.domn.097K.data" + ).absolute() + ) + ], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed & Output incorrect") +def test_heartwall(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "heartwall" / "main.c" + + verifier = SDFGVerification( + verification={'sdfgs': 50, 'MAP': 4, 'FOR': 51, 'SEQUENTIAL': 3, 'CPU_PARALLEL': 1, 'WHILE': 70, 'Memset': 6}, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-I" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "heartwall" / "AVI/" + ).absolute() + ), + "-lm", + ], + "openmp", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "heartwall" + / "AVI" + / "avilib.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "heartwall" + / "AVI" + / "avimod.c", + ], + partial( + evaluate, + args=[ + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "heartwall" + / "test.avi.data" + ).absolute() + ), + "3", + "1", + ], + dtype=np.int64, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=350) + +@pytest.mark.skip(reason="Test is flaky, needs investigation") +def test_hotspot(compiler="clang++-19"): + test_case = ( + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "hotspot" / "hotspot_openmp.cpp" + ) + + verifier = SDFGVerification( + verification={"sdfgs": 8, "Free": 2, "FOR": 8, "WHILE": 3, "Calloc": 3} + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "openmp", + [], + partial( + evaluate, + args=[ + "1024", + "1024", + "10", + "1", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "hotspot" + / "temp_1024.data" + ).absolute() + ), + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "hotspot" + / "power_1024.data" + ).absolute() + ), + "/dev/null", + ], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_hotspot3D(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "hotspot3D" / "3D.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "Memcpy": 1, + "WHILE": 2, + "MAP": 4, + "SEQUENTIAL": 4, + "FOR": 17, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "openmp", + [], + partial( + evaluate, + args=[ + "512", + "8", + "100", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "hotspot3D" + / "power_512x8.data" + ).absolute() + ), + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "hotspot3D" + / "temp_512x8.data" + ).absolute() + ), + "/dev/null", + ], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed & Output incorrect") +def test_kmeans(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "kmeans" / "kmeans.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 6, + "FOR": 16, + "MAP": 4, + "WHILE": 11, + "SEQUENTIAL": 3, + "CPU_PARALLEL": 1, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "openmp", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "kmeans" + / "kmeans_clustering.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "kmeans" / "cluster.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "kmeans" / "getopt.c", + ], + partial( + evaluate, + args=[ + "-i", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "kmeans" + / "kdd_cup.data" + ).absolute() + ), + ], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_lavaMD(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "lavaMD" / "main.c" + + verifier = SDFGVerification( + verification={'sdfgs': 4, 'FOR': 20, 'WHILE': 6}, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "openmp", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "lavaMD" + / "kernel" + / "kernel_cpu.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "lavaMD" + / "util" + / "num" + / "num.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "lavaMD" + / "util" + / "timer" + / "timer.c", + ], + partial(evaluate, args=["-boxes1d", "10"], dtype=np.float64), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.skip(reason="Timeout") +def test_lud(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "lud" / "lud.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 14, + "MAP": 11, + "CPU_PARALLEL": 3, + "WHILE": 3, + "FOR": 40, + "SEQUENTIAL": 8, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "openmp", + [ + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "lud" / "lud_omp.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "lud" / "common.c", + ], + partial(evaluate, args=["-s", "4096", "-v"], dtype=np.float32), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed & Output incorrect") +def test_nw(compiler="clang++-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "nw" / "nw.cpp" + + verifier = SDFGVerification( + verification={ + "sdfgs": 7, + "WHILE": 5, + "FOR": 28, + "SEQUENTIAL": 12, + "CPU_PARALLEL": 1, + "MAP": 13, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "openmp", + [], + partial(evaluate, args=["16384", "10", "1"], dtype=np.int64), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_particlefilter(compiler="clang-19"): + test_case = ( + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "particlefilter" + / "particlefilter.c" + ) + + verifier = SDFGVerification( + verification={ + 'sdfgs': 17, + 'WHILE': 12, + 'MAP': 20, + 'SEQUENTIAL': 7, + 'CPU_PARALLEL': 13, + 'FOR': 51, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-w", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "openmp", + [], + partial( + evaluate, + args=["-x", "2048", "-y", "2048", "-z", "100", "-np", "100"], + dtype=np.float64, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed & Output incorrect") +def test_pathfinder(compiler="clang++-19"): + test_case = ( + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "pathfinder" / "pathfinder.cpp" + ) + + verifier = SDFGVerification( + verification={"sdfgs": 1, "FOR": 7, "SEQUENTIAL": 1, "MAP": 2, "CPU_PARALLEL": 1}, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "openmp", + [], + partial(evaluate, args=["65536", "1000"], dtype=np.int64), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_srad(compiler="clang++-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "srad" / "srad.cpp" + + verifier = SDFGVerification( + verification={ + "sdfgs": 3, + "MAP": 6, + "FOR": 15, + "CPU_PARALLEL": 6, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "openmp", + [], + partial( + evaluate, + args=["2048", "2048", "0", "127", "0", "127", "1", "0.5", "2"], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed & Output incorrect") +def test_streamcluster(compiler="clang++-19"): + test_case = ( + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "streamcluster" + / "streamcluster.cpp" + ) + verifier = SDFGVerification( + verification={ + "sdfgs": 25, + "MAP": 3, + "CPU_PARALLEL": 3, + "FOR": 45, + "WHILE": 28, + "Trap": 1, + "Memset": 3, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "openmp", + [], + partial( + evaluate, + args=["2", "3", "256", "65536", "65536", "1000", "none", "/dev/null", "1"], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) diff --git a/integration/rodinia_sequential_test.py b/integration/rodinia_sequential_test.py new file mode 100644 index 00000000..cf8f0e54 --- /dev/null +++ b/integration/rodinia_sequential_test.py @@ -0,0 +1,1034 @@ +import subprocess +import numpy as np +import pandas as pd +import pytest + +from pathlib import Path +from functools import partial + +from test_runner import TestRunner, SDFGVerification + + +def evaluate(reference_file: Path, test_file: Path, args, dtype=np.float64) -> float: + cmd = [reference_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while header != "BEGIN DUMP OUTPUT": + header = data.pop(0) + assert header == "BEGIN DUMP OUTPUT" + + reference_arrays = {"output": []} + current_array = "output" + for line in data: + if line == "END DUMP OUTPUT": + break + + row = np.fromstring(line, dtype=dtype, sep=" ") + reference_arrays[current_array].append(row) + + cmd = [test_file] + args + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + data = stdout.splitlines() + header = data.pop(0) + while header != "BEGIN DUMP OUTPUT": + header = data.pop(0) + assert header == "BEGIN DUMP OUTPUT" + + test_arrays = {"output": []} + current_array = "output" + for line in data: + if line == "END DUMP OUTPUT": + break + + row = np.fromstring(line, dtype=dtype, sep=" ") + test_arrays[current_array].append(row) + + if not reference_arrays["output"]: + return + + reference_arrays["output"] = np.concatenate(reference_arrays["output"]) + test_arrays["output"] = np.concatenate(test_arrays["output"]) + + for array in reference_arrays: + assert np.array_equal( + reference_arrays[array], test_arrays[array], equal_nan=True + ) + + +def test_bplustree(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "b+tree" / "main.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 80, + "FOR": 28, + "Malloc": 14, + "WHILE": 113, + "Free": 31, + "MAP": 2, + "SEQUENTIAL": 2, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "sequential", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "b+tree" + / "kernel" + / "kernel_cpu.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "b+tree" + / "kernel" + / "kernel_cpu_2.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "b+tree" + / "util" + / "timer" + / "timer.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "b+tree" + / "util" + / "num" + / "num.c", + ], + partial( + evaluate, + args=[ + "file", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "b+tree" + / "mil.data" + ).absolute() + ), + "command", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "b+tree" + / "command.data" + ).absolute() + ), + ], + dtype=np.int64, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_backprop(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "backprop" / "backprop.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 25, + "Free": 6, + "Malloc": 5, + "MAP": 25, + "SEQUENTIAL": 25, + "FOR": 62, + "WHILE": 6, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "sequential", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "backprop" + / "backprop_kernel.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "backprop" / "facetrain.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "backprop" / "imagenet.c", + ], + partial(evaluate, args=["1024"], dtype=np.float32), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_bfs(compiler="clang++-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "bfs" / "bfs.cpp" + + verifier = SDFGVerification( + verification={'sdfgs': 3, 'Malloc': 6, 'WHILE': 7, 'FOR': 1, 'SEQUENTIAL': 1, 'MAP': 1, 'Free': 6}, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "sequential", + [], + partial( + evaluate, + args=[ + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "bfs" + / "graph1MW_6.data" + ).absolute() + ) + ], + dtype=np.int64, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_cfd(compiler="clang++-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "cfd" / "euler3d_cpu.cpp" + + verifier = SDFGVerification( + verification={"sdfgs": 6, "MAP": 15, "SEQUENTIAL": 15, "FOR": 28} + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "sequential", + [], + partial( + evaluate, + args=[ + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "cfd" + / "fvcorr.domn.097K.data" + ).absolute() + ) + ], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed & Output incorrect") +def test_heartwall(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "heartwall" / "main.c" + + verifier = SDFGVerification( + verification={'sdfgs': 50, 'Malloc': 27, 'MAP': 4, 'FOR': 51, 'SEQUENTIAL': 4, 'Free': 37, 'WHILE': 70} + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-I" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "heartwall" / "AVI/" + ).absolute() + ), + "-lm", + ], + "sequential", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "heartwall" + / "AVI" + / "avilib.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "heartwall" + / "AVI" + / "avimod.c", + ], + partial( + evaluate, + args=[ + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "heartwall" + / "test.avi.data" + ).absolute() + ), + "3", + "1", + ], + dtype=np.int64, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=350) + +@pytest.mark.skip(reason="Test is flaky, needs investigation") +def test_hotspot(compiler="clang++-19"): + test_case = ( + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "hotspot" / "hotspot_openmp.cpp" + ) + + verifier = SDFGVerification( + verification={"sdfgs": 8, "FOR": 8, "WHILE": 3, "Calloc": 3} + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "sequential", + [], + partial( + evaluate, + args=[ + "1024", + "1024", + "10", + "1", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "hotspot" + / "temp_1024.data" + ).absolute() + ), + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "hotspot" + / "power_1024.data" + ).absolute() + ), + "/dev/null", + ], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_hotspot3D(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "hotspot3D" / "3D.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 8, + "Malloc": 1, + "Free": 3, + "Calloc": 4, + "WHILE": 2, + "FOR": 17, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "sequential", + [], + partial( + evaluate, + args=[ + "512", + "8", + "100", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "hotspot3D" + / "power_512x8.data" + ).absolute() + ), + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "hotspot3D" + / "temp_512x8.data" + ).absolute() + ), + "/dev/null", + ], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed & Output incorrect") +def test_kmeans(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "kmeans" / "kmeans.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 6, + "Free": 10, + "FOR": 16, + "MAP": 4, + "Calloc": 2, + "Malloc": 6, + "WHILE": 11, + "SEQUENTIAL": 4, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "sequential", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "kmeans" + / "kmeans_clustering.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "kmeans" / "cluster.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "kmeans" / "getopt.c", + ], + partial( + evaluate, + args=[ + "-i", + str( + ( + Path(__file__).parent + / "tests" + / "rodinia" + / "data" + / "kmeans" + / "kdd_cup.data" + ).absolute() + ), + ], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_lavaMD(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "lavaMD" / "main.c" + + verifier = SDFGVerification( + verification={'sdfgs': 4, 'FOR': 20, 'WHILE': 6, 'Malloc': 8, 'Free': 8}, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "sequential", + [ + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "lavaMD" + / "kernel" + / "kernel_cpu.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "lavaMD" + / "util" + / "num" + / "num.c", + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "lavaMD" + / "util" + / "timer" + / "timer.c", + ], + partial(evaluate, args=["-boxes1d", "10"], dtype=np.float64), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.skip(reason="Timeout") +def test_lud(compiler="clang-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "lud" / "lud.c" + + verifier = SDFGVerification( + verification={ + "sdfgs": 14, + "Free": 9, + "Malloc": 2, + "FOR": 40, + "MAP": 11, + "WHILE": 3, + "SEQUENTIAL": 11, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "sequential", + [ + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "lud" / "lud_omp.c", + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "lud" / "common.c", + ], + partial(evaluate, args=["-s", "4096", "-v"], dtype=np.float32), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed & Output incorrect") +def test_nw(compiler="clang++-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "nw" / "nw.cpp" + + verifier = SDFGVerification( + verification={ + "sdfgs": 7, + "Malloc": 3, + "WHILE": 5, + "Free": 15, + "FOR": 28, + "SEQUENTIAL": 13, + "MAP": 13, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "sequential", + [], + partial(evaluate, args=["16384", "10", "1"], dtype=np.int64), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_particlefilter(compiler="clang-19"): + test_case = ( + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "particlefilter" + / "particlefilter.c" + ) + + verifier = SDFGVerification( + verification={ + 'sdfgs': 17, + 'Free': 2, + 'Malloc': 4, + 'WHILE': 12, + 'MAP': 21, + 'SEQUENTIAL': 21, + 'FOR': 52 + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-std=gnu89", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-w", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "sequential", + [], + partial( + evaluate, + args=["-x", "2048", "-y", "2048", "-z", "100", "-np", "100"], + dtype=np.float64, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed & Output incorrect") +def test_pathfinder(compiler="clang++-19"): + test_case = ( + Path(__file__).parent / "tests" / "rodinia" / "openmp" / "pathfinder" / "pathfinder.cpp" + ) + + verifier = SDFGVerification( + verification={"sdfgs": 1, "FOR": 7}, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "sequential", + [], + partial(evaluate, args=["65536", "1000"], dtype=np.int64), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +def test_srad(compiler="clang++-19"): + test_case = Path(__file__).parent / "tests" / "rodinia" / "openmp" / "srad" / "srad.cpp" + + verifier = SDFGVerification( + verification={ + "sdfgs": 3, + "Malloc": 11, + "MAP": 8, + "FOR": 17, + "SEQUENTIAL": 8, + "Free": 11, + } + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "sequential", + [], + partial( + evaluate, + args=["2048", "2048", "0", "127", "0", "127", "1", "0.5", "2"], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) + + +@pytest.mark.xfail(reason="Verifier changed & Output incorrect") +def test_streamcluster(compiler="clang++-19"): + test_case = ( + Path(__file__).parent + / "tests" + / "rodinia" + / "openmp" + / "streamcluster" + / "streamcluster.cpp" + ) + + verifier = SDFGVerification( + verification={ + "sdfgs": 25, + "MAP": 3, + "SEQUENTIAL": 3, + "FOR": 45, + "Malloc": 7, + "Free": 8, + "Calloc": 3, + "WHILE": 28, + }, + ) + runner = TestRunner( + "Rodinia", + test_case, + "docc-cpp", + compiler, + [ + "-O3", + "-g", + "-fopenmp", + "-DRODINIA_DUMP_ARRAYS", + "-D_MY_IS_NOT_CUDA_", + "-D_MY_IS_OPENMP", + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_timing.h" + ).absolute() + ), + "-include" + + str( + ( + Path(__file__).parent / "tests" / "rodinia" / "common" / "my_verification.h" + ).absolute() + ), + "-lm", + ], + "sequential", + [], + partial( + evaluate, + args=["2", "3", "256", "65536", "65536", "1000", "none", "/dev/null", "1"], + dtype=np.float32, + ), + sdfg_verification=verifier, + ) + return runner.run(timeout=240) diff --git a/integration/rtl_test.py b/integration/rtl_test.py new file mode 100644 index 00000000..d6483b7b --- /dev/null +++ b/integration/rtl_test.py @@ -0,0 +1,1056 @@ +import os +import subprocess +import pytest +import json +import numpy as np +import shutil +from datetime import datetime +from tempfile import mkdtemp + +from pathlib import Path +import sys + +def _prepend_env_path(var_name: str, path: Path) -> None: + if not path.exists(): + return + value = str(path) + current = os.environ.get(var_name, "") + if not current: + os.environ[var_name] = value + return + parts = current.split(os.pathsep) + if value in parts: + return + os.environ[var_name] = value + os.pathsep + current + +root_dir = Path(__file__).parent / "pytestOut" / "rtl" + +# Ensure generated harnesses can find the RTL headers (e.g., ). +# This is required for CI where the include path isn't implicitly configured. +_docc_root = Path(__file__).resolve().parents[1] +_daisy_rtl_include = _docc_root / "docc" / "rtl" / "include" +_prepend_env_path("CPATH", _daisy_rtl_include) +_prepend_env_path("CPLUS_INCLUDE_PATH", _daisy_rtl_include) + +def get_output_dir(test_name): + dir = root_dir / test_name + Path.mkdir(dir, parents=True, exist_ok=True) + tmp_dir = Path(mkdtemp(prefix=datetime.now().strftime('%Y-%m-%d_'), dir= dir)) + return tmp_dir + +def get_docc_work_dir(test_name): + dir = get_output_dir(test_name) + return dir / "DOCC" + + +def parse_stdout(data, dtype): + header = data.pop(0) + assert header == "==BEGIN DUMP_ARRAYS==" + + reference_arrays = {} + current_array = None + for line in data: + if line == "==END DUMP_ARRAYS==": + break + + if line.startswith("begin"): + assert current_array is None + current_array = line[line.find(":") + 1 :].strip() + reference_arrays[current_array] = [] + elif line.startswith("end"): + assert ( + current_array is not None + and current_array == line[line.find(":") + 1 :].strip() + ) + reference_arrays[current_array] = np.hstack(reference_arrays[current_array]) + current_array = None + else: + assert current_array is not None + row = np.fromstring(line, dtype=dtype, sep=" ") + reference_arrays[current_array].append(row) + + return reference_arrays + + +def parse_arg_captures(capture_path, dtype): + capture_json = list(capture_path.glob("*.json")) + + results = [] + + for i in range(len(capture_json)): + result = dict() + data = {} + with open(capture_json[i], "r") as f: + data = json.load(f) + + before_args = {} + after_args = {} + for capture in data["captures"]: + array = np.fromfile(capture_path / capture["ext_file"], dtype=dtype) + argidx = capture["arg_idx"] + if capture["after"]: + after_args[argidx] = array + else: + before_args[argidx] = array + result["before"] = before_args + result["after"] = after_args + result["data"] = data + results.append(result) + + return results + +def load_sdfg(workdir): + from docc.sdfg import StructuredSDFG + + indices = list(workdir.rglob("JSON")) + sdfg = None + for path in indices: + with open(path, "r") as f: + index = json.load(f) + for entry in index["sdfgs"]: + if entry["name"] == "main": + sdfg = StructuredSDFG.from_file( + str((path.parent / entry["file"]).absolute()) + ) + break + + if sdfg is not None: + break + + if sdfg is None: + raise RuntimeError("Could not find compiled SDFG") + + return sdfg + +def test_optional_flag_docc_instrument(): + workdir = Path(__file__).parent / "tests" / "polybench" / "datamining" / "correlation" + output_dir = get_output_dir("instrument") + doccWorkDir = output_dir / "DOCC" + + benchmark_path = workdir / "correlation.c" + output_path = output_dir / "correlation.out" + cmd = [ + "docc", + "-g", + "-O3", + "-DMEDIUM_DATASET", + "-DDATA_TYPE_IS_FLOAT", + "-I" + str(Path(__file__).parent / "tests" / "polybench" / "utilities"), + str(Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"), + str(benchmark_path), + "-o", + str(output_path), + "-docc-work-dir=" + str(doccWorkDir), + "-docc-instrument=ols", + "-lm", + ] + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + trace_file = output_dir / "data_cpu.json" + + ## THIS VERSION IS RUNNER-SPECIFIC and points to CI version of PAPI + os.environ["__DAISY_PAPI_VERSION"] = "0x07020000" + os.environ["__DAISY_INSTRUMENTATION_FILE"] = str(trace_file) + os.environ["__DAISY_INSTRUMENTATION_MODE"] = "aggregate" + os.environ["__DAISY_INSTRUMENTATION_EVENTS"] = "perf::INSTRUCTIONS,perf::CYCLES" + + # Run benchmark + process = subprocess.Popen( + [str(output_path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + with open(trace_file, "r") as f: + trace = json.load(f) + events = trace["traceEvents"] + assert len(events) > 4 + + + # Check first event as a sample + event_0 = events[0] + assert event_0["ph"] == "X" + assert event_0["cat"] == "aggregated_region,daisy" + assert event_0["name"] == "kernel_correlation [L110-118]" + + event_0_args = event_0["args"] + assert event_0_args["module"] == "correlation.c" + assert event_0_args["function"] == "kernel_correlation" + assert event_0_args["source_ranges"][0]["from"]["line"] == 110 + assert event_0_args["source_ranges"][0]["to"]["line"] == 118 + assert event_0_args["source_ranges"][0]["from"]["col"] > 0 + assert event_0_args["source_ranges"][0]["to"]["col"] > 0 + + event_0_docc = event_0_args["docc"] + assert event_0_docc["sdfg_name"] == "main" + assert Path(event_0_docc["sdfg_file"]).is_file() + assert event_0_docc["element_id"] > 0 + assert event_0_docc["element_type"] == "map" + assert event_0_docc["loopnest_index"] == 4 + + event_0_loop_info = event_0_docc["loop_info"] + assert event_0_loop_info["num_loops"] == 3 + assert event_0_loop_info["num_maps"] == 2 + assert event_0_loop_info["num_fors"] == 1 + assert event_0_loop_info["num_whiles"] == 0 + assert event_0_loop_info["max_depth"] == 3 + assert event_0_loop_info["is_perfectly_nested"] == False + assert event_0_loop_info["is_perfectly_parallel"] == False + assert event_0_loop_info["is_elementwise"] == False + assert event_0_loop_info["has_side_effects"] == False + + event_0_metrics = event_0_args["metrics"] + assert "perf::INSTRUCTIONS" in event_0_metrics + assert "mean" in event_0_metrics["perf::INSTRUCTIONS"] + assert "count" in event_0_metrics["perf::INSTRUCTIONS"] + assert event_0_metrics["perf::INSTRUCTIONS"]["count"] == 1 + + assert "perf::CYCLES" in event_0_metrics + assert "mean" in event_0_metrics["perf::CYCLES"] + assert "count" in event_0_metrics["perf::CYCLES"] + assert event_0_metrics["perf::CYCLES"]["count"] == 1 + + assert "runtime" in event_0_metrics + assert "mean" in event_0_metrics["runtime"] + assert "count" in event_0_metrics["runtime"] + assert event_0_metrics["runtime"]["count"] == 1 + +@pytest.mark.parametrize( + "event", + [ + pytest.param(""), + pytest.param( + "nvml:::NVIDIA_GeForce_RTX_5060_Ti:device_0:gpu_utilization,nvml:::NVIDIA_GeForce_RTX_5060_Ti:device_0:memory_utilization" + ), + ], +) +def test_instrumentation_cuda(event): + workdir = Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "symm" + output_dir = get_output_dir("instrument-cuda") + doccWorkDir = output_dir / "DOCC" + + benchmark_path = workdir / "symm.c" + output_path = output_dir / "symm.out" + cmd = [ + "docc", + "-docc-tune=cuda", + "-g", + "-O3", + "-DMEDIUM_DATASET", + "-DDATA_TYPE_IS_FLOAT", + "-I" + str(Path(__file__).parent / "tests" / "polybench" / "utilities"), + str(Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"), + str(benchmark_path), + "-o", + str(output_path), + "-docc-work-dir=" + str(doccWorkDir), + "-docc-instrument=ols", + "-lm", + ] + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + trace_file = output_dir / "data_cuda.json" + + ## THIS VERSION IS RUNNER-SPECIFIC and points to CI version of PAPI + os.environ["__DAISY_PAPI_VERSION"] = "0x07020000" + os.environ["__DAISY_INSTRUMENTATION_FILE"] = str(trace_file) + os.environ["__DAISY_INSTRUMENTATION_MODE"] = "" + os.environ["__DAISY_INSTRUMENTATION_EVENTS_CUDA"] = event + + # Run benchmark + process = subprocess.Popen( + [str(output_path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print(stdout) + print(stderr) + assert process.returncode == 0 + + event_names = [] + if event: + event_names = event.split(",") + + trace = json.load(open(trace_file)) + events = trace["traceEvents"] + assert len(events) > 0 + print(events) + + found_gpu_events = 0 + for i in range(len(events)): + assert events[i]["cat"] == "region,daisy" + assert events[i]["ph"] == "X" + assert events[i]["args"]["module"] == "symm.c" + assert events[i]["args"]["source_ranges"][0]["from"]["line"] > 0 + assert events[i]["args"]["source_ranges"][0]["to"]["line"] > 0 + assert events[i]["args"]["source_ranges"][0]["from"]["col"] > 0 + assert events[i]["args"]["source_ranges"][0]["to"]["col"] > 0 + + includes_events = len(event_names) > 0 + for event_name in event_names: + includes_events = includes_events and ( + event_name in events[i]["args"]["metrics"] + ) + if includes_events: + found_gpu_events += 1 + + if event_names: + assert found_gpu_events > 0 + + +def test_ci_mode(): + """ + This test checks that the CI mode works as expected. It compiles and runs + the correlation benchmark without and with instrumentation and arg-capturing enabled + and verifies that the instrumentation data is collected correctly. + """ + + workdir = Path(__file__).parent / "tests" / "polybench" / "datamining" / "correlation" + output_dir = get_output_dir("ci-mode") + normal_out = output_dir / "correlation.out" + instrumented_out = output_dir / "correlation.instrumented.out" + doccWorkDir = output_dir / "DOCC" + + benchmark_path = workdir / "correlation.c" + output_path = workdir / "correlation.out" + cmd = [ + "docc", + "-g", + "-O3", + "-DMEDIUM_DATASET", + "-DDATA_TYPE_IS_FLOAT", + "-I" + str(Path(__file__).parent / "tests" / "polybench" / "utilities"), + str(Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"), + str(benchmark_path), + "-o", + str(normal_out), + "-docc-work-dir=" + str(doccWorkDir), + "-docc-tune=none", + "-lm", + ] + + os.environ["DOCC_CI"] = "ON" + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + os.environ["DOCC_CI"] = "" + assert instrumented_out.exists() + + trace_file = output_dir / "data_linker_override.json" + + ## THIS VERSION IS RUNNER-SPECIFIC and points to CI version of PAPI + os.environ["__DAISY_PAPI_VERSION"] = "0x07020000" + os.environ["__DAISY_INSTRUMENTATION_FILE"] = str(trace_file) + os.environ["__DAISY_INSTRUMENTATION_MODE"] = "" + os.environ["__DAISY_INSTRUMENTATION_EVENTS"] = "perf::INSTRUCTIONS,perf::CYCLES" + + # Run benchmark + process = subprocess.Popen( + [str(instrumented_out)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + assert process.returncode == 0 + + # Check arg capture + # arg_capture_path = Path("/tmp/DOCC").rglob("arg_captures") + # arg_capture_path = list(arg_capture_path) + # assert len(arg_capture_path) == 0 + + # Check instrumenation + event_names = ["perf::INSTRUCTIONS", "perf::CYCLES"] + trace = json.load(open(trace_file)) + events = trace["traceEvents"] + print(events) + for i in range(len(events)): + assert events[i]["cat"] == "region,daisy" + assert events[i]["ph"] == "X" + assert events[i]["args"]["module"] == "correlation.c" + assert events[i]["args"]["source_ranges"][0]["from"]["line"] > 0 + assert events[i]["args"]["source_ranges"][0]["to"]["line"] > 0 + assert events[i]["args"]["source_ranges"][0]["from"]["col"] > 0 + assert events[i]["args"]["source_ranges"][0]["to"]["col"] > 0 + + for event_name in event_names: + assert event_name in events[i]["args"]["metrics"] + + +@pytest.mark.skip(reason="migrate to docc-auto pending.") +@pytest.mark.parametrize( + "workdir, benchmark, dtype, reference_to_arg_mapping", + [ + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "datamining" / "correlation", + "correlation.c", + "-DDATA_TYPE_IS_DOUBLE", + {"corr": 0} + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "datamining" / "correlation", + "correlation.c", + "-DDATA_TYPE_IS_FLOAT", + {"corr": 0} + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "datamining" / "covariance", + "covariance.c", + "-DDATA_TYPE_IS_DOUBLE", + {"cov": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "datamining" / "covariance", + "covariance.c", + "-DDATA_TYPE_IS_FLOAT", + {"cov": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemm", + "gemm.c", + "-DDATA_TYPE_IS_DOUBLE", + {"C": 2}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemm", + "gemm.c", + "-DDATA_TYPE_IS_FLOAT", + {"C": 2}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemver", + "gemver.c", + "-DDATA_TYPE_IS_DOUBLE", + {"w": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemver", + "gemver.c", + "-DDATA_TYPE_IS_FLOAT", + {"w": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gesummv", + "gesummv.c", + "-DDATA_TYPE_IS_DOUBLE", + {"w": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gesummv", + "gesummv.c", + "-DDATA_TYPE_IS_FLOAT", + {"w": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "symm", + "symm.c", + "-DDATA_TYPE_IS_DOUBLE", + {"w": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "symm", + "symm.c", + "-DDATA_TYPE_IS_FLOAT", + {"w": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "syr2k", + "syr2k.c", + "-DDATA_TYPE_IS_DOUBLE", + {"w": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "syr2k", + "syr2k.c", + "-DDATA_TYPE_IS_FLOAT", + {"w": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "syrk", + "syrk.c", + "-DDATA_TYPE_IS_DOUBLE", + {"w": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "syrk", + "syrk.c", + "-DDATA_TYPE_IS_FLOAT", + {"w": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "trmm", + "trmm.c", + "-DDATA_TYPE_IS_DOUBLE", + {"w": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "trmm", + "trmm.c", + "-DDATA_TYPE_IS_FLOAT", + {"w": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "2mm", + "2mm.c", + "-DDATA_TYPE_IS_DOUBLE", + {"w": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "2mm", + "2mm.c", + "-DDATA_TYPE_IS_FLOAT", + {"w": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "3mm", + "3mm.c", + "-DDATA_TYPE_IS_DOUBLE", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "3mm", + "3mm.c", + "-DDATA_TYPE_IS_FLOAT", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "atax", + "atax.c", + "-DDATA_TYPE_IS_DOUBLE", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "atax", + "atax.c", + "-DDATA_TYPE_IS_FLOAT", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "bicg", + "bicg.c", + "-DDATA_TYPE_IS_DOUBLE", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "bicg", + "bicg.c", + "-DDATA_TYPE_IS_FLOAT", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "doitgen", + "doitgen.c", + "-DDATA_TYPE_IS_DOUBLE", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "doitgen", + "doitgen.c", + "-DDATA_TYPE_IS_FLOAT", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "mvt", + "mvt.c", + "-DDATA_TYPE_IS_DOUBLE", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "mvt", + "mvt.c", + "-DDATA_TYPE_IS_FLOAT", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "cholesky", + "cholesky.c", + "-DDATA_TYPE_IS_DOUBLE", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "cholesky", + "cholesky.c", + "-DDATA_TYPE_IS_FLOAT", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "durbin", + "durbin.c", + "-DDATA_TYPE_IS_DOUBLE", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "durbin", + "durbin.c", + "-DDATA_TYPE_IS_FLOAT", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "gramschmidt", + "gramschmidt.c", + "-DDATA_TYPE_IS_DOUBLE", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "gramschmidt", + "gramschmidt.c", + "-DDATA_TYPE_IS_FLOAT", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "lu", + "lu.c", + "-DDATA_TYPE_IS_DOUBLE", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "lu", + "lu.c", + "-DDATA_TYPE_IS_FLOAT", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "ludcmp", + "ludcmp.c", + "-DDATA_TYPE_IS_DOUBLE", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "ludcmp", + "ludcmp.c", + "-DDATA_TYPE_IS_FLOAT", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "trisolv", + "trisolv.c", + "-DDATA_TYPE_IS_DOUBLE", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "solvers" / "trisolv", + "trisolv.c", + "-DDATA_TYPE_IS_FLOAT", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "medley" / "deriche", + "deriche.c", + "-DDATA_TYPE_IS_DOUBLE", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "medley" / "deriche", + "deriche.c", + "-DDATA_TYPE_IS_FLOAT", + {"G": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "medley" / "floyd-warshall", + "floyd-warshall.c", + "-DDATA_TYPE_IS_INT", + {"path": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "medley" / "nussinov", + "nussinov.c", + "-DDATA_TYPE_IS_INT", + {"table": 0} + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "stencils" / "adi", + "adi.c", + "-DDATA_TYPE_IS_DOUBLE", + {"A": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "stencils" / "adi", + "adi.c", + "-DDATA_TYPE_IS_FLOAT", + {"A": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "stencils" / "fdtd-2d", + "fdtd-2d.c", + "-DDATA_TYPE_IS_DOUBLE", + {"A": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "stencils" / "fdtd-2d", + "fdtd-2d.c", + "-DDATA_TYPE_IS_FLOAT", + {"A": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "stencils" / "heat-3d", + "heat-3d.c", + "-DDATA_TYPE_IS_DOUBLE", + {"A": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "stencils" / "heat-3d", + "heat-3d.c", + "-DDATA_TYPE_IS_FLOAT", + {"A": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "stencils" / "jacobi-1d", + "jacobi-1d.c", + "-DDATA_TYPE_IS_DOUBLE", + {"A": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "stencils" / "jacobi-1d", + "jacobi-1d.c", + "-DDATA_TYPE_IS_FLOAT", + {"A": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "stencils" / "jacobi-2d", + "jacobi-2d.c", + "-DDATA_TYPE_IS_DOUBLE", + {"A": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "stencils" / "jacobi-2d", + "jacobi-2d.c", + "-DDATA_TYPE_IS_FLOAT", + {"A": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "stencils" / "seidel-2d", + "seidel-2d.c", + "-DDATA_TYPE_IS_DOUBLE", + {"A": 0}, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "stencils" / "seidel-2d", + "seidel-2d.c", + "-DDATA_TYPE_IS_FLOAT", + {"A": 0}, + ), + ], +) +def test_arg_capture(workdir, benchmark, dtype, reference_to_arg_mapping): + from docc.sdfg import StructuredSDFG, AnalysisManager + + output_dir = get_output_dir(benchmark+dtype) + doccWorkDir = output_dir / "DOCC" + + benchmark_path = workdir / benchmark + output_path = output_dir / benchmark.replace(".c", ".out") + cmd = [ + "docc", + "-g", + "-O3", + "-DPOLYBENCH_DUMP_ARRAYS", + "-DMEDIUM_DATASET", + dtype, + "-I" + str(Path(__file__).parent / "tests" / "polybench" / "utilities"), + str(Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"), + str(benchmark_path), + "-o", + str(output_path), + "-docc-work-dir=" + str(doccWorkDir), + "-docc-capture-args", + "-lm", + ] + + # Compile + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + assert Path(output_path).exists() + + # Run + process = subprocess.Popen( + [output_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + + dtype = ( + np.float64 + if "-DDATA_TYPE_IS_DOUBLE" in dtype + else np.float32 if "-DDATA_TYPE_IS_FLOAT" in dtype else np.int32 + ) + + # Parse reference arrays from stderr + reference_arrays = parse_stdout(stderr.splitlines(), dtype) + + # Load captured args + arg_capture_path = doccWorkDir.rglob("arg_captures") + arg_capture_path = list(arg_capture_path) + print(arg_capture_path, file=sys.stderr) + assert len(arg_capture_path) == 1 + arg_capture_path = arg_capture_path[0] + parsed_captures = parse_arg_captures(arg_capture_path, dtype) + + # For every captured invocation index, exercise the cutout-runner pipeline via Python bindings. + # - element_id is taken from the capture index json + # - input SDFG is expected in the parent folder as sdfg_0.json + sdfg_path = arg_capture_path.parent / "sdfg_0.json" + assert sdfg_path.exists(), f"Expected input SDFG at {sdfg_path}" + + harness_out_dir = output_dir / "cutout_harnesses" + harness_out_dir.mkdir(parents=True, exist_ok=True) + + index_files = sorted(arg_capture_path.glob("*.index.json")) + assert index_files, f"No *.index.json files found in {arg_capture_path}" + + for index_file in index_files: + with open(index_file, "r") as f: + idx = json.load(f) + element_id = int(idx["element_id"]) + output_prefix = harness_out_dir / f"{index_file.stem}." + result = run_cutout_runner( + str(sdfg_path), + element_id, + str(index_file), + str(output_prefix), + "sequential", + ) + assert "main_cpp" in result + assert "baseline_runtime" in result + assert Path(result["main_cpp"]).exists(), f"Expected generated harness main at {result['main_cpp']}" + assert isinstance(result["baseline_runtime"], float) + assert result["baseline_runtime"] >= 0.0 + + sdfg = load_sdfg(doccWorkDir) + analysis_manager = AnalysisManager(sdfg) + loop_analysis = analysis_manager.loop_analysis() + loop_infos = loop_analysis.loop_infos() + + print(loop_infos, file=sys.stderr) + + # Verify output + for loop_info in loop_infos: + element_id = loop_info.element_id + if loop_info.num_maps == 0: + continue # We only capture args for maps + if loop_info.has_side_effects: + continue # We do not capture args for maps with side effects + capture = None + for parsed_capture in parsed_captures: + if parsed_capture["data"]["element_id"] == str(element_id): + capture = parsed_capture + break + assert capture is not None, f"No capture found for element ID {element_id}: indvar {loop_info.indvar}" + + # for ref, arg_idx in reference_to_arg_mapping.items(): + # assert ( + # arg_idx in after_args + # ), f"Argument index {arg_idx} not found in captured args" + # assert ( + # ref in reference_arrays + # ), f"Reference array '{ref}' not found in reference arrays" + + # ref_array = reference_arrays[ref] + # arg_array = after_args[arg_idx] + + # assert ( + # ref_array.shape == arg_array.shape + # ), f"Shape mismatch for '{ref}': reference shape {ref_array.shape}, captured shape {arg_array.shape}" + # assert np.array_equal( + # reference_arrays[ref], after_args[arg_idx], equal_nan=True + # ) + +@pytest.mark.parametrize( + "workdir, benchmark, dtype, optlevel, result", + [ + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "datamining" / "covariance", + "covariance.c", + "-DDATA_TYPE_IS_DOUBLE", + "-O3", + 2029315760.0 + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "datamining" / "covariance", + "covariance.c", + "-DDATA_TYPE_IS_FLOAT", + "-O3", + 2030995760.0, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "datamining" / "covariance", + "covariance.c", + "-DDATA_TYPE_IS_FLOAT", + "-O0", + 0.0, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemm", + "gemm.c", + "-DDATA_TYPE_IS_DOUBLE", + "-O3", + 3968914560.0, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemm", + "gemm.c", + "-DDATA_TYPE_IS_FLOAT", + "-O3", + 3968914560.0, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemver", + "gemver.c", + "-DDATA_TYPE_IS_DOUBLE", + "-O3", + 48208560.0, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "blas" / "gemver", + "gemver.c", + "-DDATA_TYPE_IS_FLOAT", + "-O3", + 48208560.0, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "3mm", + "3mm.c", + "-DDATA_TYPE_IS_DOUBLE", + "-O3", + 5408294560.0, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "linear-algebra" / "kernels" / "3mm", + "3mm.c", + "-DDATA_TYPE_IS_FLOAT", + "-O3", + 5408294560.0, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "medley" / "floyd-warshall", + "floyd-warshall.c", + "-DDATA_TYPE_IS_INT", + "-O3", + 4194560.0, + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "stencils" / "heat-3d", + "heat-3d.c", + "-DDATA_TYPE_IS_DOUBLE", + "-O3", + 24653150560.0 + ), + pytest.param( + Path(__file__).parent / "tests" / "polybench" / "stencils" / "heat-3d", + "heat-3d.c", + "-DDATA_TYPE_IS_FLOAT", + "-O3", + 24653150560.0, + ), + ], +) +def test_flop_values(workdir, benchmark, dtype, optlevel, result): + output_dir = get_output_dir(benchmark+dtype) + doccWorkDir = output_dir / "DOCC" + + benchmark_path = workdir / benchmark + output_path = output_dir / benchmark.replace(".c", ".out") + cmd = [ + "docc", + "-g", + optlevel, + "-DLARGE_DATASET", + "-DPOLYBENCH_TIME", + dtype, + "-I" + str(Path(__file__).parent / "tests" / "polybench" / "utilities"), + str(Path(__file__).parent / "tests" / "polybench" / "utilities" / "polybench.c"), + str(benchmark_path), + "-o", + str(output_path), + "-lm", + "-docc-work-dir=" + str(doccWorkDir), + "-docc-instrument=ols" + ] + + # Compile + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + assert Path(output_path).exists() + + (workdir / "data_cpu.json").unlink(missing_ok=True) + + ## THIS VERSION IS RUNNER-SPECIFIC and points to CI version of PAPI + os.environ["__DAISY_PAPI_VERSION"] = "0x07020000" + os.environ["__DAISY_INSTRUMENTATION_MODE"] = "aggregate" + os.environ["__DAISY_INSTRUMENTATION_FILE"] = str(workdir / "data_cpu.json") + + # Run + process = subprocess.Popen( + [output_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = process.communicate() + if process.returncode != 0: + print("STDOUT:", stdout) + print("STDERR:", stderr) + assert process.returncode == 0 + + trace = json.load(open(workdir / "data_cpu.json")) + static_flops = 0.0 + for event in trace["traceEvents"]: + assert "args" in event + assert "metrics" in event["args"] + # Currently with -O0 there are lots of while loops, which do not have a static flop count + if ("static:::flop" not in event["args"]["metrics"]) and (optlevel == "-O0"): + continue + assert "static:::flop" in event["args"]["metrics"] + assert "mean" in event["args"]["metrics"]["static:::flop"] + static_flops += float(event["args"]["metrics"]["static:::flop"]["mean"]) + assert static_flops == result diff --git a/integration/test_runner.py b/integration/test_runner.py new file mode 100644 index 00000000..405debaa --- /dev/null +++ b/integration/test_runner.py @@ -0,0 +1,217 @@ +import os +import subprocess + +from typing import Dict +from datetime import datetime +from pathlib import Path +from tempfile import mkdtemp +import threading + +class SDFGVerification: + def __init__(self, verification: dict): + self._verification = verification + + def _parse_opt_report(self, output_str: str) -> Dict[str, int]: + opt_mode = False + stats = {} + for line in output_str.splitlines(): + line = line.strip() + if line.startswith("DOCC Optimization Report Start:"): + opt_mode = True + elif line.startswith("DOCC Optimization Report End"): + opt_mode = False + elif opt_mode: + key, val = line.split(": ") + if key.strip() in ["source_language", "target_triple", "data_layout"]: + continue + if key.strip() not in stats: + stats[key.strip()] = 0 + stats[key.strip()] += int(val.strip()) + + return stats + + def verify(self, stderr: str) -> None: + stats = self._parse_opt_report(stderr) + print(stats) + for key, val in self._verification.items(): + if key not in stats: + assert val == 0, f"Key {key} not found in stats" + else: + assert stats[key] == val, f"Key {key} has value {stats[key]} but expected {val}" + print("All verifications passed.") + +class TransformationVerification: + def __init__(self, transformations: dict): + self._transformations = transformations + + def verify(self, output_dir, stderr: str = None) -> None: + import json + opt_report_files = list(Path(output_dir).rglob("sdfg_0.opt_report.json")) + if not opt_report_files: + raise FileNotFoundError(f"No sdfg_0.opt_report.json found in {output_dir}") + for opt_report_path in opt_report_files: + print(f"Using opt_report file at: {opt_report_path}") + # Load the opt_report JSON file + with open(opt_report_path, 'r') as f: + opt_report = json.load(f) + regions = opt_report.get("regions", []) + if regions != []: + break + + # Build a lookup: loopnest_index -> region + loopnest_to_region = {region.get("loopnest_index"): region for region in regions} + + for transformation, params in self._transformations.items(): + # Check for new dict format with 'loop_nests' and 'max_tuned_loops' + if isinstance(params, dict) and 'loop_nests' in params and 'tuned_loops' in params: + loop_nests = params['loop_nests'] + tuned_loops = params['tuned_loops'] + # Check that transformation is present in all specified loop_nests + for idx in loop_nests: + region = loopnest_to_region.get(idx) + assert region is not None, f"Region with loopnest_index {idx} not found." + transformations = region.get("transformations", {}) + assert transformation in transformations, ( + f"Transformation {transformation} not found in region with loopnest_index {idx}." + ) + # Count total number of times transformation was applied + applied_count = 0 + for region in regions: + transformations = region.get("transformations", {}) + if transformation in transformations: + # Check if 'applied' is True (if present) + t = transformations[transformation] + if isinstance(t, dict) and t.get('applied', True): + applied_count += 1 + elif t is True: + applied_count += 1 + assert applied_count == tuned_loops, ( + f"Transformation {transformation} applied {applied_count} times but expected {tuned_loops} times" + ) + else: + # Fallback to old behavior: params is a set of indices + for idx in params: + region = loopnest_to_region.get(idx) + assert region is not None, f"Region with loopnest_index {idx} not found." + transformations = region.get("transformations", {}) + assert transformation in transformations, ( + f"Transformation {transformation} not found in region with loopnest_index {idx}." + ) + print("All transformation verifications passed.") + +class TestRunner: + __test__ = False + + def __init__( + self, + test_suite, + test_case, + compiler, + reference_compiler, + flags, + mode, + additional_source_files, + evaluation_function, + sdfg_verification : SDFGVerification = None, + transformation_verification : TransformationVerification = None, + docc_flags: list = [], + output_dir = None + ) -> None: + self._test_suite = test_suite + self._test_case = test_case + self._compiler = compiler + self._reference_compiler = reference_compiler + self._flags = flags + self._mode = mode + self._additional_source_files = additional_source_files + self._evaluation_function = evaluation_function + self._sdfg_verification = sdfg_verification + self._transformation_verification = transformation_verification + self._docc_flags = docc_flags + if output_dir is None: + rootDir = Path(__file__).parent / "pytestOut" / (test_suite+"-"+mode) + else: + rootDir = Path(output_dir) + out_dir = rootDir / test_case.name + Path.mkdir(out_dir, parents=True, exist_ok=True) + self.output_dir_ = Path(mkdtemp(prefix=datetime.now().strftime('%Y-%m-%d_%H-%M-%S_'), dir= out_dir)) + + def run(self, timeout=600) -> None: + exception_bucket = [] + def target(): + try: + reference_file = self._compile(self._test_case, reference=True) + assert reference_file.is_file() + test_file = self._compile(self._test_case, False) + assert test_file.is_file() + self._evaluation_function(reference_file, test_file) + except Exception as e: + exception_bucket.append(e) + + t = threading.Thread(target=target) + t.start() + t.join(timeout) + + if exception_bucket: + raise exception_bucket[0] + + if t.is_alive(): + raise TimeoutError("Timeout reached") + + return + + def _compile(self, test_case, reference: bool) -> None: + if reference: + cmd = [self._reference_compiler] + else: + cmd = [self._compiler] + + cmd += [str(path.absolute()) for path in self._additional_source_files] + cmd.append(str(test_case.absolute())) + cmd += self._flags + + if reference: + os.environ["OMPI_CC"] = "clang-19" + os.environ["OMPI_CXX"] = "clang++-19" + os.environ["MPICH_CC"] = "clang-19" + os.environ["MPICH_CXX"] = "clang++-19" + output_path = self.output_dir_ / "reference" + else: + os.environ["OMPI_CC"] = "docc" + os.environ["OMPI_CXX"] = "docc-cpp" + os.environ["MPICH_CC"] = "docc" + os.environ["MPICH_CXX"] = "docc-cpp" + os.environ["DOCC_OPT_REPORT"] = "1" + output_path = self.output_dir_ / "test" + cmd += ["-docc-work-dir=" + str(self.output_dir_ / "DOCC")] + cmd += ["-docc-tune=" + self._mode] + cmd += self._docc_flags + cmd += ["-docc-dot-scheduled"] + + output_file = output_path / "a.out" + cmd += ["-o", str(output_file.absolute())] + print("\nBuild command for", "reference" if reference else "test", "executable:\n", " ".join(cmd)) + + Path.mkdir(output_path, exist_ok=False) + my_env = os.environ.copy() + my_env["LLVM_ENABLE_BACKTRACES"] = "1" + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + encoding="utf-8", + errors="backslashreplace", + env=my_env, + ) + stdout, stderr = process.communicate() + print(stdout) + print(stderr) + + if not reference and self._sdfg_verification is not None: + self._sdfg_verification.verify(stderr) + + if not reference and self._transformation_verification is not None: + self._transformation_verification.verify(self.output_dir_, stderr) + + return output_file diff --git a/integration/tests b/integration/tests new file mode 160000 index 00000000..88832ca7 --- /dev/null +++ b/integration/tests @@ -0,0 +1 @@ +Subproject commit 88832ca72d9359ca6002855f624c4dac332bbe4c