diff --git a/results/benchmark_analyzer.ipynb b/results/benchmark_analyzer.ipynb index 88c3c90..b6572c6 100644 --- a/results/benchmark_analyzer.ipynb +++ b/results/benchmark_analyzer.ipynb @@ -100,24 +100,24 @@ " \n", " if result.returncode == 0:\n", " print(\"\\n\" + \"=\" * 80)\n", - " print(\"✓ BENCHMARKS COMPLETED SUCCESSFULLY\")\n", + " print(\" BENCHMARKS COMPLETED SUCCESSFULLY\")\n", " print(\"=\" * 80)\n", " else:\n", " print(\"\\n\" + \"=\" * 80)\n", - " print(\"✗ BENCHMARK SCRIPT FAILED\")\n", + " print(\"[ERROR] BENCHMARK SCRIPT FAILED\")\n", " print(\"=\" * 80)\n", " print(f\"\\nError output:\\n{result.stderr}\")\n", " raise RuntimeError(\"Benchmark script failed. Please check the error output above.\")\n", " \n", " except subprocess.TimeoutExpired:\n", " print(\"\\n\" + \"=\" * 80)\n", - " print(\"✗ BENCHMARK SCRIPT TIMED OUT\")\n", + " print(\"[ERROR] BENCHMARK SCRIPT TIMED OUT\")\n", " print(\"=\" * 80)\n", - " print(\"\\nThe benchmark script took longer than expected.\")\n", + " print(\"The benchmark script took longer than expected.\")\n", " raise RuntimeError(\"Benchmark script timed out after 5 minutes.\")\n", " except Exception as e:\n", " print(\"\\n\" + \"=\" * 80)\n", - " print(\"✗ ERROR RUNNING BENCHMARK SCRIPT\")\n", + " print(\"[ERROR] ERROR RUNNING BENCHMARK SCRIPT\")\n", " print(\"=\" * 80)\n", " print(f\"\\nError: {e}\")\n", " raise\n", @@ -215,10 +215,20 @@ " print(\"=\" * 80)\n", " raise FileNotFoundError(\"No analysis folders found in results/\")\n", "\n", + "# ============================================================================\n", + "# FILE PATHS - UPDATED FOR DUAL-PROFILE STRUCTURE\n", + "# ============================================================================\n", "# Set up file paths based on selected folder\n", "BENCHMARK_FILE = os.path.join(RUN_FOLDER, \"benchmark_full_suite.txt\")\n", - "PROFILE_FILE = os.path.join(RUN_FOLDER, \"cpu_profile.prof\")\n", - "PROFILE_TREE_FILE = os.path.join(RUN_FOLDER, \"profile_tree.txt\")\n", + "SIMD_COMPARISON_FILE = os.path.join(RUN_FOLDER, \"simd_comparison.txt\")\n", + "\n", + "# New dual-profile structure\n", + "PROFILE_MAIN_FILE = os.path.join(RUN_FOLDER, \"cpu_profile_main.prof\")\n", + "PROFILE_SIMD_FILE = os.path.join(RUN_FOLDER, \"cpu_profile_simd.prof\")\n", + "PROFILE_MAIN_TEXT_FILE = os.path.join(RUN_FOLDER, \"profile_main_text.txt\")\n", + "PROFILE_MAIN_TREE_FILE = os.path.join(RUN_FOLDER, \"profile_main_tree.txt\")\n", + "PROFILE_SIMD_TEXT_FILE = os.path.join(RUN_FOLDER, \"profile_simd_text.txt\")\n", + "PROFILE_SIMD_TREE_FILE = os.path.join(RUN_FOLDER, \"profile_simd_tree.txt\")\n", "\n", "# ============================================================================\n", "# ANALYSIS CONFIGURATION\n", @@ -234,42 +244,41 @@ "print(\"=\" * 80)\n", "print(f\"\\nAnalysis Folder: {RUN_FOLDER}\")\n", "print(f\"Analysis Name: {ANALYSIS_NAME}\")\n", - "print(f\"Full path: {os.path.abspath(BENCHMARK_FILE)}\")\n", "\n", "if not os.path.exists(BENCHMARK_FILE):\n", - " print(f\"\\n✗ ERROR: Benchmark file not found: {BENCHMARK_FILE}\")\n", - " print(f\"Absolute path checked: {os.path.abspath(BENCHMARK_FILE)}\")\n", - " print(f\"Current directory: {os.getcwd()}\")\n", - " print(\"\\nPlease run the benchmark script first:\")\n", - " print(\" bash scripts/benchmark.sh\")\n", + " print(f\"\\n[ERROR] ERROR: Benchmark file not found: {BENCHMARK_FILE}\")\n", " raise FileNotFoundError(f\"Benchmark file not found: {BENCHMARK_FILE}\")\n", "else:\n", - " print(f\"\\n✓ Benchmark file: {os.path.basename(BENCHMARK_FILE)}\")\n", + " print(f\"\\n[OK] Main benchmarks: {os.path.basename(BENCHMARK_FILE)}\")\n", " print(f\" Size: {os.path.getsize(BENCHMARK_FILE):,} bytes\")\n", "\n", - "if PROFILE_FILE:\n", - " if not os.path.exists(PROFILE_FILE):\n", - " print(f\"\\n○ CPU Profile: Not found (analysis will be skipped)\")\n", - " PROFILE_FILE = None\n", - " else:\n", - " print(f\"\\n✓ CPU Profile: {os.path.basename(PROFILE_FILE)}\")\n", - " print(f\" Size: {os.path.getsize(PROFILE_FILE):,} bytes\")\n", + "# Check SIMD comparison file\n", + "if os.path.exists(SIMD_COMPARISON_FILE) and os.path.getsize(SIMD_COMPARISON_FILE) > 0:\n", + " print(f\"\\n[OK] SIMD comparison: {os.path.basename(SIMD_COMPARISON_FILE)}\")\n", + " print(f\" Size: {os.path.getsize(SIMD_COMPARISON_FILE):,} bytes\")\n", "else:\n", - " print(f\"\\n○ CPU Profile: None\")\n", + " print(f\"\\n[SKIP] SIMD comparison: Not found or empty (analysis will be skipped)\")\n", + " SIMD_COMPARISON_FILE = None\n", "\n", - "if PROFILE_TREE_FILE:\n", - " if not os.path.exists(PROFILE_TREE_FILE):\n", - " print(f\"\\n○ Profile Tree: Not found (analysis will be skipped)\")\n", - " PROFILE_TREE_FILE = None\n", - " else:\n", - " print(f\"\\n✓ Profile Tree: {os.path.basename(PROFILE_TREE_FILE)}\")\n", - " print(f\" Size: {os.path.getsize(PROFILE_TREE_FILE):,} bytes\")\n", + "# Check main CPU profile\n", + "if os.path.exists(PROFILE_MAIN_FILE) and os.path.getsize(PROFILE_MAIN_FILE) > 0:\n", + " print(f\"\\n[OK] Main CPU Profile: {os.path.basename(PROFILE_MAIN_FILE)}\")\n", + " print(f\" Size: {os.path.getsize(PROFILE_MAIN_FILE):,} bytes\")\n", "else:\n", - " print(f\"\\n○ Profile Tree: None\")\n", + " print(f\"\\n[SKIP] Main CPU Profile: Not found or empty (analysis will be skipped)\")\n", + " PROFILE_MAIN_FILE = None\n", + "\n", + "# Check SIMD CPU profile\n", + "if os.path.exists(PROFILE_SIMD_FILE) and os.path.getsize(PROFILE_SIMD_FILE) > 0:\n", + " print(f\"\\n[OK] SIMD CPU Profile: {os.path.basename(PROFILE_SIMD_FILE)}\")\n", + " print(f\" Size: {os.path.getsize(PROFILE_SIMD_FILE):,} bytes\")\n", + "else:\n", + " print(f\"\\n[SKIP] SIMD CPU Profile: Not found or empty (analysis will be skipped)\")\n", + " PROFILE_SIMD_FILE = None\n", "\n", "# Set up output directory - output goes inside the run folder\n", "output_dir = RUN_FOLDER\n", - "print(f\"\\n✓ Output directory: {output_dir}\")\n", + "print(f\"\\n[OK] Output directory: {output_dir}\")\n", "\n", "os.makedirs(os.path.join(output_dir, 'graphs'), exist_ok=True)\n", "os.makedirs(os.path.join(output_dir, 'data'), exist_ok=True)\n", @@ -375,10 +384,10 @@ "df = parse_benchmark_file(BENCHMARK_FILE)\n", "\n", "if df.empty:\n", - " print(\"\\n⚠ No benchmarks parsed. Cannot continue with analysis.\")\n", + " print(\"\\n[WARN]  No benchmarks parsed. Cannot continue with analysis.\")\n", " print(\"Please check the benchmark file format.\")\n", "else:\n", - " print(f\"\\n✓ Parsed {len(df)} benchmark results\")\n", + " print(f\" Parsed {len(df)} benchmark results\")\n", " print(f\"\\nColumns: {', '.join(df.columns)}\")\n", " if 'category' in df.columns:\n", " print(f\"\\nBenchmark categories: {', '.join(df['category'].unique())}\")\n", @@ -443,40 +452,60 @@ "metadata": {}, "outputs": [], "source": [ - "# Filter SIMD comparison benchmarks\n", - "simd_df = df[df['name'].str.contains('SIMDvsScalar', na=False)].copy()\n", + "# Parse SIMD comparison benchmarks from separate file\n", + "simd_df = pd.DataFrame()\n", "\n", - "if not simd_df.empty:\n", - " # Extract operation type and implementation\n", - " simd_df['operation'] = simd_df['name'].str.extract(r'/(\\w+)_Size')[0]\n", - " simd_df['implementation'] = simd_df['name'].str.extract(r'/(SIMD|Fallback)-')[0]\n", - " \n", - " # Pivot for comparison\n", - " comparison = simd_df.pivot_table(\n", - " index=['size', 'operation'],\n", - " columns='implementation',\n", - " values='ns_op'\n", - " ).reset_index()\n", + "if SIMD_COMPARISON_FILE and os.path.exists(SIMD_COMPARISON_FILE):\n", + " # Parse the SIMD comparison file\n", + " simd_df = parse_benchmark_file(SIMD_COMPARISON_FILE)\n", " \n", - " if 'SIMD' in comparison.columns and 'Fallback' in comparison.columns:\n", - " comparison['speedup'] = comparison['Fallback'] / comparison['SIMD']\n", - " comparison['improvement_pct'] = (comparison['speedup'] - 1) * 100\n", - " \n", - " print(\"\\n\" + \"=\" * 80)\n", - " print(\"SIMD PERFORMANCE ANALYSIS\")\n", - " print(\"=\" * 80)\n", - " print(f\"\\nAverage SIMD speedup: {comparison['speedup'].mean():.2f}x\")\n", - " print(f\"Best SIMD speedup: {comparison['speedup'].max():.2f}x (size {comparison.loc[comparison['speedup'].idxmax(), 'size']})\")\n", - " print(f\"Worst SIMD speedup: {comparison['speedup'].min():.2f}x (size {comparison.loc[comparison['speedup'].idxmin(), 'size']})\")\n", + " if not simd_df.empty:\n", + " # Extract operation type and implementation\n", + " simd_df['operation'] = simd_df['name'].str.extract(r'/(\\w+)_Size')[0]\n", + " simd_df['implementation'] = simd_df['name'].str.extract(r'/(SIMD|Fallback)-')[0]\n", " \n", - " print(\"\\nSpeedup by operation:\")\n", - " op_speedup = comparison.groupby('operation')['speedup'].mean().sort_values(ascending=False)\n", - " for op, speedup in op_speedup.items():\n", - " print(f\" {op:15s}: {speedup:.2f}x\")\n", + " # Pivot for comparison\n", + " comparison = simd_df.pivot_table(\n", + " index=['size', 'operation'],\n", + " columns='implementation',\n", + " values='ns_op'\n", + " ).reset_index()\n", " \n", - " display(comparison.sort_values('speedup', ascending=False))\n", + " if 'SIMD' in comparison.columns and 'Fallback' in comparison.columns:\n", + " comparison['speedup'] = comparison['Fallback'] / comparison['SIMD']\n", + " comparison['improvement_pct'] = (comparison['speedup'] - 1) * 100\n", + " \n", + " print(\"\\n\" + \"=\" * 80)\n", + " print(\"SIMD PERFORMANCE ANALYSIS\")\n", + " print(\"=\" * 80)\n", + " print(f\"\\nTotal SIMD benchmarks: {len(simd_df)}\")\n", + " print(f\"Operations tested: {', '.join(simd_df['operation'].unique())}\")\n", + " print(f\"\\nAverage SIMD speedup: {comparison['speedup'].mean():.2f}x\")\n", + " print(f\"Best SIMD speedup: {comparison['speedup'].max():.2f}x (size {comparison.loc[comparison['speedup'].idxmax(), 'size']})\")\n", + " print(f\"Worst SIMD speedup: {comparison['speedup'].min():.2f}x (size {comparison.loc[comparison['speedup'].idxmin(), 'size']})\")\n", + " \n", + " print(\"\\nSpeedup by operation:\")\n", + " op_speedup = comparison.groupby('operation')['speedup'].mean().sort_values(ascending=False)\n", + " for op, speedup in op_speedup.items():\n", + " print(f\" {op:15s}: {speedup:.2f}x\")\n", + " \n", + " print(\"\\nTop 10 SIMD improvements:\")\n", + " display(comparison.sort_values('speedup', ascending=False).head(10))\n", + " else:\n", + " print(f\"\\n[ERROR] Could not extract SIMD vs Fallback comparison\")\n", + " print(f\"Available columns: {comparison.columns.tolist()}\")\n", + " else:\n", + " print(f\"\\n[ERROR] No benchmarks parsed from {SIMD_COMPARISON_FILE}\")\n", "else:\n", - " print(\"No SIMD comparison benchmarks found in this file.\")" + " print(\"\\n\" + \"=\" * 80)\n", + " print(\"SIMD COMPARISON FILE NOT AVAILABLE\")\n", + " print(\"=\" * 80)\n", + " if SIMD_COMPARISON_FILE:\n", + " print(f\"File: {SIMD_COMPARISON_FILE}\")\n", + " print(\"File not found or empty - SIMD analysis will be skipped\")\n", + " else:\n", + " print(\"SIMD_COMPARISON_FILE is None - file not configured\")\n", + " print(\"=\" * 80)" ] }, { @@ -522,12 +551,12 @@ " # Save the figure\n", " graph_path = os.path.join(output_dir, 'graphs', 'simd_speedup_analysis.png')\n", " plt.savefig(graph_path, dpi=300, bbox_inches='tight')\n", - " print(f\"✓ Graph saved: {graph_path}\")\n", + " print(f\" Graph saved: {graph_path}\")\n", " \n", " # Also save as SVG for high-quality reports\n", " svg_path = os.path.join(output_dir, 'graphs', 'simd_speedup_analysis.svg')\n", " plt.savefig(svg_path, format='svg', bbox_inches='tight')\n", - " print(f\"✓ Graph saved (SVG): {svg_path}\")\n", + " print(f\" Graph saved (SVG): {svg_path}\")\n", " \n", " plt.show()\n", "else:\n", @@ -584,11 +613,11 @@ " # Save the figure\n", " graph_path = os.path.join(output_dir, 'graphs', 'performance_scaling.png')\n", " plt.savefig(graph_path, dpi=300, bbox_inches='tight')\n", - " print(f\"✓ Graph saved: {graph_path}\")\n", + " print(f\"Graph saved: {graph_path}\")\n", " \n", " svg_path = os.path.join(output_dir, 'graphs', 'performance_scaling.svg')\n", " plt.savefig(svg_path, format='svg', bbox_inches='tight')\n", - " print(f\"✓ Graph saved (SVG): {svg_path}\")\n", + " print(f\"Graph saved (SVG): {svg_path}\")\n", " \n", " plt.show()\n", "else:\n", @@ -636,11 +665,11 @@ " # Save the figure\n", " graph_path = os.path.join(output_dir, 'graphs', 'memory_allocation_analysis.png')\n", " plt.savefig(graph_path, dpi=300, bbox_inches='tight')\n", - " print(f\"✓ Graph saved: {graph_path}\")\n", + " print(f\"Graph saved: {graph_path}\")\n", " \n", " svg_path = os.path.join(output_dir, 'graphs', 'memory_allocation_analysis.svg')\n", " plt.savefig(svg_path, format='svg', bbox_inches='tight')\n", - " print(f\"✓ Graph saved (SVG): {svg_path}\")\n", + " print(f\"Graph saved (SVG): {svg_path}\")\n", " \n", " plt.show()\n", " \n", @@ -748,11 +777,11 @@ " # Save the figure\n", " graph_path = os.path.join(output_dir, 'graphs', 'hybrid_mode_comparison.png')\n", " plt.savefig(graph_path, dpi=300, bbox_inches='tight')\n", - " print(f\"\\n✓ Graph saved: {graph_path}\")\n", + " print(f\"\\n Graph saved: {graph_path}\")\n", " \n", " svg_path = os.path.join(output_dir, 'graphs', 'hybrid_mode_comparison.svg')\n", " plt.savefig(svg_path, format='svg', bbox_inches='tight')\n", - " print(f\"✓ Graph saved (SVG): {svg_path}\")\n", + " print(f\" Graph saved (SVG): {svg_path}\")\n", " \n", " plt.show()\n", "else:\n", @@ -815,13 +844,13 @@ "# Export CSV files\n", "csv_file = os.path.join(output_dir, 'data', 'all_benchmarks.csv')\n", "df.to_csv(csv_file, index=False)\n", - "print(f\"\\n✓ All benchmarks exported to: {csv_file}\")\n", + "print(f\"\\n All benchmarks exported to: {csv_file}\")\n", "\n", "# Export SIMD comparison if available\n", "if not simd_df.empty and 'speedup' in comparison.columns:\n", " simd_csv = os.path.join(output_dir, 'data', 'simd_comparison.csv')\n", " comparison.to_csv(simd_csv, index=False)\n", - " print(f\"✓ SIMD comparison exported to: {simd_csv}\")\n", + " print(f\" SIMD comparison exported to: {simd_csv}\")\n", " \n", " # Export summary statistics\n", " summary_data = {\n", @@ -836,13 +865,13 @@ " summary_df = pd.DataFrame(summary_data)\n", " summary_csv = os.path.join(output_dir, 'data', 'simd_summary.csv')\n", " summary_df.to_csv(summary_csv, index=False)\n", - " print(f\"✓ SIMD summary statistics exported to: {summary_csv}\")\n", + " print(f\" SIMD summary statistics exported to: {summary_csv}\")\n", "\n", "# Export hybrid mode analysis if available\n", "if not hybrid_df.empty:\n", " hybrid_csv = os.path.join(output_dir, 'data', 'hybrid_modes.csv')\n", " hybrid_df.to_csv(hybrid_csv, index=False)\n", - " print(f\"✓ Hybrid mode data exported to: {hybrid_csv}\")\n", + " print(f\" Hybrid mode data exported to: {hybrid_csv}\")\n", " \n", " # Mode comparison summary\n", " mode_comparison = pd.DataFrame({\n", @@ -858,19 +887,19 @@ " })\n", " mode_csv = os.path.join(output_dir, 'data', 'mode_comparison.csv')\n", " mode_comparison.to_csv(mode_csv, index=False)\n", - " print(f\"✓ Mode comparison summary exported to: {mode_csv}\")\n", + " print(f\" Mode comparison summary exported to: {mode_csv}\")\n", "\n", "# Export top performers\n", "top_10_fastest = df.nsmallest(10, 'ns_op')[['name', 'ns_op', 'ops_per_sec', 'bytes_op', 'allocs_op']]\n", "top_csv = os.path.join(output_dir, 'data', 'top_10_fastest.csv')\n", "top_10_fastest.to_csv(top_csv, index=False)\n", - "print(f\"✓ Top 10 fastest benchmarks exported to: {top_csv}\")\n", + "print(f\" Top 10 fastest benchmarks exported to: {top_csv}\")\n", "\n", "# Export top 10 slowest\n", "top_10_slowest = df.nlargest(10, 'ns_op')[['name', 'ns_op', 'ops_per_sec', 'bytes_op', 'allocs_op']]\n", "slow_csv = os.path.join(output_dir, 'data', 'top_10_slowest.csv')\n", "top_10_slowest.to_csv(slow_csv, index=False)\n", - "print(f\"✓ Top 10 slowest benchmarks exported to: {slow_csv}\")\n", + "print(f\" Top 10 slowest benchmarks exported to: {slow_csv}\")\n", "\n", "# Export metadata\n", "metadata = {\n", @@ -886,10 +915,10 @@ "metadata_df = pd.DataFrame([metadata])\n", "meta_csv = os.path.join(output_dir, 'data', 'metadata.csv')\n", "metadata_df.to_csv(meta_csv, index=False)\n", - "print(f\"✓ Metadata exported to: {meta_csv}\")\n", + "print(f\" Metadata exported to: {meta_csv}\")\n", "\n", - "print(f\"\\n✓ All data files saved to: {os.path.join(output_dir, 'data')}\")\n", - "print(f\"✓ All graphs have been saved to: {os.path.join(output_dir, 'graphs')}\")\n", + "print(f\"\\n All data files saved to: {os.path.join(output_dir, 'data')}\")\n", + "print(f\" All graphs have been saved to: {os.path.join(output_dir, 'graphs')}\")\n", "\n", "# Initialize profile variables\n", "prof_df = None\n", @@ -917,29 +946,26 @@ "import subprocess\n", "import re\n", "\n", - "def analyze_pprof_file(prof_file):\n", + "def analyze_pprof_file(prof_file, profile_name=\"\"):\n", " \"\"\"\n", " Analyze a Go pprof CPU profile file.\n", - " \n", + "\n", " Returns:\n", - " DataFrame with profile data\n", + " DataFrame with profile data and metadata dict\n", " \"\"\"\n", " try:\n", - " # Run go tool pprof to get top functions\n", " result = subprocess.run(\n", " ['go', 'tool', 'pprof', '-top', '-nodecount=50', prof_file],\n", " capture_output=True,\n", " text=True,\n", " timeout=30\n", " )\n", - " \n", + "\n", " if result.returncode != 0:\n", - " print(f\"Error running pprof: {result.stderr}\")\n", + " print(f\"Error running pprof on {profile_name}: {result.stderr}\")\n", " return None, None\n", - " \n", + "\n", " output = result.stdout\n", - " \n", - " # Extract metadata\n", " metadata = {}\n", " for line in output.split('\\n')[:10]:\n", " if 'Duration:' in line:\n", @@ -949,22 +975,16 @@ " metadata['type'] = line.split('Type:')[1].strip()\n", " elif 'Time:' in line:\n", " metadata['time'] = line.split('Time:')[1].strip()\n", - " \n", - " # Parse the profile data\n", + "\n", " profile_data = []\n", " in_data_section = False\n", - " \n", + "\n", " for line in output.split('\\n'):\n", - " # Start parsing after the header\n", " if 'flat flat% sum%' in line:\n", " in_data_section = True\n", " continue\n", - " \n", " if not in_data_section:\n", " continue\n", - " \n", - " # Parse data lines\n", - " # Format: flat flat% sum% cum cum% function_name\n", " match = re.match(r'\\s*(\\d+\\.?\\d*[a-z]*)\\s+(\\d+\\.?\\d+)%\\s+(\\d+\\.?\\d+)%\\s+(\\d+\\.?\\d*[a-z]*)\\s+(\\d+\\.?\\d+)%\\s+(.+)', line)\n", " if match:\n", " profile_data.append({\n", @@ -975,105 +995,122 @@ " 'cum_pct': float(match.group(5)),\n", " 'function': match.group(6).strip()\n", " })\n", - " \n", + "\n", " if not profile_data:\n", - " print(\"No profile data found\")\n", " return None, None\n", - " \n", + "\n", " df = pd.DataFrame(profile_data)\n", - " \n", - " # Simplify function names for display\n", " df['function_short'] = df['function'].apply(lambda x: x.split('/')[-1] if '/' in x else x)\n", " df['function_short'] = df['function_short'].apply(lambda x: x[:60] + '...' if len(x) > 60 else x)\n", - " \n", " return df, metadata\n", - " \n", - " except subprocess.TimeoutExpired:\n", - " print(\"pprof analysis timed out\")\n", - " return None, None\n", - " except FileNotFoundError:\n", - " print(\"Error: 'go' command not found. Make sure Go is installed and in PATH.\")\n", - " return None, None\n", " except Exception as e:\n", - " print(f\"Error analyzing pprof file: {e}\")\n", + " print(f\"Error analyzing {profile_name}: {e}\")\n", " return None, None\n", "\n", - "\n", - "# =============================================================================\n", - "# CPU PROFILE ANALYSIS (.prof files)\n", - "# =============================================================================\n", - "if PROFILE_FILE and os.path.exists(PROFILE_FILE):\n", + "# Main benchmarks profile\n", + "if PROFILE_MAIN_FILE and os.path.exists(PROFILE_MAIN_FILE):\n", " print(\"\\n\" + \"=\" * 80)\n", - " print(f\"CPU PROFILE ANALYSIS: {os.path.basename(PROFILE_FILE)}\")\n", + " print(f\"MAIN BENCHMARKS CPU PROFILE: {os.path.basename(PROFILE_MAIN_FILE)}\")\n", " print(\"=\" * 80)\n", - " \n", - " prof_df, prof_metadata = analyze_pprof_file(PROFILE_FILE)\n", - " \n", - " if prof_df is not None:\n", - " print(f\"\\nProfile Metadata:\")\n", - " for key, value in prof_metadata.items():\n", + "\n", + " prof_main_df, prof_main_metadata = analyze_pprof_file(PROFILE_MAIN_FILE, \"Main\")\n", + "\n", + " if prof_main_df is not None:\n", + " print(f\"\\nMetadata:\")\n", + " for key, value in prof_main_metadata.items():\n", " print(f\" {key}: {value}\")\n", - " \n", - " print(f\"\\n✓ Parsed {len(prof_df)} functions from profile\")\n", + " print(f\"\\n Parsed {len(prof_main_df)} functions\")\n", " print(f\"\\nTop 10 CPU Consumers:\")\n", - " display(prof_df.head(10)[['function_short', 'flat_pct', 'cum_pct']])\n", - " \n", - " # Visualize top functions\n", + " display(prof_main_df.head(10)[['function_short', 'flat_pct', 'cum_pct']])\n", + "\n", " fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 6))\n", - " \n", - " # Plot 1: Top 15 functions by flat %\n", - " top_flat = prof_df.head(15)\n", + " top_flat = prof_main_df.head(15)\n", " ax1.barh(range(len(top_flat)), top_flat['flat_pct'], color='steelblue', alpha=0.7)\n", " ax1.set_yticks(range(len(top_flat)))\n", " ax1.set_yticklabels(top_flat['function_short'], fontsize=9)\n", " ax1.set_xlabel('Flat % (self time)', fontsize=12)\n", - " ax1.set_title('Top 15 Functions by Self Time', fontsize=14, fontweight='bold')\n", + " ax1.set_title('Top 15 Functions by Self Time (Main)', fontsize=14, fontweight='bold')\n", " ax1.grid(True, alpha=0.3, axis='x')\n", " ax1.invert_yaxis()\n", - " \n", - " # Plot 2: Top 15 functions by cumulative %\n", - " top_cum = prof_df.nlargest(15, 'cum_pct')\n", + "\n", + " top_cum = prof_main_df.nlargest(15, 'cum_pct')\n", " ax2.barh(range(len(top_cum)), top_cum['cum_pct'], color='coral', alpha=0.7)\n", " ax2.set_yticks(range(len(top_cum)))\n", " ax2.set_yticklabels(top_cum['function_short'], fontsize=9)\n", - " ax2.set_xlabel('Cumulative % (including callees)', fontsize=12)\n", - " ax2.set_title('Top 15 Functions by Cumulative Time', fontsize=14, fontweight='bold')\n", + " ax2.set_xlabel('Cumulative %', fontsize=12)\n", + " ax2.set_title('Top 15 Functions by Cumulative Time (Main)', fontsize=14, fontweight='bold')\n", " ax2.grid(True, alpha=0.3, axis='x')\n", " ax2.invert_yaxis()\n", - " \n", " plt.tight_layout()\n", - " \n", - " # Save the figure\n", - " graph_path = os.path.join(output_dir, 'graphs', 'cpu_profile_analysis.png')\n", + "\n", + " graph_path = os.path.join(output_dir, 'graphs', 'cpu_profile_main_analysis.png')\n", " plt.savefig(graph_path, dpi=300, bbox_inches='tight')\n", - " print(f\"\\n✓ Graph saved: {graph_path}\")\n", - " \n", - " svg_path = os.path.join(output_dir, 'graphs', 'cpu_profile_analysis.svg')\n", - " plt.savefig(svg_path, format='svg', bbox_inches='tight')\n", - " print(f\"✓ Graph saved (SVG): {svg_path}\")\n", - " \n", + " print(f\"\\n Graph saved: {graph_path}\")\n", + " plt.savefig(os.path.join(output_dir, 'graphs', 'cpu_profile_main_analysis.svg'), format='svg', bbox_inches='tight')\n", " plt.show()\n", - " \n", - " # Export profile data\n", - " prof_csv = os.path.join(output_dir, 'data', 'cpu_profile.csv')\n", - " prof_df.to_csv(prof_csv, index=False)\n", - " print(f\"✓ CPU profile data exported to: {prof_csv}\")\n", "\n", + " prof_main_df.to_csv(os.path.join(output_dir, 'data', 'cpu_profile_main.csv'), index=False)\n", "else:\n", " print(\"\\n\" + \"=\" * 80)\n", - " print(\"CPU PROFILE ANALYSIS SKIPPED\")\n", + " print(\"MAIN CPU PROFILE SKIPPED - No file or empty\")\n", " print(\"=\" * 80)\n", - " if PROFILE_FILE is None:\n", - " print(\"\\nNo CPU profile file specified for this investigation.\")\n", - " print(\"\\nTo generate CPU profiles, run:\")\n", - " print(\" go test -bench=BenchmarkName -cpuprofile=results/cpu.prof\")\n", "\n", + "# SIMD benchmarks profile\n", + "if PROFILE_SIMD_FILE and os.path.exists(PROFILE_SIMD_FILE):\n", + " print(\"\\n\" + \"=\" * 80)\n", + " print(f\"SIMD BENCHMARKS CPU PROFILE: {os.path.basename(PROFILE_SIMD_FILE)}\")\n", + " print(\"=\" * 80)\n", + "\n", + " prof_simd_df, prof_simd_metadata = analyze_pprof_file(PROFILE_SIMD_FILE, \"SIMD\")\n", + "\n", + " if prof_simd_df is not None:\n", + " print(f\"\\nMetadata:\")\n", + " for key, value in prof_simd_metadata.items():\n", + " print(f\" {key}: {value}\")\n", + " print(f\"\\n Parsed {len(prof_simd_df)} functions\")\n", + " print(f\"\\nTop 10 CPU Consumers:\")\n", + " display(prof_simd_df.head(10)[['function_short', 'flat_pct', 'cum_pct']])\n", + "\n", + " fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 6))\n", + " top_flat = prof_simd_df.head(15)\n", + " ax1.barh(range(len(top_flat)), top_flat['flat_pct'], color='mediumseagreen', alpha=0.7)\n", + " ax1.set_yticks(range(len(top_flat)))\n", + " ax1.set_yticklabels(top_flat['function_short'], fontsize=9)\n", + " ax1.set_xlabel('Flat % (self time)', fontsize=12)\n", + " ax1.set_title('Top 15 Functions by Self Time (SIMD)', fontsize=14, fontweight='bold')\n", + " ax1.grid(True, alpha=0.3, axis='x')\n", + " ax1.invert_yaxis()\n", + "\n", + " top_cum = prof_simd_df.nlargest(15, 'cum_pct')\n", + " ax2.barh(range(len(top_cum)), top_cum['cum_pct'], color='lightcoral', alpha=0.7)\n", + " ax2.set_yticks(range(len(top_cum)))\n", + " ax2.set_yticklabels(top_cum['function_short'], fontsize=9)\n", + " ax2.set_xlabel('Cumulative %', fontsize=12)\n", + " ax2.set_title('Top 15 Functions by Cumulative Time (SIMD)', fontsize=14, fontweight='bold')\n", + " ax2.grid(True, alpha=0.3, axis='x')\n", + " ax2.invert_yaxis()\n", + " plt.tight_layout()\n", + "\n", + " graph_path = os.path.join(output_dir, 'graphs', 'cpu_profile_simd_analysis.png')\n", + " plt.savefig(graph_path, dpi=300, bbox_inches='tight')\n", + " print(f\"\\n Graph saved: {graph_path}\")\n", + " plt.savefig(os.path.join(output_dir, 'graphs', 'cpu_profile_simd_analysis.svg'), format='svg', bbox_inches='tight')\n", + " plt.show()\n", + "\n", + " prof_simd_df.to_csv(os.path.join(output_dir, 'data', 'cpu_profile_simd.csv'), index=False)\n", + "else:\n", + " print(\"\\n\" + \"=\" * 80)\n", + " print(\"SIMD CPU PROFILE SKIPPED - No file or empty\")\n", + " print(\"=\" * 80)\n", "\n", "print(\"\\n\" + \"=\" * 80)\n", "print(\"ANALYSIS COMPLETE\")\n", "print(\"=\" * 80)\n", - "print(\"\\nFor interactive profile visualization, use:\")\n", - "print(\" go tool pprof -http=:8080 results/cpu.prof\")" + "print(\"\\nFor interactive profile visualization:\")\n", + "if PROFILE_MAIN_FILE:\n", + " print(f\" Main: go tool pprof -http=:8080 {PROFILE_MAIN_FILE}\")\n", + "if PROFILE_SIMD_FILE:\n", + " print(f\" SIMD: go tool pprof -http=:8081 {PROFILE_SIMD_FILE}\")\n" ] } ], @@ -1098,4 +1135,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index 2cd7292..56f362a 100644 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -18,7 +18,7 @@ echo "" # Run full benchmark suite echo "Running full benchmark suite..." -go test -bench=. -benchmem -run=^$ -benchtime=1s > "$RESULTS_DIR/benchmark_full_suite.txt" +go test ./tests/benchmark -bench=. -benchmem -run=^$ -benchtime=1s > "$RESULTS_DIR/benchmark_full_suite.txt" echo "Saved to: $RESULTS_DIR/benchmark_full_suite.txt" # Run SIMD comparison benchmarks (integration tests) @@ -34,7 +34,7 @@ fi echo "Running benchmarks with CPU profiling..." # Exclude BenchmarkHybridMemoryAllocation and BenchmarkHybridThroughput as they take too long # Run main package benchmarks first, then integration tests with separate profiles -go test -bench='Benchmark(Cache|Insertion|Lookup|FalsePositives|Comprehensive|HybridModes|HybridCrossover)' -cpuprofile="$RESULTS_DIR/cpu_profile_main.prof" -run=^$ -benchtime=1s > "$RESULTS_DIR/profiled_benchmarks.txt" 2>&1 +go test ./tests/benchmark -bench='Benchmark(Cache|Insertion|Lookup|FalsePositives|Comprehensive|HybridModes|HybridCrossover)' -cpuprofile="$RESULTS_DIR/cpu_profile_main.prof" -run=^$ -benchtime=1s > "$RESULTS_DIR/profiled_benchmarks.txt" 2>&1 MAIN_EXIT_CODE=$? # Run SIMD benchmarks with separate profile