From 189d29ff535dd0b33f12cc2eb2eb1dd148e4027a Mon Sep 17 00:00:00 2001 From: Paul Bleess <8421069+pableess@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:12:24 -0500 Subject: [PATCH] add bench --- benchmarks/Yamux.Benchmark/Program.cs | 108 +++++++++++- benchmarks/build-go-server.ps1 | 19 +++ benchmarks/compare.ps1 | 95 +++++++++++ benchmarks/go-benchmark/go.mod | 5 + benchmarks/go-benchmark/go.sum | 2 + benchmarks/go-benchmark/main.go | 235 ++++++++++++++++++++++++++ benchmarks/go-server/go.mod | 5 + benchmarks/go-server/go.sum | 2 + benchmarks/go-server/main.go | 50 ++++++ 9 files changed, 520 insertions(+), 1 deletion(-) create mode 100644 benchmarks/build-go-server.ps1 create mode 100644 benchmarks/compare.ps1 create mode 100644 benchmarks/go-benchmark/go.mod create mode 100644 benchmarks/go-benchmark/go.sum create mode 100644 benchmarks/go-benchmark/main.go create mode 100644 benchmarks/go-server/go.mod create mode 100644 benchmarks/go-server/go.sum create mode 100644 benchmarks/go-server/main.go diff --git a/benchmarks/Yamux.Benchmark/Program.cs b/benchmarks/Yamux.Benchmark/Program.cs index f4651b6..8aaf49b 100644 --- a/benchmarks/Yamux.Benchmark/Program.cs +++ b/benchmarks/Yamux.Benchmark/Program.cs @@ -1,8 +1,10 @@ -using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Running; +using System.Diagnostics; using System.IO.Pipelines; +using System.Linq; using System.Net; using System.Net.Sockets; @@ -27,9 +29,11 @@ public static async Task Main(string[] args) } [SimpleJob(RunStrategy.Throughput, launchCount: 1, warmupCount: 3, invocationCount: 5)] + [MemoryDiagnoser] public class Yamux { private Memory _buffer; + private string? _goServerPath; [Params(1, 50, 500)] public int MBs = 50; @@ -57,6 +61,12 @@ public void Setup() _serverSock.Listen(); _port = ((IPEndPoint)_serverSock.LocalEndPoint!).Port; + + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir != null && !dir.EnumerateFiles("Yamux.sln").Any()) + dir = dir.Parent; + var repoRoot = dir?.FullName ?? throw new InvalidOperationException("Cannot find Yamux.sln"); + _goServerPath = Path.Combine(repoRoot, "benchmarks", "bin", "go-server.exe"); } [GlobalCleanup] @@ -201,6 +211,102 @@ Task RunChannelAsync(IReadOnlySessionChannel channel) await Task.WhenAll(serverTask, clientTask); } + + [Benchmark] + public async Task CsharpToGoAsync() + { + using var goServer = GoServerProcess.Start(_goServerPath!); + + var clientTask = Task.Run(async () => + { + var sock = new Socket(SocketType.Stream, ProtocolType.Tcp); + await sock.ConnectAsync(new IPEndPoint(IPAddress.Loopback, goServer.Port)); + + var opt = new SessionOptions + { + EnableKeepAlive = false, + DefaultChannelOptions = new SessionChannelOptions + { + MaxDataFrameSize = 1024 * 64, + } + }; + var session = sock!.AsYamuxSession(true, options: opt, keepOpen: false); + session.Start(); + + List channels = new List(); + int iterationsPerStream = (MBs * 32) / Streams; + + for (int i = 0; i < Streams; i++) + { + channels.Add(Task.Run(async () => + { + using var channel = await session.OpenChannelAsync(false); + + for (int j = 0; j < iterationsPerStream; j++) + { + await channel.WriteAsync(_buffer); + } + + var timeout = (await channel.WhenRemoteAckAsync(TimeSpan.FromSeconds(3)) == false); + if (timeout) + { + throw new TimeoutException("Timed out waiting for remote ack"); + } + + channel.Close(); + })); + } + + await Task.WhenAll(channels); + + sock.Close(); + }); + + await clientTask; + } + } + + internal sealed class GoServerProcess : IDisposable + { + private readonly Process _process; + public int Port { get; } + + public static GoServerProcess Start(string exePath) + { + var psi = new ProcessStartInfo(exePath) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + var proc = Process.Start(psi) ?? throw new InvalidOperationException("Failed to start Go server"); + + var line = proc.StandardOutput.ReadLine() ?? ""; + if (!line.StartsWith("LISTENING:")) + { + proc.Kill(); + throw new InvalidOperationException($"Unexpected output: {line}"); + } + var port = int.Parse(line.AsSpan("LISTENING:".Length)); + return new GoServerProcess(proc, port); + } + + private GoServerProcess(Process process, int port) + { + _process = process; + Port = port; + } + + public void Dispose() + { + if (!_process.HasExited) + { + _process.Kill(); + _process.WaitForExit(2000); + } + _process.Dispose(); + } } } } diff --git a/benchmarks/build-go-server.ps1 b/benchmarks/build-go-server.ps1 new file mode 100644 index 0000000..998c90b --- /dev/null +++ b/benchmarks/build-go-server.ps1 @@ -0,0 +1,19 @@ +param([switch]$Force) + +$ErrorActionPreference = "Stop" +$targetDir = Join-Path $PSScriptRoot "bin" +if (!(Test-Path $targetDir)) { New-Item -ItemType Directory -Path $targetDir -Force | Out-Null } + +$serverExe = Join-Path $targetDir "go-server.exe" +if (!(Test-Path $serverExe) -or $Force) { + Push-Location (Join-Path $PSScriptRoot "go-server") + try { + go mod tidy + go build -o $serverExe . + Write-Host "Built: $serverExe" + } finally { + Pop-Location + } +} else { + Write-Host "Already exists: $serverExe (use -Force to rebuild)" +} \ No newline at end of file diff --git a/benchmarks/compare.ps1 b/benchmarks/compare.ps1 new file mode 100644 index 0000000..0fb4726 --- /dev/null +++ b/benchmarks/compare.ps1 @@ -0,0 +1,95 @@ +param([switch]$Full) + +$ErrorActionPreference = "Stop" + +Write-Host "=== Building Go benchmark ===" -ForegroundColor Cyan +Push-Location (Join-Path $PSScriptRoot "go-benchmark") +try { + go build -o (Join-Path $PSScriptRoot "bin\go-benchmark.exe") . +} finally { + Pop-Location +} + +Write-Host "=== Running Go benchmark ===" -ForegroundColor Cyan +$goResults = @{} +$goOutput = & (Join-Path $PSScriptRoot "bin\go-benchmark.exe") 2>&1 +$goOutput | ForEach-Object { Write-Host $_ -ForegroundColor Gray } + +# Parse Go results +$goOutput | Select-String '^(Go\w+)\s+(\d+)\s+(\d+)\s+([\d.]+[a-z]?)\s+([\d.]+)' | ForEach-Object { + $method = $_.Matches.Groups[1].Value + $mbs = $_.Matches.Groups[2].Value + $streams = $_.Matches.Groups[3].Value + $mbps = $_.Matches.Groups[5].Value + $goResults["$method|$mbs|$streams"] = [double]$mbps +} + +Write-Host "`n=== Running C# benchmark (filtered) ===" -ForegroundColor Cyan +$csResults = @{} +$tempFile = Join-Path $env:TEMP "cs-bench-out.txt" + +if ($Full) { + $csOutput = dotnet run -c Release --project (Join-Path $PSScriptRoot "..\Yamux.Benchmark") -- --filter *CsharpToGo* 2>&1 +} else { + $csOutput = dotnet run -c Release --project (Join-Path $PSScriptRoot "..\Yamux.Benchmark") -- --filter *CsharpToGo* 2>&1 +} +$csOutput | Out-File $tempFile + +# Parse C# results - look for Mean values +$currentMethod = "" +$currentMBs = 0 +$currentStreams = 0 +Get-Content $tempFile | ForEach-Object { + if ($_ -match 'CsharpToGoAsync.*MBs:\s*(\d+).*Streams:\s*(\d+)') { + $currentMBs = [int]$Matches[1] + $currentStreams = [int]$Matches[2] + $currentMethod = "CsharpToGo" + } + if ($_ -match 'Mean\s*=\s*([\d.]+)\s*ms') { + $meanMs = [double]$Matches[1] + if ($currentMethod -eq "CsharpToGo" -and $currentMBs -gt 0) { + $totalMB = $currentMBs * $currentStreams + $mbps = ($totalMB) / ($meanMs / 1000.0) + $csResults["$currentMethod|$currentMBs|$currentStreams"] = [double]$mbps + } + } +} + +Write-Host "`n============================================" -ForegroundColor Green +Write-Host " SIDE-BY-SIDE: C# Yamux vs Go Yamux" -ForegroundColor Green +Write-Host "============================================" -ForegroundColor Green +Write-Host "" + +# Find all unique parameter combinations +$allKeys = @{} +$goResults.Keys | ForEach-Object { $allKeys[$_] = $true } +$csResults.Keys | ForEach-Object { $allKeys[$_] = $true } + +Write-Host ("{0,-10} {1,5} {2,8} {3,15} {4,15} {5,15}" -f "Method", "MBs", "Streams", "Go MB/s", "C# MB/s", "Ratio C#/Go") +Write-Host ("-" * 70) + +$sortedKeys = $allKeys.Keys | Sort-Object +foreach ($key in $sortedKeys) { + $parts = $key -split '\|' + $method = $parts[0] + $mbs = $parts[1] + $streams = $parts[2] + + $goVal = $goResults[$key] + $csVal = $csResults[$key] + + $goStr = if ($goVal) { "{0,13:F1}" -f $goVal } else { " N/A" } + $csStr = if ($csVal) { "{0,13:F1}" -f $csVal } else { " N/A" } + + $ratio = "" + if ($goVal -and $csVal -and $goVal -gt 0) { + $ratio = "{0,13:F2}x" -f ($csVal / $goVal) + } elseif ($goVal) { + $ratio = " -" + } + + Write-Host ("{0,-10} {1,5} {2,8} {3,15} {4,15} {5,15}" -f $method, $mbs, $streams, $goStr, $csStr, $ratio) +} + +Write-Host "" +Write-Host "Ratio > 1.0 means C# is faster. Ratio < 1.0 means Go is faster." \ No newline at end of file diff --git a/benchmarks/go-benchmark/go.mod b/benchmarks/go-benchmark/go.mod new file mode 100644 index 0000000..493b072 --- /dev/null +++ b/benchmarks/go-benchmark/go.mod @@ -0,0 +1,5 @@ +module github.com/paulb/Yamux/benchmarks/go-benchmark + +go 1.19 + +require github.com/hashicorp/yamux v0.1.1 diff --git a/benchmarks/go-benchmark/go.sum b/benchmarks/go-benchmark/go.sum new file mode 100644 index 0000000..956ec5d --- /dev/null +++ b/benchmarks/go-benchmark/go.sum @@ -0,0 +1,2 @@ +github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= +github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= diff --git a/benchmarks/go-benchmark/main.go b/benchmarks/go-benchmark/main.go new file mode 100644 index 0000000..6fa3c26 --- /dev/null +++ b/benchmarks/go-benchmark/main.go @@ -0,0 +1,235 @@ +package main + +import ( + "fmt" + "io" + "net" + "os" + "time" + + "github.com/hashicorp/yamux" +) + +const dataChunkSize = 32 * 1024 // 32KB, matching C# benchmark + +type result struct { + Label string + MBs int + Streams int + Duration time.Duration + MBps float64 +} + +func main() { + mbsValues := []int{1, 50, 500} + streamsValues := []int{1, 5, 20} + + results := []result{} + + for _, mbs := range mbsValues { + for _, streams := range streamsValues { + d, err := runRawTCP(mbs, streams) + if err == nil { + results = append(results, d) + } + + d, err = runGoYamux(mbs, streams) + if err == nil { + results = append(results, d) + } + } + } + + // Print results table + fmt.Printf("\n%-20s %5s %8s %12s %10s\n", "Method", "MBs", "Streams", "Duration", "MB/s") + fmt.Println("-------------------------------------------------------------") + for _, r := range results { + fmt.Printf("%-20s %5d %8d %12v %10.1f\n", r.Label, r.MBs, r.Streams, r.Duration.Round(time.Millisecond), r.MBps) + } +} + +func runRawTCP(mbs, streams int) (result, error) { + totalMB := mbs * streams + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return result{}, err + } + port := listener.Addr().(*net.TCPAddr).Port + + serverErr := make(chan error, 1) + go func() { + conn, err := listener.Accept() + if err != nil { + serverErr <- err + return + } + defer conn.Close() + listener.Close() + + totalBytes := totalMB * 1024 * 1024 + buf := make([]byte, dataChunkSize) + written := 0 + for written < totalBytes { + n := len(buf) + if remaining := totalBytes - written; remaining < n { + n = remaining + } + if _, err := conn.Write(buf[:n]); err != nil { + serverErr <- err + return + } + written += n + } + serverErr <- nil + }() + + conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", port)) + if err != nil { + return result{}, err + } + + start := time.Now() + totalBytes := totalMB * 1024 * 1024 + buf := make([]byte, dataChunkSize) + read := 0 + for read < totalBytes { + n, err := conn.Read(buf) + if err != nil { + conn.Close() + return result{}, err + } + read += n + } + elapsed := time.Since(start) + conn.Close() + + if err := <-serverErr; err != nil { + return result{}, err + } + + mbps := float64(totalMB) / elapsed.Seconds() + return result{ + Label: "GoRawTCP", + MBs: mbs, + Streams: streams, + Duration: elapsed, + MBps: mbps, + }, nil +} + +func runGoYamux(mbs, streams int) (result, error) { + totalMB := mbs * streams + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return result{}, err + } + port := listener.Addr().(*net.TCPAddr).Port + + serverErr := make(chan error, 1) + go func() { + conn, err := listener.Accept() + if err != nil { + serverErr <- err + return + } + listener.Close() + + config := yamux.DefaultConfig() + config.LogOutput = io.Discard + config.EnableKeepAlive = false + session, err := yamux.Server(conn, config) + if err != nil { + serverErr <- err + return + } + + // Accept streams and drain them + done := make(chan struct{}, streams) + for i := 0; i < streams; i++ { + stream, err := session.AcceptStream() + if err != nil { + serverErr <- err + return + } + go func(s *yamux.Stream) { + io.CopyBuffer(io.Discard, s, make([]byte, dataChunkSize)) + s.Close() + done <- struct{}{} + }(stream) + } + + // Wait for all streams to finish + for i := 0; i < streams; i++ { + <-done + } + serverErr <- nil + }() + + conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", port)) + if err != nil { + return result{}, err + } + + config := yamux.DefaultConfig() + config.LogOutput = io.Discard + config.EnableKeepAlive = false + session, err := yamux.Client(conn, config) + if err != nil { + return result{}, err + } + + iterationsPerStream := (totalMB * 1024 * 1024 / dataChunkSize) / streams + buf := make([]byte, dataChunkSize) + + start := time.Now() + errCh := make(chan error, streams) + + for i := 0; i < streams; i++ { + go func() { + stream, err := session.OpenStream() + if err != nil { + errCh <- err + return + } + for j := 0; j < iterationsPerStream; j++ { + if _, err := stream.Write(buf); err != nil { + errCh <- err + return + } + } + stream.Close() + errCh <- nil + }() + } + + for i := 0; i < streams; i++ { + if err := <-errCh; err != nil { + return result{}, err + } + } + elapsed := time.Since(start) + + conn.Close() + session.Close() + + if err := <-serverErr; err != nil { + return result{}, err + } + + mbps := float64(totalMB) / elapsed.Seconds() + return result{ + Label: "GoYamux", + MBs: mbs, + Streams: streams, + Duration: elapsed, + MBps: mbps, + }, nil +} + +func init() { + // Ensure we don't get "too many open files" errors + // by keeping the listener port free + os.Setenv("GODEBUG", os.Getenv("GODEBUG")+",netdns=go") +} \ No newline at end of file diff --git a/benchmarks/go-server/go.mod b/benchmarks/go-server/go.mod new file mode 100644 index 0000000..923add9 --- /dev/null +++ b/benchmarks/go-server/go.mod @@ -0,0 +1,5 @@ +module github.com/paulb/Yamux/benchmarks/go-server + +go 1.19 + +require github.com/hashicorp/yamux v0.1.1 diff --git a/benchmarks/go-server/go.sum b/benchmarks/go-server/go.sum new file mode 100644 index 0000000..956ec5d --- /dev/null +++ b/benchmarks/go-server/go.sum @@ -0,0 +1,2 @@ +github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= +github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= diff --git a/benchmarks/go-server/main.go b/benchmarks/go-server/main.go new file mode 100644 index 0000000..b59f55d --- /dev/null +++ b/benchmarks/go-server/main.go @@ -0,0 +1,50 @@ +package main + +import ( + "fmt" + "io" + "net" + "os" + + "github.com/hashicorp/yamux" +) + +func main() { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + fmt.Fprintf(os.Stderr, "listen error: %v\n", err) + os.Exit(1) + } + + port := listener.Addr().(*net.TCPAddr).Port + fmt.Printf("LISTENING:%d\n", port) + os.Stdout.Sync() + + conn, err := listener.Accept() + if err != nil { + fmt.Fprintf(os.Stderr, "accept error: %v\n", err) + os.Exit(1) + } + + config := yamux.DefaultConfig() + config.LogOutput = io.Discard + config.EnableKeepAlive = false + session, err := yamux.Server(conn, config) + if err != nil { + fmt.Fprintf(os.Stderr, "yamux server error: %v\n", err) + os.Exit(1) + } + + for { + stream, err := session.AcceptStream() + if err != nil { + break + } + go func() { + io.CopyBuffer(io.Discard, stream, make([]byte, 64*1024)) + stream.Close() + }() + } + + os.Exit(0) +} \ No newline at end of file