diff --git a/cmd/gomu/main.go b/cmd/gomu/main.go index 29c641b..ded3bc9 100644 --- a/cmd/gomu/main.go +++ b/cmd/gomu/main.go @@ -2,10 +2,13 @@ package main import ( + "context" "fmt" "io" "os" + "os/signal" "strings" + "syscall" "text/tabwriter" "github.com/sivchari/gomu/pkg/gomu" @@ -179,7 +182,12 @@ func listMutators(w io.Writer) error { } func main() { - if err := rootCmd.Execute(); err != nil { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + err := rootCmd.ExecuteContext(ctx) + + stop() + + if err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) os.Exit(1) } diff --git a/internal/execution/engine.go b/internal/execution/engine.go index 635baba..1adfa60 100644 --- a/internal/execution/engine.go +++ b/internal/execution/engine.go @@ -2,17 +2,29 @@ package execution import ( + "bufio" + "bytes" "context" "errors" "fmt" + "os" "os/exec" "path/filepath" + "strconv" + "strings" "sync" "time" "github.com/sivchari/gomu/internal/mutation" ) +const maxCommandOutputBytes = 1 << 20 +const defaultMaxWorkers = 4 +const defaultChildMaxRSSMiB = 2048 + +var outputTruncatedMarker = []byte("\n[gomu: command output truncated]\n") +var errChildMemoryLimit = errors.New("child process exceeded memory limit") + // Engine handles test execution using overlay-based mutation. type Engine struct { overlay *OverlayMutator @@ -46,33 +58,71 @@ func (e *Engine) RunMutations(mutants []mutation.Mutant) ([]mutation.Result, err // RunMutationsWithOptions executes tests for all mutants in parallel with custom options. func (e *Engine) RunMutationsWithOptions(mutants []mutation.Mutant, workers, timeout int) ([]mutation.Result, error) { + return e.RunMutationsWithContext(context.Background(), mutants, workers, timeout) +} + +// RunMutationsWithContext executes tests for all mutants while honoring caller cancellation. +func (e *Engine) RunMutationsWithContext( + ctx context.Context, + mutants []mutation.Mutant, + workers, timeout int, +) ([]mutation.Result, error) { if len(mutants) == 0 { return nil, nil } - results := make([]mutation.Result, len(mutants)) - resultsChan := make(chan indexedResult, len(mutants)) + if workers < 1 { + workers = 1 + } - var wg sync.WaitGroup + if workers > configuredMaxWorkers() { + workers = configuredMaxWorkers() + } + + seen := make(map[string]struct{}, len(mutants)) + for _, mutant := range mutants { + if _, ok := seen[mutant.ID]; ok { + return nil, fmt.Errorf("duplicate mutant id %q", mutant.ID) + } - semaphore := make(chan struct{}, workers) + seen[mutant.ID] = struct{}{} + } - // Start workers - no file locks needed with overlay approach - for i, mutant := range mutants { + results := make([]mutation.Result, len(mutants)) + resultsChan := make(chan indexedResult, workers) + jobs := make(chan indexedMutant) + + var wg sync.WaitGroup + for range workers { wg.Add(1) - go func(index int, m mutation.Mutant) { + go func() { defer wg.Done() - semaphore <- struct{}{} - - defer func() { <-semaphore }() + for job := range jobs { + result := e.runSingleMutationWithContext(ctx, job.mutant, timeout) - result := e.runSingleMutation(m, timeout) - resultsChan <- indexedResult{index: index, result: result} - }(i, mutant) + select { + case resultsChan <- indexedResult{index: job.index, result: result}: + case <-ctx.Done(): + return + } + } + }() } + go func() { + defer close(jobs) + + for index, mutant := range mutants { + select { + case jobs <- indexedMutant{index: index, mutant: mutant}: + case <-ctx.Done(): + return + } + } + }() + go func() { wg.Wait() close(resultsChan) @@ -82,9 +132,18 @@ func (e *Engine) RunMutationsWithOptions(mutants []mutation.Mutant, workers, tim results[indexedRes.index] = indexedRes.result } + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("mutation execution canceled: %w", err) + } + return results, nil } +type indexedMutant struct { + index int + mutant mutation.Mutant +} + type indexedResult struct { index int result mutation.Result @@ -92,11 +151,22 @@ type indexedResult struct { // runSingleMutation executes tests for a single mutant using overlay. func (e *Engine) runSingleMutation(mutant mutation.Mutant, timeout int) mutation.Result { + return e.runSingleMutationWithContext(context.Background(), mutant, timeout) +} + +func (e *Engine) runSingleMutationWithContext( + ctx context.Context, + mutant mutation.Mutant, + timeout int, +) mutation.Result { result := mutation.Result{ Mutant: mutant, Status: mutation.StatusError, } + mutationCtx, cancel := context.WithTimeout(ctx, time.Duration(timeout)*time.Second) + defer cancel() + // 1. Prepare mutation (create mutated file + overlay.json) mutCtx, err := e.overlay.PrepareMutation(mutant) if err != nil { @@ -112,7 +182,14 @@ func (e *Engine) runSingleMutation(mutant mutation.Mutant, timeout int) mutation }() // 2. Check if the mutated code compiles using overlay - if err := e.checkCompilationWithOverlay(mutCtx); err != nil { + if err := e.checkCompilationWithOverlay(mutationCtx, mutCtx); err != nil { + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, errChildMemoryLimit) { + result.Status = mutation.StatusTimedOut + result.Error = "Mutation execution timed out during compilation" + + return result + } + result.Status = mutation.StatusNotViable result.Error = fmt.Sprintf("Compilation failed: %v", err) result.Output = err.Error() @@ -121,58 +198,52 @@ func (e *Engine) runSingleMutation(mutant mutation.Mutant, timeout int) mutation } // 3. Run tests using overlay - return e.runTestWithOverlay(mutCtx, mutant, timeout) + return e.runTestWithOverlay(mutationCtx, mutCtx, mutant) } // checkCompilationWithOverlay verifies that the mutated code compiles using overlay. -// No timeout is applied because compilation always terminates. -func (e *Engine) checkCompilationWithOverlay(mutCtx *MutationContext) error { +func (e *Engine) checkCompilationWithOverlay(ctx context.Context, mutCtx *MutationContext) error { // Get the directory containing the original file for compilation compileDir := filepath.Dir(mutCtx.OriginalPath) // Build the entire package with overlay to properly resolve dependencies - cmd := exec.Command("go", "build", "-overlay="+mutCtx.OverlayPath, ".") - cmd.Dir = compileDir - - output, err := cmd.CombinedOutput() + output, err := runBoundedCommand(ctx, compileDir, "go", "build", "-overlay="+mutCtx.OverlayPath, ".") if err != nil { - return fmt.Errorf("compilation error: %s", string(output)) + return fmt.Errorf("compilation error: %s: %w", output, err) } return nil } // runTestWithOverlay runs tests using the overlay configuration. -func (e *Engine) runTestWithOverlay(mutCtx *MutationContext, mutant mutation.Mutant, timeout int) mutation.Result { +func (e *Engine) runTestWithOverlay( + ctx context.Context, + mutCtx *MutationContext, + mutant mutation.Mutant, +) mutation.Result { result := mutation.Result{ Mutant: mutant, Status: mutation.StatusError, } - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) - defer cancel() - // Get the directory containing the original file for running tests testDir := filepath.Dir(mutCtx.OriginalPath) - cmd := exec.CommandContext(ctx, "go", "test", "-overlay="+mutCtx.OverlayPath, ".") - cmd.Dir = testDir - - output, err := cmd.CombinedOutput() + output, err := runBoundedCommand(ctx, testDir, "go", "test", "-overlay="+mutCtx.OverlayPath, ".") // Analyze test results - if errors.Is(ctx.Err(), context.DeadlineExceeded) { + if errors.Is(ctx.Err(), context.DeadlineExceeded) || errors.Is(err, errChildMemoryLimit) { result.Status = mutation.StatusTimedOut result.Error = "Test execution timed out" return result } - result.Output = string(output) + result.Output = output if err != nil { // Tests failed - check if it's because the mutant was killed - if cmd.ProcessState != nil && cmd.ProcessState.ExitCode() != 0 { + if !errors.Is(err, context.Canceled) { result.Status = mutation.StatusKilled } else { result.Status = mutation.StatusError @@ -185,3 +256,149 @@ func (e *Engine) runTestWithOverlay(mutCtx *MutationContext, mutant mutation.Mut return result } + +type limitedBuffer struct { + buffer bytes.Buffer + remaining int + truncated bool +} + +func newLimitedBuffer(limit int) *limitedBuffer { + return &limitedBuffer{remaining: limit} +} + +func (b *limitedBuffer) Write(data []byte) (int, error) { + written := len(data) + if len(data) > b.remaining { + data = data[:b.remaining] + b.truncated = true + } + + _, _ = b.buffer.Write(data) + b.remaining -= len(data) + + return written, nil +} + +func (b *limitedBuffer) String() string { + if b.truncated { + return b.buffer.String() + string(outputTruncatedMarker) + } + + return b.buffer.String() +} + +func runBoundedCommand(ctx context.Context, dir, name string, args ...string) (string, error) { + cmd := newProcessGroupCommand(name, args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "PWD="+dir, "GOMEMLIMIT="+childMemoryLimit()) + output := newLimitedBuffer(maxCommandOutputBytes) + cmd.Stdout = output + cmd.Stderr = output + + if err := cmd.Start(); err != nil { + return output.String(), err + } + + done := make(chan error, 1) + stopMonitor := make(chan struct{}) + memoryLimit := childMaxRSSBytes() + memoryExceeded := make(chan struct{}, 1) + + go monitorProcessGroup(cmd.Process.Pid, memoryLimit, stopMonitor, memoryExceeded) + + defer close(stopMonitor) + + go func() { done <- cmd.Wait() }() + + select { + case err := <-done: + return output.String(), err + case <-ctx.Done(): + killProcessGroup(cmd) + <-done + + return output.String(), fmt.Errorf("command canceled: %w", ctx.Err()) + case <-memoryExceeded: + killProcessGroup(cmd) + <-done + + return output.String(), errChildMemoryLimit + } +} + +func configuredMaxWorkers() int { + value, err := strconv.Atoi(os.Getenv("GOMU_MAX_WORKERS")) + if err == nil && value > 0 { + return value + } + + return defaultMaxWorkers +} + +func childMaxRSSBytes() int64 { + value, err := strconv.ParseInt(os.Getenv("GOMU_CHILD_MAX_RSS_MIB"), 10, 64) + if err == nil && value > 0 { + return value * 1024 * 1024 + } + + return defaultChildMaxRSSMiB * 1024 * 1024 +} + +func monitorProcessGroup(pid int, limit int64, stop <-chan struct{}, exceeded chan<- struct{}) { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-stop: + return + case <-ticker.C: + rss, err := processGroupRSS(pid) + if err == nil && rss > limit { + exceeded <- struct{}{} + + return + } + } + } +} + +func processGroupRSS(pgid int) (int64, error) { + output, err := exec.Command("ps", "-axo", "pgid=,rss=").Output() + if err != nil { + return 0, fmt.Errorf("read process RSS: %w", err) + } + + var total int64 + + scanner := bufio.NewScanner(strings.NewReader(string(output))) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + + if len(fields) != 2 { + continue + } + + group, groupErr := strconv.Atoi(fields[0]) + + rss, rssErr := strconv.ParseInt(fields[1], 10, 64) + if groupErr == nil && rssErr == nil && group == pgid { + total += rss * 1024 + } + } + + if err := scanner.Err(); err != nil { + return 0, fmt.Errorf("scan process RSS: %w", err) + } + + return total, nil +} + +func childMemoryLimit() string { + if value := os.Getenv("GOMU_CHILD_GOMEMLIMIT"); value != "" { + return value + } + + return "2GiB" +} diff --git a/internal/execution/engine_test.go b/internal/execution/engine_test.go index a1cd766..0565cc2 100644 --- a/internal/execution/engine_test.go +++ b/internal/execution/engine_test.go @@ -1,6 +1,7 @@ package execution import ( + "context" "os" "path/filepath" "strings" @@ -421,7 +422,7 @@ func TestCheckCompilationWithOverlay(t *testing.T) { } defer engine.overlay.CleanupMutation(ctx) - err = engine.checkCompilationWithOverlay(ctx) + err = engine.checkCompilationWithOverlay(context.Background(), ctx) if err != nil { t.Errorf("unexpected compilation error: %v", err) } diff --git a/internal/execution/overlay.go b/internal/execution/overlay.go index 63e30a7..271df28 100644 --- a/internal/execution/overlay.go +++ b/internal/execution/overlay.go @@ -49,9 +49,9 @@ func NewOverlayMutator() (*OverlayMutator, error) { // PrepareMutation prepares the mutation execution by creating mutated file and overlay.json. func (om *OverlayMutator) PrepareMutation(mutant mutation.Mutant) (*MutationContext, error) { - // Create unique directory for this mutant - mutantDir := filepath.Join(om.baseDir, fmt.Sprintf("mutant_%s", mutant.ID)) - if err := os.MkdirAll(mutantDir, 0750); err != nil { + // Use filesystem-generated uniqueness; report IDs are not filesystem locks. + mutantDir, err := os.MkdirTemp(om.baseDir, "mutant-*") + if err != nil { return nil, fmt.Errorf("failed to create mutant directory: %w", err) } diff --git a/internal/execution/process_other.go b/internal/execution/process_other.go new file mode 100644 index 0000000..706d17b --- /dev/null +++ b/internal/execution/process_other.go @@ -0,0 +1,15 @@ +//go:build !unix + +package execution + +import "os/exec" + +func newProcessGroupCommand(name string, args ...string) *exec.Cmd { + return exec.Command(name, args...) +} + +func killProcessGroup(cmd *exec.Cmd) { + if cmd.Process != nil { + _ = cmd.Process.Kill() + } +} diff --git a/internal/execution/process_unix.go b/internal/execution/process_unix.go new file mode 100644 index 0000000..1b9b4cf --- /dev/null +++ b/internal/execution/process_unix.go @@ -0,0 +1,24 @@ +//go:build unix + +package execution + +import ( + "os/exec" + "syscall" +) + +func newProcessGroupCommand(name string, args ...string) *exec.Cmd { + cmd := exec.Command(name, args...) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + + return cmd +} + +func killProcessGroup(cmd *exec.Cmd) { + if cmd.Process == nil { + return + } + + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + _ = cmd.Process.Kill() +} diff --git a/internal/execution/process_unix_test.go b/internal/execution/process_unix_test.go new file mode 100644 index 0000000..14c25af --- /dev/null +++ b/internal/execution/process_unix_test.go @@ -0,0 +1,150 @@ +//go:build unix + +package execution + +import ( + "context" + "errors" + "os" + "runtime" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +func TestRunBoundedCommandKillsDescendantsOnTimeout(t *testing.T) { + pidFile := t.TempDir() + "/child.pid" + t.Setenv("CHILD_PID_FILE", pidFile) + + ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond) + defer cancel() + + _, err := runBoundedCommand( + ctx, + "", + "sh", + "-c", + `sh -c 'while :; do sleep 1; done' & echo $! > "$CHILD_PID_FILE"; wait`, + ) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("runBoundedCommand error = %v, want deadline exceeded", err) + } + + pidBytes, err := os.ReadFile(pidFile) + if err != nil { + t.Fatalf("read child pid: %v", err) + } + + pid, err := strconv.Atoi(strings.TrimSpace(string(pidBytes))) + if err != nil { + t.Fatalf("parse child pid: %v", err) + } + + deadline := time.Now().Add(2 * time.Second) + for processExists(pid) && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + + if processExists(pid) { + t.Fatalf("descendant process %d survived command timeout", pid) + } +} + +func TestRunBoundedCommandCapsCombinedOutput(t *testing.T) { + output, err := runBoundedCommand( + context.Background(), + "", + "sh", + "-c", + "yes x | head -c 2097152", + ) + if err != nil { + t.Fatalf("runBoundedCommand: %v", err) + } + + if len(output) > maxCommandOutputBytes+len(outputTruncatedMarker) { + t.Fatalf("output length = %d, exceeds cap", len(output)) + } + + if !strings.HasSuffix(output, string(outputTruncatedMarker)) { + t.Fatalf("truncated output missing marker") + } +} + +func TestRunBoundedCommandSetsChildMemoryLimit(t *testing.T) { + t.Setenv("GOMU_CHILD_GOMEMLIMIT", "768MiB") + + output, err := runBoundedCommand(context.Background(), "", "sh", "-c", `printf %s "$GOMEMLIMIT"`) + if err != nil { + t.Fatalf("runBoundedCommand: %v", err) + } + + if output != "768MiB" { + t.Fatalf("GOMEMLIMIT = %q, want %q", output, "768MiB") + } +} + +func TestResourceLimitsAreConfigurable(t *testing.T) { + t.Setenv("GOMU_MAX_WORKERS", "") + + if got := configuredMaxWorkers(); got != defaultMaxWorkers { + t.Fatalf("default max workers = %d, want %d", got, defaultMaxWorkers) + } + + t.Setenv("GOMU_MAX_WORKERS", "12") + + if got := configuredMaxWorkers(); got != 12 { + t.Fatalf("configured max workers = %d, want 12", got) + } + + t.Setenv("GOMU_CHILD_MAX_RSS_MIB", "768") + + if got := childMaxRSSBytes(); got != 768*1024*1024 { + t.Fatalf("configured RSS limit = %d, want %d", got, 768*1024*1024) + } +} + +func TestProcessGroupRSS(t *testing.T) { + group, err := syscall.Getpgid(os.Getpid()) + if err != nil { + t.Fatalf("get process group: %v", err) + } + + rss, err := processGroupRSS(group) + if err != nil { + t.Fatalf("process group RSS: %v", err) + } + + if rss <= 0 { + t.Fatalf("process group RSS = %d, want positive", rss) + } +} + +func TestRunBoundedCommandStopsOnRSSLimit(t *testing.T) { + if os.Getenv("GOMU_RESOURCE_LIMIT_HELPER") == "1" { + buffer := make([]byte, 32*1024*1024) + buffer[0] = 1 + + time.Sleep(2 * time.Second) + + runtime.KeepAlive(buffer) + + return + } + + t.Setenv("GOMU_CHILD_MAX_RSS_MIB", "1") + t.Setenv("GOMU_RESOURCE_LIMIT_HELPER", "1") + + _, err := runBoundedCommand(context.Background(), "", os.Args[0], "-test.run=TestRunBoundedCommandStopsOnRSSLimit") + if !errors.Is(err, errChildMemoryLimit) { + t.Fatalf("runBoundedCommand error = %v, want RSS limit error", err) + } +} + +func processExists(pid int) bool { + err := syscall.Kill(pid, 0) + + return err == nil || errors.Is(err, syscall.EPERM) +} diff --git a/internal/mutation/engine.go b/internal/mutation/engine.go index c92448b..aac9e66 100644 --- a/internal/mutation/engine.go +++ b/internal/mutation/engine.go @@ -134,10 +134,9 @@ func (e *Engine) GenerateMutants(filePath string) ([]Mutant, error) { // Filter mutants based on type information for i := range mutants { mutants[i].FilePath = filePath - mutants[i].ID = fmt.Sprintf("%s_%d", filePath, len(allMutants)+i) - // Only add mutant if it passes type check if typeChecker == nil || typeChecker.IsValidMutation(node, mutants[i]) { + mutants[i].ID = fmt.Sprintf("%s_%d", filePath, len(allMutants)) allMutants = append(allMutants, mutants[i]) } } diff --git a/pkg/gomu/engine.go b/pkg/gomu/engine.go index 11aceaf..0297f37 100644 --- a/pkg/gomu/engine.go +++ b/pkg/gomu/engine.go @@ -302,7 +302,10 @@ func (e *Engine) Run(ctx context.Context, path string, opts *RunOptions) error { return nil } - allResults, totalMutants, processedFiles := e.processFiles(files, opts) + allResults, totalMutants, processedFiles, err := e.processFiles(ctx, files, opts) + if err != nil { + return err + } if err := e.cleanupAndSave(opts); err != nil { return err @@ -326,7 +329,11 @@ func (e *Engine) Run(ctx context.Context, path string, opts *RunOptions) error { } // processFiles processes all files for mutation testing. -func (e *Engine) processFiles(files []string, opts *RunOptions) ([]mutation.Result, int, int) { +func (e *Engine) processFiles( + ctx context.Context, + files []string, + opts *RunOptions, +) ([]mutation.Result, int, int, error) { var ( allResults []mutation.Result totalMutants int @@ -374,8 +381,12 @@ func (e *Engine) processFiles(files []string, opts *RunOptions) ([]mutation.Resu log.Printf("Generated %d mutants for %s", len(mutants), file) } - results, err := e.executor.RunMutationsWithOptions(mutants, opts.Workers, opts.Timeout) + results, err := e.executor.RunMutationsWithContext(ctx, mutants, opts.Workers, opts.Timeout) if err != nil { + if ctx.Err() != nil { + return nil, totalMutants, processedFiles, fmt.Errorf("mutation run canceled: %w", ctx.Err()) + } + fmt.Printf("(execution error: %v)\n", err) if opts.Verbose { @@ -413,7 +424,7 @@ func (e *Engine) processFiles(files []string, opts *RunOptions) ([]mutation.Resu processedFiles++ } - return allResults, totalMutants, processedFiles + return allResults, totalMutants, processedFiles, nil } // dryRun reports the mutants that would be generated for each file without