[User Story] Create Multi-layer Validation Pipeline

[prodigy]

User Story

As a conflict resolution system, I want to validate all AI-generated conflict resolutions through multiple security and correctness checks, so that only safe and correct code changes are applied to repositories.

Acceptance Criteria

• Implement syntax validation for all major programming languages
• Integrate security scanning for common vulnerabilities (Semgrep/CodeQL)
• Perform semantic validation to ensure logical correctness
• Execute unit tests in isolated environment when available
• Validate that resolution maintains existing code style
• Check for breaking changes in APIs/interfaces
• Support custom validation rules per repository
• Generate detailed validation reports
• Implement fallback strategies for validation failures
• Track validation metrics and success rates

Technical Implementation

Validation Pipeline Architecture

// pkg/validation/pipeline.go
package validation

import (
    "context"
    "sync"
)

type ValidationPipeline struct {
    validators   []Validator
    config       *PipelineConfig
    metrics      *MetricsCollector
    fallback     *FallbackStrategy
}

type PipelineConfig struct {
    RunInParallel      bool
    StopOnFirstFailure bool
    TimeoutPerStage    time.Duration
    EnabledValidators  []string
}

type Validator interface {
    Name() string
    Validate(ctx context.Context, input ValidationInput) ValidationResult
    Priority() int
}

type ValidationInput struct {
    ConflictType   string
    OriginalCode   string
    ResolvedCode   string
    FilePath       string
    FileType       string
    Context        map[string]interface{}
}

type ValidationResult struct {
    Valid       bool
    Score       float64
    Issues      []ValidationIssue
    Suggestions []string
    Metadata    map[string]interface{}
}

func (p *ValidationPipeline) Validate(ctx context.Context, input ValidationInput) (*PipelineResult, error) {
    results := make([]ValidationResult, 0, len(p.validators))
    
    // Sort validators by priority
    validators := p.sortValidators()
    
    if p.config.RunInParallel {
        results = p.runParallel(ctx, validators, input)
    } else {
        results = p.runSequential(ctx, validators, input)
    }
    
    // Aggregate results
    finalResult := p.aggregateResults(results)
    
    // Apply fallback if needed
    if !finalResult.Valid && p.fallback != nil {
        fallbackResult := p.fallback.Apply(input, finalResult)
        return fallbackResult, nil
    }
    
    // Record metrics
    p.metrics.RecordValidation(input.FileType, finalResult)
    
    return finalResult, nil
}

func (p *ValidationPipeline) runParallel(ctx context.Context, validators []Validator, input ValidationInput) []ValidationResult {
    var wg sync.WaitGroup
    results := make([]ValidationResult, len(validators))
    
    for i, validator := range validators {
        wg.Add(1)
        go func(idx int, v Validator) {
            defer wg.Done()
            
            ctx, cancel := context.WithTimeout(ctx, p.config.TimeoutPerStage)
            defer cancel()
            
            results[idx] = v.Validate(ctx, input)
        }(i, validator)
    }
    
    wg.Wait()
    return results
}

Syntax Validation

// pkg/validation/syntax_validator.go
type SyntaxValidator struct {
    parsers map[string]LanguageParser
}

type LanguageParser interface {
    Parse(code string) error
    GetAST(code string) (interface{}, error)
}

func (v *SyntaxValidator) Validate(ctx context.Context, input ValidationInput) ValidationResult {
    parser, exists := v.parsers[input.FileType]
    if !exists {
        return ValidationResult{
            Valid: true,
            Score: 0.5,
            Issues: []ValidationIssue{{
                Type:     "warning",
                Message:  fmt.Sprintf("No syntax validator for %s", input.FileType),
                Severity: SeverityLow,
            }},
        }
    }
    
    // Parse resolved code
    err := parser.Parse(input.ResolvedCode)
    if err != nil {
        return ValidationResult{
            Valid: false,
            Score: 0.0,
            Issues: []ValidationIssue{{
                Type:     "syntax_error",
                Message:  err.Error(),
                Line:     extractLineNumber(err),
                Severity: SeverityCritical,
            }},
            Suggestions: v.getSyntaxSuggestions(err, input),
        }
    }
    
    // Additional AST-based checks
    ast, err := parser.GetAST(input.ResolvedCode)
    if err == nil {
        issues := v.performASTChecks(ast, input)
        if len(issues) > 0 {
            return ValidationResult{
                Valid:  false,
                Score:  0.3,
                Issues: issues,
            }
        }
    }
    
    return ValidationResult{
        Valid: true,
        Score: 1.0,
    }
}

// Language-specific parsers
type PythonParser struct {
    astModule *python.ASTModule
}

func (p *PythonParser) Parse(code string) error {
    _, err := p.astModule.Parse(code, "exec", python.PyCFDontImplyDedent)
    return err
}

type GoParser struct{}

func (p *GoParser) Parse(code string) error {
    fset := token.NewFileSet()
    _, err := parser.ParseFile(fset, "", code, parser.AllErrors)
    return err
}

type JavaScriptParser struct {
    parser *js.Parser
}

func (p *JavaScriptParser) Parse(code string) error {
    _, err := p.parser.ParseFile(nil, "", code, 0)
    return err
}

Security Validation

// pkg/validation/security_validator.go
type SecurityValidator struct {
    scanner  *SecurityScanner
    rules    *RuleEngine
    patterns *PatternMatcher
}

type SecurityScanner interface {
    Scan(code string, language string) []SecurityIssue
}

type SemgrepScanner struct {
    configPath string
    rules      []Rule
}

func (s *SemgrepScanner) Scan(code string, language string) []SecurityIssue {
    // Create temporary file
    tmpFile := createTempFile(code)
    defer os.Remove(tmpFile)
    
    // Run semgrep
    cmd := exec.Command("semgrep", 
        "--config", s.configPath,
        "--json",
        "--no-git-ignore",
        tmpFile,
    )
    
    output, err := cmd.Output()
    if err != nil {
        return []SecurityIssue{{
            Type:        "scan_error",
            Description: err.Error(),
            Severity:    "high",
        }}
    }
    
    // Parse results
    var results SemgrepResults
    json.Unmarshal(output, &results)
    
    return s.convertToSecurityIssues(results)
}

func (v *SecurityValidator) Validate(ctx context.Context, input ValidationInput) ValidationResult {
    issues := []ValidationIssue{}
    
    // Run security scanner
    securityIssues := v.scanner.Scan(input.ResolvedCode, input.FileType)
    for _, issue := range securityIssues {
        issues = append(issues, ValidationIssue{
            Type:     "security",
            Message:  issue.Description,
            Line:     issue.Line,
            Severity: v.mapSeverity(issue.Severity),
            Rule:     issue.RuleID,
        })
    }
    
    // Check for hardcoded secrets
    secrets := v.patterns.FindSecrets(input.ResolvedCode)
    for _, secret := range secrets {
        issues = append(issues, ValidationIssue{
            Type:     "secret",
            Message:  fmt.Sprintf("Potential %s found", secret.Type),
            Line:     secret.Line,
            Severity: SeverityCritical,
        })
    }
    
    // Custom security rules
    customIssues := v.rules.Apply(input)
    issues = append(issues, customIssues...)
    
    // Calculate score based on severity
    score := v.calculateSecurityScore(issues)
    
    return ValidationResult{
        Valid:  len(issues) == 0,
        Score:  score,
        Issues: issues,
        Metadata: map[string]interface{}{
            "scanner_version": v.scanner.Version(),
            "rules_count":     len(v.rules.GetRules()),
        },
    }
}

Semantic Validation

// pkg/validation/semantic_validator.go
type SemanticValidator struct {
    analyzer *SemanticAnalyzer
    checker  *TypeChecker
}

func (v *SemanticValidator) Validate(ctx context.Context, input ValidationInput) ValidationResult {
    // Parse original and resolved code
    originalAST, err1 := v.analyzer.Parse(input.OriginalCode, input.FileType)
    resolvedAST, err2 := v.analyzer.Parse(input.ResolvedCode, input.FileType)
    
    if err1 != nil || err2 != nil {
        return ValidationResult{
            Valid: false,
            Score: 0.0,
            Issues: []ValidationIssue{{
                Type:    "parse_error",
                Message: "Failed to parse code for semantic analysis",
            }},
        }
    }
    
    issues := []ValidationIssue{}
    
    // Check for breaking changes
    breakingChanges := v.analyzer.FindBreakingChanges(originalAST, resolvedAST)
    for _, change := range breakingChanges {
        issues = append(issues, ValidationIssue{
            Type:     "breaking_change",
            Message:  change.Description,
            Severity: SeverityHigh,
        })
    }
    
    // Type checking (for typed languages)
    if v.checker.SupportsLanguage(input.FileType) {
        typeErrors := v.checker.Check(resolvedAST)
        for _, err := range typeErrors {
            issues = append(issues, ValidationIssue{
                Type:     "type_error",
                Message:  err.Message,
                Line:     err.Line,
                Severity: SeverityHigh,
            })
        }
    }
    
    // Check variable usage
    unusedVars := v.analyzer.FindUnusedVariables(resolvedAST)
    for _, varName := range unusedVars {
        issues = append(issues, ValidationIssue{
            Type:     "unused_variable",
            Message:  fmt.Sprintf("Variable '%s' is defined but never used", varName),
            Severity: SeverityLow,
        })
    }
    
    return ValidationResult{
        Valid:  len(issues) == 0,
        Score:  v.calculateSemanticScore(issues),
        Issues: issues,
    }
}

Test Execution Validation

// pkg/validation/test_validator.go
type TestValidator struct {
    executor  *TestExecutor
    container *ContainerRuntime
    timeout   time.Duration
}

type TestExecutor interface {
    RunTests(ctx context.Context, projectPath string, testPattern string) TestResults
}

func (v *TestValidator) Validate(ctx context.Context, input ValidationInput) ValidationResult {
    // Check if tests exist
    testFiles := v.findTestFiles(input.FilePath)
    if len(testFiles) == 0 {
        return ValidationResult{
            Valid: true,
            Score: 0.7,
            Issues: []ValidationIssue{{
                Type:     "info",
                Message:  "No tests found for this file",
                Severity: SeverityInfo,
            }},
        }
    }
    
    // Create isolated test environment
    env := v.container.CreateTestEnvironment(IsolatedEnvironment{
        BaseImage:    v.getTestImage(input.FileType),
        Code:         input.ResolvedCode,
        TestFiles:    testFiles,
        Timeout:      v.timeout,
        MemoryLimit:  "512m",
        CPULimit:     "1.0",
        NetworkMode:  "none",
    })
    defer env.Cleanup()
    
    // Run tests
    results := v.executor.RunTests(ctx, env.Path, v.getTestPattern(input.FileType))
    
    // Analyze results
    issues := []ValidationIssue{}
    for _, failure := range results.Failures {
        issues = append(issues, ValidationIssue{
            Type:     "test_failure",
            Message:  failure.Message,
            Severity: SeverityHigh,
            Metadata: map[string]interface{}{
                "test_name": failure.TestName,
                "duration":  failure.Duration,
            },
        })
    }
    
    return ValidationResult{
        Valid: results.Failed == 0,
        Score: float64(results.Passed) / float64(results.Total),
        Issues: issues,
        Metadata: map[string]interface{}{
            "tests_run":    results.Total,
            "tests_passed": results.Passed,
            "tests_failed": results.Failed,
            "duration":     results.Duration,
        },
    }
}

Fallback Strategies

// pkg/validation/fallback.go
type FallbackStrategy struct {
    strategies []FallbackOption
}

type FallbackOption interface {
    CanApply(input ValidationInput, result *PipelineResult) bool
    Apply(input ValidationInput) *Resolution
}

type SafeMergeStrategy struct{}

func (s *SafeMergeStrategy) Apply(input ValidationInput) *Resolution {
    // Take the incoming changes (usually safer)
    return &Resolution{
        Code:   input.Context["incoming_code"].(string),
        Method: "safe_merge_incoming",
        Reason: "Validation failed, using incoming changes as safer option",
    }
}

type ManualResolutionStrategy struct {
    notifier *Notifier
}

func (s *ManualResolutionStrategy) Apply(input ValidationInput) *Resolution {
    // Mark for manual resolution
    s.notifier.NotifyManualResolutionRequired(input)
    
    return &Resolution{
        Code:   input.OriginalCode,
        Method: "manual_required",
        Reason: "Complex conflict requires manual intervention",
    }
}

Architecture References

Multi-layer Validation Design

Reference: /docs/02-system-components.md:309-343

The validation system implements comprehensive checks:

async def validate_resolution(self, conflict: Conflict, solution: str) -> ValidationResult:
    """Multi-layer validation of conflict resolution"""
    results = []
    
    # 1. Syntax validation
    syntax_valid = await self._validate_syntax(conflict.file_path, solution)
    results.append(("syntax", syntax_valid))
    
    # 2. Security scan
    security_valid = await self._security_scan(solution)
    results.append(("security", security_valid))
    
    # 3. Semantic validation
    semantic_valid = await self._validate_semantics(conflict, solution)
    results.append(("semantics", semantic_valid))
    
    # 4. Test execution (if tests exist)
    if await self._has_tests(conflict.file_path):
        test_valid = await self._run_tests(conflict.file_path, solution)
        results.append(("tests", test_valid))

Security Scanning Implementation

Reference: /docs/02-system-components.md:351-367

async def _security_scan(self, code: str) -> bool:
    """Scan for security vulnerabilities"""
    suspicious_patterns = [
        r'exec\s*\(',
        r'eval\s*\(',
        r'__import__',
        r'subprocess\.call.*shell=True',
        r'os\.system',
        r'pickle\.loads',
    ]

Container-based Test Execution

Reference: /docs/02-system-components.md:813-825

Worker containers are configured for secure test execution:

apiVersion: batch/v1
kind: Job
spec:
  template:
    spec:
      securityContext:
        runAsNonRoot: true
        runAsUser: 1000
        fsGroup: 1000
        seccompProfile:
          type: Localhost
          localhostProfile: claude-worker.json

Dependencies

• Semgrep: Security scanning
• tree-sitter: Multi-language parsing
• Docker/Kubernetes: Isolated test execution
• PostgreSQL: Validation metrics storage

Definition of Done

• Unit tests cover all validators with 90%+ coverage
• Integration tests verify pipeline with real conflicts
• Security scanner detects OWASP Top 10 vulnerabilities
• Test execution works in isolated containers
• Validation reports are comprehensive and actionable
• Performance: <5s for typical validation pipeline
• Documentation includes custom rule examples

Effort Estimate

21 Story Points - Very complex with multiple validation layers

Labels

• backend
• validation
• security
• testing
• epic-5