feat(roadmap-planner): DORA Lead Time 3-stage attribution (Phase 2 backend)

roadmap-planner/backend/cmd/server/main.go

@@ -80,24 +80,38 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	// Initialize metrics system if enabled
+	// Open the persistent store once and share it across both the
+	// metrics collector (DORA Lead Time needs PR data) and the
+	// team-analytics path. nil when storage is disabled — both consumers
+	// soft-fail in that mode.
+	var sharedStore storage.Store
+	if cfg.Storage.Enabled {
+		s, err := openStore(cfg.Storage)
+		if err != nil {
+			logger.Error("Failed to open store", zap.Error(err))
+		} else if err := s.Migrate(ctx); err != nil {
+			logger.Error("Failed to migrate store", zap.Error(err))
+		} else {
+			logger.Info("Persistent store ready",
+				zap.String("type", cfg.Storage.Type),
+				zap.Int("backfill_days", cfg.Storage.BackfillDays))
+			sharedStore = s
+		}
+	}
 
+	// Initialize metrics system if enabled.
 	if cfg.Metrics.Enabled {
 		logger.Info("Initializing metrics system")
-		err = initMetrics(ctx, router, cfg)
-		if err != nil {
+		if err := initMetrics(ctx, router, cfg, sharedStore); err != nil {
 			logger.Error("Failed to initialize metrics system", zap.Error(err))
 		}
 	}
 
-	// Initialize team-analytics storage + GitHub sync if enabled.
-	//
-	// We open the store once and let it close on process exit; the
-	// GitHub syncer runs on a configurable interval in its own goroutine.
-	// Both are no-ops when the corresponding config blocks are off, so
-	// existing deployments stay unchanged.
-	if cfg.Storage.Enabled {
-		if err := initTeamAnalytics(ctx, router, cfg); err != nil {
+	// Initialize team-analytics + GitHub sync if both storage is open
+	// and the config block is enabled. The store handle is shared with
+	// the metrics collector above.
+	if cfg.Storage.Enabled && sharedStore != nil {
+		if err := initTeamAnalytics(ctx, router, cfg, sharedStore); err != nil {
 			logger.Error("Failed to initialize team analytics", zap.Error(err))
 		}
 	}
@@ -136,8 +150,10 @@ func main() {
 	logger.Info("Server exited")
 }
 
-// initMetrics initializes the metrics system if enabled in config
-func initMetrics(ctx context.Context, router *gin.Engine, cfg *config.Config) error {
+// initMetrics initializes the metrics system if enabled in config.
+// store may be nil (storage disabled) — in that case PR-backed metrics
+// like DORA Lead Time stage attribution silently degrade to empty.
+func initMetrics(ctx context.Context, router *gin.Engine, cfg *config.Config, store storage.Store) error {
 	if cfg.Jira.BaseURL == "" || cfg.Jira.Username == "" || cfg.Jira.Password == "" {
 		logger.Warn("Metrics enabled but Jira credentials not configured in config file")
 		return nil
@@ -153,8 +169,9 @@ func initMetrics(ctx context.Context, router *gin.Engine, cfg *config.Config) er
 		logger.Error("Failed to create Jira client for metrics", zap.Error(err))
 		return err
 	}
-	// Create collector and service
-	collector := metrics.NewCollector(jiraClient, &cfg.Metrics)
+	// Create collector and service. The store handle (may be nil) gives
+	// the collector access to pull_requests rows for DORA Lead Time.
+	collector := metrics.NewCollector(jiraClient, store, &cfg.Metrics)
 	metricsService := metrics.NewService(&cfg.Metrics, collector)
 
 	// Register calculators
@@ -189,22 +206,11 @@ func initMetrics(ctx context.Context, router *gin.Engine, cfg *config.Config) er
 	return nil
 }
 
-// initTeamAnalytics opens the persistent store, runs migrations, mounts
-// the contributions REST endpoints, and (if configured) starts the
-// GitHub sync loop. Failures here are non-fatal — the rest of the app
-// continues to serve roadmap + metrics.
-func initTeamAnalytics(ctx context.Context, router *gin.Engine, cfg *config.Config) error {
-	store, err := openStore(cfg.Storage)
-	if err != nil {
-		return fmt.Errorf("open store: %w", err)
-	}
-	if err := store.Migrate(ctx); err != nil {
-		return fmt.Errorf("migrate: %w", err)
-	}
-	logger.Info("Team analytics store ready",
-		zap.String("type", cfg.Storage.Type),
-		zap.Int("backfill_days", cfg.Storage.BackfillDays))
-
+// initTeamAnalytics mounts the contributions REST endpoints and (if
+// configured) starts the GitHub sync loop, using a store opened by the
+// caller. Failures here are non-fatal — the rest of the app continues
+// to serve roadmap + metrics.
+func initTeamAnalytics(ctx context.Context, router *gin.Engine, cfg *config.Config, store storage.Store) error {
 	service := contributions.NewService(store)
 	pillarMap := contributions.NewPillarMap(cfg.TeamAnalytics)
 	service.SetPillarMap(pillarMap)

roadmap-planner/backend/config.example.yaml

@@ -49,6 +49,15 @@ metrics:
     path: "/metrics"
     namespace: "roadmap"         # Metric prefix: roadmap_dora_*
 
+  # Components dropped from all component dimensions (D6 — v3-era
+  # plugins must not dilute v4 team metrics). Exact component names as
+  # parsed from version names (e.g. katanomi-v3.1.0 -> katanomi).
+  exclude_plugins:
+    - katanomi
+    - knative
+    - jenkins
+    - tekton-operator
+
   filters:
     - name: "releases"
       enabled: true

roadmap-planner/backend/internal/api/handlers/metrics.go

@@ -56,6 +56,10 @@ func (h *MetricsHandler) ListMetrics(c *gin.Context) {
 
 // GetMetric returns a specific metric with optional filters
 // GET /api/metrics/:name
+//
+// Recognised query parameters in addition to component/pillar/quarter:
+//   include_bots — bool, Lead Time only (default false)
+//   with_trend   — bool, Lead Time only (default true)
 func (h *MetricsHandler) GetMetric(c *gin.Context) {
 	name := c.Param("name")
 
@@ -69,7 +73,11 @@ func (h *MetricsHandler) GetMetric(c *gin.Context) {
 	// Parse time range
 	timeRange := h.parseTimeRange(c)
 
-	results, err := h.service.CalculateMetric(c.Request.Context(), name, filters, timeRange)
+	// Per-request options (Lead Time uses these; other calculators
+	// ignore them silently).
+	opts := parseMetricOptions(c)
+
+	results, err := h.service.CalculateMetric(c.Request.Context(), name, filters, timeRange, opts)
 	if err != nil {
 		h.logger.Error("Failed to calculate metric",
 			zap.String("metric", name),
@@ -141,7 +149,33 @@ func (h *MetricsHandler) GetCollectorStatus(c *gin.Context) {
 	})
 }
 
-// parseTimeRange parses the from/to query parameters into a TimeRange
+// parseMetricOptions extracts calculator-specific query flags (only the
+// keys known today are surfaced; new flags are added as calculators
+// learn to read them from data.Options).
+func parseMetricOptions(c *gin.Context) map[string]interface{} {
+	opts := map[string]interface{}{}
+	if v := c.Query("include_bots"); v != "" {
+		opts["include_bots"] = v == "true" || v == "1" || v == "yes"
+	}
+	if v := c.Query("with_trend"); v != "" {
+		opts["with_trend"] = v == "true" || v == "1" || v == "yes"
+	}
+	if len(opts) == 0 {
+		return nil
+	}
+	return opts
+}
+
+// parseTimeRange parses the from/to query parameters into a TimeRange.
+//
+// When `from` is omitted, the default Start mirrors
+// metrics.historical_days so the API window matches what the collector
+// actually has in memory. A previous implementation hard-coded one
+// year, which caused under-reporting on deployments configured with a
+// smaller historical window: the collector dropped PRs outside
+// HistoricalDays + lookback, but the API still asked for a full year,
+// so 270d–365d-old releases were classified as "no linked PRs" and
+// Lead Time was distorted (DORA P2 review).
 func (h *MetricsHandler) parseTimeRange(c *gin.Context) models.TimeRange {
 	var timeRange models.TimeRange
 
@@ -169,9 +203,15 @@ func (h *MetricsHandler) parseTimeRange(c *gin.Context) models.TimeRange {
 		timeRange.End = time.Now()
 	}
 
-	// Default to 1 year ago if start is not specified
+	// Default Start = HistoricalDays back. Falls back to 365 if
+	// HistoricalDays is unset or non-positive (matches the historical
+	// default and the value in config.example.yaml).
 	if timeRange.Start.IsZero() {
-		timeRange.Start = timeRange.End.AddDate(-1, 0, 0)
+		days := 365
+		if h.config != nil && h.config.Metrics.HistoricalDays > 0 {
+			days = h.config.Metrics.HistoricalDays
+		}
+		timeRange.Start = timeRange.End.AddDate(0, 0, -days)
 	}
 
 	return timeRange

roadmap-planner/backend/internal/api/handlers/metrics_test.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2024 The AlaudaDevops Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+*/
+
+package handlers
+
+import (
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/AlaudaDevops/toolbox/roadmap-planner/backend/internal/config"
+	"github.com/gin-gonic/gin"
+)
+
+// TestParseTimeRange_DefaultMatchesHistoricalDays guards the P2 fix:
+// when `from` is omitted, the API default window must mirror
+// metrics.historical_days, not a hard-coded 1 year. The collector
+// loads PRs over a window sized to HistoricalDays + lookback, so a
+// 1-year API default against HistoricalDays=90 silently under-reports
+// Lead Time for releases 270d–365d ago whose PRs were never loaded.
+func TestParseTimeRange_DefaultMatchesHistoricalDays(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+
+	cases := []struct {
+		name           string
+		historicalDays int
+		wantDays       int
+	}{
+		{"90-day deployment", 90, 90},
+		{"default 365-day deployment", 365, 365},
+		{"180-day deployment", 180, 180},
+		{"zero falls back to 365", 0, 365},
+		{"negative falls back to 365", -1, 365},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			h := &MetricsHandler{
+				config: &config.Config{
+					Metrics: config.Metrics{HistoricalDays: tc.historicalDays},
+				},
+			}
+			ctx, _ := gin.CreateTestContext(httptest.NewRecorder())
+			ctx.Request = httptest.NewRequest("GET", "/api/metrics/lead_time_to_release", nil)
+
+			before := time.Now()
+			got := h.parseTimeRange(ctx)
+			after := time.Now()
+
+			// End defaults to now — between `before` and `after`.
+			if got.End.Before(before) || got.End.After(after) {
+				t.Errorf("End = %v, want between %v and %v", got.End, before, after)
+			}
+			// Start = End - HistoricalDays. Use a small tolerance for
+			// the few microseconds between End and our `before` capture.
+			wantStart := got.End.AddDate(0, 0, -tc.wantDays)
+			delta := got.Start.Sub(wantStart)
+			if delta < -time.Second || delta > time.Second {
+				t.Errorf("Start = %v, want %v (HistoricalDays=%d → %d days back)",
+					got.Start, wantStart, tc.historicalDays, tc.wantDays)
+			}
+		})
+	}
+}
+
+// TestParseTimeRange_ExplicitFromOverridesHistoricalDays makes sure
+// callers can still query outside HistoricalDays by passing `from`
+// explicitly — the default only fires when the param is absent.
+func TestParseTimeRange_ExplicitFromOverridesHistoricalDays(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+
+	h := &MetricsHandler{
+		config: &config.Config{Metrics: config.Metrics{HistoricalDays: 30}},
+	}
+	ctx, _ := gin.CreateTestContext(httptest.NewRecorder())
+	ctx.Request = httptest.NewRequest("GET", "/x?from=2025-01-15&to=2025-06-15", nil)
+
+	got := h.parseTimeRange(ctx)
+	wantStart := time.Date(2025, 1, 15, 0, 0, 0, 0, time.UTC)
+	wantEnd := time.Date(2025, 6, 15, 0, 0, 0, 0, time.UTC)
+	if !got.Start.Equal(wantStart) || !got.End.Equal(wantEnd) {
+		t.Errorf("explicit range = (%v, %v), want (%v, %v) — caller-supplied from/to must win over the HistoricalDays default",
+			got.Start, got.End, wantStart, wantEnd)
+	}
+}

roadmap-planner/backend/internal/config/config.go

@@ -274,6 +274,22 @@ type Metrics struct {
 	Prometheus         PrometheusConfig `mapstructure:"prometheus"`
 	Filters            []OptionsConfig  `mapstructure:"filters"`
 	Calculators        []OptionsConfig  `mapstructure:"calculators"`
+	// ExcludePlugins lists component names dropped from all component
+	// dimensions (releases and issue components) during collection.
+	// Implements plan D6 — v3-era plugins (katanomi, knative, jenkins,
+	// tekton-operator) must not dilute v4 team metrics. Exact match.
+	ExcludePlugins []string `mapstructure:"exclude_plugins"`
+}
+
+// IsPluginExcluded reports whether a component name is in the
+// exclude_plugins list.
+func (c *Metrics) IsPluginExcluded(name string) bool {
+	for _, p := range c.ExcludePlugins {
+		if p == name {
+			return true
+		}
+	}
+	return false
 }
 
 // PrometheusConfig represents Prometheus exporter configuration

roadmap-planner/backend/internal/github/client.go

@@ -187,6 +187,37 @@ func (c *Client) ListReviews(ctx context.Context, owner, repo string, number int
 	return out, nil
 }
 
+// PRCommit is the slice of the PR-commits resource we use for Lead Time.
+// We only persist the earliest commit's author date (see Lead Time
+// calculator), so the struct stays narrow on purpose.
+type PRCommit struct {
+	SHA    string `json:"sha"`
+	Commit struct {
+		Author struct {
+			Date time.Time `json:"date"`
+		} `json:"author"`
+	} `json:"commit"`
+}
+
+// ListPRCommits returns the commits associated with one PR. We use this
+// to derive `pull_requests.first_commit_at` for the DORA Lead Time
+// calculator — the commit author date is the only field needed, but the
+// endpoint returns the full graph from base to head so we read it all
+// and let the caller pick min(author.date).
+//
+// We fetch a single page of up to 100 commits. PRs with more than 100
+// commits are vanishingly rare; if we ever need to handle them, paginate
+// here. GitHub's default per_page on this endpoint is 30, so the
+// per_page=100 override matters.
+func (c *Client) ListPRCommits(ctx context.Context, owner, repo string, number int) ([]PRCommit, error) {
+	path := fmt.Sprintf("/repos/%s/%s/pulls/%d/commits?per_page=100", owner, repo, number)
+	var out []PRCommit
+	if err := c.do(ctx, "GET", path, nil, &out); err != nil {
+		return nil, fmt.Errorf("list PR commits %s/%s#%d: %w", owner, repo, number, err)
+	}
+	return out, nil
+}
+
 // do is the request engine. It:
 //
 //  1. Sleeps before issuing if the rate-limit budget is near-exhausted.