Skip to content

Release v1.1.0: Confidence-aware machine detection and storage-class … #9

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gbudjeakp merged 1 commit into from
Jun 26, 2026
Merged

Release v1.1.0: Confidence-aware machine detection and storage-class … #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions action.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,18 @@ outputs:
detected-machine:
description: "Machine type detected at runtime"
value: ${{ steps.recommend.outputs.detected_machine }}
detected-machine-confidence:
description: "Confidence score (0..1) for machine detection"
value: ${{ steps.recommend.outputs.detected_machine_confidence }}
detected-machine-confidence-level:
description: "Confidence label for machine detection: high, medium, low, unknown"
value: ${{ steps.recommend.outputs.detected_machine_confidence_level }}
detected-storage-type:
description: "Storage type from the detected catalog machine (for example ebs, ssd, pd-ssd)"
value: ${{ steps.recommend.outputs.detected_storage_type }}
runtime-storage-class:
description: "Best-effort runtime storage class probe from the runner OS (ssd, hdd, unknown)"
value: ${{ steps.recommend.outputs.runtime_storage_class }}
suggested-machine:
description: "Top recommended machine ID"
value: ${{ steps.recommend.outputs.suggested_machine }}
Expand Down
126 changes: 103 additions & 23 deletions cmd/runright/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"text/tabwriter"
"time"
Expand Down Expand Up @@ -63,17 +64,20 @@ Examples:
}

var (
monitorDuration time.Duration
monitorInterval time.Duration
monitorDuration time.Duration
monitorInterval time.Duration
monitorExpensiveSampleEvery int
monitorExport string
monitorOutputDir string
monitorJobID string
monitorPromPort int
monitorHTTPURL string
monitorSlackURL string
monitorTeamsURL string
monitorDryRun bool
monitorExport string
monitorOutputDir string
monitorJobID string
monitorPromPort int
monitorHTTPURL string
monitorSlackURL string
monitorTeamsURL string
monitorDryRun bool
monitorAllowedMachineIDs string
monitorAllowedSeries string
monitorAllowedFamilies string
)

func init() {
Expand All @@ -88,6 +92,9 @@ func init() {
monitorCmd.Flags().StringVar(&monitorSlackURL, "slack-webhook", "", "Slack incoming webhook URL for slack export (or set RUNRIGHT_SLACK_WEBHOOK)")
monitorCmd.Flags().StringVar(&monitorTeamsURL, "teams-webhook", "", "Microsoft Teams incoming webhook URL for teams export (or set RUNRIGHT_TEAMS_WEBHOOK)")
monitorCmd.Flags().BoolVar(&monitorDryRun, "dry-run", false, "Print recommendation and exit non-zero if machine is not right-sized")
monitorCmd.Flags().StringVar(&monitorAllowedMachineIDs, "allowed-machine-ids", "", "Comma-separated machine allow-list (or RUNRIGHT_ALLOWED_MACHINE_IDS)")
monitorCmd.Flags().StringVar(&monitorAllowedSeries, "allowed-series", "", "Comma-separated series allow-list, e.g. c7g,m7i,n2,e2 (or RUNRIGHT_ALLOWED_SERIES)")
monitorCmd.Flags().StringVar(&monitorAllowedFamilies, "allowed-families", "", "Comma-separated family prefixes, e.g. c,m,r,n,e (or RUNRIGHT_ALLOWED_FAMILIES)")
}

func runMonitor(_ *cobra.Command, _ []string) error {
Expand All @@ -100,6 +107,18 @@ func runMonitor(_ *cobra.Command, _ []string) error {
if monitorTeamsURL == "" {
monitorTeamsURL = os.Getenv("RUNRIGHT_TEAMS_WEBHOOK")
}
if monitorAllowedMachineIDs == "" {
monitorAllowedMachineIDs = os.Getenv("RUNRIGHT_ALLOWED_MACHINE_IDS")
}
if monitorAllowedSeries == "" {
monitorAllowedSeries = os.Getenv("RUNRIGHT_ALLOWED_SERIES")
}
if monitorAllowedFamilies == "" {
monitorAllowedFamilies = os.Getenv("RUNRIGHT_ALLOWED_FAMILIES")
}
allowedMachineIDs := parseCSV(monitorAllowedMachineIDs)
allowedSeries := parseCSV(monitorAllowedSeries)
allowedFamilies := parseCSV(monitorAllowedFamilies)

ctx, cancel := context.WithCancel(context.Background())
if monitorDuration > 0 {
Expand All @@ -122,12 +141,15 @@ func runMonitor(_ *cobra.Command, _ []string) error {
defer mgr.Shutdown(context.Background())

col := agent.NewCollector(agent.Config{
Interval: monitorInterval,
Interval: monitorInterval,
ExpensiveSampleEvery: monitorExpensiveSampleEvery,
OutputDir: monitorOutputDir,
JobID: monitorJobID,
HeartbeatFilePath: filepath.Join(monitorOutputDir, "metrics-heartbeat.json"),
OutputDir: monitorOutputDir,
JobID: monitorJobID,
HeartbeatFilePath: filepath.Join(monitorOutputDir, "metrics-heartbeat.json"),
FlushFn: func(summary types.MetricsSummary) error {
summary.AllowedMachineIDs = allowedMachineIDs
summary.AllowedSeries = allowedSeries
summary.AllowedFamilies = allowedFamilies
// Compute recommendations and post to the HTTP backend (if configured).
// This runs on every heartbeat AND on the final completed flush, so
// the backend always has an up-to-date record for this run.
Expand Down Expand Up @@ -173,15 +195,21 @@ Examples:
}

var (
recommendMetrics string
recommendProvider string
recommendFormat string
recommendMetrics string
recommendProvider string
recommendFormat string
recommendAllowedMachineIDs string
recommendAllowedSeries string
recommendAllowedFamilies string
)

func init() {
recommendCmd.Flags().StringVar(&recommendMetrics, "metrics", "metrics-summary.json", "Path to metrics-summary.json")
recommendCmd.Flags().StringVar(&recommendProvider, "provider", "", "Filter by provider: aws, gcp, or github (default: all)")
recommendCmd.Flags().StringVar(&recommendFormat, "format", "table", "Output format: table, json, markdown")
recommendCmd.Flags().StringVar(&recommendAllowedMachineIDs, "allowed-machine-ids", "", "Comma-separated machine allow-list (or RUNRIGHT_ALLOWED_MACHINE_IDS)")
recommendCmd.Flags().StringVar(&recommendAllowedSeries, "allowed-series", "", "Comma-separated series allow-list, e.g. c7g,m7i,n2,e2 (or RUNRIGHT_ALLOWED_SERIES)")
recommendCmd.Flags().StringVar(&recommendAllowedFamilies, "allowed-families", "", "Comma-separated family prefixes, e.g. c,m,r,n,e (or RUNRIGHT_ALLOWED_FAMILIES)")
}

func runRecommend(_ *cobra.Command, _ []string) error {
Expand All @@ -193,6 +221,24 @@ func runRecommend(_ *cobra.Command, _ []string) error {
if err := json.Unmarshal(data, &summary); err != nil {
return fmt.Errorf("parsing metrics summary: %w", err)
}
if recommendAllowedMachineIDs == "" {
recommendAllowedMachineIDs = os.Getenv("RUNRIGHT_ALLOWED_MACHINE_IDS")
}
if recommendAllowedSeries == "" {
recommendAllowedSeries = os.Getenv("RUNRIGHT_ALLOWED_SERIES")
}
if recommendAllowedFamilies == "" {
recommendAllowedFamilies = os.Getenv("RUNRIGHT_ALLOWED_FAMILIES")
}
if ids := parseCSV(recommendAllowedMachineIDs); len(ids) > 0 {
summary.AllowedMachineIDs = ids
}
if series := parseCSV(recommendAllowedSeries); len(series) > 0 {
summary.AllowedSeries = series
}
if families := parseCSV(recommendAllowedFamilies); len(families) > 0 {
summary.AllowedFamilies = families
}

opts := catalog.QueryOptions{Provider: types.Provider(recommendProvider)}
machines := catalog.Query(opts)
Expand All @@ -212,8 +258,18 @@ func runRecommend(_ *cobra.Command, _ []string) error {
}

func printTable(recs []types.Recommendation, s types.MetricsSummary) {
fmt.Printf("\nJob: %s | CPU p95: %.1f%% | Mem p95: %.2f GiB | Duration: %.0fs\n\n",
fmt.Printf("\nJob: %s | CPU p95: %.1f%% | Mem p95: %.2f GiB | Duration: %.0fs\n",
s.JobID, s.CPUPercentP95, s.MemUsedGiBP95, s.DurationSeconds)

// Print the detected machine block so users can see the current baseline.
if s.DetectedMachine != nil {
m := s.DetectedMachine
currentMonthly := m.OnDemandPricePerHour * 720.0
fmt.Printf("\nDetected: %s (%s, %d vCPUs, %.1f GiB) $%.4f/hr $%.2f/mo\n",
m.ID, m.Provider, m.VCPUs, m.MemoryGiB, m.OnDemandPricePerHour, currentMonthly)
}
fmt.Println()

w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
fmt.Fprintln(w, "TIER\tMACHINE\tPROVIDER\tvCPUs\tMEMORY\t$/HR\t$/MO\tDELTA")
fmt.Fprintln(w, "----\t-------\t--------\t-----\t------\t----\t----\t-----")
Expand Down Expand Up @@ -266,6 +322,30 @@ func printMarkdown(recs []types.Recommendation, s types.MetricsSummary) {
}
}

func parseCSV(in string) []string {
if strings.TrimSpace(in) == "" {
return nil
}
parts := strings.Split(in, ",")
out := make([]string, 0, len(parts))
seen := make(map[string]struct{}, len(parts))
for _, p := range parts {
norm := strings.TrimSpace(strings.ToLower(p))
if norm == "" {
continue
}
if _, ok := seen[norm]; ok {
continue
}
seen[norm] = struct{}{}
out = append(out, norm)
}
if len(out) == 0 {
return nil
}
return out
}

// ── catalog ──────────────────────────────────────────────────────────────────

var catalogCmd = &cobra.Command{
Expand Down Expand Up @@ -350,7 +430,7 @@ Examples:
}

var (
verifyPreviousRec string
verifyPreviousRec string
verifyActualMetrics string
)

Expand Down Expand Up @@ -389,12 +469,12 @@ func runVerify(_ *cobra.Command, _ []string) error {
fmt.Printf("Recommended machine: %s (%d vCPUs, %.1f GiB)\n\n",
top.Machine.ID, top.Machine.VCPUs, top.Machine.MemoryGiB)

cpuOK := actual.CPUPercentP95 <= float64(top.Machine.VCPUs)/float64(top.RequiredVCPUs)*float64(top.RequiredVCPUs)
memOK := actual.MemUsedGiBP95 <= top.Machine.MemoryGiB
cpuOK := actual.CPUPercentP95 <= float64(top.Machine.VCPUs)/float64(top.RequiredVCPUs)*float64(top.RequiredVCPUs)
memOK := actual.MemUsedGiBP95 <= top.Machine.MemoryGiB

// Headroom checks: p95 usage should be within the recommended machine's capacity.
cpuFit := actual.CPUPercentP95 <= 80.0
memFit := actual.MemUsedGiBP95 <= top.Machine.MemoryGiB*0.85
cpuFit := actual.CPUPercentP95 <= 80.0
memFit := actual.MemUsedGiBP95 <= top.Machine.MemoryGiB*0.85

fmt.Fprintln(w, "CHECK\tRESULT\tACTUAL\tRECOMMENDED")
fmt.Fprintln(w, "-----\t------\t------\t-----------")
Expand Down
136 changes: 127 additions & 9 deletions internal/agent/collector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"math"
"os"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
Expand Down Expand Up @@ -149,9 +150,8 @@ func (c *Collector) sendHeartbeat() {
summary.RunID = c.runID
summary.Status = "heartbeat"
summary.CIPlatform = detectCIPlatform()
if v, m := detectResources(); v > 0 && m > 0 {
summary.DetectedMachine = catalog.DetectMachine(v, m, ciPlatformToProvider(summary.CIPlatform))
}
applyDetectedMachineMetadata(&summary)
summary.RuntimeStorageClass = detectRuntimeStorageClass()

// Write partial file so OOM-killed jobs still have data for recommendations.
if c.cfg.HeartbeatFilePath != "" {
Expand Down Expand Up @@ -264,12 +264,8 @@ func (c *Collector) Flush() error {
summary.RunID = c.runID
summary.Status = "completed"
summary.CIPlatform = detectCIPlatform()

// Detect current machine. Cgroup-aware: reads container/pod limits first
// (K8s pods, DinD), falls back to OS-level counts on bare VMs.
if v, m := detectResources(); v > 0 && m > 0 {
summary.DetectedMachine = catalog.DetectMachine(v, m, ciPlatformToProvider(summary.CIPlatform))
}
applyDetectedMachineMetadata(&summary)
summary.RuntimeStorageClass = detectRuntimeStorageClass()

summaryPath := filepath.Join(c.cfg.OutputDir, "metrics-summary.json")
sf, err := os.Create(summaryPath)
Expand Down Expand Up @@ -453,6 +449,128 @@ func detectResources() (vcpus int, memGiB float64) {
return
}

func applyDetectedMachineMetadata(summary *types.MetricsSummary) {
if summary == nil {
return
}
v, m := detectResources()
if v <= 0 || m <= 0 {
summary.DetectedMachineConfidenceLevel = "unknown"
summary.DetectedMachineMatchReason = "insufficient runtime resources for machine detection"
return
}

machine, confidence, reason := catalog.DetectMachineWithConfidence(v, m, ciPlatformToProvider(summary.CIPlatform))
summary.DetectedMachine = machine
summary.DetectedMachineConfidence = confidence
summary.DetectedMachineMatchReason = reason
summary.DetectedMachineConfidenceLevel = confidenceLevel(confidence)
if machine == nil {
summary.DetectedMachineConfidenceLevel = "unknown"
}
}

func confidenceLevel(score float64) string {
switch {
case score >= 0.85:
return "high"
case score >= 0.65:
return "medium"
case score > 0:
return "low"
default:
return "unknown"
}
}

// detectRuntimeStorageClass probes the local runtime device type as a
// best-effort hint. It does not identify cloud provider volume products.
func detectRuntimeStorageClass() string {
if runtime.GOOS != "linux" {
return "unknown"
}

dev, ok := linuxRootBlockDeviceName()
if !ok || dev == "" {
return "unknown"
}

rotPath := filepath.Join("/sys/class/block", dev, "queue/rotational")
b, err := os.ReadFile(rotPath)
if err != nil {
return "unknown"
}
v := strings.TrimSpace(string(b))
switch v {
case "0":
return "ssd"
case "1":
return "hdd"
default:
return "unknown"
}
}

func linuxRootBlockDeviceName() (string, bool) {
b, err := os.ReadFile("/proc/self/mounts")
if err != nil {
return "", false
}
lines := strings.Split(string(b), "\n")
for _, line := range lines {
fields := strings.Fields(line)
if len(fields) < 2 || fields[1] != "/" {
continue
}
src := fields[0]
if !strings.HasPrefix(src, "/dev/") {
return "", false
}
name := strings.TrimPrefix(src, "/dev/")
if strings.HasPrefix(name, "mapper/") {
return "", false
}
base := trimLinuxPartitionSuffix(name)
if base == "" {
return "", false
}
return base, true
}
return "", false
}

func trimLinuxPartitionSuffix(name string) string {
// NVMe partitions end with pN (example: nvme0n1p1).
if i := strings.LastIndex(name, "p"); i > 0 {
suffix := name[i+1:]
if suffix != "" && isDigits(suffix) {
return name[:i]
}
}

// SATA/xvd/virtio partition names typically end with digits (example: xvda1).
i := len(name) - 1
for i >= 0 && name[i] >= '0' && name[i] <= '9' {
i--
}
if i < len(name)-1 {
return name[:i+1]
}
return name
}

func isDigits(s string) bool {
if s == "" {
return false
}
for i := 0; i < len(s); i++ {
if s[i] < '0' || s[i] > '9' {
return false
}
}
return true
}

// cgroupV2 reads CPU quota and memory limit from the cgroup v2 unified
// hierarchy (/sys/fs/cgroup). Used by Kubernetes pods on modern kernels.
func cgroupV2() (vcpus int, memGiB float64, ok bool) {
Expand Down
Loading
Loading