diff --git a/docs/measurements.md b/docs/measurements.md index 5f8feeada..e86979eb5 100644 --- a/docs/measurements.md +++ b/docs/measurements.md @@ -140,3 +140,71 @@ Here is a working example: }] } ``` + +### From the Agent Command Line + +The agent can be configured to record custom measurements for every benchmark it runs using the `--record` (or `-r`) command line option. This is useful for recording system information or configuration details that should be captured for all benchmarks. + +#### Usage + +```bash +crank-agent --record "name=value" +``` + +The option can be specified multiple times to record multiple measurements: + +```bash +crank-agent --record "system/openssl=$(openssl version)" --record "system/kernel=$(uname -r)" +``` + +#### Format + +- The name and value are separated by the first `=` character +- The name portion becomes the measurement name +- The value can be a literal string + +#### Command Substitution + +Command substitution is handled by the shell before the arguments reach the agent. Use your shell's command substitution syntax: + +**Linux/macOS example (using bash/sh):** +```bash +crank-agent --record "system/openssl=$(openssl version)" \ + --record "system/kernel=$(uname -r)" \ + --record "system/hostname=$(hostname)" +``` + +**Windows example (using PowerShell):** +```powershell +crank-agent --record "system/dotnet=$(dotnet --version)" ` + --record "system/os=$(systeminfo | Select-String 'OS Name')" +``` + +#### Behavior + +- Custom measurements are automatically added to every job that the agent runs +- Each measurement includes metadata with: + - `Source`: "Agent" + - `Aggregate`: First + - `Reduce`: First + - `ShortDescription`: The measurement name + - `LongDescription`: "Custom measurement: {name}" +- Invalid formats (missing `=` or empty name) will be logged as warnings and skipped + +#### Example + +Start an agent that records the OpenSSL version: + +```bash +crank-agent --record "system/openssl=$(openssl version)" +``` + +When this agent runs a benchmark, the measurement will automatically include: + +```json +{ + "name": "system/openssl", + "timestamp": "2024-02-23T13:01:56.12Z", + "value": "OpenSSL 3.0.13 30 Jan 2024" +} +``` diff --git a/src/Microsoft.Crank.Agent/Startup.cs b/src/Microsoft.Crank.Agent/Startup.cs index c302a6f87..859387526 100644 --- a/src/Microsoft.Crank.Agent/Startup.cs +++ b/src/Microsoft.Crank.Agent/Startup.cs @@ -170,9 +170,13 @@ private static CommandOption _certClientId, _certTenantId, _certThumbprint, - _certSniAuth + _certSniAuth, + _recordOption ; + // Stores custom measurements to record for each job + private static readonly Dictionary _customMeasurements = new Dictionary(); + internal static Serilog.Core.Logger Logger { get; private set; } private static readonly string[] _powershellCommands = ["pwsh", "powershell"]; @@ -266,6 +270,7 @@ public static int Main(string[] args) _certPath = app.Option("--cert-path", "Location of the certificate to be used for auth.", CommandOptionType.SingleValue); _certPassword = app.Option("--cert-pwd", "Password of the certificate to be used for auth.", CommandOptionType.SingleValue); _certSniAuth = app.Option("--cert-sni", "Enable subject name / issuer based authentication (SNI).", CommandOptionType.NoValue); + _recordOption = app.Option("-r|--record", "Records a custom measurement for each benchmark. Format: 'name=value'. Can be specified multiple times.", CommandOptionType.MultipleValue); app.OnExecute(() => { @@ -381,6 +386,37 @@ public static int Main(string[] args) } } + // Process custom measurement records + if (_recordOption.HasValue()) + { + foreach (var recordValue in _recordOption.Values) + { + if (string.IsNullOrWhiteSpace(recordValue)) + { + continue; + } + + var separatorIndex = recordValue.IndexOf('='); + if (separatorIndex <= 0) + { + Log.Warning($"Invalid --record format: '{recordValue}'. Expected format: 'name=value'"); + continue; + } + + var name = recordValue.Substring(0, separatorIndex).Trim(); + var value = recordValue.Substring(separatorIndex + 1).Trim(); + + if (string.IsNullOrEmpty(name)) + { + Log.Warning($"Invalid --record format: '{recordValue}'. Name cannot be empty."); + continue; + } + + _customMeasurements[name] = value; + Log.Info($"Recording custom measurement: {name} = {value}"); + } + } + return Run(url, hostname, dockerHostname).Result; }); @@ -5581,6 +5617,34 @@ private static bool MarkAsRunning(string hostname, Job job, Stopwatch stopwatch) }); BenchmarksEventSource.Start(); + // Add custom measurements from --record options + foreach (var customMeasurement in _customMeasurements) + { + job.Measurements.Enqueue(new Measurement + { + Name = customMeasurement.Key, + Timestamp = DateTime.UtcNow, + Value = customMeasurement.Value + }); + + // Also add metadata for the custom measurement if it doesn't exist + if (!job.Metadata.Any(x => x.Name == customMeasurement.Key)) + { + job.Metadata.Enqueue(new MeasurementMetadata + { + Source = "Agent", + Name = customMeasurement.Key, + Aggregate = Operation.First, + Reduce = Operation.First, + Format = "", + LongDescription = $"Custom measurement: {customMeasurement.Key}", + ShortDescription = customMeasurement.Key + }); + } + + Log.Info($"Added custom measurement to job {job.Id}: {customMeasurement.Key} = {customMeasurement.Value}"); + } + Log.Info($"Running job '{job.Service}' ({job.Id})"); job.Url = ComputeServerUrl(hostname, job);