From eb41e7c1e36548cc7cc82164caead3d3d44072d9 Mon Sep 17 00:00:00 2001 From: Grant Birchmeier Date: Thu, 9 Apr 2026 17:13:15 -0500 Subject: [PATCH] new settings RedactFieldsInLogs & RedactionLogText resolves #949 --- AcceptanceTest/cfg/at_40.cfg | 3 +- QuickFIXn/Session.cs | 20 ++++-- QuickFIXn/SessionFactory.cs | 4 ++ QuickFIXn/SessionSettings.cs | 2 + QuickFIXn/SettingsDictionary.cs | 32 +++++++++ QuickFIXn/SocketReader.cs | 5 +- QuickFIXn/Util/LogAssist.cs | 97 ++++++++++++++++++++++++++++ README.md | 2 +- RELEASE_NOTES.md | 1 + UnitTests/SettingsDictionaryTests.cs | 39 ++++++++++- UnitTests/Util/LogAssistTests.cs | 41 ++++++++++++ 11 files changed, 235 insertions(+), 11 deletions(-) create mode 100644 QuickFIXn/Util/LogAssist.cs create mode 100644 UnitTests/Util/LogAssistTests.cs diff --git a/AcceptanceTest/cfg/at_40.cfg b/AcceptanceTest/cfg/at_40.cfg index fa8dc4893..0ae298288 100644 --- a/AcceptanceTest/cfg/at_40.cfg +++ b/AcceptanceTest/cfg/at_40.cfg @@ -9,7 +9,8 @@ EndTime=00:00:00 SenderCompID=ISLD TargetCompID=TW ResetOnLogon=Y -FileStorePath=store +FileStorePath=store + [SESSION] BeginString=FIX.4.0 DataDictionary=spec\fix\FIX40.xml diff --git a/QuickFIXn/Session.cs b/QuickFIXn/Session.cs index 4896fbce6..27ab8f1a1 100755 --- a/QuickFIXn/Session.cs +++ b/QuickFIXn/Session.cs @@ -6,6 +6,7 @@ using QuickFix.Fields.Converters; using QuickFix.Logger; using QuickFix.Store; +using QuickFix.Util; namespace QuickFix { @@ -223,6 +224,9 @@ public TimeStampPrecision TimeStampPrecision public bool CmeEnhancedResend { get; set; } + public int[] RedactFieldsInLogs { get; set; } = []; + public string RedactionLogText { get; set; } = ""; + #endregion internal Session( @@ -368,10 +372,11 @@ public bool Send(string message) { using (Log.BeginScope(new Dictionary { - {"MessageType", Message.GetMsgType(message)} + { "MessageType", Message.GetMsgType(message) } })) { - Log.Log(MessagesLogLevel, LogEventIds.OutgoingMessage, "{Message}", message); + Log.Log(MessagesLogLevel, LogEventIds.OutgoingMessage, "{Message}", + LogAssist.RedactSensitiveFields(message, RedactFieldsInLogs, RedactionLogText)); } } @@ -541,16 +546,17 @@ private void NextMessage(string msgStr) { using (Log.BeginScope(new Dictionary { - {"MessageType", Message.GetMsgType(msgStr)} + { "MessageType", Message.GetMsgType(msgStr) } })) { - Log.Log(MessagesLogLevel, LogEventIds.IncomingMessage, "{Message}", msgStr); + Log.Log(MessagesLogLevel, LogEventIds.IncomingMessage, "{Message}", + LogAssist.RedactSensitiveFields(msgStr, RedactFieldsInLogs, RedactionLogText)); } } - } - catch (Exception) + } catch (Exception) { - Log.Log(MessagesLogLevel, LogEventIds.IncomingMessage, "{Message}", msgStr); + Log.Log(MessagesLogLevel, LogEventIds.IncomingMessage, "{Message}", + LogAssist.RedactSensitiveFields(msgStr, RedactFieldsInLogs, RedactionLogText)); } MessageBuilder msgBuilder = new MessageBuilder( diff --git a/QuickFIXn/SessionFactory.cs b/QuickFIXn/SessionFactory.cs index ec8ff4e0b..29660ff7c 100755 --- a/QuickFIXn/SessionFactory.cs +++ b/QuickFIXn/SessionFactory.cs @@ -162,6 +162,10 @@ public Session Create(SessionID sessionId, SettingsDictionary settings) session.RequiresOrigSendingTime = settings.GetBool(SessionSettings.RESETSEQUENCE_MESSAGE_REQUIRES_ORIGSENDINGTIME); if (settings.Has(SessionSettings.CME_ENHANCED_RESEND)) session.CmeEnhancedResend = settings.GetBool(SessionSettings.CME_ENHANCED_RESEND); + if (settings.Has(SessionSettings.REDACT_FIELDS_IN_LOGS)) + session.RedactFieldsInLogs = settings.GetIntArray(SessionSettings.REDACT_FIELDS_IN_LOGS); + if (settings.Has(SessionSettings.REDACTION_LOG_TEXT)) + session.RedactionLogText = settings.GetString(SessionSettings.REDACTION_LOG_TEXT); return session; } diff --git a/QuickFIXn/SessionSettings.cs b/QuickFIXn/SessionSettings.cs index 863896860..46e20c72d 100755 --- a/QuickFIXn/SessionSettings.cs +++ b/QuickFIXn/SessionSettings.cs @@ -73,6 +73,8 @@ public class SessionSettings public const string SOCKET_IGNORE_PROXY = "SocketIgnoreProxy"; public const string ENCODING = "Encoding"; public const string CME_ENHANCED_RESEND = "CmeEnhancedResend"; + public const string REDACT_FIELDS_IN_LOGS = "RedactFieldsInLogs"; + public const string REDACTION_LOG_TEXT = "RedactionLogText"; public const string SSL_ENABLE = "SSLEnable"; public const string SSL_SERVERNAME = "SSLServerName"; diff --git a/QuickFIXn/SettingsDictionary.cs b/QuickFIXn/SettingsDictionary.cs index 8ec7a09d9..62cf9ada4 100755 --- a/QuickFIXn/SettingsDictionary.cs +++ b/QuickFIXn/SettingsDictionary.cs @@ -51,6 +51,12 @@ private SettingsDictionary(string name, Dictionary dataSource) .ToDictionary(x => x.k, x => x.v); } + /// + /// Get a string value by case-insensitive key + /// + /// used for case-insensitive lookup + /// + /// if key is not found public string GetString(string key) { if (_data.TryGetValue(key.ToUpperInvariant(), out var val)) @@ -58,6 +64,7 @@ public string GetString(string key) throw new ConfigError($"No value for key: {key}"); } + [Obsolete("Will be removed in a future release because the engine doesn't use it")] public String GetString(string key, bool capitalize) { string s = GetString(key); @@ -144,6 +151,31 @@ public bool GetBool(string key) } } + public int[] GetIntArray(string key) + { + try + { + string[] items = GetString(key).Split(","); + List rvList = []; + foreach (string item in items) + { + string it = item.Trim(); + if (it == "") + continue; + rvList.Add(int.Parse(it)); + } + return rvList.Distinct().OrderBy(s=>s).ToArray(); + } + catch (FormatException) + { + throw new ConfigError("Incorrect data type"); + } + catch (QuickFIXException) + { + throw new ConfigError("No value for key: " + key); + } + } + /// /// Return true if key is present AND value is true, else false /// diff --git a/QuickFIXn/SocketReader.cs b/QuickFIXn/SocketReader.cs index 15985978d..51a803ccd 100755 --- a/QuickFIXn/SocketReader.cs +++ b/QuickFIXn/SocketReader.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using QuickFix.Logger; +using QuickFix.Util; namespace QuickFix; @@ -135,7 +136,9 @@ private void OnMessageFound(string msg) if (_qfSession.HasResponder) { - _qfSession.Log.Log(LogLevel.Information, LogEventIds.IncomingMessage, "{Message}", msg); + _qfSession.Log.Log(LogLevel.Information, LogEventIds.IncomingMessage, "{Message}", + LogAssist.RedactSensitiveFields( + msg, _qfSession.RedactFieldsInLogs, _qfSession.RedactionLogText)); _qfSession.Log.Log(LogLevel.Error, "Multiple logons/connections for this session are not allowed ({Endpoint})", _tcpClient.Client.RemoteEndPoint); diff --git a/QuickFIXn/Util/LogAssist.cs b/QuickFIXn/Util/LogAssist.cs new file mode 100644 index 000000000..5cd75febd --- /dev/null +++ b/QuickFIXn/Util/LogAssist.cs @@ -0,0 +1,97 @@ +using System.Collections.Generic; +using System.Text; + +namespace QuickFix.Util; + +public class LogAssist +{ + public static string RedactSensitiveFields(string msg, int[] tagsToRedact, string redactionText = "") + { + // This long text processor is much faster than Regex. + // Since this is used on every message log, it needs to be fast. + + if (tagsToRedact.Length == 0 || string.IsNullOrEmpty(msg)) + return msg; + + var sensitiveTags = new HashSet(tagsToRedact); + StringBuilder? sb = null; + + int len = msg.Length; + int lastCopiedPos = 0; + int currentPos = 0; + + while (currentPos < len) + { + // 1. Identify the tag start (skip leading SOH if present) + int tagStart = currentPos; + if (msg[currentPos] == Message.SOH) + tagStart++; + + // 2. Find the '=' delimiter + int eqPos = -1; + for (int i = tagStart; i < len; i++) + { + if (msg[i] == '=') + { + eqPos = i; + break; + } + if (msg[i] == Message.SOH) + break; // Malformed field + } + + if (eqPos == -1) + break; + + // 3. Parse tag as integer (no substring allocation) + int tag = 0; + bool validTag = tagStart != eqPos; + for (int i = tagStart; i < eqPos; i++) + { + char c = msg[i]; + if (c is >= '0' and <= '9') + tag = tag * 10 + (c - '0'); + else + { + validTag = false; + break; + } + } + + // 4. Find field end (next SOH or end of string) + int nextSoh = -1; + for (int i = eqPos + 1; i < len; i++) + { + if (msg[i] == '\x01') + { + nextSoh = i; + break; + } + } + int fieldEnd = nextSoh == -1 ? len : nextSoh; + + // 5. Redact if sensitive + if (validTag && sensitiveTags.Contains(tag)) + { + sb ??= new StringBuilder(len); + + // Append everything up to '=' + sb.Append(msg, lastCopiedPos, eqPos + 1 - lastCopiedPos); + sb.Append(redactionText); + + lastCopiedPos = fieldEnd; + } + + currentPos = fieldEnd; + } + + if (sb == null) + return msg; + + // Append remaining part of the message + if (lastCopiedPos < len) + sb.Append(msg, lastCopiedPos, len - lastCopiedPos); + + return sb.ToString(); + } +} diff --git a/README.md b/README.md index 545e13c1a..b034d7e90 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ To run a specific suite, use ``--filter``, e.g. * `dotnet test --filter Fix44Test AcceptanceTest` (`Fix44Test` is the `TestCaseSource` function in Fix44.cs) -AcceptanceTest logs are output to `bin/Debug/net6.0/log`. +AcceptanceTest logs are output to `bin/Debug/net8.0/log`. Credits diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 4c12d3893..601ec3251 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -35,6 +35,7 @@ What's New * #961 - new GenerateKeys app to create Example-app SSL certs (dckorben/gbirchmeier) * #1001 - AcceptanceTests bug: double.Parse with InvariantCulture (gbirchmeier) * #562 - deprecate Message.IsHeaderField without transport DD param (gbirchmeier) +* #949 - new settings RedactFieldsInLogs & RedactionLogText (gbirchmeier) ### v1.14.0 diff --git a/UnitTests/SettingsDictionaryTests.cs b/UnitTests/SettingsDictionaryTests.cs index 3c0968ab5..b09360ec7 100755 --- a/UnitTests/SettingsDictionaryTests.cs +++ b/UnitTests/SettingsDictionaryTests.cs @@ -21,8 +21,11 @@ public void TestSetGetString() d.SetString("STRINGKEY2", "stringvalue2"); Assert.That(d.GetString("STRINGKEY1"), Is.EqualTo("STRINGVALUE1")); Assert.That(d.GetString("STRINGKEY2"), Is.EqualTo("stringvalue2")); - Assert.That(d.GetString("STRINGKEY2", true), Is.EqualTo("STRINGVALUE2")); Assert.Throws(delegate { d.GetString("STRINGKEY3"); }); + + #pragma warning disable CS0618 + Assert.That(d.GetString("STRINGKEY2", true), Is.EqualTo("STRINGVALUE2")); + #pragma warning restore CS0618 } [Test] @@ -188,4 +191,38 @@ public void TestCopyCtor() { Assert.That(dupe.GetString("uNo"), Is.EqualTo("One")); Assert.That(dupe.GetString("DOs"), Is.EqualTo("2")); } + + [Test] + public void TestGetIntArray() { + SettingsDictionary settings = new("test"); + + // empty string becomes empty array + settings.SetString("myintarray", ""); + Assert.That(settings.GetIntArray("myintarray"), Is.Empty); + + int[] expected = [1, 33, 999]; + + // the intended input format + settings.SetString("myintarray", "1,33,999"); + Assert.That(settings.GetIntArray("myintarray"), Is.EqualTo(expected)); + + // trailing & repeated commas ignored + settings.SetString("myintarray", "1,33,,999,"); + Assert.That(settings.GetIntArray("myintarray"), Is.EqualTo(expected)); + + // spaces ignored + settings.SetString("myintarray", "1, 33, , 999, "); + Assert.That(settings.GetIntArray("myintarray"), Is.EqualTo(expected)); + + // sorted, dupes consolidated + settings.SetString("myintarray", "33, 1, , 999, 33, 1"); + Assert.That(settings.GetIntArray("myintarray"), Is.EqualTo(expected)); + + // key not found + Assert.Throws(delegate { settings.GetIntArray("nope"); }); + + // invalid values + settings.SetString("myintarray", "1, 33, , 999, fourteen, "); + Assert.Throws(delegate { settings.GetIntArray("myintarray"); }); + } } diff --git a/UnitTests/Util/LogAssistTests.cs b/UnitTests/Util/LogAssistTests.cs new file mode 100644 index 000000000..ec662a123 --- /dev/null +++ b/UnitTests/Util/LogAssistTests.cs @@ -0,0 +1,41 @@ +using NUnit.Framework; +using QuickFix.Util; + +namespace UnitTests.Util; + +[TestFixture] +public class LogAssistTests { + + private static string Sohize(string s) { + return s.Replace('|', QuickFix.Message.SOH); + } + + [Test] + public void RedactSensitiveFieldsTest() + { + // no change + string inp = Sohize("8=FIX.4.3|9=100|35=B|34=2|49=ISLD|52=20260116-03:12:25.547|56=TW|148=blah|10=126|"); + Assert.That(inp, Is.EqualTo(LogAssist.RedactSensitiveFields(inp, []))); // no fields to redact + Assert.That(inp, Is.EqualTo(LogAssist.RedactSensitiveFields(inp, [3,48]))); // fields are not in message + + // redact 1 tag + string outp = Sohize("8=FIX.4.3|9=100|35=B|34=2|49=|52=20260116-03:12:25.547|56=TW|148=blah|10=126|"); + Assert.That(outp, Is.EqualTo(LogAssist.RedactSensitiveFields(inp, [49]))); + + // redact 2 tags + outp = Sohize("8=FIX.4.3|9=100|35=B|34=2|49=|52=|56=TW|148=blah|10=126|"); + Assert.That(outp, Is.EqualTo(LogAssist.RedactSensitiveFields(inp, [49,52]))); + + // redact 2 tags and specify alt redaction text + outp = Sohize("8=FIX.4.3|9=100|35=B|34=2|49=hidden|52=hidden|56=TW|148=blah|10=126|"); + Assert.That(outp, Is.EqualTo(LogAssist.RedactSensitiveFields(inp, [49,52], "hidden"))); + + // redact multiple occurrences of a tag + inp = Sohize("8=FIX.4.2|9=91|35=B|34=2|49=TW|52=20111011-15:06:23.103|56=ISLD|148=headline|33=3|" + + "58=line1|354=3|355=uno|58=line2|354=9|355=dos|58=line3|354=4|355=tres|10=193|"); + + outp = Sohize("8=FIX.4.2|9=91|35=B|34=2|49=TW|52=20111011-15:06:23.103|56=ISLD|148=|33=3|" + + "58=line1|354=3|355=|58=line2|354=9|355=|58=line3|354=4|355=|10=193|"); + Assert.That(outp, Is.EqualTo(LogAssist.RedactSensitiveFields(inp, [148,355]))); + } +}