new settings RedactFieldsInLogs & RedactionLogText

AcceptanceTest/cfg/at_40.cfg

@@ -9,7 +9,8 @@ EndTime=00:00:00
 SenderCompID=ISLD 
 TargetCompID=TW 
 ResetOnLogon=Y 
-FileStorePath=store 
+FileStorePath=store
+
 [SESSION] 
 BeginString=FIX.4.0 
 DataDictionary=spec\fix\FIX40.xml 

QuickFIXn/Session.cs

@@ -6,6 +6,7 @@
 using QuickFix.Fields.Converters;
 using QuickFix.Logger;
 using QuickFix.Store;
+using QuickFix.Util;
 
 namespace QuickFix
 {
@@ -223,6 +224,9 @@ public TimeStampPrecision TimeStampPrecision
 
         public bool CmeEnhancedResend { get; set; }
 
+        public int[] RedactFieldsInLogs { get; set; } = [];
+        public string RedactionLogText { get; set; } = "<redacted>";
+
         #endregion
 
         internal Session(
@@ -368,10 +372,11 @@ public bool Send(string message)
                 {
                     using (Log.BeginScope(new Dictionary<string, object>
                            {
-                               {"MessageType", Message.GetMsgType(message)}
+                               { "MessageType", Message.GetMsgType(message) }
                            }))
                     {
-                        Log.Log(MessagesLogLevel, LogEventIds.OutgoingMessage, "{Message}", message);
+                        Log.Log(MessagesLogLevel, LogEventIds.OutgoingMessage, "{Message}",
+                            LogAssist.RedactSensitiveFields(message, RedactFieldsInLogs, RedactionLogText));
                     }
                 }
 
@@ -541,16 +546,17 @@ private void NextMessage(string msgStr)
                 {
                     using (Log.BeginScope(new Dictionary<string, object>
                            {
-                               {"MessageType", Message.GetMsgType(msgStr)}
+                               { "MessageType", Message.GetMsgType(msgStr) }
                            }))
                     {
-                        Log.Log(MessagesLogLevel, LogEventIds.IncomingMessage, "{Message}", msgStr);
+                        Log.Log(MessagesLogLevel, LogEventIds.IncomingMessage, "{Message}",
+                            LogAssist.RedactSensitiveFields(msgStr, RedactFieldsInLogs, RedactionLogText));
                     }
                 }
-            }
-            catch (Exception)
+            } catch (Exception)
             {
-                Log.Log(MessagesLogLevel, LogEventIds.IncomingMessage, "{Message}", msgStr);
+                Log.Log(MessagesLogLevel, LogEventIds.IncomingMessage, "{Message}",
+                    LogAssist.RedactSensitiveFields(msgStr, RedactFieldsInLogs, RedactionLogText));
             }
 
             MessageBuilder msgBuilder = new MessageBuilder(

QuickFIXn/SessionFactory.cs

@@ -162,6 +162,10 @@ public Session Create(SessionID sessionId, SettingsDictionary settings)
             session.RequiresOrigSendingTime = settings.GetBool(SessionSettings.RESETSEQUENCE_MESSAGE_REQUIRES_ORIGSENDINGTIME);
         if (settings.Has(SessionSettings.CME_ENHANCED_RESEND))
             session.CmeEnhancedResend = settings.GetBool(SessionSettings.CME_ENHANCED_RESEND);
+        if (settings.Has(SessionSettings.REDACT_FIELDS_IN_LOGS))
+            session.RedactFieldsInLogs = settings.GetIntArray(SessionSettings.REDACT_FIELDS_IN_LOGS);
+        if (settings.Has(SessionSettings.REDACTION_LOG_TEXT))
+            session.RedactionLogText = settings.GetString(SessionSettings.REDACTION_LOG_TEXT);
 
         return session;
     }

QuickFIXn/SessionSettings.cs

@@ -73,6 +73,8 @@ public class SessionSettings
     public const string SOCKET_IGNORE_PROXY = "SocketIgnoreProxy";
     public const string ENCODING = "Encoding";
     public const string CME_ENHANCED_RESEND = "CmeEnhancedResend";
+    public const string REDACT_FIELDS_IN_LOGS = "RedactFieldsInLogs";
+    public const string REDACTION_LOG_TEXT = "RedactionLogText";
 
     public const string SSL_ENABLE = "SSLEnable";
     public const string SSL_SERVERNAME = "SSLServerName";

QuickFIXn/SettingsDictionary.cs

@@ -51,13 +51,20 @@ private SettingsDictionary(string name, Dictionary<string,string> dataSource)
             .ToDictionary(x => x.k, x => x.v);
     }
 
+    /// <summary>
+    /// Get a string value by case-insensitive key
+    /// </summary>
+    /// <param name="key">used for case-insensitive lookup</param>
+    /// <returns></returns>
+    /// <exception cref="ConfigError">if key is not found</exception>
     public string GetString(string key)
     {
         if (_data.TryGetValue(key.ToUpperInvariant(), out var val))
             return val;
         throw new ConfigError($"No value for key: {key}");
     }
 
+    [Obsolete("Will be removed in a future release because the engine doesn't use it")]
     public String GetString(string key, bool capitalize)
     {
         string s = GetString(key);
@@ -144,6 +151,31 @@ public bool GetBool(string key)
         }
     }
 
+    public int[] GetIntArray(string key)
+    {
+        try
+        {
+            string[] items = GetString(key).Split(",");
+            List<int> rvList = [];
+            foreach (string item in items)
+            {
+                string it = item.Trim();
+                if (it == "")
+                    continue;
+                rvList.Add(int.Parse(it));
+            }
+            return rvList.Distinct().OrderBy(s=>s).ToArray();
+        }
+        catch (FormatException)
+        {
+            throw new ConfigError("Incorrect data type");
+        }
+        catch (QuickFIXException)
+        {
+            throw new ConfigError("No value for key: " + key);
+        }
+    }
+
     /// <summary>
     /// Return true if key is present AND value is true, else false
     /// </summary>

QuickFIXn/SocketReader.cs

@@ -6,6 +6,7 @@
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using QuickFix.Logger;
+using QuickFix.Util;
 
 namespace QuickFix;
 
@@ -135,7 +136,9 @@ private void OnMessageFound(string msg)
 
                 if (_qfSession.HasResponder)
                 {
-                    _qfSession.Log.Log(LogLevel.Information, LogEventIds.IncomingMessage, "{Message}", msg);
+                    _qfSession.Log.Log(LogLevel.Information, LogEventIds.IncomingMessage, "{Message}",
+                        LogAssist.RedactSensitiveFields(
+                            msg, _qfSession.RedactFieldsInLogs, _qfSession.RedactionLogText));
                     _qfSession.Log.Log(LogLevel.Error,
                         "Multiple logons/connections for this session are not allowed ({Endpoint})",
                         _tcpClient.Client.RemoteEndPoint);

QuickFIXn/Util/LogAssist.cs

@@ -0,0 +1,97 @@
+using System.Collections.Generic;
+using System.Text;
+
+namespace QuickFix.Util;
+
+public class LogAssist
+{
+    public static string RedactSensitiveFields(string msg, int[] tagsToRedact, string redactionText = "<redacted>")
+    {
+        // This long text processor is much faster than Regex.
+        // Since this is used on every message log, it needs to be fast.
+
+        if (tagsToRedact.Length == 0 || string.IsNullOrEmpty(msg))
+            return msg;
+
+        var sensitiveTags = new HashSet<int>(tagsToRedact);
+        StringBuilder? sb = null;
+
+        int len = msg.Length;
+        int lastCopiedPos = 0;
+        int currentPos = 0;
+
+        while (currentPos < len)
+        {
+            // 1. Identify the tag start (skip leading SOH if present)
+            int tagStart = currentPos;
+            if (msg[currentPos] == Message.SOH)
+                tagStart++;
+
+            // 2. Find the '=' delimiter
+            int eqPos = -1;
+            for (int i = tagStart; i < len; i++)
+            {
+                if (msg[i] == '=')
+                {
+                    eqPos = i;
+                    break;
+                }
+                if (msg[i] == Message.SOH)
+                    break; // Malformed field
+            }
+
+            if (eqPos == -1)
+                break;
+
+            // 3. Parse tag as integer (no substring allocation)
+            int tag = 0;
+            bool validTag = tagStart != eqPos;
+            for (int i = tagStart; i < eqPos; i++)
+            {
+                char c = msg[i];
+                if (c is >= '0' and <= '9')
+                    tag = tag * 10 + (c - '0');
+                else
+                {
+                    validTag = false;
+                    break;
+                }
+            }
+
+            // 4. Find field end (next SOH or end of string)
+            int nextSoh = -1;
+            for (int i = eqPos + 1; i < len; i++)
+            {
+                if (msg[i] == '\x01')
+                {
+                    nextSoh = i;
+                    break;
+                }
+            }
+            int fieldEnd = nextSoh == -1 ? len : nextSoh;
+
+            // 5. Redact if sensitive
+            if (validTag && sensitiveTags.Contains(tag))
+            {
+                sb ??= new StringBuilder(len);
+
+                // Append everything up to '='
+                sb.Append(msg, lastCopiedPos, eqPos + 1 - lastCopiedPos);
+                sb.Append(redactionText);
+
+                lastCopiedPos = fieldEnd;
+            }
+
+            currentPos = fieldEnd;
+        }
+
+        if (sb == null)
+            return msg;
+
+        // Append remaining part of the message
+        if (lastCopiedPos < len)
+            sb.Append(msg, lastCopiedPos, len - lastCopiedPos);
+
+        return sb.ToString();
+    }
+}

README.md

@@ -48,7 +48,7 @@ To run a specific suite, use ``--filter``, e.g.
 * `dotnet test --filter Fix44Test AcceptanceTest`
 (`Fix44Test` is the `TestCaseSource` function in Fix44.cs)
 
-AcceptanceTest logs are output to `bin/Debug/net6.0/log`.
+AcceptanceTest logs are output to `bin/Debug/net8.0/log`.
 
 
 Credits

RELEASE_NOTES.md

@@ -35,6 +35,7 @@ What's New
 * #961 - new GenerateKeys app to create Example-app SSL certs (dckorben/gbirchmeier)
 * #1001 - AcceptanceTests bug: double.Parse with InvariantCulture (gbirchmeier)
 * #562 - deprecate Message.IsHeaderField without transport DD param (gbirchmeier)
+* #949 - new settings RedactFieldsInLogs & RedactionLogText (gbirchmeier)
 
 
 ### v1.14.0

UnitTests/SettingsDictionaryTests.cs

@@ -21,8 +21,11 @@ public void TestSetGetString()
         d.SetString("STRINGKEY2", "stringvalue2");
         Assert.That(d.GetString("STRINGKEY1"), Is.EqualTo("STRINGVALUE1"));
         Assert.That(d.GetString("STRINGKEY2"), Is.EqualTo("stringvalue2"));
-        Assert.That(d.GetString("STRINGKEY2", true), Is.EqualTo("STRINGVALUE2"));
         Assert.Throws<ConfigError>(delegate { d.GetString("STRINGKEY3"); });
+
+        #pragma warning disable CS0618
+        Assert.That(d.GetString("STRINGKEY2", true), Is.EqualTo("STRINGVALUE2"));
+        #pragma warning restore CS0618
     }
 
     [Test]
@@ -188,4 +191,38 @@ public void TestCopyCtor() {
         Assert.That(dupe.GetString("uNo"), Is.EqualTo("One"));
         Assert.That(dupe.GetString("DOs"), Is.EqualTo("2"));
     }
+
+    [Test]
+    public void TestGetIntArray() {
+        SettingsDictionary settings = new("test");
+
+        // empty string becomes empty array
+        settings.SetString("myintarray", "");
+        Assert.That(settings.GetIntArray("myintarray"), Is.Empty);
+
+        int[] expected = [1, 33, 999];
+
+        // the intended input format
+        settings.SetString("myintarray", "1,33,999");
+        Assert.That(settings.GetIntArray("myintarray"), Is.EqualTo(expected));
+
+        // trailing & repeated commas ignored
+        settings.SetString("myintarray", "1,33,,999,");
+        Assert.That(settings.GetIntArray("myintarray"), Is.EqualTo(expected));
+
+        // spaces ignored
+        settings.SetString("myintarray", "1, 33, , 999, ");
+        Assert.That(settings.GetIntArray("myintarray"), Is.EqualTo(expected));
+
+        // sorted, dupes consolidated
+        settings.SetString("myintarray", "33, 1, , 999, 33, 1");
+        Assert.That(settings.GetIntArray("myintarray"), Is.EqualTo(expected));
+
+        // key not found
+        Assert.Throws<ConfigError>(delegate { settings.GetIntArray("nope"); });
+
+        // invalid values
+        settings.SetString("myintarray", "1, 33, , 999, fourteen, ");
+        Assert.Throws<ConfigError>(delegate { settings.GetIntArray("myintarray"); });
+    }
 }

UnitTests/Util/LogAssistTests.cs

@@ -0,0 +1,41 @@
+using NUnit.Framework;
+using QuickFix.Util;
+
+namespace UnitTests.Util;
+
+[TestFixture]
+public class LogAssistTests {
+
+    private static string Sohize(string s) {
+        return s.Replace('|', QuickFix.Message.SOH);
+    }
+
+    [Test]
+    public void RedactSensitiveFieldsTest()
+    {
+        // no change
+        string inp = Sohize("8=FIX.4.3|9=100|35=B|34=2|49=ISLD|52=20260116-03:12:25.547|56=TW|148=blah|10=126|");
+        Assert.That(inp, Is.EqualTo(LogAssist.RedactSensitiveFields(inp, []))); // no fields to redact
+        Assert.That(inp, Is.EqualTo(LogAssist.RedactSensitiveFields(inp, [3,48]))); // fields are not in message
+
+        // redact 1 tag
+        string outp = Sohize("8=FIX.4.3|9=100|35=B|34=2|49=<redacted>|52=20260116-03:12:25.547|56=TW|148=blah|10=126|");
+        Assert.That(outp, Is.EqualTo(LogAssist.RedactSensitiveFields(inp, [49])));
+
+        // redact 2 tags
+        outp = Sohize("8=FIX.4.3|9=100|35=B|34=2|49=<redacted>|52=<redacted>|56=TW|148=blah|10=126|");
+        Assert.That(outp, Is.EqualTo(LogAssist.RedactSensitiveFields(inp, [49,52])));
+
+        // redact 2 tags and specify alt redaction text
+        outp = Sohize("8=FIX.4.3|9=100|35=B|34=2|49=hidden|52=hidden|56=TW|148=blah|10=126|");
+        Assert.That(outp, Is.EqualTo(LogAssist.RedactSensitiveFields(inp, [49,52], "hidden")));
+
+        // redact multiple occurrences of a tag
+        inp = Sohize("8=FIX.4.2|9=91|35=B|34=2|49=TW|52=20111011-15:06:23.103|56=ISLD|148=headline|33=3|"
+                     + "58=line1|354=3|355=uno|58=line2|354=9|355=dos|58=line3|354=4|355=tres|10=193|");
+
+        outp = Sohize("8=FIX.4.2|9=91|35=B|34=2|49=TW|52=20111011-15:06:23.103|56=ISLD|148=<redacted>|33=3|"
+                      + "58=line1|354=3|355=<redacted>|58=line2|354=9|355=<redacted>|58=line3|354=4|355=<redacted>|10=193|");
+        Assert.That(outp, Is.EqualTo(LogAssist.RedactSensitiveFields(inp, [148,355])));
+    }
+}