feat(#804): var checks

src/lib/convert-forms/handle-var-checks.js

@@ -0,0 +1,202 @@
+const { getNodes, XPATH_MODEL, XML_ATT_NODESET } = require('../forms-utils');
+const { info, warn} = require('../log');
+
+const DEFAULT = {
+  warn_length: 100,
+  error_length: 138
+};
+
+function processLengthInput(n) {
+  if(typeof n !== 'number' || !Number.isFinite(n) || !Number.isInteger(n) || n < 0 ){
+    throw new Error('Please ensure that the warn/error length value is a positive integer');
+  }
+
+  return n;
+}
+
+function processListInput(e) {
+  const set = new Set([]);
+  const invalidPaths = [];
+
+  if(!Array.isArray(e)){
+    return [new Set(), []];
+  }
+
+  for(const entry of e){
+    if(/[`'"]/.test(entry)){
+      invalidPaths.push(entry);
+    }
+    else {
+      set.add(entry);
+    }
+  }
+
+  return [set, invalidPaths];
+}
+
+function formatFeedbackMsg(title, items, footer){
+  return `${title}\n${items.join('\n')}\n${footer}`;
+}
+
+function checkLengthEntries(warnLength, errorLength){
+  if(errorLength && warnLength >= errorLength){
+    throw new Error('The error length needs to be larger than the warn length.');
+  }
+}
+
+function checkInvalidListEntries(entries, label){
+  if(entries.length > 0){
+    throw new Error(formatFeedbackMsg(
+      `The following ${label} entries are invalid:`, 
+      entries, 
+      'Please fix or remove where appropriate.'
+    ));
+  }
+}
+
+function checkListOverlap(ignoreSet, reservedSet){
+  if(!ignoreSet.size || !reservedSet.size){
+    return;
+  }
+
+  const overlap = [];
+  for(const ignore of ignoreSet){
+    if(reservedSet.has(ignore)){
+      overlap.push(ignore);
+    }
+  }
+
+  if(overlap.length > 0){
+    throw new Error(formatFeedbackMsg(
+      'Overlap between reserved and ignore lists:',
+      overlap,
+      'Please remove where appropriate.'
+    ));
+  }
+}
+
+function processPropData(props){
+  const warnLength = 'warn_length' in props ? processLengthInput(props.warn_length) : null;
+  const errorLength = 'error_length' in props ? processLengthInput(props.error_length) : null;
+  const [ignoreSet, invalidIgnoreEntries] = processListInput(props.ignore_list);
+  const [reservedSet, invalidReservedEntries] = processListInput(props.reserved_list);
+
+  if(!warnLength && !errorLength && reservedSet.size === 0){
+    info('Warn and error lengths and reserved list not provided. Skipping var checks.');
+    return;
+  }
+
+  checkLengthEntries(warnLength, errorLength);
+  checkInvalidListEntries(invalidIgnoreEntries, 'ignored');
+  checkInvalidListEntries(invalidReservedEntries, 'reserved');
+  checkListOverlap(ignoreSet, reservedSet);
+
+  return { warnLength, errorLength, ignoreSet, reservedSet };
+}
+
+function buildExclusionPath(set){
+  if(!set.size){
+    return '';
+  }
+  const conditions = Array.from(set).map(v => `@${XML_ATT_NODESET} = "${v}"`).join(' or ');
+  return `and not(${conditions})`;
+}
+
+function getBindNodes(xmlDoc, ignoreSet){
+  try {
+    return getNodes(
+      xmlDoc,
+      `${XPATH_MODEL}/bind[starts-with(@${XML_ATT_NODESET}, "/data/") ${buildExclusionPath(ignoreSet)}]`
+    );
+  }
+  catch (e){
+    const key = 'Unterminated string literal: "';
+    if(e.message?.includes(key)){
+      const problemPath = e.message.substring(e.message.indexOf(key) + key.length, e.message.indexOf(')'));
+      throw new Error(`Unable to find path: ${problemPath}`);
+    }
+    throw e;
+  }
+}
+
+function processBindNodes(bindNodes, warnLength, errorLength, reservedSet){
+  const reserved = [];
+  const errorNodes = [];
+  const warnNodes = [];
+
+  function classifyNode(nodeset) {
+    const length = nodeset.length;
+    if (reservedSet.has(nodeset)){
+      return 'reserved';
+    }
+    if (errorLength > 0 && length >= errorLength) {
+      return 'error';
+    }
+    if (warnLength > 0 && length >= warnLength) {
+      return 'warn';
+    }
+    return null;
+  }
+
+  for (const bind of bindNodes) {
+    const nodeset = bind.getAttribute(XML_ATT_NODESET);
+    switch (classifyNode(nodeset)) {
+    case 'reserved':
+      reserved.push(nodeset);
+      break;
+    case 'error':
+      errorNodes.push(nodeset);
+      break;
+    case 'warn':
+      warnNodes.push(nodeset);
+      break;
+    }
+  }
+
+  return { reserved, errorNodes, warnNodes };
+}
+
+function handleFormVarResults(reserved, warnObj, errorObj){
+  if(reserved.length > 0){
+    throw new Error(formatFeedbackMsg(
+      'The following reserved entries were found in the form:',
+      reserved,
+      'Please remove or rename as appropriate.'
+    ));
+  }
+  if(errorObj.errorNodes.length > 0){
+    throw new Error(formatFeedbackMsg(
+      `The following vars are longer than the acceptable var length (${errorObj.errorLength}):`,
+      errorObj.errorNodes,
+      'Please simplify nesting or remove verbosity.'
+    ));
+  }
+  else if(warnObj.warnNodes.length > 0){
+    warn(formatFeedbackMsg(
+      `The following vars are longer than the acceptable var length (${warnObj.warnLength}):`,
+      warnObj.warnNodes,
+      'Please consider simplifying nesting or removing verbosity.'
+    ));
+  }
+}
+
+function checkVars(xmlDoc, props) {
+  const varConfig = processPropData(props ?? DEFAULT);
+  if(!varConfig){
+    return;
+  }
+  const { warnLength, errorLength, ignoreSet, reservedSet } = varConfig;
+
+  const bindNodes = getBindNodes(xmlDoc, ignoreSet);
+  if(!bindNodes || bindNodes.length === 0){
+    info('Form did not contain any bind nodes');
+    return;
+  }
+
+  const { reserved, errorNodes, warnNodes } = processBindNodes(bindNodes, warnLength, errorLength, reservedSet);
+  handleFormVarResults(reserved, { warnNodes, warnLength }, { errorNodes, errorLength } );
+}
+
+module.exports = {
+  checkVars
+};

src/lib/convert-forms/index.js

@@ -2,10 +2,7 @@ const argsFormFilter = require('../args-form-filter');
 const exec = require('../exec-promise');
 const fs = require('../sync-fs');
 const nodeFs = require('node:fs');
-const {
-  getFormDir,
-  escapeWhitespacesInPath,
-} = require('../forms-utils');
+const { getFormDir, escapeWhitespacesInPath, } = require('../forms-utils');
 const { info, trace, warn, LEVEL_NONE } = require('../log');
 const path = require('node:path');
 const { DOMParser, XMLSerializer } = require('@xmldom/xmldom');
@@ -15,6 +12,7 @@ const { removeNoLabelNodes } = require('./handle-no-label-placeholders');
 const { removeExtraRepeatInstance, addRepeatCount } = require('./handle-repeat');
 const { handleDbDocRefs } = require('./handle-db-doc-ref');
 const { handleFormId } = require('./handle-form-id');
+const { checkVars } = require('./handle-var-checks');
 
 const domParser = new DOMParser();
 const serializer = new XMLSerializer();
@@ -52,8 +50,8 @@ const execute = async (projectDir, subDirectory, options = {}) => {
 
     try {
       await xls2xform(escapeWhitespacesInPath(sourcePath), escapeWhitespacesInPath(xmlSwpPath), xls);
-      const hiddenFields = await getHiddenFields(`${fs.withoutExtension(sourcePath)}.properties.json`);
-      fixXml(xmlSwpPath, hiddenFields, options.transformer, options.enketo);
+      const propsData = getPropsData(`${fs.withoutExtension(sourcePath)}.properties.json`);
+      fixXml(xmlSwpPath, propsData, options.transformer, options.enketo);
     } catch (e) {
       nodeFs.rmSync(xmlSwpPath, { force: true });
       throw e;
@@ -123,7 +121,7 @@ const xls2xform = async (sourcePath, targetPath, xlsxFileName) => {
 
 // here we fix the form content in arcane ways.  Seeing as we have out own fork
 // of pyxform, we should probably be doing this fixing there.
-const fixXml = (path, hiddenFields, transformer, enketo) => {
+const fixXml = (path, propsData, transformer, enketo) => {
   // This is not how you should modify XML, but we have reasonable control over
   // the input and so far this works OK.  Keep an eye on the tests, and any
   // future changes to the output of xls2xform.
@@ -140,8 +138,8 @@ const fixXml = (path, hiddenFields, transformer, enketo) => {
     xml = xml.replaceAll('default="true()"', '');
   }
 
-  if (hiddenFields) {
-    const r = new RegExp(`<(${hiddenFields.join('|')})(/?)>`, 'g');
+  if (propsData[FORM_PROPERTIES_HIDDEN_FIELDS]) {
+    const r = new RegExp(`<(${propsData[FORM_PROPERTIES_HIDDEN_FIELDS].join('|')})(/?)>`, 'g');
     xml = xml.replace(r, '<$1 tag="hidden"$2>');
   }
 
@@ -170,19 +168,31 @@ const fixXml = (path, hiddenFields, transformer, enketo) => {
     lineSeparator: '\n'
   }).replaceAll(/\s+<\/value>/g, '</value>'); // Ignoring the 'value' path results in extra trailing whitespace
 
+  if(propsData[FORM_PROPERTIES_VAR_RESTRICTIONS]){
+    checkVars(xmlDoc, propsData[FORM_PROPERTIES_VAR_RESTRICTIONS]);
+  }
+
   if (transformer) {
     xml = transformer(xml, path);
   }
 
   fs.write(path, xml);
 };
 
-function getHiddenFields(propsJson) {
-  if (fs.exists(propsJson)) {
-    return fs.readJson(propsJson).hidden_fields;
+const FORM_PROPERTIES_HIDDEN_FIELDS = 'hidden_fields';
+const FORM_PROPERTIES_VAR_RESTRICTIONS = 'var_restrictions';
+function getPropsData(propsJson) {
+  if(fs.exists(propsJson)){
+    const json = fs.readJson(propsJson);
+    return {
+      [FORM_PROPERTIES_HIDDEN_FIELDS]: json[FORM_PROPERTIES_HIDDEN_FIELDS],
+      [FORM_PROPERTIES_VAR_RESTRICTIONS]: json[FORM_PROPERTIES_VAR_RESTRICTIONS]
+    };
   }
-
-  return [];
+  return {
+    [FORM_PROPERTIES_HIDDEN_FIELDS]: [],
+    [FORM_PROPERTIES_VAR_RESTRICTIONS]: {} 
+  };
 }
 
 const META_XML_SECTION = `<inputs>

src/lib/forms-utils.js

@@ -3,6 +3,7 @@ const fs = require('./sync-fs');
 
 const XPATH_MODEL = '/h:html/h:head/model';
 const XPATH_BODY = '/h:html/h:body';
+const XML_ATT_NODESET = 'nodeset';
 
 const getNode = (currentNode, path) =>
   xpath.parse(path).select1({ node: currentNode, allowAnyNamespaceForNoPrefix: true });
@@ -22,6 +23,8 @@ module.exports = {
   XPATH_MODEL,
   XPATH_BODY,
 
+  XML_ATT_NODESET,
+
   /**
    * Matches XPath expressions that are only paths to a node (either absolute or relative) without any
    * predicates, functions, operators, etc.

test/fn/convert-forms.utils.js

@@ -12,6 +12,7 @@ const serializer = new XMLSerializer();
 const createXformString = ({
   itext = '',
   primaryInstance = '',
+  bindNodes = [],
   model = `
     <itext>
       ${itext}
@@ -21,6 +22,7 @@ const createXformString = ({
         ${primaryInstance}
       </data>
     </instance>
+    ${bindNodes.join('\n')}
   `,
   body = ''
 }) => `