🎨 Changed and improved the slug generation

packages/string/lib/slugify.js

@@ -1,4 +1,4 @@
-const unidecode = require('unidecode');
+const anyAscii = require('any-ascii').default;
 const stripInvisibleChars = require('./strip-invisible-chars');
 
 /**
@@ -9,40 +9,59 @@ const stripInvisibleChars = require('./strip-invisible-chars');
  * @param {String} string - the string we want to slugify
  * @param {object} options - filter options
  * @param {bool} [options.requiredChangesOnly] - don't perform optional cleanup, e.g. removing extra dashes
+ * @param {bool} [options.unicodeSlugs] - don't perform optional transliteration, e.g. keep smörgåsbord as it is instead of turning it into smorgasbord
+ * @param {string} [options.slugSeparator] - separator to be used for the slugs, can be ` `, `_` or `-`, defaults to `-`
  * @returns {String} slugified string
  */
 module.exports = function (string, options = {}) {
+    // If the separator is invalid or unset, replace it with the default `-`
+    const separator = ['-', '_', ' '].includes(options.slugSeparator) ? options.slugSeparator : '-';
+
     // Ensure we have a string
     string = string || '';
 
     // Strip all characters that cannot be printed
-    string = stripInvisibleChars(string);
-
-    // Handle the £ symbol separately, since it needs to be removed before the unicode conversion.
-    string = string.replace(/£/g, '-');
+    string = stripInvisibleChars(string)
+    // Normalize the input to make sure accent marks, etc. are part of the characters before continuing
+        .normalize('NFC')
+        // Remove the most common forms of apostrophes, to turn contractions like "what’s" into "whats"
+        .replace(/['‘’´]/g, '')
+        // Remove anything that's not a letter, number or a separator
+        .replace(/[^\p{L}\p{N}\p{Mn}\p{Mc}\s_-]/gu, separator)
+        // Remove potential misuse of combining marks, like Zalgo text, by limiting the number of marks,
+        // a limit of 3 marks should allow pretty much any natural language usage, so in case there's 4
+        // marks or more, we remove all marks. Combining marks in the beginning of a word shouldn't
+        // exist at all in natural language, so these are just removed.
+        .replace(/([\p{L}\p{N}][\p{Mn}\p{Mc}]*)|[\p{Mn}\p{Mc}]+/gu, '$1')
+        .replace(/([^\p{Mn}\p{Mc}])[\p{Mn}\p{Mc}]{4,}/gu, '$1');
 
-    // Remove non ascii characters
-    string = unidecode(string);
+    // Perform the transliteration if requested
+    if (!options.unicodeSlugs) {
+        string = anyAscii(string);
+    }
 
-    // Replace URL reserved chars: `@:/?#[]!$&()*+,;=` as well as `\%<>|^~£"{}` and \`
-    string = string.replace(/(\s|\.|@|:|\/|\?|#|\[|\]|!|\$|&|\(|\)|\*|\+|,|;|=|\\|%|<|>|\||\^|~|"|\{|\}|`|–|—)/g, '-')
-    // Remove apostrophes
-        .replace(/'/g, '')
+    // Replace spaces, URL reserved chars: `@:/?#[]!$&()*+,;=` as well as `\%<>|^~£"{}-` and \` with the selected separator.
+    // Should only be needed in case the transliteration added something, but it's safer to always run in case the regex filter missed something.
+    string = string.replace(/(\s|\.|@|:|\/|\?|#|\[|\]|!|\$|&|\(|\)|\*|\+|,|;|=|\\|%|<|>|\||\^|~|£|"|\{|\}|`|–|—)/g, separator)
+    // Remove apostrophes (again, in case the transliteration added some)
+        .replace(/['‘’´]/g, '')
+        // camelCase looking text after initial cleanup and transliteration are most likely separate words, so we add separators between the parts
+        .replace(/([a-z])([A-Z])/g, `$1${separator}$2`)
         // Make the whole thing lowercase
         .toLowerCase();
 
     // These changes are optional changes, we can enable/disable these
     if (!options.requiredChangesOnly) {
-        // Convert 2 or more dashes into a single dash
-        string = string.replace(/-+/g, '-')
-        // Remove trailing dash
-            .replace(/-$/, '')
-            // Remove any dashes at the beginning
-            .replace(/^-/, '');
+        // Convert 2 or more separators into a single separator
+        string = string.replace(/[\s_-]{2,}/g, separator)
+        // Remove trailing separators
+            .replace(/[\s_-]$/, '')
+            // Remove any separators at the beginning
+            .replace(/^[\s_-]/, '');
+    } else {
+        // Handle whitespace at the beginning or end.
+        string = string.trim();
     }
 
-    // Handle whitespace at the beginning or end.
-    string = string.trim();
-
     return string;
 };

packages/string/package.json

@@ -30,6 +30,6 @@
     "sinon": "22.0.0"
   },
   "dependencies": {
-    "unidecode": "1.1.0"
+    "any-ascii": "^0.3.3"
   }
 }

packages/string/test/slugify.test.js

@@ -15,7 +15,7 @@ describe('Slugify', function () {
         result.should.equal('');
     });
 
-    it('should remove non ascii characters', function () {
+    it('should remove non-letter characters', function () {
         var result = slugify('howtowin✓', options);
         result.should.equal('howtowin');
     });
@@ -33,13 +33,13 @@ describe('Slugify', function () {
     it('should replace all of the html4 compat symbols in ascii except hyphen and underscore', function () {
         // note: This is missing the soft-hyphen char that isn't much-liked by linters/browsers/etc,
         // it passed the test before it was removed
-        var result = slugify('!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿');
-        result.should.equal('_-c-y-ss-c-a-r-deg-23up-1o-1-41-23-4');
+        var result = slugify('!"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~¡¢£¤¥¦§¨©ª«¬®¯°±²_³´µ¶·¸¹º»¼½¾¿');
+        result.should.equal('a-2_3u-1o-1-41-23-4');
     });
 
     it('should replace all of the foreign chars in ascii', function () {
         var result = slugify('ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ');
-        result.should.equal('aaaaaaaeceeeeiiiidnoooooxouuuuythssaaaaaaaeceeeeiiiidnooooo-ouuuuythy');
+        result.should.equal('aaaaaaae-ceeeeiiiidnooooo-ouuuuythssaaaaaaaeceeeeiiiidnooooo-ouuuuythy');
     });
 
     it('should remove control characters', function () {
@@ -83,8 +83,68 @@ describe('Slugify', function () {
     });
 
     it('should properly handle unicode punctuation conversion', function () {
+        // note: the previous unidecode transformation handled this differently than anyascii, so this is
+        // a compromise that's "good enough" and gives the most optimal results for most languages
+        // result using unidecode was: nijian-wei-iganaika-zai-du-que-ren-sitekudasai-zai-du-miip-misitekudasai
         var result = slugify('に間違いがないか、再度確認してください。再読み込みしてください。', options);
-        result.should.equal('nijian-wei-iganaika-zai-du-que-ren-sitekudasai-zai-du-miip-misitekudasai');
+        result.should.equal('ni-jian-weiiganaika-zai-du-que-renshitekudasai-zai-dumi-yumishitekudasai');
+    });
+
+    it('should not transliterate the slugs if the unicodeSlugs flag is passed', function () {
+        var result;
+        options = {unicodeSlugs: true};
+        result = slugify('Ett smörgåsbord från Sydkorea: 스뫼르고스보르드', options);
+        result.should.equal('ett-smörgåsbord-från-sydkorea-스뫼르고스보르드');
+    });
+
+    it('should normalize characters with combining marks before creating the slugs', function () {
+        var result;
+        options = {unicodeSlugs: true};
+        result = slugify('café'.normalize('NFD'), options);
+        result.should.equal('café'.normalize('NFC'));
+    });
+
+    it('should permit words in languages that rely on combining marks without a normalized form', function () {
+        var result;
+        options = {unicodeSlugs: true};
+        result = slugify('น้ำ (water)', options);
+        result.should.equal('น้ำ-water');
+    });
+
+    it('should remove potential misuse of combining marks, like Zalgo text', function () {
+        // note that this might break some text editors, so it might need to be removed
+        var result;
+        options = {unicodeSlugs: true};
+        result = slugify('G̸̛̦̼̜̱̹̦̲̩̰̀̓̆̇̔̎̒̎h̸͕̹̤̿͌́͊͋̈̂͗̕o̶̠͑̍s̷̝̭̰̳̖̣͉̈́̌̐́̈́̒͂̚t̴̩̦̫̟̲̘̆̔̑̅͘̕͠͝͠ ̶̜̺͚̆̈ͅb̸̰͕͔͈̤̾̉͒̂̎ͅl̵̳͚̘̯̀̎o̵̯͝ǵ̴̨̛͍̞͙̲̦̗̖͍̂̈́͆͝', options);
+        result.should.equal('ghost-blo̵̯͝ǵ');
+    });
+
+    it('should remove any loose combining marks in the beginning of a text', function () {
+        var result;
+        options = {unicodeSlugs: true};
+        result = slugify('\u0303\u0301\u0302ผีในวัฒนธรรมไทย', options);
+        result.should.equal('ผีในวัฒนธรรมไทย');
+    });
+
+    it('should remove any loose combining marks in the beginning of a word', function () {
+        var result;
+        options = {unicodeSlugs: true};
+        result = slugify('ghost-\u0301\u0302blog', options);
+        result.should.equal('ghost-blog');
+    });
+
+    it('should replace an invalid separator with -', function () {
+        var result;
+        options = {slugSeparator: '%'};
+        result = slugify('Another day, another post', options);
+        result.should.equal('another-day-another-post');
+    });
+
+    it('should not replace existing dashes and underscores when the separator is set to spaces', function () {
+        var result;
+        options = {slugSeparator: ' '};
+        result = slugify('Herr./Klaus-Jürgen_44', options);
+        result.should.equal('herr klaus-jurgen_44');
     });
 
     it('should not lose or convert dashes if options are passed with truthy importing flag', function () {
@@ -98,6 +158,6 @@ describe('Slugify', function () {
         var result;
         options = {requiredChangesOnly: true};
         result = slugify('-slug-&with-✓-invalid-characters-に\'', options);
-        result.should.equal('-slug--with--invalid-characters-ni');
+        result.should.equal('-slug--with---invalid-characters-ni');
     });
 });