diff --git a/packages/string/lib/slugify.js b/packages/string/lib/slugify.js index 1f5563439..653e640ce 100644 --- a/packages/string/lib/slugify.js +++ b/packages/string/lib/slugify.js @@ -1,4 +1,4 @@ -const unidecode = require('unidecode'); +const anyAscii = require('any-ascii').default; const stripInvisibleChars = require('./strip-invisible-chars'); /** @@ -9,40 +9,59 @@ const stripInvisibleChars = require('./strip-invisible-chars'); * @param {String} string - the string we want to slugify * @param {object} options - filter options * @param {bool} [options.requiredChangesOnly] - don't perform optional cleanup, e.g. removing extra dashes + * @param {bool} [options.unicodeSlugs] - don't perform optional transliteration, e.g. keep smörgåsbord as it is instead of turning it into smorgasbord + * @param {string} [options.slugSeparator] - separator to be used for the slugs, can be ` `, `_` or `-`, defaults to `-` * @returns {String} slugified string */ module.exports = function (string, options = {}) { + // If the separator is invalid or unset, replace it with the default `-` + const separator = ['-', '_', ' '].includes(options.slugSeparator) ? options.slugSeparator : '-'; + // Ensure we have a string string = string || ''; // Strip all characters that cannot be printed - string = stripInvisibleChars(string); - - // Handle the £ symbol separately, since it needs to be removed before the unicode conversion. - string = string.replace(/£/g, '-'); + string = stripInvisibleChars(string) + // Normalize the input to make sure accent marks, etc. are part of the characters before continuing + .normalize('NFC') + // Remove the most common forms of apostrophes, to turn contractions like "what’s" into "whats" + .replace(/['‘’´]/g, '') + // Remove anything that's not a letter, number or a separator + .replace(/[^\p{L}\p{N}\p{Mn}\p{Mc}\s_-]/gu, separator) + // Remove potential misuse of combining marks, like Zalgo text, by limiting the number of marks, + // a limit of 3 marks should allow pretty much any natural language usage, so in case there's 4 + // marks or more, we remove all marks. Combining marks in the beginning of a word shouldn't + // exist at all in natural language, so these are just removed. + .replace(/([\p{L}\p{N}][\p{Mn}\p{Mc}]*)|[\p{Mn}\p{Mc}]+/gu, '$1') + .replace(/([^\p{Mn}\p{Mc}])[\p{Mn}\p{Mc}]{4,}/gu, '$1'); - // Remove non ascii characters - string = unidecode(string); + // Perform the transliteration if requested + if (!options.unicodeSlugs) { + string = anyAscii(string); + } - // Replace URL reserved chars: `@:/?#[]!$&()*+,;=` as well as `\%<>|^~£"{}` and \` - string = string.replace(/(\s|\.|@|:|\/|\?|#|\[|\]|!|\$|&|\(|\)|\*|\+|,|;|=|\\|%|<|>|\||\^|~|"|\{|\}|`|–|—)/g, '-') - // Remove apostrophes - .replace(/'/g, '') + // Replace spaces, URL reserved chars: `@:/?#[]!$&()*+,;=` as well as `\%<>|^~£"{}-` and \` with the selected separator. + // Should only be needed in case the transliteration added something, but it's safer to always run in case the regex filter missed something. + string = string.replace(/(\s|\.|@|:|\/|\?|#|\[|\]|!|\$|&|\(|\)|\*|\+|,|;|=|\\|%|<|>|\||\^|~|£|"|\{|\}|`|–|—)/g, separator) + // Remove apostrophes (again, in case the transliteration added some) + .replace(/['‘’´]/g, '') + // camelCase looking text after initial cleanup and transliteration are most likely separate words, so we add separators between the parts + .replace(/([a-z])([A-Z])/g, `$1${separator}$2`) // Make the whole thing lowercase .toLowerCase(); // These changes are optional changes, we can enable/disable these if (!options.requiredChangesOnly) { - // Convert 2 or more dashes into a single dash - string = string.replace(/-+/g, '-') - // Remove trailing dash - .replace(/-$/, '') - // Remove any dashes at the beginning - .replace(/^-/, ''); + // Convert 2 or more separators into a single separator + string = string.replace(/[\s_-]{2,}/g, separator) + // Remove trailing separators + .replace(/[\s_-]$/, '') + // Remove any separators at the beginning + .replace(/^[\s_-]/, ''); + } else { + // Handle whitespace at the beginning or end. + string = string.trim(); } - // Handle whitespace at the beginning or end. - string = string.trim(); - return string; }; diff --git a/packages/string/package.json b/packages/string/package.json index 2995ee4a8..59c631eb8 100644 --- a/packages/string/package.json +++ b/packages/string/package.json @@ -30,6 +30,6 @@ "sinon": "22.0.0" }, "dependencies": { - "unidecode": "1.1.0" + "any-ascii": "^0.3.3" } } diff --git a/packages/string/test/slugify.test.js b/packages/string/test/slugify.test.js index cdb79de77..9c508e611 100644 --- a/packages/string/test/slugify.test.js +++ b/packages/string/test/slugify.test.js @@ -15,7 +15,7 @@ describe('Slugify', function () { result.should.equal(''); }); - it('should remove non ascii characters', function () { + it('should remove non-letter characters', function () { var result = slugify('howtowin✓', options); result.should.equal('howtowin'); }); @@ -33,13 +33,13 @@ describe('Slugify', function () { it('should replace all of the html4 compat symbols in ascii except hyphen and underscore', function () { // note: This is missing the soft-hyphen char that isn't much-liked by linters/browsers/etc, // it passed the test before it was removed - var result = slugify('!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿'); - result.should.equal('_-c-y-ss-c-a-r-deg-23up-1o-1-41-23-4'); + var result = slugify('!"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~¡¢£¤¥¦§¨©ª«¬®¯°±²_³´µ¶·¸¹º»¼½¾¿'); + result.should.equal('a-2_3u-1o-1-41-23-4'); }); it('should replace all of the foreign chars in ascii', function () { var result = slugify('ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖרÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ'); - result.should.equal('aaaaaaaeceeeeiiiidnoooooxouuuuythssaaaaaaaeceeeeiiiidnooooo-ouuuuythy'); + result.should.equal('aaaaaaae-ceeeeiiiidnooooo-ouuuuythssaaaaaaaeceeeeiiiidnooooo-ouuuuythy'); }); it('should remove control characters', function () { @@ -83,8 +83,68 @@ describe('Slugify', function () { }); it('should properly handle unicode punctuation conversion', function () { + // note: the previous unidecode transformation handled this differently than anyascii, so this is + // a compromise that's "good enough" and gives the most optimal results for most languages + // result using unidecode was: nijian-wei-iganaika-zai-du-que-ren-sitekudasai-zai-du-miip-misitekudasai var result = slugify('に間違いがないか、再度確認してください。再読み込みしてください。', options); - result.should.equal('nijian-wei-iganaika-zai-du-que-ren-sitekudasai-zai-du-miip-misitekudasai'); + result.should.equal('ni-jian-weiiganaika-zai-du-que-renshitekudasai-zai-dumi-yumishitekudasai'); + }); + + it('should not transliterate the slugs if the unicodeSlugs flag is passed', function () { + var result; + options = {unicodeSlugs: true}; + result = slugify('Ett smörgåsbord från Sydkorea: 스뫼르고스보르드', options); + result.should.equal('ett-smörgåsbord-från-sydkorea-스뫼르고스보르드'); + }); + + it('should normalize characters with combining marks before creating the slugs', function () { + var result; + options = {unicodeSlugs: true}; + result = slugify('café'.normalize('NFD'), options); + result.should.equal('café'.normalize('NFC')); + }); + + it('should permit words in languages that rely on combining marks without a normalized form', function () { + var result; + options = {unicodeSlugs: true}; + result = slugify('น้ำ (water)', options); + result.should.equal('น้ำ-water'); + }); + + it('should remove potential misuse of combining marks, like Zalgo text', function () { + // note that this might break some text editors, so it might need to be removed + var result; + options = {unicodeSlugs: true}; + result = slugify('G̸̛̦̼̜̱̹̦̲̩̰̀̓̆̇̔̎̒̎h̸͕̹̤̿͌́͊͋̈̂͗̕o̶̠͑̍s̷̝̭̰̳̖̣͉̈́̌̐́̈́̒͂̚t̴̩̦̫̟̲̘̆̔̑̅͘̕͠͝͠ ̶̜̺͚̆̈ͅb̸̰͕͔͈̤̾̉͒̂̎ͅl̵̳͚̘̯̀̎o̵̯͝ǵ̴̨̛͍̞͙̲̦̗̖͍̂̈́͆͝', options); + result.should.equal('ghost-blo̵̯͝ǵ'); + }); + + it('should remove any loose combining marks in the beginning of a text', function () { + var result; + options = {unicodeSlugs: true}; + result = slugify('\u0303\u0301\u0302ผีในวัฒนธรรมไทย', options); + result.should.equal('ผีในวัฒนธรรมไทย'); + }); + + it('should remove any loose combining marks in the beginning of a word', function () { + var result; + options = {unicodeSlugs: true}; + result = slugify('ghost-\u0301\u0302blog', options); + result.should.equal('ghost-blog'); + }); + + it('should replace an invalid separator with -', function () { + var result; + options = {slugSeparator: '%'}; + result = slugify('Another day, another post', options); + result.should.equal('another-day-another-post'); + }); + + it('should not replace existing dashes and underscores when the separator is set to spaces', function () { + var result; + options = {slugSeparator: ' '}; + result = slugify('Herr./Klaus-Jürgen_44', options); + result.should.equal('herr klaus-jurgen_44'); }); it('should not lose or convert dashes if options are passed with truthy importing flag', function () { @@ -98,6 +158,6 @@ describe('Slugify', function () { var result; options = {requiredChangesOnly: true}; result = slugify('-slug-&with-✓-invalid-characters-に\'', options); - result.should.equal('-slug--with--invalid-characters-ni'); + result.should.equal('-slug--with---invalid-characters-ni'); }); });