From 8cf91ee8534c04091508c7fbbad81ddfd4df9069 Mon Sep 17 00:00:00 2001 From: dittnamn Date: Sun, 19 Jul 2026 09:07:41 +0200 Subject: [PATCH 1/6] =?UTF-8?q?=F0=9F=8E=A8=20Changed=20and=20improved=20t?= =?UTF-8?q?he=20slug=20generation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ref towards https://github.com/TryGhost/Ghost/issues/3224 etc. The old slug generation using unidecode had a lot of issues with erroneous transliteration for many languages, which has been the reason for many discussions and requests for change over the years. By replacing unidecode with anyascii, a lot of these issues should be solved, however not perfectly. Over the years, the initial reasons for not allowing anything other than ascii letters and numbers in the slugs have been fixed. The Ghost databases, routes, links, loading, etc. now support unicode characters in URL:s. Browser support is fully working and many large sites, including Wikipedia, use unicode characters in URL:s. With just a few modifications in the Ghost sources, an option for full unicode slug support could therefore be added for users who want it. To work towards this, an extra option has been added to the slugify function, to allow the disabling of the transliteration part of the slug generation. Another option, allowing the change of slug part separator was also added. Due to how the filtering work, the possible options are currently just spaces, dashes and underscores, but more options could possibly be added in the future. Dots as separators could in theory make good looking slugs, but should be avoided due to the risk of filename mixups. Due to the slightly different transliteration method, some of the tests have been revised and some new ones were added as well. Note that anyascii has some quirks compared to unidecode, but in total this is an improvement for most languages. --- packages/string/lib/slugify.js | 49 +++++++++++++++++----------- packages/string/package.json | 2 +- packages/string/test/slugify.test.js | 29 ++++++++++++---- 3 files changed, 54 insertions(+), 26 deletions(-) diff --git a/packages/string/lib/slugify.js b/packages/string/lib/slugify.js index 1f5563439..603aa9776 100644 --- a/packages/string/lib/slugify.js +++ b/packages/string/lib/slugify.js @@ -1,4 +1,4 @@ -const unidecode = require('unidecode'); +const anyAscii = require('any-ascii').default; const stripInvisibleChars = require('./strip-invisible-chars'); /** @@ -9,40 +9,51 @@ const stripInvisibleChars = require('./strip-invisible-chars'); * @param {String} string - the string we want to slugify * @param {object} options - filter options * @param {bool} [options.requiredChangesOnly] - don't perform optional cleanup, e.g. removing extra dashes + * @param {bool} [options.noTransliteration] - don't perform optional transliteration, e.g. keep smörgåsbord as it is instead of turning it into smorgasbord + * @param {string} [options.separator] - separator to be used for the slugs, can be ` `, `_` or `-`, defaults to `-` * @returns {String} slugified string */ module.exports = function (string, options = {}) { + // If the separator isn't set, default to `-` + const separator = options.separator || '-'; + // Ensure we have a string string = string || ''; // Strip all characters that cannot be printed - string = stripInvisibleChars(string); - - // Handle the £ symbol separately, since it needs to be removed before the unicode conversion. - string = string.replace(/£/g, '-'); + string = stripInvisibleChars(string) + // Remove apostrophes + .replace(/'/g, '') + // Remove anything that's not a letter, number or a separator + .replace(/[^\p{L}\p{N}\s_-]/gu, separator); - // Remove non ascii characters - string = unidecode(string); + // Perform the transliteration if requested + if (!options.noTransliteration) { + string = anyAscii(string); + } - // Replace URL reserved chars: `@:/?#[]!$&()*+,;=` as well as `\%<>|^~£"{}` and \` - string = string.replace(/(\s|\.|@|:|\/|\?|#|\[|\]|!|\$|&|\(|\)|\*|\+|,|;|=|\\|%|<|>|\||\^|~|"|\{|\}|`|–|—)/g, '-') - // Remove apostrophes + // Replace spaces, URL reserved chars: `@:/?#[]!$&()*+,;=` as well as `\%<>|^~£"{}-` and \` with the selected separator. + // Should only be needed in case the transliteration added something, but it's safer to always run in case the regex filter missed something. + string = string.replace(/(\s|\.|@|:|\/|\?|#|\[|\]|!|\$|&|\(|\)|\*|\+|,|;|=|\\|%|<|>|\||\^|~|£|"|\{|\}|`|–|—)/g, separator) + // Remove apostrophes (again, in case the transliteration added some) .replace(/'/g, '') + // camelCase looking text after initial cleanup and transliteration are most likely separate words, so we add separators between the parts + .replace(/([a-z])([A-Z])/g, `$1${separator}$2`) // Make the whole thing lowercase .toLowerCase(); // These changes are optional changes, we can enable/disable these if (!options.requiredChangesOnly) { - // Convert 2 or more dashes into a single dash - string = string.replace(/-+/g, '-') - // Remove trailing dash - .replace(/-$/, '') - // Remove any dashes at the beginning - .replace(/^-/, ''); + // Convert 2 or more separators into a single separator + string = string.replace(/[\s_-]{2,}/g, separator) + // Remove trailing separators + .replace(/[\s_-]$/, '') + // Remove any separators at the beginning + .replace(/^[\s_-]/, ''); + } else { + // Handle whitespace at the beginning or end. + string = string.trim(); } - // Handle whitespace at the beginning or end. - string = string.trim(); - return string; }; diff --git a/packages/string/package.json b/packages/string/package.json index 2995ee4a8..59c631eb8 100644 --- a/packages/string/package.json +++ b/packages/string/package.json @@ -30,6 +30,6 @@ "sinon": "22.0.0" }, "dependencies": { - "unidecode": "1.1.0" + "any-ascii": "^0.3.3" } } diff --git a/packages/string/test/slugify.test.js b/packages/string/test/slugify.test.js index cdb79de77..890e2d7d4 100644 --- a/packages/string/test/slugify.test.js +++ b/packages/string/test/slugify.test.js @@ -15,7 +15,7 @@ describe('Slugify', function () { result.should.equal(''); }); - it('should remove non ascii characters', function () { + it('should remove non-letter characters', function () { var result = slugify('howtowin✓', options); result.should.equal('howtowin'); }); @@ -33,13 +33,13 @@ describe('Slugify', function () { it('should replace all of the html4 compat symbols in ascii except hyphen and underscore', function () { // note: This is missing the soft-hyphen char that isn't much-liked by linters/browsers/etc, // it passed the test before it was removed - var result = slugify('!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿'); - result.should.equal('_-c-y-ss-c-a-r-deg-23up-1o-1-41-23-4'); + var result = slugify('!"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~¡¢£¤¥¦§¨©ª«¬®¯°±²_³´µ¶·¸¹º»¼½¾¿'); + result.should.equal('a-2_3-u-1o-1-41-23-4'); }); it('should replace all of the foreign chars in ascii', function () { var result = slugify('ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖרÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ'); - result.should.equal('aaaaaaaeceeeeiiiidnoooooxouuuuythssaaaaaaaeceeeeiiiidnooooo-ouuuuythy'); + result.should.equal('aaaaaaae-ceeeeiiiidnooooo-ouuuuythssaaaaaaaeceeeeiiiidnooooo-ouuuuythy'); }); it('should remove control characters', function () { @@ -83,8 +83,25 @@ describe('Slugify', function () { }); it('should properly handle unicode punctuation conversion', function () { + // note: the previous unidecode transformation handled this differently than anyascii, so this is + // a compromise that's "good enough" and gives the most optimal results for most languages + // result using unidecode was: nijian-wei-iganaika-zai-du-que-ren-sitekudasai-zai-du-miip-misitekudasai var result = slugify('に間違いがないか、再度確認してください。再読み込みしてください。', options); - result.should.equal('nijian-wei-iganaika-zai-du-que-ren-sitekudasai-zai-du-miip-misitekudasai'); + result.should.equal('ni-jian-weiiganaika-zai-du-que-renshitekudasai-zai-dumi-yumishitekudasai'); + }); + + it('should not transliterate the slugs if the noTransliteration flag is passed', function () { + var result; + options = {noTransliteration: true}; + result = slugify('Ett smörgåsbord från Sydkorea: 스뫼르고스보르드', options); + result.should.equal('ett-smörgåsbord-från-sydkorea-스뫼르고스보르드'); + }); + + it('should not replace existing dashes and underscores when the separator is set to spaces', function () { + var result; + options = {separator: ' '}; + result = slugify('Herr./Klaus-Jürgen_44', options); + result.should.equal('herr klaus-jurgen_44'); }); it('should not lose or convert dashes if options are passed with truthy importing flag', function () { @@ -98,6 +115,6 @@ describe('Slugify', function () { var result; options = {requiredChangesOnly: true}; result = slugify('-slug-&with-✓-invalid-characters-に\'', options); - result.should.equal('-slug--with--invalid-characters-ni'); + result.should.equal('-slug--with---invalid-characters-ni'); }); }); From 05f11a811957b27414f36a9e0ac487cf6c687b83 Mon Sep 17 00:00:00 2001 From: dittnamn Date: Thu, 30 Jul 2026 20:30:59 +0200 Subject: [PATCH 2/6] Changed slugify options and made a few adjustments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The slugify options were renamed to describe their usage better. Also, a NFC normalization has been added to make sure combining marks aren't lost in the conversion and to make sure seemingly identical slugs won't be generated when the unicode slugs are enabled. Extra types of common apostrophes are also now just removed instead of turned into separators, to make sure "what‘s" is turned into "whats" instead of "what-s". --- packages/string/lib/slugify.js | 17 ++++++++++------- packages/string/test/slugify.test.js | 8 ++++---- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/packages/string/lib/slugify.js b/packages/string/lib/slugify.js index 603aa9776..a07cf7d41 100644 --- a/packages/string/lib/slugify.js +++ b/packages/string/lib/slugify.js @@ -9,26 +9,29 @@ const stripInvisibleChars = require('./strip-invisible-chars'); * @param {String} string - the string we want to slugify * @param {object} options - filter options * @param {bool} [options.requiredChangesOnly] - don't perform optional cleanup, e.g. removing extra dashes - * @param {bool} [options.noTransliteration] - don't perform optional transliteration, e.g. keep smörgåsbord as it is instead of turning it into smorgasbord - * @param {string} [options.separator] - separator to be used for the slugs, can be ` `, `_` or `-`, defaults to `-` + * @param {bool} [options.unicodeSlugs] - don't perform optional transliteration, e.g. keep smörgåsbord as it is instead of turning it into smorgasbord + * @param {string} [options.slugSeparator] - separator to be used for the slugs, can be ` `, `_` or `-`, defaults to `-` * @returns {String} slugified string */ module.exports = function (string, options = {}) { // If the separator isn't set, default to `-` - const separator = options.separator || '-'; + const separator = options.slugSeparator || '-'; // Ensure we have a string string = string || ''; // Strip all characters that cannot be printed string = stripInvisibleChars(string) - // Remove apostrophes - .replace(/'/g, '') + // Normalize the input to make sure accent marks, etc. are part of the characters before continuing + .normalize('NFC') + // Remove the most common forms of apostrophes, to turn contractions like "what’s" into "whats" + .replace(/['‘’´]/g, '') + // Remove anything that's not a letter, number or a separator .replace(/[^\p{L}\p{N}\s_-]/gu, separator); // Perform the transliteration if requested - if (!options.noTransliteration) { + if (!options.unicodeSlugs) { string = anyAscii(string); } @@ -36,7 +39,7 @@ module.exports = function (string, options = {}) { // Should only be needed in case the transliteration added something, but it's safer to always run in case the regex filter missed something. string = string.replace(/(\s|\.|@|:|\/|\?|#|\[|\]|!|\$|&|\(|\)|\*|\+|,|;|=|\\|%|<|>|\||\^|~|£|"|\{|\}|`|–|—)/g, separator) // Remove apostrophes (again, in case the transliteration added some) - .replace(/'/g, '') + .replace(/['‘’´]/g, '') // camelCase looking text after initial cleanup and transliteration are most likely separate words, so we add separators between the parts .replace(/([a-z])([A-Z])/g, `$1${separator}$2`) // Make the whole thing lowercase diff --git a/packages/string/test/slugify.test.js b/packages/string/test/slugify.test.js index 890e2d7d4..37a801c32 100644 --- a/packages/string/test/slugify.test.js +++ b/packages/string/test/slugify.test.js @@ -34,7 +34,7 @@ describe('Slugify', function () { // note: This is missing the soft-hyphen char that isn't much-liked by linters/browsers/etc, // it passed the test before it was removed var result = slugify('!"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~¡¢£¤¥¦§¨©ª«¬®¯°±²_³´µ¶·¸¹º»¼½¾¿'); - result.should.equal('a-2_3-u-1o-1-41-23-4'); + result.should.equal('a-2_3u-1o-1-41-23-4'); }); it('should replace all of the foreign chars in ascii', function () { @@ -90,16 +90,16 @@ describe('Slugify', function () { result.should.equal('ni-jian-weiiganaika-zai-du-que-renshitekudasai-zai-dumi-yumishitekudasai'); }); - it('should not transliterate the slugs if the noTransliteration flag is passed', function () { + it('should not transliterate the slugs if the unicodeSlugs flag is passed', function () { var result; - options = {noTransliteration: true}; + options = {unicodeSlugs: true}; result = slugify('Ett smörgåsbord från Sydkorea: 스뫼르고스보르드', options); result.should.equal('ett-smörgåsbord-från-sydkorea-스뫼르고스보르드'); }); it('should not replace existing dashes and underscores when the separator is set to spaces', function () { var result; - options = {separator: ' '}; + options = {slugSeparator: ' '}; result = slugify('Herr./Klaus-Jürgen_44', options); result.should.equal('herr klaus-jurgen_44'); }); From 5a7c544fec4fab77c2c786882a5c154c5deef0b6 Mon Sep 17 00:00:00 2001 From: dittnamn Date: Thu, 30 Jul 2026 22:07:04 +0200 Subject: [PATCH 3/6] Added more tests and made sure the separators are valid. --- packages/string/lib/slugify.js | 4 ++-- packages/string/test/slugify.test.js | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/string/lib/slugify.js b/packages/string/lib/slugify.js index a07cf7d41..32a603365 100644 --- a/packages/string/lib/slugify.js +++ b/packages/string/lib/slugify.js @@ -14,8 +14,8 @@ const stripInvisibleChars = require('./strip-invisible-chars'); * @returns {String} slugified string */ module.exports = function (string, options = {}) { - // If the separator isn't set, default to `-` - const separator = options.slugSeparator || '-'; + // If the separator is invalid or unset, replace it with the default `-` + const separator = ['-', '_', ' '].includes(options.slugSeparator) ? options.slugSeparator : '-'; // Ensure we have a string string = string || ''; diff --git a/packages/string/test/slugify.test.js b/packages/string/test/slugify.test.js index 37a801c32..3acb5728f 100644 --- a/packages/string/test/slugify.test.js +++ b/packages/string/test/slugify.test.js @@ -97,6 +97,20 @@ describe('Slugify', function () { result.should.equal('ett-smörgåsbord-från-sydkorea-스뫼르고스보르드'); }); + it('should normalize characters with combining marks before creating the slugs', function () { + var result; + options = {unicodeSlugs: true}; + result = slugify('café'.normalize('NFD'), options); + result.should.equal('café'.normalize('NFC')); + }); + + it('should replace an invalid separator with -', function () { + var result; + options = {slugSeparator: '%'}; + result = slugify('Another day, another post', options); + result.should.equal('another-day-another-post'); + }); + it('should not replace existing dashes and underscores when the separator is set to spaces', function () { var result; options = {slugSeparator: ' '}; From e9a450bca968fdc55b3a98fe62c6c3a31fb81cd8 Mon Sep 17 00:00:00 2001 From: dittnamn Date: Thu, 30 Jul 2026 23:35:15 +0200 Subject: [PATCH 4/6] Added more tests and allowed combining marks to be used to some extent This makes sure Thai script, etc. can be used as slugs, but removes overuse of combining marks that creates Zalgo text and similar. --- packages/string/lib/slugify.js | 7 +++++-- packages/string/test/slugify.test.js | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/string/lib/slugify.js b/packages/string/lib/slugify.js index 32a603365..a1d945e1a 100644 --- a/packages/string/lib/slugify.js +++ b/packages/string/lib/slugify.js @@ -26,9 +26,12 @@ module.exports = function (string, options = {}) { .normalize('NFC') // Remove the most common forms of apostrophes, to turn contractions like "what’s" into "whats" .replace(/['‘’´]/g, '') - // Remove anything that's not a letter, number or a separator - .replace(/[^\p{L}\p{N}\s_-]/gu, separator); + .replace(/[^\p{L}\p{N}\p{Mn}\p{Mc}\s_-]/gu, separator) + // Remove potential misuse of combining marks, like Zalgo text, by limiting the number of marks, + // a limit of 3 marks should allow pretty much any natural language usage, so in case there's 4 + // marks or more, we remove all marks. + .replace(/([^\p{Mn}\p{Mc}])[\p{Mn}\p{Mc}]{4,}/gu, '$1'); // Perform the transliteration if requested if (!options.unicodeSlugs) { diff --git a/packages/string/test/slugify.test.js b/packages/string/test/slugify.test.js index 3acb5728f..2536e9ee4 100644 --- a/packages/string/test/slugify.test.js +++ b/packages/string/test/slugify.test.js @@ -104,6 +104,21 @@ describe('Slugify', function () { result.should.equal('café'.normalize('NFC')); }); + it('should permit words in languages that rely on combining marks without a normalized form', function () { + var result; + options = {unicodeSlugs: true}; + result = slugify('น้ำ (water)', options); + result.should.equal('น้ำ-water'); + }); + + it('should remove potential misuse of combining marks, like Zalgo text', function () { + // note that this might break some text editors, so it might need to be removed + var result; + options = {unicodeSlugs: true}; + result = slugify('G̸̛̦̼̜̱̹̦̲̩̰̀̓̆̇̔̎̒̎h̸͕̹̤̿͌́͊͋̈̂͗̕o̶̠͑̍s̷̝̭̰̳̖̣͉̈́̌̐́̈́̒͂̚t̴̩̦̫̟̲̘̆̔̑̅͘̕͠͝͠ ̶̜̺͚̆̈ͅb̸̰͕͔͈̤̾̉͒̂̎ͅl̵̳͚̘̯̀̎o̵̯͝ǵ̴̨̛͍̞͙̲̦̗̖͍̂̈́͆͝', options); + result.should.equal('ghost-blo̵̯͝ǵ'); + }); + it('should replace an invalid separator with -', function () { var result; options = {slugSeparator: '%'}; From 2eb98afa66732b438e0608e2c2d6b52e0c67c27c Mon Sep 17 00:00:00 2001 From: dittnamn Date: Fri, 31 Jul 2026 00:00:14 +0200 Subject: [PATCH 5/6] Added a filter that removes leading combining marks, with extra testing --- packages/string/lib/slugify.js | 4 +++- packages/string/test/slugify.test.js | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/string/lib/slugify.js b/packages/string/lib/slugify.js index a1d945e1a..dd7b88d48 100644 --- a/packages/string/lib/slugify.js +++ b/packages/string/lib/slugify.js @@ -30,7 +30,9 @@ module.exports = function (string, options = {}) { .replace(/[^\p{L}\p{N}\p{Mn}\p{Mc}\s_-]/gu, separator) // Remove potential misuse of combining marks, like Zalgo text, by limiting the number of marks, // a limit of 3 marks should allow pretty much any natural language usage, so in case there's 4 - // marks or more, we remove all marks. + // marks or more, we remove all marks. Combining marks in the beginning of a text shouldn't + // exist at all in natural language, so these are just removed. + .replace(/^[\p{Mn}\p{Mc}]+/gu, '') .replace(/([^\p{Mn}\p{Mc}])[\p{Mn}\p{Mc}]{4,}/gu, '$1'); // Perform the transliteration if requested diff --git a/packages/string/test/slugify.test.js b/packages/string/test/slugify.test.js index 2536e9ee4..17c867d3f 100644 --- a/packages/string/test/slugify.test.js +++ b/packages/string/test/slugify.test.js @@ -119,6 +119,13 @@ describe('Slugify', function () { result.should.equal('ghost-blo̵̯͝ǵ'); }); + it('should remove any loose combining marks in the beginning of a text', function () { + var result; + options = {unicodeSlugs: true}; + result = slugify('\u0303\u0301\u0302ผีในวัฒนธรรมไทย', options); + result.should.equal('ผีในวัฒนธรรมไทย'); + }); + it('should replace an invalid separator with -', function () { var result; options = {slugSeparator: '%'}; From fe010ac68a2bd4c68901ce82c72b64db39e6b57f Mon Sep 17 00:00:00 2001 From: dittnamn Date: Fri, 31 Jul 2026 16:22:14 +0200 Subject: [PATCH 6/6] Changed the filter to remove leading combining marks from the beginning of words --- packages/string/lib/slugify.js | 4 ++-- packages/string/test/slugify.test.js | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/string/lib/slugify.js b/packages/string/lib/slugify.js index dd7b88d48..653e640ce 100644 --- a/packages/string/lib/slugify.js +++ b/packages/string/lib/slugify.js @@ -30,9 +30,9 @@ module.exports = function (string, options = {}) { .replace(/[^\p{L}\p{N}\p{Mn}\p{Mc}\s_-]/gu, separator) // Remove potential misuse of combining marks, like Zalgo text, by limiting the number of marks, // a limit of 3 marks should allow pretty much any natural language usage, so in case there's 4 - // marks or more, we remove all marks. Combining marks in the beginning of a text shouldn't + // marks or more, we remove all marks. Combining marks in the beginning of a word shouldn't // exist at all in natural language, so these are just removed. - .replace(/^[\p{Mn}\p{Mc}]+/gu, '') + .replace(/([\p{L}\p{N}][\p{Mn}\p{Mc}]*)|[\p{Mn}\p{Mc}]+/gu, '$1') .replace(/([^\p{Mn}\p{Mc}])[\p{Mn}\p{Mc}]{4,}/gu, '$1'); // Perform the transliteration if requested diff --git a/packages/string/test/slugify.test.js b/packages/string/test/slugify.test.js index 17c867d3f..9c508e611 100644 --- a/packages/string/test/slugify.test.js +++ b/packages/string/test/slugify.test.js @@ -126,6 +126,13 @@ describe('Slugify', function () { result.should.equal('ผีในวัฒนธรรมไทย'); }); + it('should remove any loose combining marks in the beginning of a word', function () { + var result; + options = {unicodeSlugs: true}; + result = slugify('ghost-\u0301\u0302blog', options); + result.should.equal('ghost-blog'); + }); + it('should replace an invalid separator with -', function () { var result; options = {slugSeparator: '%'};