diff --git a/conformance-tests/2023-09/EXPR/jobs/expr2.2.4--string-classification-unicode.test.yaml b/conformance-tests/2023-09/EXPR/jobs/expr2.2.4--string-classification-unicode.test.yaml new file mode 100644 index 0000000..005145e --- /dev/null +++ b/conformance-tests/2023-09/EXPR/jobs/expr2.2.4--string-classification-unicode.test.yaml @@ -0,0 +1,94 @@ +# String classification functions must match Python str method semantics on +# Unicode (non-ASCII) input, and title/capitalize must match Python's case +# mapping rules, per the notes in section 2.2.4. Expected values are +# CPython's answers for the same inputs. +# +# Characters used (as \uXXXX escapes so this file stays ASCII): +# U+0663 ARABIC-INDIC DIGIT THREE Nd, Numeric_Type=Decimal +# U+00B2 SUPERSCRIPT TWO No, Numeric_Type=Digit +# U+00BD VULGAR FRACTION ONE HALF No, Numeric_Type=Numeric +# U+216B ROMAN NUMERAL TWELVE Nl, Numeric_Type=Numeric +# U+0345 COMBINING GREEK YPOGEGRAMMENI Mn, Other_Alphabetic +# U+24B6 CIRCLED LATIN CAPITAL LETTER A So, Other_Alphabetic +# U+4E94 CJK UNIFIED IDEOGRAPH-4E94 (five) Lo, uncased letter +# U+01C5 LATIN CAPITAL D WITH SMALL Z CARON Lt, titlecase (cased) +# U+0295 LATIN LETTER PHARYNGEAL VOICED FRICATIVE Ll +# U+001C-U+001F information separators Cc, Python isspace +template: + specificationVersion: jobtemplate-2023-09 + extensions: + - EXPR + name: TestJob + steps: + - name: Step1 + script: + actions: + onRun: + command: python + args: + - -c + - | + print(r'ISDIGIT_ARABIC:[{{ "\u0663".isdigit() }}]') + print(r'ISDIGIT_MIXED:[{{ "12\u0663".isdigit() }}]') + print(r'ISDIGIT_SUPER2:[{{ "\u00b2".isdigit() }}]') + print(r'ISDIGIT_HALF:[{{ "\u00bd".isdigit() }}]') + print(r'ISDIGIT_ROMAN:[{{ "\u216b".isdigit() }}]') + print(r'ISALPHA_ROMAN:[{{ "\u216b".isalpha() }}]') + print(r'ISALPHA_COMBINING:[{{ "\u0345".isalpha() }}]') + print(r'ISALPHA_CIRCLED:[{{ "\u24b6".isalpha() }}]') + print(r'ISALPHA_CJK:[{{ "\u4e94".isalpha() }}]') + print(r'ISALNUM_ARABIC:[{{ "\u0663".isalnum() }}]') + print(r'ISALNUM_HALF:[{{ "\u00bd".isalnum() }}]') + print(r'ISALNUM_ROMAN:[{{ "\u216b".isalnum() }}]') + print(r'ISALNUM_CIRCLED:[{{ "\u24b6".isalnum() }}]') + print(r'ISSPACE_SEPARATORS:[{{ "\u001c\u001d\u001e\u001f".isspace() }}]') + print(r'ISSPACE_NBSP:[{{ "\u00a0".isspace() }}]') + print(r'ISUPPER_WITH_UNCASED:[{{ "A\u4e94".isupper() }}]') + print(r'ISUPPER_TITLECASE:[{{ "\u01c5".isupper() }}]') + print(r'ISLOWER_WITH_UNCASED:[{{ "a\u4e94".islower() }}]') + print(r'ISLOWER_TITLECASE:[{{ "\u01c5".islower() }}]') + print(r'ISLOWER_PHARYNGEAL:[{{ "\u0295".islower() }}]') + print(r'ISLOWER_UNCASED_ONLY:[{{ "\u4e94".islower() }}]') + print(r'TITLE_DIGIT_BOUNDARY:[{{ title("1st") == "1St" }}]') + print(r'TITLE_MID_DIGIT:[{{ title("ab2cd") == "Ab2Cd" }}]') + print(r'TITLE_DZ_DIGRAPH:[{{ title("\u01c6ab") == "\u01c5ab" }}]') + print(r'TITLE_SHARP_S_MID:[{{ title("ss\u00df") == "Ss\u00df" }}]') + print(r'TITLE_UNCASED_BOUNDARY:[{{ title("a\u4e94b") == "A\u4e94B" }}]') + print(r'TITLE_FINAL_SIGMA:[{{ title("O\u03a3 K") == "O\u03c2 K" }}]') + print(r'CAP_DZ_DIGRAPH:[{{ capitalize("\u01c6ab") == "\u01c5ab" }}]') + print(r'CAP_SHARP_S:[{{ capitalize("\u00dfx") == "Ssx" }}]') + print(r'CAP_UNCASED_FIRST:[{{ capitalize("1st") == "1st" }}]') + print(r'CAP_FINAL_SIGMA:[{{ capitalize("O\u03a3 K") == "O\u03c2 k" }}]') +expected: + output: + - ISDIGIT_ARABIC:[true] + - ISDIGIT_MIXED:[true] + - ISDIGIT_SUPER2:[true] + - ISDIGIT_HALF:[false] + - ISDIGIT_ROMAN:[false] + - ISALPHA_ROMAN:[false] + - ISALPHA_COMBINING:[false] + - ISALPHA_CIRCLED:[false] + - ISALPHA_CJK:[true] + - ISALNUM_ARABIC:[true] + - ISALNUM_HALF:[true] + - ISALNUM_ROMAN:[true] + - ISALNUM_CIRCLED:[false] + - ISSPACE_SEPARATORS:[true] + - ISSPACE_NBSP:[true] + - ISUPPER_WITH_UNCASED:[true] + - ISUPPER_TITLECASE:[false] + - ISLOWER_WITH_UNCASED:[true] + - ISLOWER_TITLECASE:[false] + - ISLOWER_PHARYNGEAL:[true] + - ISLOWER_UNCASED_ONLY:[false] + - TITLE_DIGIT_BOUNDARY:[true] + - TITLE_MID_DIGIT:[true] + - TITLE_DZ_DIGRAPH:[true] + - TITLE_SHARP_S_MID:[true] + - TITLE_UNCASED_BOUNDARY:[true] + - TITLE_FINAL_SIGMA:[true] + - CAP_DZ_DIGRAPH:[true] + - CAP_SHARP_S:[true] + - CAP_UNCASED_FIRST:[true] + - CAP_FINAL_SIGMA:[true] diff --git a/rfcs/0006-expression-function-library.md b/rfcs/0006-expression-function-library.md index 10a433f..e12f616 100644 --- a/rfcs/0006-expression-function-library.md +++ b/rfcs/0006-expression-function-library.md @@ -444,8 +444,8 @@ The `list[nulltype]` overloads handle empty lists, matching Python semantics. |-----------|-------------| | `upper(s: string) -> string` | Convert to uppercase | | `lower(s: string) -> string` | Convert to lowercase | -| `capitalize(s: string) -> string` | Capitalize first character, lowercase rest | -| `title(s: string) -> string` | Capitalize first character of each word | +| `capitalize(s: string) -> string` | Titlecase first character, lowercase rest, per Python `str.capitalize` (see the case mapping note below) | +| `title(s: string) -> string` | Titlecase the first character of each word, lowercase the rest, per Python `str.title`: a word starts after any uncased character (see the case mapping note below) | | `strip(s: string) -> string` | Remove leading/trailing whitespace | | `strip(s: string, chars: string) -> string` | Remove leading/trailing characters in `chars` | | `lstrip(s: string) -> string` | Remove leading whitespace | @@ -456,12 +456,12 @@ The `list[nulltype]` overloads handle empty lists, matching Python semantics. | `removesuffix(s: string, suffix: string) -> string` | Remove suffix if present, otherwise return unchanged | | `startswith(s: string, prefix: string) -> bool` | Test if string starts with prefix | | `endswith(s: string, suffix: string) -> bool` | Test if string ends with suffix | -| `isdigit(s: string) -> bool` | True if all characters are digits and string is non-empty | -| `isalpha(s: string) -> bool` | True if all characters are alphabetic and string is non-empty | -| `isalnum(s: string) -> bool` | True if all characters are alphanumeric and string is non-empty | -| `isspace(s: string) -> bool` | True if all characters are whitespace and string is non-empty | -| `isupper(s: string) -> bool` | True if all cased characters are uppercase and there is at least one cased character | -| `islower(s: string) -> bool` | True if all cased characters are lowercase and there is at least one cased character | +| `isdigit(s: string) -> bool` | True if all characters are digits and string is non-empty, per Python `str.isdigit` (Unicode `Numeric_Type` of `Decimal` or `Digit` — includes non-ASCII decimal digits and superscripts) | +| `isalpha(s: string) -> bool` | True if all characters are alphabetic and string is non-empty, per Python `str.isalpha` (Unicode general category `Lu`/`Ll`/`Lt`/`Lm`/`Lo`) | +| `isalnum(s: string) -> bool` | True if all characters are alphanumeric and string is non-empty, per Python `str.isalnum` (alphabetic per `isalpha`, or any Unicode `Numeric_Type`) | +| `isspace(s: string) -> bool` | True if all characters are whitespace and string is non-empty, per Python `str.isspace` (category `Zs`/`Zl`/`Zp` or bidirectional class WS/B/S — includes U+001C–U+001F) | +| `isupper(s: string) -> bool` | True if all cased characters are uppercase and there is at least one cased character, per Python `str.isupper` (see the cased-character note below) | +| `islower(s: string) -> bool` | True if all cased characters are lowercase and there is at least one cased character, per Python `str.islower` (see the cased-character note below) | | `isascii(s: string) -> bool` | True if all characters are ASCII (U+0000–U+007F), or string is empty | | `count(s: string, sub: string) -> int` | Count non-overlapping occurrences of substring. The `sub` argument must be non-empty; an empty `sub` is an error. | | `find(s: string, sub: string) -> int` | Return lowest index of substring, or -1 if not found. The `sub` argument must be non-empty; an empty `sub` is an error. | @@ -508,6 +508,36 @@ Note: The `join` function intentionally differs from Python's `str.join()`. In P This design enables natural method chaining like `items.split(';').join(',')` and matches the convention used by JavaScript and Ruby. +Note: The character classification functions (`isdigit`, `isalpha`, `isalnum`, `isspace`, +`isupper`, `islower`) have exactly the semantics of the Python `str` methods of the same +name, evaluated over the Unicode Character Database. They are not ASCII-only, and they do +not correspond to other languages' character predicates (for example, Rust's `Alphabetic` +property is a superset of `isalpha`'s `L*` categories). In particular: +- `isdigit` is true for any decimal digit, e.g. U+0663 ARABIC-INDIC DIGIT THREE. +- `isalnum` is broader than `isalpha` OR `isdigit`: characters whose `Numeric_Type` is + `Numeric`, such as U+00BD VULGAR FRACTION ONE HALF, are alphanumeric but neither + alphabetic nor digits. +- A character is *cased* if it has the Unicode `Uppercase` or `Lowercase` property or + general category `Lt` (titlecase). `isupper` and `islower` ignore uncased characters + (digits, ideographs, punctuation), and titlecase characters are cased but neither + uppercase nor lowercase. +Implementations must derive these classifications from Unicode Character Database data +equivalent to a current CPython release. Differences between Unicode versions for newly +assigned code points are permitted. + +Note: The case mapping functions `title` and `capitalize` likewise have exactly the +semantics of Python's `str.title` and `str.capitalize`: +- Word boundaries in `title` are determined by cased-ness, not alphanumeric-ness: a + character is titlecased when the preceding character is not cased, so digits, + punctuation, and uncased letters all start a new word (`title("1st")` is `"1St"`). +- Word-start characters use the full Unicode titlecase mapping, not the uppercase + mapping: `title("džab")` is `"Džab"` (U+01C5, titlecase) and `capitalize("ßx")` is + `"Ssx"`. +- `capitalize` titlecases the first character and lowercases the rest (Python ≥ 3.8 + semantics). +- Lowercasing applies the Unicode Final_Sigma context rule: `title("OΣ K")` is + `"Oς K"`. + #### Regular Expression Functions | Signature | Description | diff --git a/wiki/2026-02-Expression-Language.md b/wiki/2026-02-Expression-Language.md index af9e318..f598455 100644 --- a/wiki/2026-02-Expression-Language.md +++ b/wiki/2026-02-Expression-Language.md @@ -1497,8 +1497,8 @@ Examples: |-----------|-------------| | `upper(s: string) -> string` | Convert to uppercase | | `lower(s: string) -> string` | Convert to lowercase | -| `capitalize(s: string) -> string` | Capitalize first character, lowercase rest | -| `title(s: string) -> string` | Capitalize first character of each word | +| `capitalize(s: string) -> string` | Titlecase first character, lowercase rest, per Python `str.capitalize` (see the case mapping note below) | +| `title(s: string) -> string` | Titlecase the first character of each word, lowercase the rest, per Python `str.title`: a word starts after any uncased character (see the case mapping note below) | | `strip(s: string) -> string` | Remove leading/trailing whitespace | | `strip(s: string, chars: string) -> string` | Remove leading/trailing characters in `chars` | | `lstrip(s: string) -> string` | Remove leading whitespace | @@ -1509,12 +1509,12 @@ Examples: | `removesuffix(s: string, suffix: string) -> string` | Remove suffix if present, otherwise return unchanged | | `startswith(s: string, prefix: string) -> bool` | Test if string starts with prefix | | `endswith(s: string, suffix: string) -> bool` | Test if string ends with suffix | -| `isdigit(s: string) -> bool` | True if all characters are digits and string is non-empty | -| `isalpha(s: string) -> bool` | True if all characters are alphabetic and string is non-empty | -| `isalnum(s: string) -> bool` | True if all characters are alphanumeric and string is non-empty | -| `isspace(s: string) -> bool` | True if all characters are whitespace and string is non-empty | -| `isupper(s: string) -> bool` | True if all cased characters are uppercase and there is at least one cased character | -| `islower(s: string) -> bool` | True if all cased characters are lowercase and there is at least one cased character | +| `isdigit(s: string) -> bool` | True if all characters are digits and string is non-empty, per Python `str.isdigit` (Unicode `Numeric_Type` of `Decimal` or `Digit` — includes non-ASCII decimal digits and superscripts) | +| `isalpha(s: string) -> bool` | True if all characters are alphabetic and string is non-empty, per Python `str.isalpha` (Unicode general category `Lu`/`Ll`/`Lt`/`Lm`/`Lo`) | +| `isalnum(s: string) -> bool` | True if all characters are alphanumeric and string is non-empty, per Python `str.isalnum` (alphabetic per `isalpha`, or any Unicode `Numeric_Type`) | +| `isspace(s: string) -> bool` | True if all characters are whitespace and string is non-empty, per Python `str.isspace` (category `Zs`/`Zl`/`Zp` or bidirectional class WS/B/S — includes U+001C–U+001F) | +| `isupper(s: string) -> bool` | True if all cased characters are uppercase and there is at least one cased character, per Python `str.isupper` (see the cased-character note below) | +| `islower(s: string) -> bool` | True if all cased characters are lowercase and there is at least one cased character, per Python `str.islower` (see the cased-character note below) | | `isascii(s: string) -> bool` | True if all characters are ASCII (U+0000–U+007F), or string is empty | | `count(s: string, sub: string) -> int` | Count non-overlapping occurrences of substring. The `sub` argument must be non-empty; an empty `sub` is an error. | | `find(s: string, sub: string) -> int` | Return lowest index of substring, or -1 if not found. The `sub` argument must be non-empty; an empty `sub` is an error. | @@ -1559,6 +1559,36 @@ Note: Method calls on integer and float literals require parentheses around the (e.g., `(42).zfill(5)` not `42.zfill(5)`) because the parser interprets `42.` as the start of a float literal. +Note: The character classification functions (`isdigit`, `isalpha`, `isalnum`, `isspace`, +`isupper`, `islower`) have exactly the semantics of the Python `str` methods of the same +name, evaluated over the Unicode Character Database. They are not ASCII-only, and they do +not correspond to other languages' character predicates (for example, Rust's `Alphabetic` +property is a superset of `isalpha`'s `L*` categories). In particular: +- `isdigit` is true for any decimal digit, e.g. U+0663 ARABIC-INDIC DIGIT THREE. +- `isalnum` is broader than `isalpha` OR `isdigit`: characters whose `Numeric_Type` is + `Numeric`, such as U+00BD VULGAR FRACTION ONE HALF, are alphanumeric but neither + alphabetic nor digits. +- A character is *cased* if it has the Unicode `Uppercase` or `Lowercase` property or + general category `Lt` (titlecase). `isupper` and `islower` ignore uncased characters + (digits, ideographs, punctuation), and titlecase characters are cased but neither + uppercase nor lowercase. +Implementations must derive these classifications from Unicode Character Database data +equivalent to a current CPython release. Differences between Unicode versions for newly +assigned code points are permitted. + +Note: The case mapping functions `title` and `capitalize` likewise have exactly the +semantics of Python's `str.title` and `str.capitalize`: +- Word boundaries in `title` are determined by cased-ness, not alphanumeric-ness: a + character is titlecased when the preceding character is not cased, so digits, + punctuation, and uncased letters all start a new word (`title("1st")` is `"1St"`). +- Word-start characters use the full Unicode titlecase mapping, not the uppercase + mapping: `title("džab")` is `"Džab"` (U+01C5, titlecase) and `capitalize("ßx")` is + `"Ssx"`. +- `capitalize` titlecases the first character and lowercases the rest (Python ≥ 3.8 + semantics). +- Lowercasing applies the Unicode Final_Sigma context rule: `title("OΣ K")` is + `"Oς K"`. + #### 2.2.5. Regular Expression Functions | Signature | Description |