From a36ad21720f346e4188b481b276b0a9466518df3 Mon Sep 17 00:00:00 2001 From: sevenzing Date: Sun, 1 Dec 2024 14:07:34 +0700 Subject: [PATCH 1/3] cargo doc --- README.md | 2 ++ src/code_points/specs.rs | 1 + src/code_points/types.rs | 2 ++ src/error.rs | 3 +++ src/lib.rs | 2 +- src/normalizer.rs | 14 ++++++++++++++ src/tokens/tokenize.rs | 8 -------- src/tokens/types.rs | 16 +++++++++++++++- 8 files changed, 38 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index b90c24d..49532c4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # ens-normalize-rs ![tests](https://github.com/sevenzing/ens-normalize-rs/actions/workflows/tests.yml/badge.svg) +![Crates.io Version](https://img.shields.io/crates/v/ens-normalize-rs) + A Rust implementation of ENS (Ethereum Name Service) name normalization. diff --git a/src/code_points/specs.rs b/src/code_points/specs.rs index df71e2a..fb13144 100644 --- a/src/code_points/specs.rs +++ b/src/code_points/specs.rs @@ -48,6 +48,7 @@ impl CodePointsSpecs { .collect(); let valid = compute_valid(&groups, &decomp); let whole_map = compute_whole_map(spec.whole_map); + let emoji_str_list = emoji .iter() .map(|cps| utils::cps2str(cps)) diff --git a/src/code_points/types.rs b/src/code_points/types.rs index 3a4e0b2..db4a172 100644 --- a/src/code_points/types.rs +++ b/src/code_points/types.rs @@ -42,6 +42,7 @@ impl ParsedGroup { pub type ParsedWholeMap = HashMap; pub enum ParsedWholeValue { + #[allow(dead_code)] Number(u32), WholeObject(ParsedWholeObject), } @@ -59,6 +60,7 @@ impl TryFrom for ParsedWholeValue { } pub struct ParsedWholeObject { + #[allow(dead_code)] pub v: HashSet, pub m: HashMap>, } diff --git a/src/error.rs b/src/error.rs index 013e28f..91da31c 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,5 +1,6 @@ use crate::CodePoint; +/// Errors that can occur during processing of an ENS name. #[derive(Debug, Clone, thiserror::Error, PartialEq, Eq)] pub enum ProcessError { #[error("contains visually confusing characters from multiple scripts: {0}")] @@ -17,6 +18,7 @@ pub enum ProcessError { DisallowedSequence(#[from] DisallowedSequence), } +/// Errors that can be cured by the normalizer. #[derive(Debug, Clone, thiserror::Error, PartialEq, Eq)] pub enum CurrableError { #[error("underscore in middle")] @@ -35,6 +37,7 @@ pub enum CurrableError { FencedConsecutive, } +/// Errors regarding disallowed sequences. #[derive(Debug, Clone, thiserror::Error, PartialEq, Eq)] pub enum DisallowedSequence { #[error("disallowed character: {0}")] diff --git a/src/lib.rs b/src/lib.rs index c0de560..92eb01c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,7 +9,7 @@ mod tokens; mod utils; mod validate; -pub use code_points::*; +pub(crate) use code_points::*; pub use error::{CurrableError, DisallowedSequence, ProcessError}; pub use normalizer::{beautify, normalize, process, tokenize, EnsNameNormalizer, ProcessedName}; pub use tokens::*; diff --git a/src/normalizer.rs b/src/normalizer.rs index 0113727..1f9bcf5 100644 --- a/src/normalizer.rs +++ b/src/normalizer.rs @@ -3,11 +3,16 @@ use crate::{ ProcessError, TokenizedName, ValidatedLabel, }; +/// Main struct to handle ENS name normalization including +/// tokenization, validation, beautification and normalization #[derive(Default)] pub struct EnsNameNormalizer { specs: CodePointsSpecs, } +/// Result of processing an ENS name. +/// Contains tokenized name as intermediate processing result and validated labels. +/// Validated labels can be normalized and beautified. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ProcessedName { pub labels: Vec, @@ -19,10 +24,13 @@ impl EnsNameNormalizer { Self { specs } } + /// Tokenize the input string, return a `TokenizedName` object with `Vec` inside pub fn tokenize(&self, input: impl AsRef) -> Result { TokenizedName::from_input(input.as_ref(), &self.specs, true) } + /// Process the input string, return a `ProcessedName` object with `Vec` inside + /// This function will tokenize and validate the name. Processed name can be normalized and beautified. pub fn process(&self, input: impl AsRef) -> Result { let input = input.as_ref(); let tokenized = self.tokenize(input)?; @@ -30,10 +38,12 @@ impl EnsNameNormalizer { Ok(ProcessedName { tokenized, labels }) } + /// Normalize the input string, return a normalized version of ENS name pub fn normalize(&self, input: impl AsRef) -> Result { self.process(input).map(|processed| processed.normalize()) } + /// Beautify the input string, return a beautified version of ENS name/// Beautify the input string, return a beautified version of ENS name pub fn beautify(&self, input: impl AsRef) -> Result { self.process(input).map(|processed| processed.beautify()) } @@ -49,18 +59,22 @@ impl ProcessedName { } } +/// `no-cache` version of [`EnsNameNormalizer::tokenize`] pub fn tokenize(input: impl AsRef) -> Result { EnsNameNormalizer::default().tokenize(input) } +/// `no-cache` version of [`EnsNameNormalizer::process`] pub fn process(input: impl AsRef) -> Result { EnsNameNormalizer::default().process(input) } +/// `no-cache` version of [`EnsNameNormalizer::normalize`] pub fn normalize(input: impl AsRef) -> Result { EnsNameNormalizer::default().normalize(input) } +/// `no-cache` version of [`EnsNameNormalizer::beautify`] pub fn beautify(input: impl AsRef) -> Result { EnsNameNormalizer::default().beautify(input) } diff --git a/src/tokens/tokenize.rs b/src/tokens/tokenize.rs index 2d097e0..555f8e2 100644 --- a/src/tokens/tokenize.rs +++ b/src/tokens/tokenize.rs @@ -7,9 +7,6 @@ use crate::{ }; /// Represents a full ENS name, including the original input and the sequence of tokens -/// vitalik.eth -/// ^^^^^^^^^^^ -/// name #[derive(Debug, Clone, PartialEq, Eq)] pub struct TokenizedName { pub input: String, @@ -17,11 +14,6 @@ pub struct TokenizedName { } /// Represents a tokenized ENS label (part of a name separated by periods), including sequence of tokens -/// vitalik.eth -/// ^^^^^^^ -/// label 1 -/// ^^^ -/// label 2 #[derive(Debug, Clone, PartialEq, Eq)] pub struct TokenizedLabel<'a> { pub tokens: &'a [EnsNameToken], diff --git a/src/tokens/types.rs b/src/tokens/types.rs index 6e6a592..55e4cb6 100644 --- a/src/tokens/types.rs +++ b/src/tokens/types.rs @@ -1,7 +1,7 @@ use crate::{constants, utils, CodePoint}; /// Represents a token in an ENS name. -/// see https://docs.ens.domains/ensip/15#tokenize for more details. +/// see for more details. #[derive(Debug, Clone, PartialEq, Eq)] pub enum EnsNameToken { Valid(TokenValid), @@ -72,35 +72,48 @@ impl EnsNameToken { } } +/// A valid vector of code points #[derive(Debug, Clone, PartialEq, Eq)] pub struct TokenValid { pub cps: Vec, } + +/// Code point should be mapped to vector of code points #[derive(Debug, Clone, PartialEq, Eq)] pub struct TokenMapped { pub cps: Vec, pub cp: CodePoint, } +/// Code point should be ignored #[derive(Debug, Clone, PartialEq, Eq)] pub struct TokenIgnored { pub cp: CodePoint, } +/// Code point is disallowed #[derive(Debug, Clone, PartialEq, Eq)] pub struct TokenDisallowed { pub cp: CodePoint, } + +/// Represents a stop token (.) #[derive(Debug, Clone, PartialEq, Eq)] pub struct TokenStop { pub cp: CodePoint, } + +/// Represents a vector of code points that should be normalized using NFC #[derive(Debug, Clone, PartialEq, Eq)] pub struct TokenNfc { pub cps: Vec, pub input: Vec, } +/// Represents a vector of code points of emoji +/// `cps_input` contains vector of code from input string +/// `emoji` contains vector of beautified emoji code points +/// `cps_no_fe0f` contains vector of code points of emoji without `FE0F` #[derive(Debug, Clone, PartialEq, Eq)] pub struct TokenEmoji { pub input: String, @@ -109,6 +122,7 @@ pub struct TokenEmoji { pub cps_no_fe0f: Vec, } +/// Represents a collapsed token in an ENS name: either text or emoji #[derive(Debug, Clone, PartialEq, Eq)] pub enum CollapsedEnsNameToken { Text(TokenValid), From 4fb080d316901a45b16500044ab8a685ffba4a61 Mon Sep 17 00:00:00 2001 From: sevenzing Date: Thu, 5 Dec 2024 14:28:53 +0700 Subject: [PATCH 2/3] change errors --- examples/tokens.rs | 2 +- src/code_points/specs.rs | 2 +- src/error.rs | 8 ++--- src/normalizer.rs | 6 ++-- src/tokens/tokenize.rs | 30 +++++++---------- src/validate.rs | 71 +++++++++++++++++++++++++++++----------- 6 files changed, 72 insertions(+), 47 deletions(-) diff --git a/examples/tokens.rs b/examples/tokens.rs index c3dce75..7edc7a6 100644 --- a/examples/tokens.rs +++ b/examples/tokens.rs @@ -4,7 +4,7 @@ fn main() { let normalizer = EnsNameNormalizer::default(); let name = "Nàme‍🧙‍♂.eth"; - let result = normalizer.tokenize(name).unwrap(); + let result = normalizer.tokenize(name); for token in result.tokens { if token.is_disallowed() { diff --git a/src/code_points/specs.rs b/src/code_points/specs.rs index fb13144..cb05bf2 100644 --- a/src/code_points/specs.rs +++ b/src/code_points/specs.rs @@ -96,7 +96,7 @@ impl CodePointsSpecs { .unwrap_or(false) } - pub fn finditer_emoji<'a>(&'a self, s: &'a str) -> impl Iterator> { + pub fn finditer_emoji<'a>(&'a self, s: &'a str) -> impl Iterator> { self.emoji_regex.find_iter(s) } diff --git a/src/error.rs b/src/error.rs index 91da31c..ade4e64 100644 --- a/src/error.rs +++ b/src/error.rs @@ -3,10 +3,6 @@ use crate::CodePoint; /// Errors that can occur during processing of an ENS name. #[derive(Debug, Clone, thiserror::Error, PartialEq, Eq)] pub enum ProcessError { - #[error("contains visually confusing characters from multiple scripts: {0}")] - Confused(String), - #[error("contains visually confusing characters from {group1} and {group2} scripts")] - ConfusedGroups { group1: String, group2: String }, #[error("invalid character ('{sequence}') at position {index}: {inner}")] CurrableError { inner: CurrableError, @@ -35,6 +31,8 @@ pub enum CurrableError { FencedTrailing, #[error("consecutive sequence of fenced characters")] FencedConsecutive, + #[error("contains visually confusing characters from multiple scripts: character with code '{cp}' not in group '{group_name}'")] + Confused { group_name: String, cp: CodePoint }, } /// Errors regarding disallowed sequences. @@ -50,4 +48,6 @@ pub enum DisallowedSequence { NsmTooMany, #[error("nsm repeated")] NsmRepeated, + #[error("contains visually confusing characters from {group1} and {group2} scripts")] + ConfusedGroups { group1: String, group2: String }, } diff --git a/src/normalizer.rs b/src/normalizer.rs index 1f9bcf5..4fbea78 100644 --- a/src/normalizer.rs +++ b/src/normalizer.rs @@ -25,7 +25,7 @@ impl EnsNameNormalizer { } /// Tokenize the input string, return a `TokenizedName` object with `Vec` inside - pub fn tokenize(&self, input: impl AsRef) -> Result { + pub fn tokenize(&self, input: impl AsRef) -> TokenizedName { TokenizedName::from_input(input.as_ref(), &self.specs, true) } @@ -33,7 +33,7 @@ impl EnsNameNormalizer { /// This function will tokenize and validate the name. Processed name can be normalized and beautified. pub fn process(&self, input: impl AsRef) -> Result { let input = input.as_ref(); - let tokenized = self.tokenize(input)?; + let tokenized = self.tokenize(input); let labels = validate_name(&tokenized, &self.specs)?; Ok(ProcessedName { tokenized, labels }) } @@ -60,7 +60,7 @@ impl ProcessedName { } /// `no-cache` version of [`EnsNameNormalizer::tokenize`] -pub fn tokenize(input: impl AsRef) -> Result { +pub fn tokenize(input: impl AsRef) -> TokenizedName { EnsNameNormalizer::default().tokenize(input) } diff --git a/src/tokens/tokenize.rs b/src/tokens/tokenize.rs index 555f8e2..4f366b8 100644 --- a/src/tokens/tokenize.rs +++ b/src/tokens/tokenize.rs @@ -3,7 +3,7 @@ use crate::{ CollapsedEnsNameToken, EnsNameToken, TokenDisallowed, TokenEmoji, TokenIgnored, TokenMapped, TokenNfc, TokenStop, TokenValid, }, - utils, CodePoint, CodePointsSpecs, ProcessError, + utils, CodePoint, CodePointsSpecs, }; /// Represents a full ENS name, including the original input and the sequence of tokens @@ -28,11 +28,7 @@ impl TokenizedName { } /// Tokenizes an input string, applying NFC normalization if requested. - pub fn from_input( - input: impl AsRef, - specs: &CodePointsSpecs, - apply_nfc: bool, - ) -> Result { + pub fn from_input(input: impl AsRef, specs: &CodePointsSpecs, apply_nfc: bool) -> Self { tokenize_name(input, specs, apply_nfc) } @@ -135,27 +131,23 @@ where } } -fn tokenize_name( - name: impl AsRef, - specs: &CodePointsSpecs, - apply_nfc: bool, -) -> Result { +fn tokenize_name(name: impl AsRef, specs: &CodePointsSpecs, apply_nfc: bool) -> TokenizedName { let name = name.as_ref(); if name.is_empty() { - return Ok(TokenizedName::empty()); + return TokenizedName::empty(); } - let tokens = tokenize_input(name, specs, apply_nfc)?; - Ok(TokenizedName { + let tokens = tokenize_input(name, specs, apply_nfc); + TokenizedName { input: name.to_string(), tokens, - }) + } } fn tokenize_input( input: impl AsRef, specs: &CodePointsSpecs, apply_nfc: bool, -) -> Result, ProcessError> { +) -> Vec { let input = input.as_ref(); let emojis = specs.finditer_emoji(input).collect::>(); @@ -184,7 +176,7 @@ fn tokenize_input( perform_nfc_transform(&mut tokens, specs); } collapse_valid_tokens(&mut tokens); - Ok(tokens) + tokens } fn perform_nfc_transform(tokens: &mut Vec, specs: &CodePointsSpecs) { @@ -462,7 +454,7 @@ mod tests { #[case] expected: Vec, specs: &CodePointsSpecs, ) { - let tokens = tokenize_input(input, specs, apply_nfc).expect("tokenize"); + let tokens = tokenize_input(input, specs, apply_nfc); assert_eq!(tokens, expected); } @@ -486,7 +478,7 @@ mod tests { #[case] expected: Vec, specs: &CodePointsSpecs, ) { - let tokens = tokenize_input(input, specs, true).expect("tokenize"); + let tokens = tokenize_input(input, specs, true); let label = TokenizedLabel::from(&tokens); let result = label.collapse_into_text_or_emoji(); assert_eq!(result, expected); diff --git a/src/validate.rs b/src/validate.rs index 0a3d35d..9cb4e18 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -212,7 +212,7 @@ fn check_and_get_group( .collect::>() .into_iter() .collect::>(); - let group = determine_group(&unique_cps, specs).cloned()?; + let group = determine_group(label, &unique_cps, specs).cloned()?; check_group(&group, &cps, specs)?; check_whole(&group, &unique_cps, specs)?; Ok(group) @@ -223,13 +223,17 @@ fn check_group( cps: &[CodePoint], specs: &CodePointsSpecs, ) -> Result<(), ProcessError> { - for cp in cps.iter() { + for (i, cp) in cps.iter().enumerate() { if !group.contains_cp(*cp) { - return Err(ProcessError::Confused(format!( - "symbol {} not present in group {}", - utils::cp2str(*cp), - group.name - ))); + return Err(ProcessError::CurrableError { + inner: CurrableError::Confused { + group_name: group.name.to_string(), + cp: *cp, + }, + index: i, + sequence: utils::cps2str(cps), + maybe_suggest: Some("".to_string()), + }); } } if group.cm_absent { @@ -271,10 +275,12 @@ fn check_whole( for group_name in maker { let confused_group_candidate = specs.group_by_name(group_name).expect("group must exist"); if confused_group_candidate.contains_all_cps(&shared) { - return Err(ProcessError::ConfusedGroups { - group1: group.name.to_string(), - group2: confused_group_candidate.name.to_string(), - }); + return Err(ProcessError::DisallowedSequence( + DisallowedSequence::ConfusedGroups { + group1: group.name.to_string(), + group2: confused_group_candidate.name.to_string(), + }, + )); } } Ok(()) @@ -317,16 +323,43 @@ fn get_groups_candidates_and_shared_cps( } fn determine_group<'a>( + label: &TokenizedLabel, unique_cps: &'a [CodePoint], specs: &'a CodePointsSpecs, ) -> Result<&'a ParsedGroup, ProcessError> { - specs - .groups_for_cps(unique_cps) - .next() - .ok_or(ProcessError::Confused(format!( - "no group found for {:?}", - unique_cps - ))) + if let Some(group) = specs.groups_for_cps(unique_cps).next() { + Ok(group) + } else { + let mut maybe_group = None; + for last_cp_index in 0..unique_cps.len() { + let cps = &unique_cps[..last_cp_index + 1]; + if let Some(group) = specs.groups_for_cps(cps).next() { + maybe_group = Some(group); + continue; + } else { + let cp = unique_cps[last_cp_index]; + let index_of_cp = label + .iter_cps() + .position(|cp_in_label| cp_in_label == cp) + .expect("cp must exist in label"); + match maybe_group { + Some(group) => { + return Err(ProcessError::CurrableError { + inner: CurrableError::Confused { + group_name: group.name.to_string(), + cp, + }, + index: index_of_cp, + sequence: utils::cp2str(cp), + maybe_suggest: Some("".to_string()), + }); + } + None => unreachable!(), + } + } + } + unreachable!("") + } } #[cfg(test)] @@ -386,7 +419,7 @@ mod tests { #[case] expected: Result, specs: &CodePointsSpecs, ) { - let name = TokenizedName::from_input(input, specs, true).unwrap(); + let name = TokenizedName::from_input(input, specs, true); let label = name.iter_labels().next().unwrap(); let result = validate_label(label, specs); assert_eq!( From 0784b0ef263b86d18d598d33293a396690a3d439 Mon Sep 17 00:00:00 2001 From: sevenzing Date: Thu, 5 Dec 2024 15:14:49 +0700 Subject: [PATCH 3/3] fix cargo test --- src/error.rs | 2 ++ src/validate.rs | 29 ++++++++++++++--------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/error.rs b/src/error.rs index ade4e64..2dd1146 100644 --- a/src/error.rs +++ b/src/error.rs @@ -33,6 +33,8 @@ pub enum CurrableError { FencedConsecutive, #[error("contains visually confusing characters from multiple scripts: character with code '{cp}' not in group '{group_name}'")] Confused { group_name: String, cp: CodePoint }, + #[error("contains a disallowed character")] + Disallowed, } /// Errors regarding disallowed sequences. diff --git a/src/validate.rs b/src/validate.rs index 9cb4e18..80c0bd7 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -342,23 +342,22 @@ fn determine_group<'a>( .iter_cps() .position(|cp_in_label| cp_in_label == cp) .expect("cp must exist in label"); - match maybe_group { - Some(group) => { - return Err(ProcessError::CurrableError { - inner: CurrableError::Confused { - group_name: group.name.to_string(), - cp, - }, - index: index_of_cp, - sequence: utils::cp2str(cp), - maybe_suggest: Some("".to_string()), - }); - } - None => unreachable!(), - } + let inner = match maybe_group { + Some(group) => CurrableError::Confused { + group_name: group.name.to_string(), + cp, + }, + None => CurrableError::Disallowed, + }; + return Err(ProcessError::CurrableError { + inner, + index: index_of_cp, + sequence: utils::cp2str(cp), + maybe_suggest: Some("".to_string()), + }); } } - unreachable!("") + unreachable!() } }