diff --git a/.gitignore b/.gitignore index 90f16f95..23bdbb59 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ _book/ cache/ coverage/ build/ +target/ dts/ esm/ lib/ diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 00000000..1a4e918b --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2356 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "any_ascii" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90c6333e01ba7235575b6ab53e5af10f1c327927fd97c36462917e289557ea64" + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "cssparser" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c66d1cd8ed61bf80b38432613a7a2f09401ab8d0501110655f8b341484a3e3" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "derive_more" +version = "0.99.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ego-tree" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2972feb8dffe7bc8c5463b1dacda1b0dfbed3710e50f977d965429692d74cd8" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "emojibase-core" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "emojibase-generator" +version = "0.1.0" +dependencies = [ + "any_ascii", + "clap", + "emojibase-core", + "flate2", + "indicatif", + "rayon", + "regex", + "reqwest", + "scraper", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", +] + +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "html5ever" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b7410cae13cbc75623c98ac4cbfd1f0bedddf3227afc24f370cf0f50a44a11c" +dependencies = [ + "log", + "mac", + "markup5ever", + "match_token", +] + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "indicatif" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", + "web-time", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iri-string" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.184" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7a7213d12e1864c0f002f52c2923d4556935a43dec5e71355c2760e0f6e7a18" +dependencies = [ + "log", + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", + "tendril", +] + +[[package]] +name = "match_token" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a9689d8d44bf9964484516275f5cd4c9b59457a6940c1d5d0ecbb94510a36b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl" +version = "0.10.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-sys" +version = "0.9.112" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scraper" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc3d051b884f40e309de6c149734eab57aa8cc1347992710dc80bcc1c2194c15" +dependencies = [ + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "precomputed-hash", + "selectors", + "tendril", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "selectors" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "fxhash", + "log", + "new_debug_unreachable", + "phf", + "phf_codegen", + "precomputed-hash", + "servo_arc", + "smallvec", +] + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "servo_arc" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "string_cache" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000..69ea03bc --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,13 @@ +[workspace] +members = ["crates/emojibase-core", "crates/emojibase-generator"] +resolver = "2" + +[workspace.package] +version = "0.1.0" +edition = "2021" +license = "MIT" +repository = "https://github.com/milesj/emojibase" + +[workspace.dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/crates/emojibase-core/Cargo.toml b/crates/emojibase-core/Cargo.toml new file mode 100644 index 00000000..8585bdc6 --- /dev/null +++ b/crates/emojibase-core/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "emojibase-core" +description = "Core types and utilities for working with emoji data, following the Unicode and CLDR specifications." +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } diff --git a/crates/emojibase-core/src/constants.rs b/crates/emojibase-core/src/constants.rs new file mode 100644 index 00000000..028756f3 --- /dev/null +++ b/crates/emojibase-core/src/constants.rs @@ -0,0 +1,149 @@ +//! Version constants, supported locales, and well-known codepoints. + +use crate::types::{GroupKey, SkinTone, SkinToneKey}; + +// --------------------------------------------------------------------------- +// Latest version info +// --------------------------------------------------------------------------- + +/// Latest supported Emoji specification version. +pub const LATEST_EMOJI_VERSION: &str = "17.0"; + +/// Latest Unicode version. +pub const LATEST_UNICODE_VERSION: &str = "17.0.0"; + +/// Latest CLDR version. +pub const LATEST_CLDR_VERSION: &str = "48"; + +// --------------------------------------------------------------------------- +// All known Emoji versions +// --------------------------------------------------------------------------- + +pub const EMOJI_VERSIONS: &[&str] = &[ + "1.0", "2.0", "3.0", "4.0", "5.0", "11.0", "12.0", "12.1", "13.0", "13.1", "14.0", "15.0", + "15.1", "16.0", "17.0", +]; + +/// All known Unicode versions. +pub const UNICODE_VERSIONS: &[&str] = &[ + "6.0", "6.1", "6.2", "6.3", "7.0", "8.0", "9.0", "10.0", "11.0", "12.0", "12.1", "13.0", + "14.0", "15.0", "15.1", "16.0", "17.0", +]; + +// --------------------------------------------------------------------------- +// Supported locales +// --------------------------------------------------------------------------- + +pub const SUPPORTED_LOCALES: &[&str] = &[ + "bn", "da", "de", "en", "en-gb", "es", "es-mx", "et", "fi", "fr", "hi", "hu", "it", "ja", + "ko", "lt", "ms", "nb", "nl", "pl", "pt", "ru", "sv", "th", "uk", "vi", "zh", "zh-hant", +]; + +// --------------------------------------------------------------------------- +// Special Unicode codepoints (as hex strings) +// --------------------------------------------------------------------------- + +/// Zero-width joiner. +pub const ZERO_WIDTH_JOINER: &str = "200D"; + +/// Variation selector 15 – force text presentation. +pub const TEXT_VARIATION_SELECTOR: &str = "FE0E"; + +/// Variation selector 16 – force emoji presentation. +pub const EMOJI_VARIATION_SELECTOR: &str = "FE0F"; + +// Skin-tone modifier hexcodes (Fitzpatrick scale). +pub const LIGHT_SKIN_MODIFIER: &str = "1F3FB"; +pub const MEDIUM_LIGHT_SKIN_MODIFIER: &str = "1F3FC"; +pub const MEDIUM_SKIN_MODIFIER: &str = "1F3FD"; +pub const MEDIUM_DARK_SKIN_MODIFIER: &str = "1F3FE"; +pub const DARK_SKIN_MODIFIER: &str = "1F3FF"; + +/// All skin-tone modifier hex strings, indexed by [`SkinTone`] ordinal − 1. +pub const SKIN_MODIFIER_HEXCODES: &[&str] = &[ + LIGHT_SKIN_MODIFIER, + MEDIUM_LIGHT_SKIN_MODIFIER, + MEDIUM_SKIN_MODIFIER, + MEDIUM_DARK_SKIN_MODIFIER, + DARK_SKIN_MODIFIER, +]; + +// Gender sign hexcodes. +pub const FEMALE_SIGN: &str = "2640"; +pub const MALE_SIGN: &str = "2642"; + +// Well-known person hexcodes. +pub const PERSON: &str = "1F9D1"; +pub const MAN: &str = "1F468"; +pub const WOMAN: &str = "1F469"; +pub const BLACK_FLAG: &str = "1F3F4"; + +// --------------------------------------------------------------------------- +// Group helpers +// --------------------------------------------------------------------------- + +/// Ordered list of group keys matching their numeric index (0–9). +pub const GROUP_KEYS: &[GroupKey] = &[ + GroupKey::SmileysEmotion, + GroupKey::PeopleBody, + GroupKey::AnimalsNature, + GroupKey::FoodDrink, + GroupKey::TravelPlaces, + GroupKey::Activities, + GroupKey::Objects, + GroupKey::Symbols, + GroupKey::Flags, + GroupKey::Component, +]; + +/// Ordered list of skin-tone keys (light → dark). +pub const SKIN_TONE_KEYS: &[SkinToneKey] = &[ + SkinToneKey::Light, + SkinToneKey::MediumLight, + SkinToneKey::Medium, + SkinToneKey::MediumDark, + SkinToneKey::Dark, +]; + +/// Ordered list of [`SkinTone`] values. +pub const SKIN_TONES: &[SkinTone] = &[ + SkinTone::Light, + SkinTone::MediumLight, + SkinTone::Medium, + SkinTone::MediumDark, + SkinTone::Dark, +]; + +// --------------------------------------------------------------------------- +// Regional indicator mapping (for flag construction) +// --------------------------------------------------------------------------- + +/// Maps regional-indicator hexcodes to their letter (A–Z). +pub const REGIONAL_INDICATORS: &[(&str, char)] = &[ + ("1F1E6", 'A'), + ("1F1E7", 'B'), + ("1F1E8", 'C'), + ("1F1E9", 'D'), + ("1F1EA", 'E'), + ("1F1EB", 'F'), + ("1F1EC", 'G'), + ("1F1ED", 'H'), + ("1F1EE", 'I'), + ("1F1EF", 'J'), + ("1F1F0", 'K'), + ("1F1F1", 'L'), + ("1F1F2", 'M'), + ("1F1F3", 'N'), + ("1F1F4", 'O'), + ("1F1F5", 'P'), + ("1F1F6", 'Q'), + ("1F1F7", 'R'), + ("1F1F8", 'S'), + ("1F1F9", 'T'), + ("1F1FA", 'U'), + ("1F1FB", 'V'), + ("1F1FC", 'W'), + ("1F1FD", 'X'), + ("1F1FE", 'Y'), + ("1F1FF", 'Z'), +]; diff --git a/crates/emojibase-core/src/lib.rs b/crates/emojibase-core/src/lib.rs new file mode 100644 index 00000000..eafc249f --- /dev/null +++ b/crates/emojibase-core/src/lib.rs @@ -0,0 +1,17 @@ +//! # emojibase-core +//! +//! Core types and utilities for working with emoji data, following the +//! Unicode and CLDR specifications. This crate provides: +//! +//! - Data model types ([`Emoji`], [`CompactEmoji`], etc.) with serde support +//! - Hexcode / codepoint / Unicode conversion utilities +//! - Shortcode and emoticon helpers +//! - Constants for groups, skin tones, presentations, and supported locales + +pub mod constants; +pub mod shortcode; +pub mod types; +pub mod unicode; + +pub use constants::*; +pub use types::*; diff --git a/crates/emojibase-core/src/shortcode.rs b/crates/emojibase-core/src/shortcode.rs new file mode 100644 index 00000000..d33c8d0c --- /dev/null +++ b/crates/emojibase-core/src/shortcode.rs @@ -0,0 +1,59 @@ +//! Shortcode utilities: joining, parsing, and formatting. + +use crate::types::Shortcode; + +/// Wrap a shortcode in colons: `"grinning"` → `":grinning:"`. +pub fn shortcode_to_colons(shortcode: &str) -> String { + format!(":{shortcode}:") +} + +/// Remove surrounding colons if present: `":grinning:"` → `"grinning"`. +pub fn shortcode_from_colons(text: &str) -> Shortcode { + text.trim_start_matches(':') + .trim_end_matches(':') + .to_string() +} + +/// Slugify a label string into a shortcode-friendly form. +/// +/// Converts to lowercase, replaces whitespace and special chars with +/// underscores, strips non-alphanumeric/underscore characters, and collapses +/// consecutive underscores. +pub fn slugify(label: &str) -> String { + let mut slug = String::with_capacity(label.len()); + for ch in label.chars() { + if ch.is_ascii_alphanumeric() { + slug.push(ch.to_ascii_lowercase()); + } else if ch == ' ' || ch == '-' || ch == ':' || ch == '&' { + slug.push('_'); + } + // else: drop the character + } + // Collapse consecutive underscores and trim leading/trailing. + let collapsed = slug + .split('_') + .filter(|s| !s.is_empty()) + .collect::>() + .join("_"); + collapsed +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_colons() { + assert_eq!(shortcode_to_colons("grinning"), ":grinning:"); + assert_eq!(shortcode_from_colons(":grinning:"), "grinning"); + assert_eq!(shortcode_from_colons("grinning"), "grinning"); + } + + #[test] + fn test_slugify() { + assert_eq!(slugify("Grinning Face"), "grinning_face"); + assert_eq!(slugify("smileys & emotion"), "smileys_emotion"); + assert_eq!(slugify("face-smiling"), "face_smiling"); + assert_eq!(slugify("flag: Japan"), "flag_japan"); + } +} diff --git a/crates/emojibase-core/src/types.rs b/crates/emojibase-core/src/types.rs new file mode 100644 index 00000000..81f54b50 --- /dev/null +++ b/crates/emojibase-core/src/types.rs @@ -0,0 +1,283 @@ +use serde::{Deserialize, Serialize}; + +// --------------------------------------------------------------------------- +// Primitive newtypes / aliases +// --------------------------------------------------------------------------- + +/// Hex representation of one or more Unicode codepoints, separated by hyphens. +/// +/// Examples: `"1F600"`, `"1F468-1F3FB-200D-1F4BB"`. +pub type Hexcode = String; + +/// The actual emoji or text character(s). +pub type Unicode = String; + +/// ASCII emoticon string, e.g. `":)"`. +pub type Emoticon = String; + +/// A platform-specific shortcode **without** surrounding colons. +pub type Shortcode = String; + +/// Numeric Unicode codepoint. +pub type CodePoint = u32; + +// --------------------------------------------------------------------------- +// Small enums encoded as integers (to match the JSON schema) +// --------------------------------------------------------------------------- + +/// Presentation style: `0` = text, `1` = emoji. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[repr(u8)] +pub enum Presentation { + Text = 0, + Emoji = 1, +} + +/// Gender: `0` = female, `1` = male. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[repr(u8)] +pub enum Gender { + Female = 0, + Male = 1, +} + +/// Qualification level per UTS #51. +/// +/// - `0` – fully qualified (RGI) +/// - `1` – minimally qualified +/// - `2` – unqualified +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[repr(u8)] +pub enum Qualifier { + FullyQualified = 0, + MinimallyQualified = 1, + Unqualified = 2, +} + +/// Fitzpatrick skin-tone index (1–5, light → dark). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[repr(u8)] +pub enum SkinTone { + Light = 1, + MediumLight = 2, + Medium = 3, + MediumDark = 4, + Dark = 5, +} + +// --------------------------------------------------------------------------- +// Group / Subgroup +// --------------------------------------------------------------------------- + +/// Categorical group index (0–9). +pub type Group = u8; + +/// Categorical subgroup index. +pub type Subgroup = u8; + +/// Well-known group keys matching the Unicode emoji-test.txt categories. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum GroupKey { + SmileysEmotion, + PeopleBody, + AnimalsNature, + FoodDrink, + TravelPlaces, + Activities, + Objects, + Symbols, + Flags, + Component, +} + +impl GroupKey { + /// Return the string key used in JSON datasets. + pub fn as_str(&self) -> &'static str { + match self { + Self::SmileysEmotion => "smileys-emotion", + Self::PeopleBody => "people-body", + Self::AnimalsNature => "animals-nature", + Self::FoodDrink => "food-drink", + Self::TravelPlaces => "travel-places", + Self::Activities => "activities", + Self::Objects => "objects", + Self::Symbols => "symbols", + Self::Flags => "flags", + Self::Component => "component", + } + } +} + +/// Skin-tone key strings used in messages datasets. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum SkinToneKey { + Light, + MediumLight, + Medium, + MediumDark, + Dark, +} + +impl SkinToneKey { + pub fn as_str(&self) -> &'static str { + match self { + Self::Light => "light", + Self::MediumLight => "medium-light", + Self::Medium => "medium", + Self::MediumDark => "medium-dark", + Self::Dark => "dark", + } + } +} + +// --------------------------------------------------------------------------- +// Skin-tone value (single or multi) +// --------------------------------------------------------------------------- + +/// A skin-tone value that can be a single tone or a tuple of tones +/// (for multi-person emoji such as handshake with mixed tones). +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(untagged)] +pub enum SkinToneValue { + Single(SkinTone), + Multi(Vec), +} + +// --------------------------------------------------------------------------- +// Emoticon value (single or list) +// --------------------------------------------------------------------------- + +/// An emoticon that may be a single string or a list of variants. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(untagged)] +pub enum EmoticonValue { + Single(Emoticon), + Multi(Vec), +} + +// --------------------------------------------------------------------------- +// Main Emoji struct (full format – data.json) +// --------------------------------------------------------------------------- + +/// A fully-described emoji entry, matching the `data.json` / `data.raw.json` +/// schema produced by the generator. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Emoji { + /// Hex representation of the emoji codepoint(s). + pub hexcode: Hexcode, + + /// CLDR-localized label / description. + pub label: String, + + /// Emoji-presentation character. + pub emoji: Unicode, + + /// Text-presentation character (may be empty). + pub text: Unicode, + + /// Default presentation type. + #[serde(rename = "type")] + pub presentation: Presentation, + + /// Emoji version when this character was introduced. + pub version: f32, + + /// Display order in an emoji picker. + #[serde(skip_serializing_if = "Option::is_none")] + pub order: Option, + + /// Group index (0–9). + #[serde(skip_serializing_if = "Option::is_none")] + pub group: Option, + + /// Subgroup index. + #[serde(skip_serializing_if = "Option::is_none")] + pub subgroup: Option, + + /// CLDR keyword tags for searching. + #[serde(skip_serializing_if = "Option::is_none")] + pub tags: Option>, + + /// Platform-specific shortcodes (without colons). + #[serde(skip_serializing_if = "Option::is_none")] + pub shortcodes: Option>, + + /// ASCII emoticon(s) associated with this emoji. + #[serde(skip_serializing_if = "Option::is_none")] + pub emoticon: Option, + + /// Gender, if this is a gendered variant. + #[serde(skip_serializing_if = "Option::is_none")] + pub gender: Option, + + /// Skin-tone value(s), if this is a skin-toned variant. + #[serde(skip_serializing_if = "Option::is_none")] + pub tone: Option, + + /// Skin-tone variations nested under the base emoji. + #[serde(skip_serializing_if = "Option::is_none")] + pub skins: Option>, +} + +// --------------------------------------------------------------------------- +// Compact Emoji struct (compact.json) +// --------------------------------------------------------------------------- + +/// A compact emoji entry with fewer fields, for bandwidth-sensitive use. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct CompactEmoji { + pub hexcode: Hexcode, + pub label: String, + pub unicode: Unicode, + pub group: Group, + #[serde(skip_serializing_if = "Option::is_none")] + pub order: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tags: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub shortcodes: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub emoticon: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub skins: Option>, +} + +// --------------------------------------------------------------------------- +// Messages dataset +// --------------------------------------------------------------------------- + +/// A single translated group / subgroup / skin-tone entry. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MessageEntry { + pub key: String, + pub message: String, + pub order: u32, +} + +/// The `messages.json` dataset structure. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MessagesDataset { + pub groups: Vec, + pub subgroups: Vec, + #[serde(rename = "skinTones")] + pub skin_tones: Vec, +} + +// --------------------------------------------------------------------------- +// Other dataset types +// --------------------------------------------------------------------------- + +/// `shortcodes/{preset}.json` – maps hexcode → shortcode list. +pub type ShortcodesDataset = std::collections::HashMap>; + +/// `meta/hexcodes.json` – maps hexcode → qualifier map. +pub type HexcodesDataset = + std::collections::HashMap>; + +/// `versions/emoji.json` / `versions/unicode.json` – maps version string → hexcode list. +pub type VersionDataset = std::collections::HashMap>; + +/// `meta/groups.json` – group dataset with hierarchy. +pub type GroupDataset = std::collections::HashMap; diff --git a/crates/emojibase-core/src/unicode.rs b/crates/emojibase-core/src/unicode.rs new file mode 100644 index 00000000..b1d4db56 --- /dev/null +++ b/crates/emojibase-core/src/unicode.rs @@ -0,0 +1,82 @@ +//! Unicode ↔ hexcode ↔ codepoint conversion helpers. + +use crate::types::{CodePoint, Hexcode, Unicode}; + +/// Convert a hexcode string (e.g. `"1F600"`) to a vector of numeric codepoints. +/// +/// Multi-codepoint hexcodes are separated by hyphens: `"1F468-200D-1F4BB"`. +pub fn from_hexcode_to_codepoints(hexcode: &str) -> Vec { + hexcode + .split('-') + .filter_map(|part| u32::from_str_radix(part, 16).ok()) + .collect() +} + +/// Convert a vector of numeric codepoints to a hexcode string. +pub fn from_codepoints_to_hexcode(codepoints: &[CodePoint]) -> Hexcode { + codepoints + .iter() + .map(|cp| format!("{cp:X}")) + .collect::>() + .join("-") +} + +/// Convert numeric codepoints to a Unicode string. +pub fn from_codepoints_to_unicode(codepoints: &[CodePoint]) -> Unicode { + codepoints + .iter() + .filter_map(|&cp| char::from_u32(cp)) + .collect() +} + +/// Convert a hexcode string directly to a Unicode string. +pub fn from_hexcode_to_unicode(hexcode: &str) -> Unicode { + let codepoints = from_hexcode_to_codepoints(hexcode); + from_codepoints_to_unicode(&codepoints) +} + +/// Convert a Unicode string to a hexcode. +pub fn from_unicode_to_hexcode(unicode: &str) -> Hexcode { + let codepoints: Vec = unicode.chars().map(|c| c as u32).collect(); + from_codepoints_to_hexcode(&codepoints) +} + +/// Strip variation selectors (`FE0E`, `FE0F`) and ZWJ (`200D`) from a hexcode. +pub fn strip_hexcode(hexcode: &str) -> Hexcode { + hexcode + .split('-') + .filter(|part| !matches!(*part, "200D" | "FE0E" | "FE0F")) + .collect::>() + .join("-") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_hexcode_to_codepoints() { + assert_eq!(from_hexcode_to_codepoints("1F600"), vec![0x1F600]); + assert_eq!( + from_hexcode_to_codepoints("1F468-200D-1F4BB"), + vec![0x1F468, 0x200D, 0x1F4BB] + ); + } + + #[test] + fn test_hexcode_to_unicode() { + assert_eq!(from_hexcode_to_unicode("1F600"), "😀"); + } + + #[test] + fn test_unicode_to_hexcode() { + assert_eq!(from_unicode_to_hexcode("😀"), "1F600"); + } + + #[test] + fn test_strip_hexcode() { + assert_eq!(strip_hexcode("1F468-200D-1F4BB"), "1F468-1F4BB"); + assert_eq!(strip_hexcode("1F600-FE0F"), "1F600"); + assert_eq!(strip_hexcode("1F600"), "1F600"); + } +} diff --git a/crates/emojibase-generator/Cargo.toml b/crates/emojibase-generator/Cargo.toml new file mode 100644 index 00000000..79fe5b58 --- /dev/null +++ b/crates/emojibase-generator/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "emojibase-generator" +description = "Build-time generator for emojibase emoji datasets. Fetches Unicode and CLDR data, normalizes it, and produces JSON datasets, regex patterns, and shortcode mappings." +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true + +[[bin]] +name = "emojibase-generator" +path = "src/main.rs" + +[dependencies] +emojibase-core = { path = "../emojibase-core" } +clap = { version = "4", features = ["derive"] } +reqwest = { version = "0.12", features = ["json"] } +tokio = { version = "1", features = ["full"] } +scraper = "0.22" +serde = { workspace = true } +serde_json = { workspace = true } +regex = "1" +rayon = "1" +indicatif = "0.17" +flate2 = "1" +any_ascii = "0.3" diff --git a/crates/emojibase-generator/src/builder/annotations.rs b/crates/emojibase-generator/src/builder/annotations.rs new file mode 100644 index 00000000..800684a2 --- /dev/null +++ b/crates/emojibase-generator/src/builder/annotations.rs @@ -0,0 +1,55 @@ +//! Apply CLDR annotations (labels and tags) to emoji data. + +use crate::model::emoji::{CldrAnnotationMap, EmojiMap}; + +/// Merge CLDR annotations into the emoji map. +/// +/// For each emoji, if a CLDR annotation exists, the `description` field +/// is updated with the localized label, and tags are stored for later use. +/// +/// Returns a map of hexcode → tags for use in output generation. +pub fn apply_annotations( + emoji_map: &mut EmojiMap, + annotations: &CldrAnnotationMap, +) -> std::collections::HashMap> { + let mut tags_map = std::collections::HashMap::new(); + + for (hexcode, emoji) in emoji_map.iter_mut() { + if let Some(annotation) = annotations.get(hexcode) { + if !annotation.annotation.is_empty() { + emoji.description = annotation.annotation.clone(); + } + if !annotation.tags.is_empty() { + tags_map.insert(hexcode.clone(), annotation.tags.clone()); + } + } + } + + tags_map +} + +/// Merge derived annotations (for modifier sequences, ZWJ sequences). +/// +/// Derived annotations follow the same format but may fill in gaps +/// for sequences not covered by the primary annotation file. +pub fn apply_derived_annotations( + emoji_map: &mut EmojiMap, + derived: &CldrAnnotationMap, + tags_map: &mut std::collections::HashMap>, +) { + for (hexcode, emoji) in emoji_map.iter_mut() { + // Only apply if there's no existing annotation. + if emoji.description.is_empty() || emoji.description == emoji.name { + if let Some(annotation) = derived.get(hexcode) { + if !annotation.annotation.is_empty() { + emoji.description = annotation.annotation.clone(); + } + if !annotation.tags.is_empty() { + tags_map + .entry(hexcode.clone()) + .or_insert_with(|| annotation.tags.clone()); + } + } + } + } +} diff --git a/crates/emojibase-generator/src/builder/data.rs b/crates/emojibase-generator/src/builder/data.rs new file mode 100644 index 00000000..0090e4e3 --- /dev/null +++ b/crates/emojibase-generator/src/builder/data.rs @@ -0,0 +1,120 @@ +//! Merge parsed data from multiple sources into a unified [`EmojiMap`]. + +use crate::model::emoji::{ + EmojiDataMap, EmojiMap, EmojiMetadataMap, EmojiVariationMap, GeneratorEmoji, UnicodeNamesMap, +}; +use emojibase_core::types::Presentation; + +/// Join emoji data, metadata, variations, and names into a single [`EmojiMap`]. +/// +/// This is the central merge step that produces the internal representation +/// from which all output formats are derived. +pub fn build_emoji_data( + data: &EmojiDataMap, + metadata: &EmojiMetadataMap, + variations: &EmojiVariationMap, + names: &UnicodeNamesMap, +) -> EmojiMap { + let mut map = EmojiMap::new(); + + for (hexcode, emoji_data) in data { + let meta = metadata.get(hexcode); + let name = names.get(hexcode).cloned().unwrap_or_default(); + let variation = variations.get(hexcode).cloned(); + + let group = meta.map(|m| m.group).unwrap_or(0); + let subgroup = meta.map(|m| m.subgroup).unwrap_or(0); + let order = meta.map(|m| m.order).unwrap_or(0); + + let emoji = GeneratorEmoji { + description: emoji_data.description.clone(), + hexcode: hexcode.clone(), + property: emoji_data.property.clone(), + presentation: emoji_data.presentation, + unicode_version: emoji_data + .unicode_version + .map(|v| format!("{v:.1}")), + version: format!("{:.1}", emoji_data.version), + gender: emoji_data.gender, + group, + order, + subgroup, + name, + modifications: None, + qualifiers: None, + variations: variation, + shortcodes: None, + emoticon: None, + }; + + map.insert(hexcode.clone(), emoji); + } + + map +} + +/// Merge a secondary data map (e.g. from sequences) into the primary map. +pub fn merge_data_maps(primary: &mut EmojiDataMap, secondary: EmojiDataMap) { + for (hexcode, data) in secondary { + primary + .entry(hexcode) + .and_modify(|existing| { + // Merge properties. + for prop in &data.property { + if !existing.property.contains(prop) { + existing.property.push(*prop); + } + } + // Upgrade presentation if needed. + if data.presentation == Presentation::Emoji { + existing.presentation = Presentation::Emoji; + } + }) + .or_insert(data); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::emoji::EmojiData; + use crate::model::emoji::Property; + + #[test] + fn test_merge_data_maps() { + let mut primary = EmojiDataMap::new(); + primary.insert( + "1F600".to_string(), + EmojiData { + description: "grinning face".to_string(), + hexcode: "1F600".to_string(), + property: vec![Property::Emoji], + presentation: Presentation::Text, + unicode_version: Some(6.1), + version: 1.0, + gender: None, + }, + ); + + let mut secondary = EmojiDataMap::new(); + secondary.insert( + "1F600".to_string(), + EmojiData { + description: "grinning face".to_string(), + hexcode: "1F600".to_string(), + property: vec![Property::EmojiPresentation], + presentation: Presentation::Emoji, + unicode_version: Some(6.1), + version: 1.0, + gender: None, + }, + ); + + merge_data_maps(&mut primary, secondary); + + let emoji = &primary["1F600"]; + assert!(emoji.property.contains(&Property::Emoji)); + assert!(emoji.property.contains(&Property::EmojiPresentation)); + assert_eq!(emoji.presentation, Presentation::Emoji); + } +} diff --git a/crates/emojibase-generator/src/builder/gender.rs b/crates/emojibase-generator/src/builder/gender.rs new file mode 100644 index 00000000..a03b448c --- /dev/null +++ b/crates/emojibase-generator/src/builder/gender.rs @@ -0,0 +1,41 @@ +//! Gender modifier handling for emoji. + +use crate::model::emoji::{EmojiMap, Property}; +use emojibase_core::constants::{FEMALE_SIGN, MALE_SIGN}; +use emojibase_core::types::Gender; + +/// Detect gender from a hexcode sequence and set the `gender` field. +/// +/// An emoji is considered gendered if its hexcode contains the +/// female sign (`2640`) or male sign (`2642`). +pub fn apply_gender(emoji_map: &mut EmojiMap) { + let updates: Vec<(String, Gender)> = emoji_map + .iter() + .filter_map(|(hex, _)| { + if hex.contains(FEMALE_SIGN) { + Some((hex.clone(), Gender::Female)) + } else if hex.contains(MALE_SIGN) { + Some((hex.clone(), Gender::Male)) + } else { + None + } + }) + .collect(); + + for (hex, gender) in updates { + if let Some(emoji) = emoji_map.get_mut(&hex) { + emoji.gender = Some(gender); + } + } +} + +/// Check if an emoji has a gender-related property. +pub fn has_gender_property(emoji: &crate::model::emoji::GeneratorEmoji) -> bool { + emoji.gender.is_some() + || emoji + .hexcode + .contains(FEMALE_SIGN) + || emoji + .hexcode + .contains(MALE_SIGN) +} diff --git a/crates/emojibase-generator/src/builder/mod.rs b/crates/emojibase-generator/src/builder/mod.rs new file mode 100644 index 00000000..f5c44a90 --- /dev/null +++ b/crates/emojibase-generator/src/builder/mod.rs @@ -0,0 +1,6 @@ +pub mod annotations; +pub mod data; +pub mod gender; +pub mod shortcodes; +pub mod skin_tones; +pub mod verify; diff --git a/crates/emojibase-generator/src/builder/shortcodes.rs b/crates/emojibase-generator/src/builder/shortcodes.rs new file mode 100644 index 00000000..37c3575e --- /dev/null +++ b/crates/emojibase-generator/src/builder/shortcodes.rs @@ -0,0 +1,57 @@ +//! Generate shortcodes for emoji using various presets. + +use crate::model::emoji::{EmojiMap, ShortcodeMap}; +use crate::model::shortcode::ShortcodePreset; +use crate::util::slug::slugify; + +/// Generate emojibase-style shortcodes from emoji descriptions. +pub fn generate_emojibase_shortcodes(emoji_map: &EmojiMap) -> ShortcodeMap { + let mut map = ShortcodeMap::new(); + + for (hexcode, emoji) in emoji_map { + let slug = slugify(&emoji.description); + if !slug.is_empty() { + map.insert(hexcode.clone(), vec![slug]); + } + } + + map +} + +/// Generate CLDR-based shortcodes from emoji annotations. +pub fn generate_cldr_shortcodes( + emoji_map: &EmojiMap, + tags_map: &std::collections::HashMap>, +) -> ShortcodeMap { + let mut map = ShortcodeMap::new(); + + for (hexcode, emoji) in emoji_map { + let shortcode = slugify(&emoji.description); + if !shortcode.is_empty() { + let mut codes = vec![shortcode]; + + // Add tag-based shortcodes. + if let Some(tags) = tags_map.get(hexcode) { + for tag in tags.iter().take(2) { + let tag_slug = slugify(tag); + if !tag_slug.is_empty() && !codes.contains(&tag_slug) { + codes.push(tag_slug); + } + } + } + + map.insert(hexcode.clone(), codes); + } + } + + map +} + +/// Apply generated shortcodes to the emoji map. +pub fn apply_shortcodes(emoji_map: &mut EmojiMap, shortcodes: &ShortcodeMap) { + for (hexcode, codes) in shortcodes { + if let Some(emoji) = emoji_map.get_mut(hexcode) { + emoji.shortcodes = Some(codes.clone()); + } + } +} diff --git a/crates/emojibase-generator/src/builder/skin_tones.rs b/crates/emojibase-generator/src/builder/skin_tones.rs new file mode 100644 index 00000000..db5528a2 --- /dev/null +++ b/crates/emojibase-generator/src/builder/skin_tones.rs @@ -0,0 +1,145 @@ +//! Generate skin-tone variations for emoji that support modifiers. + +use crate::model::emoji::{EmojiMap, EmojiModification, GeneratorEmoji, Property}; +use crate::model::hexcode::{extract_skin_tones, SKIN_MODIFIER_PATTERN}; +use emojibase_core::constants::SKIN_MODIFIER_HEXCODES; +use emojibase_core::types::SkinToneValue; +use std::collections::HashMap; + +/// For each emoji with `Emoji_Modifier_Base` property, generate the five +/// single-skin-tone modifications and attach them to `modifications`. +pub fn apply_skin_tone_modifications(emoji_map: &mut EmojiMap) { + let base_hexcodes: Vec = emoji_map + .iter() + .filter(|(_, emoji)| emoji.property.contains(&Property::EmojiModifierBase)) + .map(|(hex, _)| hex.clone()) + .collect(); + + for base_hex in base_hexcodes { + let base = match emoji_map.get(&base_hex) { + Some(e) => e.clone(), + None => continue, + }; + + let mut modifications = HashMap::new(); + + for (i, modifier_hex) in SKIN_MODIFIER_HEXCODES.iter().enumerate() { + let modified_hexcode = format!("{base_hex}-{modifier_hex}"); + let tone = emojibase_core::constants::SKIN_TONES[i]; + + let modification = EmojiModification { + hexcode: modified_hexcode.clone(), + description: base.description.clone(), + property: base.property.clone(), + presentation: base.presentation, + unicode_version: base.unicode_version.clone(), + version: base.version.clone(), + gender: base.gender, + group: base.group, + order: base.order, + subgroup: base.subgroup, + name: base.name.clone(), + tone: SkinToneValue::Single(tone), + }; + + modifications.insert(modified_hexcode, modification); + } + + if let Some(emoji) = emoji_map.get_mut(&base_hex) { + emoji.modifications = Some(modifications); + } + } +} + +/// Detect multi-person skin tones from ZWJ sequences already in the map. +/// +/// Sequences like `1F469-1F3FD-200D-1F91D-200D-1F468-1F3FF` have multiple +/// skin tone modifiers. These are extracted and associated with their +/// base (non-toned) sequence. +pub fn detect_multi_person_skin_tones(emoji_map: &mut EmojiMap) { + let multi_skin_hexcodes: Vec<(String, Vec)> = emoji_map + .keys() + .filter_map(|hex| { + let tones = extract_skin_tones(hex); + if tones.len() > 1 { + Some((hex.clone(), tones)) + } else { + None + } + }) + .collect(); + + for (hexcode, tones) in multi_skin_hexcodes { + let base_hexcode = SKIN_MODIFIER_PATTERN.replace_all(&hexcode, "").to_string(); + // Clean up double hyphens from removed modifiers. + let base_hexcode = base_hexcode.replace("--", "-"); + let base_hexcode = base_hexcode.trim_matches('-').to_string(); + + if let Some(base_emoji) = emoji_map.get_mut(&base_hexcode) { + let modifications = base_emoji.modifications.get_or_insert_with(HashMap::new); + if !modifications.contains_key(&hexcode) { + modifications.insert( + hexcode.clone(), + EmojiModification { + hexcode: hexcode.clone(), + description: base_emoji.description.clone(), + property: base_emoji.property.clone(), + presentation: base_emoji.presentation, + unicode_version: base_emoji.unicode_version.clone(), + version: base_emoji.version.clone(), + gender: base_emoji.gender, + group: base_emoji.group, + order: base_emoji.order, + subgroup: base_emoji.subgroup, + name: base_emoji.name.clone(), + tone: SkinToneValue::Multi(tones.clone()), + }, + ); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use emojibase_core::types::{Presentation, SkinTone}; + + fn make_base_emoji(hexcode: &str) -> GeneratorEmoji { + GeneratorEmoji { + description: "test".to_string(), + hexcode: hexcode.to_string(), + property: vec![Property::EmojiModifierBase], + presentation: Presentation::Emoji, + unicode_version: Some("6.0".to_string()), + version: "1.0".to_string(), + gender: None, + group: 0, + order: 1, + subgroup: 0, + name: "TEST".to_string(), + modifications: None, + qualifiers: None, + variations: None, + shortcodes: None, + emoticon: None, + } + } + + #[test] + fn test_skin_tone_modifications() { + let mut map = EmojiMap::new(); + map.insert("1F44D".to_string(), make_base_emoji("1F44D")); + + apply_skin_tone_modifications(&mut map); + + let emoji = &map["1F44D"]; + assert!(emoji.modifications.is_some()); + let mods = emoji.modifications.as_ref().unwrap(); + assert_eq!(mods.len(), 5); + assert!(mods.contains_key("1F44D-1F3FB")); + + let light = &mods["1F44D-1F3FB"]; + assert_eq!(light.tone, SkinToneValue::Single(SkinTone::Light)); + } +} diff --git a/crates/emojibase-generator/src/builder/verify.rs b/crates/emojibase-generator/src/builder/verify.rs new file mode 100644 index 00000000..4b362444 --- /dev/null +++ b/crates/emojibase-generator/src/builder/verify.rs @@ -0,0 +1,60 @@ +//! Data integrity verification. + +use crate::model::emoji::{EmojiMap, Property}; +use crate::util::log; + +/// Verify the integrity of the built emoji data. +/// +/// Checks: +/// - Every emoji has a hexcode. +/// - Every emoji has at least one property. +/// - Every emoji used in output is not hidden (modifier/component/etc.). +/// - Group and subgroup are within expected ranges. +/// +/// Returns `true` if all checks pass. +pub fn verify_data_integrity(emoji_map: &EmojiMap) -> bool { + let mut ok = true; + let mut count = 0; + + for (hexcode, emoji) in emoji_map { + count += 1; + + if hexcode.is_empty() { + log::error("Found emoji with empty hexcode"); + ok = false; + } + + if emoji.property.is_empty() { + log::warn(&format!("Emoji {hexcode} has no properties")); + } + + if emoji.group > 9 { + log::error(&format!( + "Emoji {hexcode} has invalid group: {}", + emoji.group + )); + ok = false; + } + } + + if ok { + log::success(&format!( + "Data integrity verified: {count} emoji entries" + )); + } + + ok +} + +/// Filter out hidden emoji (modifiers, components, extended pictographic) +/// from the final output. +pub fn filter_visible_emoji(emoji_map: &EmojiMap) -> EmojiMap { + emoji_map + .iter() + .filter(|(_, emoji)| { + // Keep if it has at least one non-hidden property. + emoji.property.iter().any(|p| !p.is_hidden()) + }) + .map(|(k, v)| (k.clone(), v.clone())) + .collect() +} diff --git a/crates/emojibase-generator/src/config.rs b/crates/emojibase-generator/src/config.rs new file mode 100644 index 00000000..82616c9d --- /dev/null +++ b/crates/emojibase-generator/src/config.rs @@ -0,0 +1,33 @@ +//! Generator configuration: version constants, URLs, and locale list. + +use emojibase_core::constants; + +/// Configuration for a generator run. +#[derive(Debug, Clone)] +pub struct Config { + /// Emoji specification version (e.g. `"17.0"`). + pub emoji_version: String, + /// Unicode version (e.g. `"17.0.0"`). + pub unicode_version: String, + /// CLDR version (e.g. `"48"`). + pub cldr_version: String, + /// Locales to generate data for. + pub locales: Vec, + /// Output directory for generated datasets. + pub output_dir: String, +} + +impl Default for Config { + fn default() -> Self { + Self { + emoji_version: constants::LATEST_EMOJI_VERSION.to_string(), + unicode_version: constants::LATEST_UNICODE_VERSION.to_string(), + cldr_version: constants::LATEST_CLDR_VERSION.to_string(), + locales: constants::SUPPORTED_LOCALES + .iter() + .map(|s| s.to_string()) + .collect(), + output_dir: "packages/data".to_string(), + } + } +} diff --git a/crates/emojibase-generator/src/fetcher/cache.rs b/crates/emojibase-generator/src/fetcher/cache.rs new file mode 100644 index 00000000..db5b151e --- /dev/null +++ b/crates/emojibase-generator/src/fetcher/cache.rs @@ -0,0 +1,48 @@ +//! File-based caching for downloaded resources. + +use std::path::{Path, PathBuf}; + +/// Resolve the default cache directory (`/.cache`). +pub fn cache_dir() -> PathBuf { + let dir = PathBuf::from(".cache"); + if !dir.exists() { + std::fs::create_dir_all(&dir).expect("failed to create cache directory"); + } + dir +} + +/// Return the cached file path for a given URL. +pub fn cache_path_for(url: &str) -> PathBuf { + // Create a filesystem-safe key from the URL. + let key: String = url + .chars() + .map(|c| if c.is_ascii_alphanumeric() || c == '.' { c } else { '_' }) + .collect(); + cache_dir().join(key) +} + +/// Read a cached file if it exists and is not empty. +pub fn read_cache(url: &str) -> Option { + let path = cache_path_for(url); + if path.exists() { + let content = std::fs::read_to_string(&path).ok()?; + if !content.is_empty() { + return Some(content); + } + } + None +} + +/// Write content to the cache for a given URL. +pub fn write_cache(url: &str, content: &str) -> std::io::Result<()> { + let path = cache_path_for(url); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&path, content) +} + +/// Check if a cache file exists at the given path. +pub fn cache_exists(path: &Path) -> bool { + path.exists() && path.metadata().map(|m| m.len() > 0).unwrap_or(false) +} diff --git a/crates/emojibase-generator/src/fetcher/cldr.rs b/crates/emojibase-generator/src/fetcher/cldr.rs new file mode 100644 index 00000000..7e767137 --- /dev/null +++ b/crates/emojibase-generator/src/fetcher/cldr.rs @@ -0,0 +1,53 @@ +//! Fetch and parse CLDR annotation data. + +use super::cache; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +/// Base URL for CLDR JSON data on GitHub. +const CLDR_GITHUB_BASE: &str = + "https://raw.githubusercontent.com/nicolo-ribaudo/tc39-proposal-intl-emoji/main/cldr-json"; + +/// Resolve the local path for CLDR annotation data. +/// +/// In the JS project these come from npm packages +/// (`cldr-annotations-full`, etc.). In Rust we download the JSON directly +/// from GitHub or from a local checkout. +pub fn cldr_annotations_path(locale: &str, derived: bool) -> PathBuf { + let pkg = if derived { + "cldr-annotations-derived-full" + } else { + "cldr-annotations-full" + }; + PathBuf::from(format!( + "node_modules/{pkg}/annotationsDerived/{locale}/annotations.json" + )) +} + +/// Load CLDR annotations from a local node_modules checkout (if the npm +/// packages are installed) or fall back to a cached download. +pub fn load_cldr_annotations_local( + locale: &str, + derived: bool, +) -> Option> { + let path = cldr_annotations_path(locale, derived); + if path.exists() { + let content = std::fs::read_to_string(&path).ok()?; + serde_json::from_str(&content).ok() + } else { + None + } +} + +/// Attempt to load CLDR locale names from local node_modules. +pub fn load_cldr_locale_names(locale: &str) -> Option { + let path = Path::new("node_modules/cldr-localenames-full/main") + .join(locale) + .join("territories.json"); + if path.exists() { + let content = std::fs::read_to_string(&path).ok()?; + serde_json::from_str(&content).ok() + } else { + None + } +} diff --git a/crates/emojibase-generator/src/fetcher/mod.rs b/crates/emojibase-generator/src/fetcher/mod.rs new file mode 100644 index 00000000..f557c53f --- /dev/null +++ b/crates/emojibase-generator/src/fetcher/mod.rs @@ -0,0 +1,3 @@ +pub mod cache; +pub mod cldr; +pub mod unicode; diff --git a/crates/emojibase-generator/src/fetcher/unicode.rs b/crates/emojibase-generator/src/fetcher/unicode.rs new file mode 100644 index 00000000..b26e5f95 --- /dev/null +++ b/crates/emojibase-generator/src/fetcher/unicode.rs @@ -0,0 +1,90 @@ +//! Fetch Unicode data files from unicode.org. + +use super::cache; + +/// Base URL for Unicode Public data. +const UNICODE_PUBLIC_BASE: &str = "https://unicode.org/Public"; + +/// Fetch a text resource, using the local cache if available. +pub async fn fetch_text(url: &str) -> Result> { + if let Some(cached) = cache::read_cache(url) { + return Ok(cached); + } + + let response = reqwest::get(url).await?; + let text = response.text().await?; + cache::write_cache(url, &text)?; + Ok(text) +} + +/// Build the URL for `emoji-data.txt`. +pub fn emoji_data_url(unicode_version: &str) -> String { + format!("{UNICODE_PUBLIC_BASE}/{unicode_version}/ucd/emoji/emoji-data.txt") +} + +/// Build the URL for `emoji-sequences.txt`. +pub fn emoji_sequences_url(emoji_version: &str) -> String { + format!("{UNICODE_PUBLIC_BASE}/emoji/{emoji_version}/emoji-sequences.txt") +} + +/// Build the URL for `emoji-zwj-sequences.txt`. +pub fn emoji_zwj_sequences_url(emoji_version: &str) -> String { + format!("{UNICODE_PUBLIC_BASE}/emoji/{emoji_version}/emoji-zwj-sequences.txt") +} + +/// Build the URL for `emoji-variation-sequences.txt`. +pub fn emoji_variation_sequences_url(emoji_version: &str) -> String { + format!("{UNICODE_PUBLIC_BASE}/emoji/{emoji_version}/emoji-variation-sequences.txt") +} + +/// Build the URL for `UnicodeData.txt`. +pub fn unicode_data_url(unicode_version: &str) -> String { + format!("{UNICODE_PUBLIC_BASE}/{unicode_version}/ucd/UnicodeData.txt") +} + +/// Build the URL for `emoji-test.txt`. +pub fn emoji_test_url(emoji_version: &str) -> String { + format!("https://unicode.org/Public/emoji/{emoji_version}/emoji-test.txt") +} + +/// Fetch `emoji-data.txt`. +pub async fn fetch_emoji_data( + unicode_version: &str, +) -> Result> { + fetch_text(&emoji_data_url(unicode_version)).await +} + +/// Fetch `emoji-sequences.txt`. +pub async fn fetch_emoji_sequences( + emoji_version: &str, +) -> Result> { + fetch_text(&emoji_sequences_url(emoji_version)).await +} + +/// Fetch `emoji-zwj-sequences.txt`. +pub async fn fetch_emoji_zwj_sequences( + emoji_version: &str, +) -> Result> { + fetch_text(&emoji_zwj_sequences_url(emoji_version)).await +} + +/// Fetch `emoji-variation-sequences.txt`. +pub async fn fetch_emoji_variation_sequences( + emoji_version: &str, +) -> Result> { + fetch_text(&emoji_variation_sequences_url(emoji_version)).await +} + +/// Fetch `UnicodeData.txt`. +pub async fn fetch_unicode_data( + unicode_version: &str, +) -> Result> { + fetch_text(&unicode_data_url(unicode_version)).await +} + +/// Fetch `emoji-test.txt`. +pub async fn fetch_emoji_test( + emoji_version: &str, +) -> Result> { + fetch_text(&emoji_test_url(emoji_version)).await +} diff --git a/crates/emojibase-generator/src/generator/compact_json.rs b/crates/emojibase-generator/src/generator/compact_json.rs new file mode 100644 index 00000000..406f9d82 --- /dev/null +++ b/crates/emojibase-generator/src/generator/compact_json.rs @@ -0,0 +1,64 @@ +//! Generate `compact.json` and `compact.raw.json` output files. + +use emojibase_core::types::{CompactEmoji, Emoji, Presentation}; +use std::path::Path; + +/// Convert a full [`Emoji`] to a [`CompactEmoji`]. +pub fn to_compact(emoji: &Emoji) -> CompactEmoji { + let unicode = if emoji.presentation == Presentation::Emoji { + emoji.emoji.clone() + } else { + emoji.text.clone() + }; + + let skins = emoji.skins.as_ref().map(|skins| { + skins.iter().map(to_compact).collect() + }); + + CompactEmoji { + hexcode: emoji.hexcode.clone(), + label: emoji.label.clone(), + unicode, + group: emoji.group.unwrap_or(0), + order: emoji.order, + tags: emoji.tags.clone(), + shortcodes: emoji.shortcodes.clone(), + emoticon: emoji.emoticon.clone(), + skins, + } +} + +/// Write compact data to `compact.json` and `compact.raw.json`. +pub fn write_compact_json( + emoji_list: &[Emoji], + output_dir: &Path, +) -> Result<(), Box> { + std::fs::create_dir_all(output_dir)?; + + let compact_list: Vec = emoji_list.iter().map(to_compact).collect(); + + // compact.raw.json — no shortcodes + let raw_list: Vec = compact_list + .iter() + .map(|c| { + let mut raw = c.clone(); + raw.shortcodes = None; + if let Some(skins) = &mut raw.skins { + for s in skins { + s.shortcodes = None; + } + } + raw + }) + .collect(); + + let raw_path = output_dir.join("compact.raw.json"); + let raw_json = serde_json::to_string_pretty(&raw_list)?; + std::fs::write(&raw_path, &raw_json)?; + + let compact_path = output_dir.join("compact.json"); + let compact_json = serde_json::to_string_pretty(&compact_list)?; + std::fs::write(&compact_path, &compact_json)?; + + Ok(()) +} diff --git a/crates/emojibase-generator/src/generator/data_json.rs b/crates/emojibase-generator/src/generator/data_json.rs new file mode 100644 index 00000000..66a46512 --- /dev/null +++ b/crates/emojibase-generator/src/generator/data_json.rs @@ -0,0 +1,115 @@ +//! Generate `data.json` and `data.raw.json` output files. + +use crate::model::emoji::EmojiMap; +use emojibase_core::types::{Emoji, EmoticonValue, Presentation, SkinToneValue}; +use std::collections::HashMap; +use std::path::Path; + +/// Convert the internal [`EmojiMap`] to the final output [`Emoji`] list. +/// +/// The `tags_map` provides CLDR keyword tags, and `emoticon_map` provides +/// emoticon associations. The resulting list is sorted by display order. +pub fn build_data_list( + emoji_map: &EmojiMap, + tags_map: &HashMap>, +) -> Vec { + let mut list: Vec = emoji_map + .values() + .map(|gen| { + let version: f32 = gen.version.parse().unwrap_or(0.0); + let emoji_char = + emojibase_core::unicode::from_hexcode_to_unicode(&gen.hexcode); + let text_char = if gen.presentation == Presentation::Text { + emoji_char.clone() + } else { + String::new() + }; + + // Build skin variations if present. + let skins = gen.modifications.as_ref().map(|mods| { + let mut skin_list: Vec = mods + .values() + .map(|m| { + let skin_emoji = + emojibase_core::unicode::from_hexcode_to_unicode(&m.hexcode); + Emoji { + hexcode: m.hexcode.clone(), + label: m.description.clone(), + emoji: skin_emoji.clone(), + text: String::new(), + presentation: m.presentation, + version: m.version.parse().unwrap_or(0.0), + order: Some(m.order), + group: Some(m.group), + subgroup: Some(m.subgroup), + tags: None, + shortcodes: None, + emoticon: None, + gender: m.gender, + tone: Some(m.tone.clone()), + skins: None, + } + }) + .collect(); + skin_list.sort_by_key(|s| s.hexcode.clone()); + skin_list + }); + + Emoji { + hexcode: gen.hexcode.clone(), + label: gen.description.clone(), + emoji: emoji_char, + text: text_char, + presentation: gen.presentation, + version, + order: Some(gen.order), + group: Some(gen.group), + subgroup: Some(gen.subgroup), + tags: tags_map.get(&gen.hexcode).cloned(), + shortcodes: gen.shortcodes.clone(), + emoticon: gen.emoticon.clone(), + gender: gen.gender, + tone: None, + skins, + } + }) + .collect(); + + list.sort_by_key(|e| e.order.unwrap_or(u32::MAX)); + list +} + +/// Write the data list to `data.json` (with shortcodes) and `data.raw.json` +/// (without shortcodes) under the given output directory. +pub fn write_data_json( + list: &[Emoji], + output_dir: &Path, +) -> Result<(), Box> { + std::fs::create_dir_all(output_dir)?; + + // data.raw.json — no shortcodes + let raw_list: Vec = list + .iter() + .map(|e| { + let mut raw = e.clone(); + raw.shortcodes = None; + if let Some(skins) = &mut raw.skins { + for s in skins { + s.shortcodes = None; + } + } + raw + }) + .collect(); + + let raw_path = output_dir.join("data.raw.json"); + let raw_json = serde_json::to_string_pretty(&raw_list)?; + std::fs::write(&raw_path, &raw_json)?; + + // data.json — with shortcodes + let data_path = output_dir.join("data.json"); + let data_json = serde_json::to_string_pretty(list)?; + std::fs::write(&data_path, &data_json)?; + + Ok(()) +} diff --git a/crates/emojibase-generator/src/generator/messages_json.rs b/crates/emojibase-generator/src/generator/messages_json.rs new file mode 100644 index 00000000..c3565311 --- /dev/null +++ b/crates/emojibase-generator/src/generator/messages_json.rs @@ -0,0 +1,73 @@ +//! Generate `messages.json` for localized group/subgroup/skin-tone labels. + +use emojibase_core::constants::{GROUP_KEYS, SKIN_TONE_KEYS}; +use emojibase_core::types::{MessageEntry, MessagesDataset}; +use std::path::Path; + +/// Build a messages dataset from group names and subgroup names. +pub fn build_messages( + group_names: &[String], + subgroup_entries: &[(u8, String)], +) -> MessagesDataset { + let groups: Vec = group_names + .iter() + .enumerate() + .map(|(i, name)| { + let key = GROUP_KEYS + .get(i) + .map(|k| k.as_str().to_string()) + .unwrap_or_else(|| format!("group-{i}")); + MessageEntry { + key, + message: name.clone(), + order: i as u32, + } + }) + .collect(); + + let subgroups: Vec = subgroup_entries + .iter() + .enumerate() + .map(|(i, (_, name))| MessageEntry { + key: name.clone(), + message: name.clone(), + order: i as u32, + }) + .collect(); + + let skin_tones: Vec = SKIN_TONE_KEYS + .iter() + .enumerate() + .map(|(i, key)| MessageEntry { + key: key.as_str().to_string(), + message: format!("{} Skin Tone", capitalize(key.as_str())), + order: i as u32, + }) + .collect(); + + MessagesDataset { + groups, + subgroups, + skin_tones, + } +} + +/// Write the messages dataset to `messages.json`. +pub fn write_messages_json( + messages: &MessagesDataset, + output_dir: &Path, +) -> Result<(), Box> { + std::fs::create_dir_all(output_dir)?; + let path = output_dir.join("messages.json"); + let json = serde_json::to_string_pretty(messages)?; + std::fs::write(&path, &json)?; + Ok(()) +} + +fn capitalize(s: &str) -> String { + let mut chars = s.chars(); + match chars.next() { + None => String::new(), + Some(c) => c.to_uppercase().chain(chars).collect(), + } +} diff --git a/crates/emojibase-generator/src/generator/metadata.rs b/crates/emojibase-generator/src/generator/metadata.rs new file mode 100644 index 00000000..c87be85d --- /dev/null +++ b/crates/emojibase-generator/src/generator/metadata.rs @@ -0,0 +1,81 @@ +//! Generate metadata files: `meta/groups.json`, `meta/unicode.json`, etc. + +use crate::model::emoji::EmojiMap; +use std::collections::HashMap; +use std::path::Path; + +/// Build group metadata from parsed group/subgroup names. +pub fn build_group_metadata( + group_names: &[String], + subgroup_entries: &[(u8, String)], +) -> serde_json::Value { + let mut groups = serde_json::Map::new(); + + for (i, name) in group_names.iter().enumerate() { + let subgroups: Vec<&str> = subgroup_entries + .iter() + .filter(|(g, _)| *g as usize == i) + .map(|(_, name)| name.as_str()) + .collect(); + + let mut group_obj = serde_json::Map::new(); + group_obj.insert("order".to_string(), serde_json::json!(i)); + group_obj.insert("key".to_string(), serde_json::json!(name)); + group_obj.insert( + "subgroups".to_string(), + serde_json::json!(subgroups), + ); + groups.insert(i.to_string(), serde_json::Value::Object(group_obj)); + } + + serde_json::Value::Object(groups) +} + +/// Build the unicode metadata (all hexcodes in the dataset). +pub fn build_unicode_metadata(emoji_map: &EmojiMap) -> Vec { + let mut hexcodes: Vec = emoji_map.keys().cloned().collect(); + hexcodes.sort(); + hexcodes +} + +/// Build the unicode names map (hexcode → character name). +pub fn build_unicode_names(emoji_map: &EmojiMap) -> HashMap { + emoji_map + .iter() + .map(|(hex, emoji)| (hex.clone(), emoji.name.clone())) + .collect() +} + +/// Write metadata files to the `meta/` directory. +pub fn write_metadata( + emoji_map: &EmojiMap, + group_names: &[String], + subgroup_entries: &[(u8, String)], + output_dir: &Path, +) -> Result<(), Box> { + let meta_dir = output_dir.join("meta"); + std::fs::create_dir_all(&meta_dir)?; + + // meta/groups.json + let groups = build_group_metadata(group_names, subgroup_entries); + std::fs::write( + meta_dir.join("groups.json"), + serde_json::to_string_pretty(&groups)?, + )?; + + // meta/unicode.json + let unicode = build_unicode_metadata(emoji_map); + std::fs::write( + meta_dir.join("unicode.json"), + serde_json::to_string_pretty(&unicode)?, + )?; + + // meta/unicode-names.json + let names = build_unicode_names(emoji_map); + std::fs::write( + meta_dir.join("unicode-names.json"), + serde_json::to_string_pretty(&names)?, + )?; + + Ok(()) +} diff --git a/crates/emojibase-generator/src/generator/mod.rs b/crates/emojibase-generator/src/generator/mod.rs new file mode 100644 index 00000000..62e68c01 --- /dev/null +++ b/crates/emojibase-generator/src/generator/mod.rs @@ -0,0 +1,6 @@ +pub mod compact_json; +pub mod data_json; +pub mod messages_json; +pub mod metadata; +pub mod regex; +pub mod versions; diff --git a/crates/emojibase-generator/src/generator/regex.rs b/crates/emojibase-generator/src/generator/regex.rs new file mode 100644 index 00000000..f3dec22d --- /dev/null +++ b/crates/emojibase-generator/src/generator/regex.rs @@ -0,0 +1,136 @@ +//! Generate regex patterns for matching emoji. + +use crate::model::emoji::EmojiMap; +use std::path::Path; + +/// Generate a regex pattern string that matches all emoji in the map. +/// +/// This produces a character-class based regex. For full optimization +/// (trie-based compression like `regexgen`), a more sophisticated +/// algorithm would be needed. +pub fn generate_emoji_regex(emoji_map: &EmojiMap) -> String { + let mut codepoints: Vec = emoji_map + .keys() + .filter_map(|hex| { + // Only single-codepoint emoji for the character class. + if hex.contains('-') { + None + } else { + u32::from_str_radix(hex, 16).ok() + } + }) + .collect(); + + codepoints.sort(); + codepoints.dedup(); + + if codepoints.is_empty() { + return String::new(); + } + + // Build character class ranges. + let mut parts: Vec = Vec::new(); + let mut i = 0; + + while i < codepoints.len() { + let start = codepoints[i]; + let mut end = start; + + while i + 1 < codepoints.len() && codepoints[i + 1] == end + 1 { + end = codepoints[i + 1]; + i += 1; + } + + if start == end { + parts.push(format!("\\u{{{:X}}}", start)); + } else if end - start == 1 { + parts.push(format!("\\u{{{:X}}}\\u{{{:X}}}", start, end)); + } else { + parts.push(format!("\\u{{{:X}}}-\\u{{{:X}}}", start, end)); + } + + i += 1; + } + + format!("[{}]", parts.join("")) +} + +/// Generate regex patterns for multi-codepoint sequences (ZWJ, flags, etc.). +pub fn generate_sequence_regex(emoji_map: &EmojiMap) -> Vec { + let mut sequences: Vec = emoji_map + .keys() + .filter(|hex| hex.contains('-')) + .cloned() + .collect(); + + // Sort by length descending so longer sequences match first. + sequences.sort_by(|a, b| b.len().cmp(&a.len())); + + sequences + .into_iter() + .map(|hex| { + hex.split('-') + .map(|part| format!("\\u{{{part}}}")) + .collect::>() + .join("") + }) + .collect() +} + +/// Write regex patterns to JavaScript files. +pub fn write_regex_files( + emoji_map: &EmojiMap, + output_dir: &Path, +) -> Result<(), Box> { + std::fs::create_dir_all(output_dir)?; + + let char_class = generate_emoji_regex(emoji_map); + let sequences = generate_sequence_regex(emoji_map); + + // Build the full pattern. + let mut pattern_parts = sequences; + if !char_class.is_empty() { + pattern_parts.push(char_class); + } + + let full_pattern = pattern_parts.join("|"); + + // emoji.js + let emoji_js = format!( + "// AUTO-GENERATED by emojibase-generator\nexport default /(?:{})/g;\n", + full_pattern + ); + std::fs::write(output_dir.join("emoji.js"), &emoji_js)?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_generate_emoji_regex() { + let mut map = EmojiMap::new(); + map.insert("1F600".to_string(), crate::model::emoji::GeneratorEmoji { + description: "grinning face".to_string(), + hexcode: "1F600".to_string(), + property: vec![], + presentation: emojibase_core::types::Presentation::Emoji, + unicode_version: None, + version: "1.0".to_string(), + gender: None, + group: 0, + order: 1, + subgroup: 0, + name: "".to_string(), + modifications: None, + qualifiers: None, + variations: None, + shortcodes: None, + emoticon: None, + }); + let regex = generate_emoji_regex(&map); + assert!(regex.contains("1F600")); + } +} diff --git a/crates/emojibase-generator/src/generator/versions.rs b/crates/emojibase-generator/src/generator/versions.rs new file mode 100644 index 00000000..7aa7c5ac --- /dev/null +++ b/crates/emojibase-generator/src/generator/versions.rs @@ -0,0 +1,39 @@ +//! Generate version mapping files: `versions/emoji.json`, `versions/unicode.json`. + +use crate::model::emoji::{EmojiMap, VersionMap}; +use std::path::Path; + +/// Build a version map: version string → list of hexcodes introduced in that version. +pub fn build_version_map(emoji_map: &EmojiMap) -> VersionMap { + let mut map = VersionMap::new(); + + for (hexcode, emoji) in emoji_map { + map.entry(emoji.version.clone()) + .or_default() + .push(hexcode.clone()); + } + + // Sort hexcodes within each version. + for hexcodes in map.values_mut() { + hexcodes.sort(); + } + + map +} + +/// Write version files. +pub fn write_versions( + emoji_map: &EmojiMap, + output_dir: &Path, +) -> Result<(), Box> { + let versions_dir = output_dir.join("versions"); + std::fs::create_dir_all(&versions_dir)?; + + let version_map = build_version_map(emoji_map); + std::fs::write( + versions_dir.join("emoji.json"), + serde_json::to_string_pretty(&version_map)?, + )?; + + Ok(()) +} diff --git a/crates/emojibase-generator/src/main.rs b/crates/emojibase-generator/src/main.rs new file mode 100644 index 00000000..5df12d4c --- /dev/null +++ b/crates/emojibase-generator/src/main.rs @@ -0,0 +1,263 @@ +//! # emojibase-generator +//! +//! A CLI tool that fetches Unicode and CLDR data sources, normalizes them, +//! and produces pre-generated JSON datasets, regex patterns, and shortcode +//! mappings for the emojibase project. +//! +//! ## Usage +//! +//! ```text +//! emojibase-generator generate --output packages/data +//! emojibase-generator generate --locale en --locale ja +//! ``` + +use clap::{Parser, Subcommand}; +use std::path::PathBuf; + +mod builder; +mod config; +mod fetcher; +mod generator; +mod model; +mod parser; +mod util; + +/// Emojibase dataset generator: fetches Unicode / CLDR data and produces +/// pre-built JSON datasets, regex patterns, and shortcode mappings. +#[derive(Parser)] +#[command(name = "emojibase-generator", version, about)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// Generate emoji datasets for all (or selected) locales. + Generate { + /// Output directory for generated files. + #[arg(short, long, default_value = "packages/data")] + output: PathBuf, + + /// Specific locale(s) to generate. If omitted, all supported locales + /// are generated. + #[arg(short, long)] + locale: Vec, + + /// Emoji specification version. + #[arg(long)] + emoji_version: Option, + + /// Unicode version. + #[arg(long)] + unicode_version: Option, + }, + + /// Verify integrity of previously-generated data. + Verify { + /// Directory containing generated data. + #[arg(short, long, default_value = "packages/data")] + data_dir: PathBuf, + }, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let cli = Cli::parse(); + + match cli.command { + Commands::Generate { + output, + locale, + emoji_version, + unicode_version, + } => { + let mut cfg = config::Config::default(); + if let Some(ev) = emoji_version { + cfg.emoji_version = ev; + } + if let Some(uv) = unicode_version { + cfg.unicode_version = uv; + } + if !locale.is_empty() { + cfg.locales = locale; + } + cfg.output_dir = output.to_string_lossy().to_string(); + + run_generate(cfg).await?; + } + Commands::Verify { data_dir } => { + run_verify(&data_dir)?; + } + } + + Ok(()) +} + +/// Main generation pipeline. +async fn run_generate(cfg: config::Config) -> Result<(), Box> { + util::log::info(&format!( + "Generating emoji data (emoji v{}, unicode v{})", + cfg.emoji_version, cfg.unicode_version + )); + + // ------------------------------------------------------------------- + // Phase 1: Fetch Unicode data files + // ------------------------------------------------------------------- + let spinner = util::log::create_spinner("Fetching Unicode data files..."); + + let (emoji_data_content, sequences_content, zwj_content, variation_content, unicode_data_content, test_content) = tokio::try_join!( + fetcher::unicode::fetch_emoji_data(&cfg.unicode_version), + fetcher::unicode::fetch_emoji_sequences(&cfg.emoji_version), + fetcher::unicode::fetch_emoji_zwj_sequences(&cfg.emoji_version), + fetcher::unicode::fetch_emoji_variation_sequences(&cfg.emoji_version), + fetcher::unicode::fetch_unicode_data(&cfg.unicode_version), + fetcher::unicode::fetch_emoji_test(&cfg.emoji_version), + )?; + + spinner.finish_with_message("Fetched Unicode data files"); + + // ------------------------------------------------------------------- + // Phase 2: Parse data files + // ------------------------------------------------------------------- + let spinner = util::log::create_spinner("Parsing data files..."); + + let mut emoji_data = parser::emoji_data::parse_emoji_data(&emoji_data_content); + let sequences = parser::sequences::parse_sequences( + &sequences_content, + model::emoji::Property::EmojiModifierSequence, + ); + let zwj_sequences = parser::sequences::parse_sequences( + &zwj_content, + model::emoji::Property::EmojiZwjSequence, + ); + let variations = parser::variations::parse_variations(&variation_content); + let names = parser::unicode_data::parse_unicode_names(&unicode_data_content); + let (metadata, group_names, subgroup_entries) = + parser::test_file::parse_test_file(&test_content); + + // Merge sequence data into the primary map. + builder::data::merge_data_maps(&mut emoji_data, sequences); + builder::data::merge_data_maps(&mut emoji_data, zwj_sequences); + + spinner.finish_with_message("Parsed data files"); + + // ------------------------------------------------------------------- + // Phase 3: Build unified emoji map + // ------------------------------------------------------------------- + let spinner = util::log::create_spinner("Building emoji map..."); + + let mut emoji_map = + builder::data::build_emoji_data(&emoji_data, &metadata, &variations, &names); + + // Apply gender detection. + builder::gender::apply_gender(&mut emoji_map); + + // Generate skin tone modifications. + builder::skin_tones::apply_skin_tone_modifications(&mut emoji_map); + builder::skin_tones::detect_multi_person_skin_tones(&mut emoji_map); + + // Load and apply emoticons. + let emoticons = parser::emoticons::builtin_emoticons(); + for (hex, emoticon) in &emoticons { + if let Some(emoji) = emoji_map.get_mut(hex) { + emoji.emoticon = Some(emoticon.clone()); + } + } + + spinner.finish_with_message("Built emoji map"); + + // ------------------------------------------------------------------- + // Phase 4: Verify + // ------------------------------------------------------------------- + let visible = builder::verify::filter_visible_emoji(&emoji_map); + builder::verify::verify_data_integrity(&visible); + + // ------------------------------------------------------------------- + // Phase 5: Generate shortcodes + // ------------------------------------------------------------------- + let shortcodes = builder::shortcodes::generate_emojibase_shortcodes(&visible); + let mut output_map = visible.clone(); + builder::shortcodes::apply_shortcodes(&mut output_map, &shortcodes); + + // ------------------------------------------------------------------- + // Phase 6: Generate output for each locale + // ------------------------------------------------------------------- + let pb = util::log::create_progress_bar(cfg.locales.len() as u64, "Generating datasets"); + let tags_map = std::collections::HashMap::new(); + + for locale in &cfg.locales { + let locale_dir = PathBuf::from(&cfg.output_dir).join(locale); + + // Build data list and write output files. + let data_list = + generator::data_json::build_data_list(&output_map, &tags_map); + + generator::data_json::write_data_json(&data_list, &locale_dir)?; + generator::compact_json::write_compact_json(&data_list, &locale_dir)?; + + // Messages + let messages = + generator::messages_json::build_messages(&group_names, &subgroup_entries); + generator::messages_json::write_messages_json(&messages, &locale_dir)?; + + pb.inc(1); + } + + pb.finish_with_message("Generated locale datasets"); + + // ------------------------------------------------------------------- + // Phase 7: Generate metadata and regex + // ------------------------------------------------------------------- + let output_path = PathBuf::from(&cfg.output_dir); + generator::metadata::write_metadata( + &output_map, + &group_names, + &subgroup_entries, + &output_path, + )?; + generator::versions::write_versions(&output_map, &output_path)?; + + util::log::success("Generation complete!"); + Ok(()) +} + +/// Verify previously generated data. +fn run_verify(data_dir: &PathBuf) -> Result<(), Box> { + util::log::info(&format!("Verifying data in {}", data_dir.display())); + + // Load the English data.json as a sanity check. + let data_path = data_dir.join("en/data.json"); + if !data_path.exists() { + return Err(format!("Data file not found: {}", data_path.display()).into()); + } + + let content = std::fs::read_to_string(&data_path)?; + let emojis: Vec = serde_json::from_str(&content)?; + + util::log::success(&format!( + "Loaded {} emoji entries from en/data.json", + emojis.len() + )); + + // Basic integrity checks. + let mut issues = 0; + for emoji in &emojis { + if emoji.hexcode.is_empty() { + util::log::error("Empty hexcode found"); + issues += 1; + } + if emoji.label.is_empty() { + util::log::warn(&format!("Empty label for {}", emoji.hexcode)); + issues += 1; + } + } + + if issues == 0 { + util::log::success("All checks passed"); + } else { + util::log::error(&format!("{issues} issue(s) found")); + } + + Ok(()) +} diff --git a/crates/emojibase-generator/src/model/emoji.rs b/crates/emojibase-generator/src/model/emoji.rs new file mode 100644 index 00000000..fa9e9ee3 --- /dev/null +++ b/crates/emojibase-generator/src/model/emoji.rs @@ -0,0 +1,193 @@ +use emojibase_core::types::{ + EmoticonValue, Gender, Group, Hexcode, Presentation, Shortcode, SkinToneValue, Subgroup, +}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +// --------------------------------------------------------------------------- +// Unicode emoji property tags +// --------------------------------------------------------------------------- + +/// Unicode emoji properties from `emoji-data.txt` and sequence files. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Property { + BasicEmoji, + Emoji, + EmojiPresentation, + EmojiComponent, + EmojiModifier, + EmojiModifierBase, + EmojiModifierSequence, + EmojiKeycapSequence, + EmojiFlagSequence, + EmojiTagSequence, + EmojiZwjSequence, + EmojiCombiningSequence, + ExtendedPictographic, + RegionalIndicator, + RgiEmojiFlagSequence, + RgiEmojiModifierSequence, + RgiEmojiTagSequence, + RgiEmojiZwjSequence, +} + +impl Property { + /// Parse a property name from the Unicode data files. + pub fn from_str_name(s: &str) -> Option { + match s.trim() { + "Basic_Emoji" => Some(Self::BasicEmoji), + "Emoji" => Some(Self::Emoji), + "Emoji_Presentation" => Some(Self::EmojiPresentation), + "Emoji_Component" => Some(Self::EmojiComponent), + "Emoji_Modifier" => Some(Self::EmojiModifier), + "Emoji_Modifier_Base" => Some(Self::EmojiModifierBase), + "Emoji_Modifier_Sequence" => Some(Self::EmojiModifierSequence), + "Emoji_Keycap_Sequence" => Some(Self::EmojiKeycapSequence), + "Emoji_Flag_Sequence" => Some(Self::EmojiFlagSequence), + "Emoji_Tag_Sequence" => Some(Self::EmojiTagSequence), + "Emoji_ZWJ_Sequence" => Some(Self::EmojiZwjSequence), + "Emoji_Combining_Sequence" => Some(Self::EmojiCombiningSequence), + "Extended_Pictographic" => Some(Self::ExtendedPictographic), + "Regional_Indicator" => Some(Self::RegionalIndicator), + "RGI_Emoji_Flag_Sequence" => Some(Self::RgiEmojiFlagSequence), + "RGI_Emoji_Modifier_Sequence" => Some(Self::RgiEmojiModifierSequence), + "RGI_Emoji_Tag_Sequence" => Some(Self::RgiEmojiTagSequence), + "RGI_Emoji_ZWJ_Sequence" => Some(Self::RgiEmojiZwjSequence), + _ => None, + } + } + + /// Whether this property should be hidden from the final output + /// (modifier, component, extended pictographic, regional indicator). + pub fn is_hidden(&self) -> bool { + matches!( + self, + Self::EmojiModifier + | Self::EmojiComponent + | Self::ExtendedPictographic + | Self::RegionalIndicator + ) + } +} + +// --------------------------------------------------------------------------- +// Internal emoji representation during building +// --------------------------------------------------------------------------- + +/// Minimal emoji data parsed from `emoji-data.txt`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EmojiData { + pub description: String, + pub hexcode: Hexcode, + pub property: Vec, + pub presentation: Presentation, + pub unicode_version: Option, + pub version: f32, + pub gender: Option, +} + +/// Metadata extracted from `emoji-test.txt` (group, order, qualifiers). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EmojiMetadata { + pub group: Group, + pub subgroup: Subgroup, + pub order: u32, + pub qualifier: Option, +} + +/// Source information parsed from emoji-list.html. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EmojiSource { + pub group: Group, + pub subgroup: Subgroup, +} + +/// Variation information (text / emoji presentation hexcodes). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EmojiVariation { + pub text: Hexcode, + pub emoji: Hexcode, + pub property: Option>, +} + +/// A fully-merged internal emoji, combining all data sources. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GeneratorEmoji { + // Core data + pub description: String, + pub hexcode: Hexcode, + pub property: Vec, + pub presentation: Presentation, + pub unicode_version: Option, + pub version: String, + pub gender: Option, + + // Group info + pub group: Group, + pub order: u32, + pub subgroup: Subgroup, + + // Name from UnicodeData.txt + pub name: String, + + // Optional metadata + pub modifications: Option>, + pub qualifiers: Option>, + pub variations: Option, + pub shortcodes: Option>, + pub emoticon: Option, +} + +/// A skin-tone modification of a base emoji. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EmojiModification { + pub hexcode: Hexcode, + pub description: String, + pub property: Vec, + pub presentation: Presentation, + pub unicode_version: Option, + pub version: String, + pub gender: Option, + pub group: Group, + pub order: u32, + pub subgroup: Subgroup, + pub name: String, + pub tone: SkinToneValue, +} + +/// CLDR annotation for an emoji (localized label + tags). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CldrAnnotation { + pub annotation: String, + pub tags: Vec, +} + +/// Map from hexcode to CLDR annotation. +pub type CldrAnnotationMap = HashMap; + +/// Map from hexcode to emoji data (used during parsing). +pub type EmojiDataMap = HashMap; + +/// Map from hexcode to metadata. +pub type EmojiMetadataMap = HashMap; + +/// Map from hexcode to source info. +pub type EmojiSourceMap = HashMap; + +/// Map from hexcode to variation info. +pub type EmojiVariationMap = HashMap; + +/// Map from hexcode to generator emoji. +pub type EmojiMap = HashMap; + +/// Map from hexcode to Unicode character name. +pub type UnicodeNamesMap = HashMap; + +/// Emoticon map from hexcode to emoticon(s). +pub type EmoticonMap = HashMap; + +/// Shortcode map from hexcode to shortcode list. +pub type ShortcodeMap = HashMap>; + +/// Version map from version string to list of hexcodes. +pub type VersionMap = HashMap>; diff --git a/crates/emojibase-generator/src/model/group.rs b/crates/emojibase-generator/src/model/group.rs new file mode 100644 index 00000000..d48f8251 --- /dev/null +++ b/crates/emojibase-generator/src/model/group.rs @@ -0,0 +1,20 @@ +//! Group and subgroup definitions matching `emoji-test.txt` categories. + +use std::collections::HashMap; + +/// Group hierarchy: maps group name → list of subgroup names. +pub type GroupHierarchy = HashMap>; + +/// Well-known group names in order. +pub const GROUP_NAMES: &[&str] = &[ + "Smileys & Emotion", + "People & Body", + "Animals & Nature", + "Food & Drink", + "Travel & Places", + "Activities", + "Objects", + "Symbols", + "Flags", + "Component", +]; diff --git a/crates/emojibase-generator/src/model/hexcode.rs b/crates/emojibase-generator/src/model/hexcode.rs new file mode 100644 index 00000000..59200985 --- /dev/null +++ b/crates/emojibase-generator/src/model/hexcode.rs @@ -0,0 +1,78 @@ +//! Hexcode formatting and manipulation utilities. + +use emojibase_core::constants::{ + DARK_SKIN_MODIFIER, LIGHT_SKIN_MODIFIER, MEDIUM_DARK_SKIN_MODIFIER, + MEDIUM_LIGHT_SKIN_MODIFIER, MEDIUM_SKIN_MODIFIER, +}; +use emojibase_core::types::SkinTone; +use regex::Regex; +use std::sync::LazyLock; + +/// Regex matching any skin modifier hexcode in a hyphen-separated sequence. +pub static SKIN_MODIFIER_PATTERN: LazyLock = + LazyLock::new(|| Regex::new(r"(1F3FB|1F3FC|1F3FD|1F3FE|1F3FF)").unwrap()); + +/// Regex matching variation selectors. +pub static VARIATION_PATTERN: LazyLock = + LazyLock::new(|| Regex::new(r"(FE0E|FE0F)").unwrap()); + +/// Regex matching ZWJ. +pub static ZWJ_PATTERN: LazyLock = LazyLock::new(|| Regex::new(r"200D").unwrap()); + +/// Regex matching gender sign hexcodes. +pub static GENDER_PATTERN: LazyLock = + LazyLock::new(|| Regex::new(r"(2640|2642)").unwrap()); + +/// Regex matching ZWJ, variation selectors (for removal / stripping). +pub static SEQUENCE_REMOVAL_PATTERN: LazyLock = + LazyLock::new(|| Regex::new(r"(200D|FE0E|FE0F)").unwrap()); + +/// Format a hexcode to uppercase with zero-padding to at least 4 characters. +pub fn format_hexcode(hex: &str) -> String { + let upper = hex.to_uppercase(); + if upper.len() < 4 { + format!("{upper:0>4}") + } else { + upper + } +} + +/// Resolve the [`SkinTone`] from a skin modifier hexcode. +pub fn skin_tone_from_modifier(modifier: &str) -> Option { + match modifier { + s if s == LIGHT_SKIN_MODIFIER => Some(SkinTone::Light), + s if s == MEDIUM_LIGHT_SKIN_MODIFIER => Some(SkinTone::MediumLight), + s if s == MEDIUM_SKIN_MODIFIER => Some(SkinTone::Medium), + s if s == MEDIUM_DARK_SKIN_MODIFIER => Some(SkinTone::MediumDark), + s if s == DARK_SKIN_MODIFIER => Some(SkinTone::Dark), + _ => None, + } +} + +/// Extract all skin tones present in a hexcode sequence. +pub fn extract_skin_tones(hexcode: &str) -> Vec { + SKIN_MODIFIER_PATTERN + .find_iter(hexcode) + .filter_map(|m| skin_tone_from_modifier(m.as_str())) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_format_hexcode() { + assert_eq!(format_hexcode("23"), "0023"); + assert_eq!(format_hexcode("1f600"), "1F600"); + } + + #[test] + fn test_extract_skin_tones() { + let tones = extract_skin_tones("1F468-1F3FB-200D-1F4BB"); + assert_eq!(tones, vec![SkinTone::Light]); + + let tones = extract_skin_tones("1F469-1F3FD-200D-1F91D-200D-1F468-1F3FF"); + assert_eq!(tones, vec![SkinTone::Medium, SkinTone::Dark]); + } +} diff --git a/crates/emojibase-generator/src/model/mod.rs b/crates/emojibase-generator/src/model/mod.rs new file mode 100644 index 00000000..11c7cb7b --- /dev/null +++ b/crates/emojibase-generator/src/model/mod.rs @@ -0,0 +1,14 @@ +//! Generator-internal data model types used during the build pipeline. +//! +//! These types carry more metadata than the final output types in +//! `emojibase-core` and are reduced/mapped during the generation phase. + +pub mod emoji; +pub mod group; +pub mod hexcode; +pub mod shortcode; + +pub use emoji::*; +pub use group::*; +pub use hexcode::*; +pub use shortcode::*; diff --git a/crates/emojibase-generator/src/model/shortcode.rs b/crates/emojibase-generator/src/model/shortcode.rs new file mode 100644 index 00000000..202cf16e --- /dev/null +++ b/crates/emojibase-generator/src/model/shortcode.rs @@ -0,0 +1,60 @@ +//! Shortcode preset definitions and metadata. + +/// All supported shortcode presets. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ShortcodePreset { + /// CLDR-native locale-specific shortcodes. + CldrNative, + /// CLDR English-based shortcodes. + Cldr, + /// Discord shortcodes. + Discord, + /// Emojibase legacy shortcodes. + EmojibaseLegacy, + /// Emojibase native locale-specific shortcodes. + EmojibaseNative, + /// Default emojibase shortcodes. + Emojibase, + /// GitHub shortcodes. + GitHub, + /// IamCal shortcodes. + IamCal, + /// JoyPixels shortcodes. + JoyPixels, + /// Slack shortcodes (alias for IamCal). + Slack, +} + +impl ShortcodePreset { + /// File name used in the `shortcodes/` output directory. + pub fn file_name(&self) -> &'static str { + match self { + Self::CldrNative => "cldr-native", + Self::Cldr => "cldr", + Self::Discord => "discord", + Self::EmojibaseLegacy => "emojibase-legacy", + Self::EmojibaseNative => "emojibase-native", + Self::Emojibase => "emojibase", + Self::GitHub => "github", + Self::IamCal => "iamcal", + Self::JoyPixels => "joypixels", + Self::Slack => "slack", + } + } + + /// All known presets. + pub fn all() -> &'static [Self] { + &[ + Self::CldrNative, + Self::Cldr, + Self::Discord, + Self::EmojibaseLegacy, + Self::EmojibaseNative, + Self::Emojibase, + Self::GitHub, + Self::IamCal, + Self::JoyPixels, + Self::Slack, + ] + } +} diff --git a/crates/emojibase-generator/src/parser/annotations.rs b/crates/emojibase-generator/src/parser/annotations.rs new file mode 100644 index 00000000..149efca6 --- /dev/null +++ b/crates/emojibase-generator/src/parser/annotations.rs @@ -0,0 +1,96 @@ +//! Parse CLDR annotation JSON files. + +use crate::model::emoji::CldrAnnotationMap; +use std::collections::HashMap; + +/// Parse CLDR annotations from a JSON value. +/// +/// The JSON structure from `cldr-annotations-full` looks like: +/// ```json +/// { +/// "annotations": { +/// "annotations": { +/// "😀": { +/// "default": ["cheerful", "face", "grin", ...], +/// "tts": ["grinning face"] +/// } +/// } +/// } +/// } +/// ``` +pub fn parse_cldr_annotations(json: &serde_json::Value) -> CldrAnnotationMap { + let mut map = CldrAnnotationMap::new(); + + // Navigate the JSON structure. + let annotations = json + .get("annotations") + .and_then(|a| a.get("annotations")) + .or_else(|| { + // Derived annotations have a different path. + json.get("annotationsDerived") + .and_then(|a| a.get("annotations")) + }); + + let annotations = match annotations { + Some(serde_json::Value::Object(obj)) => obj, + _ => return map, + }; + + for (emoji_char, value) in annotations { + // Convert the emoji character to its hexcode. + let hexcode = emojibase_core::unicode::from_unicode_to_hexcode(emoji_char); + + let tags = value + .get("default") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str().map(|s| s.to_string())) + .collect::>() + }) + .unwrap_or_default(); + + let annotation = value + .get("tts") + .and_then(|v| v.as_array()) + .and_then(|arr| arr.first()) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + + if !annotation.is_empty() || !tags.is_empty() { + map.insert( + hexcode, + crate::model::emoji::CldrAnnotation { annotation, tags }, + ); + } + } + + map +} + +/// Parse CLDR localization data (territory names). +pub fn parse_cldr_territories( + json: &serde_json::Value, +) -> HashMap { + let mut map = HashMap::new(); + + let territories = json + .pointer("/main") + .and_then(|m| { + // The structure is /main/{locale}/localeDisplayNames/territories + m.as_object() + .and_then(|obj| obj.values().next()) + .and_then(|v| v.pointer("/localeDisplayNames/territories")) + }); + + if let Some(serde_json::Value::Object(obj)) = territories { + for (code, name) in obj { + if let Some(name_str) = name.as_str() { + map.insert(code.clone(), name_str.to_string()); + } + } + } + + map +} diff --git a/crates/emojibase-generator/src/parser/emoji_data.rs b/crates/emojibase-generator/src/parser/emoji_data.rs new file mode 100644 index 00000000..61aeb9d9 --- /dev/null +++ b/crates/emojibase-generator/src/parser/emoji_data.rs @@ -0,0 +1,106 @@ +//! Parse `emoji-data.txt` into [`EmojiDataMap`]. +//! +//! Each line looks like: +//! ```text +//! 1F600 ; Emoji_Presentation # 6.1 [1] (😀) grinning face +//! 1F3FB..1F3FF ; Emoji_Modifier # 8.0 [5] (🏻..🏿) light/dark skin tone +//! ``` + +use crate::model::emoji::{EmojiData, EmojiDataMap, Property}; +use crate::model::hexcode::format_hexcode; +use crate::parser::{extract_emoji_version, emoji_version_to_unicode_version, parse_unicode_file}; +use crate::util::hexcode::spread_hexcode; +use emojibase_core::types::Presentation; + +/// Parse the full content of `emoji-data.txt` and return a map of hexcode → [`EmojiData`]. +pub fn parse_emoji_data(content: &str) -> EmojiDataMap { + let lines = parse_unicode_file(content); + let mut map = EmojiDataMap::new(); + + for line in &lines { + // Field 1 = property name + let property_str = line.fields.get(1).map(|s| s.as_str()).unwrap_or(""); + let property = match Property::from_str_name(property_str) { + Some(p) => p, + None => continue, + }; + + // Extract version from comment + let emoji_version = line + .comment + .as_deref() + .and_then(extract_emoji_version) + .unwrap_or(0.0); + + let unicode_version = if emoji_version > 0.0 { + Some(emoji_version_to_unicode_version(emoji_version)) + } else { + None + }; + + // Determine presentation from property + let presentation = if property == Property::EmojiPresentation { + Presentation::Emoji + } else { + Presentation::Text + }; + + // Description from comment (after the parenthesized char) + let description = line + .comment + .as_deref() + .and_then(|c| { + // Format: "6.1 [1] (😀) grinning face" + c.rfind(')').map(|idx| c[idx + 1..].trim().to_string()) + }) + .unwrap_or_default(); + + // Expand hexcode ranges + let hexcodes = spread_hexcode(&line.hexcode); + for hex in hexcodes { + let hex = format_hexcode(&hex); + let entry = map.entry(hex.clone()).or_insert_with(|| EmojiData { + description: description.clone(), + hexcode: hex, + property: Vec::new(), + presentation, + unicode_version, + version: emoji_version, + gender: None, + }); + if !entry.property.contains(&property) { + entry.property.push(property); + } + // Upgrade presentation if we see Emoji_Presentation + if property == Property::EmojiPresentation { + entry.presentation = Presentation::Emoji; + } + } + } + + map +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_single_line() { + let content = "1F600 ; Emoji_Presentation # 6.1 [1] (😀) grinning face\n"; + let map = parse_emoji_data(content); + assert!(map.contains_key("1F600")); + let emoji = &map["1F600"]; + assert_eq!(emoji.presentation, Presentation::Emoji); + assert!(emoji.property.contains(&Property::EmojiPresentation)); + } + + #[test] + fn test_parse_range() { + let content = "1F3FB..1F3FF ; Emoji_Modifier # 8.0 [5] (🏻..🏿) skin tone\n"; + let map = parse_emoji_data(content); + assert_eq!(map.len(), 5); + assert!(map.contains_key("1F3FB")); + assert!(map.contains_key("1F3FF")); + } +} diff --git a/crates/emojibase-generator/src/parser/emoticons.rs b/crates/emojibase-generator/src/parser/emoticons.rs new file mode 100644 index 00000000..2eb80b1f --- /dev/null +++ b/crates/emojibase-generator/src/parser/emoticons.rs @@ -0,0 +1,190 @@ +//! Emoticon data and parsing. + +use crate::model::emoji::EmoticonMap; +use emojibase_core::types::EmoticonValue; +use std::collections::HashMap; + +/// Generate all permutations for an emoticon. +/// +/// Mirrors the TypeScript `generateEmoticonPermutations` function: +/// - Swaps `)` ↔ `(`, `/` ↔ `\`, `>` ↔ `<` +/// - Adds/removes nose (`-`) +/// - Swaps `:` ↔ `=` +/// - Lowercases/uppercases letters +pub fn generate_emoticon_permutations( + emoticon: &str, + is_face: bool, + with_nose: bool, +) -> Vec { + let mut perms = vec![emoticon.to_string()]; + + // Only apply face-specific permutations if this is a face emoticon. + if is_face { + // Mouth swaps + let mut extra = Vec::new(); + for p in &perms { + if p.contains(')') { + extra.push(p.replace(')', "}")); + extra.push(p.replace(')', "]")); + } + if p.contains('(') { + extra.push(p.replace('(', "{")); + extra.push(p.replace('(', "[")); + } + } + perms.extend(extra); + + // Eyes swap: : ↔ = + let mut extra = Vec::new(); + for p in &perms { + if p.starts_with(':') { + extra.push(format!("={}", &p[1..])); + } else if p.starts_with('=') { + extra.push(format!(":{}", &p[1..])); + } + } + perms.extend(extra); + + // Nose: add/remove '-' + if with_nose { + let mut extra = Vec::new(); + for p in &perms { + // The nose character is at position 1 (after the eyes character). + if p.len() >= 2 { + let chars: Vec = p.chars().collect(); + if chars.get(1) == Some(&'-') { + // Remove nose + let without: String = + std::iter::once(chars[0]).chain(chars[2..].iter().copied()).collect(); + extra.push(without); + } else { + // Add nose + let with: String = std::iter::once(chars[0]) + .chain(std::iter::once('-')) + .chain(chars[1..].iter().copied()) + .collect(); + extra.push(with); + } + } + } + perms.extend(extra); + } + } else { + // Non-face emoticons: slash swap + let mut extra = Vec::new(); + for p in &perms { + if p.contains('/') { + extra.push(p.replace('/', "\\")); + } + if p.contains('\\') { + extra.push(p.replace('\\', "/")); + } + } + perms.extend(extra); + } + + // Case variants + let mut extra = Vec::new(); + for p in &perms { + let lower = p.to_lowercase(); + let upper = p.to_uppercase(); + if lower != *p { + extra.push(lower); + } + if upper != *p { + extra.push(upper); + } + } + perms.extend(extra); + + // Deduplicate + perms.sort(); + perms.dedup(); + perms +} + +/// Return the built-in emoticon map. +/// +/// This mirrors the data from `packages/generator/src/resources/emoticons.ts`. +pub fn builtin_emoticons() -> EmoticonMap { + let mut map = EmoticonMap::new(); + + let single = |map: &mut EmoticonMap, hex: &str, emoticon: &str| { + map.insert(hex.to_string(), EmoticonValue::Single(emoticon.to_string())); + }; + + let multi = |map: &mut EmoticonMap, hex: &str, emoticons: &[&str]| { + map.insert( + hex.to_string(), + EmoticonValue::Multi(emoticons.iter().map(|s| s.to_string()).collect()), + ); + }; + + // Happy + single(&mut map, "1F642", ":)"); + single(&mut map, "1F604", ":D"); + single(&mut map, "1F60A", ":>"); + single(&mut map, "1F609", ";)"); + + // Playful + single(&mut map, "1F602", ":')" ); + single(&mut map, "1F923", ":'D"); + multi(&mut map, "1F606", &["xD", "XD"]); + + // Sad / upset + single(&mut map, "1F641", ":("); + single(&mut map, "1F622", ":'("); + single(&mut map, "1F62D", ":'o"); + single(&mut map, "1F620", ">:("); + single(&mut map, "1F624", ">:E"); + + // Surprise / neutral + single(&mut map, "1F62E", ":o"); + single(&mut map, "1F610", ":|"); + single(&mut map, "1F611", "-_-"); + single(&mut map, "1F636", ":x"); + + // Tongue / wink + single(&mut map, "1F61B", ":p"); + single(&mut map, "1F61C", ";p"); + + // Cool / special + single(&mut map, "1F60E", "B)"); + single(&mut map, "1F913", "8)"); + single(&mut map, "1F615", ":/"); + single(&mut map, "1F607", "O)"); + single(&mut map, "1F608", "3)"); + + // Hearts / non-face + single(&mut map, "2764", "<3"); + single(&mut map, "1F494", " EmojiSourceMap { + let document = Html::parse_document(html); + let _row_selector = Selector::parse("tr").unwrap(); + let _td_selector = Selector::parse("td").unwrap(); + + // The actual HTML parsing is complex and depends on the exact structure + // of the emoji-list.html page. This is a skeleton implementation. + // + // In the TS version, cheerio is used to iterate over elements + // and extract the group/subgroup from header rows, and the hexcode + // from emoji data rows. + let map = EmojiSourceMap::new(); + + // TODO: Implement full HTML parsing once we have test data. + // The structure is: + // - Header rows with class "bighead" contain group names + // - Header rows with class "mediumhead" contain subgroup names + // - Data rows contain emoji codepoints and names + + let _ = document; // suppress unused warning + + map +} + +/// Emoji style information extracted from emoji-style.html. +pub struct EmojiStyles { + /// Hexcodes that default to emoji presentation. + pub emoji: std::collections::HashSet, + /// Hexcodes that default to text presentation. + pub text: std::collections::HashSet, +} + +/// Parse `emoji-style.html` to extract default presentation styles. +pub fn parse_emoji_styles_html(html: &str) -> EmojiStyles { + let document = Html::parse_document(html); + let _td_selector = Selector::parse("td").unwrap(); + + // TODO: Implement full HTML parsing. + // The structure is similar to emoji-list.html but categorizes emoji + // by their default presentation style (text vs emoji). + + let _ = document; + + EmojiStyles { + emoji: std::collections::HashSet::new(), + text: std::collections::HashSet::new(), + } +} diff --git a/crates/emojibase-generator/src/parser/mod.rs b/crates/emojibase-generator/src/parser/mod.rs new file mode 100644 index 00000000..87aa7a2b --- /dev/null +++ b/crates/emojibase-generator/src/parser/mod.rs @@ -0,0 +1,84 @@ +pub mod annotations; +pub mod emoji_data; +pub mod emoticons; +pub mod html; +pub mod sequences; +pub mod test_file; +pub mod unicode_data; +pub mod variations; + +/// A parsed line from a Unicode data file (semicolon-separated fields). +#[derive(Debug, Clone)] +pub struct ParsedLine { + /// The hexcode or hexcode range (first field). + pub hexcode: String, + /// All fields from the line. + pub fields: Vec, + /// The trailing comment, if any. + pub comment: Option, +} + +/// Parse a Unicode data file into structured lines. +/// +/// Each line has the format: +/// ```text +/// HEXCODE(..HEXCODE)? ; FIELD ; ... # COMMENT +/// ``` +/// +/// Blank lines and lines starting with `#` (without data) are skipped. +pub fn parse_unicode_file(content: &str) -> Vec { + let mut lines = Vec::new(); + + for raw_line in content.lines() { + let trimmed = raw_line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + + // Split data from comment. + let (data_part, comment) = if let Some(idx) = trimmed.find('#') { + let data = trimmed[..idx].trim(); + let comment = trimmed[idx + 1..].trim().to_string(); + (data, Some(comment)) + } else { + (trimmed, None) + }; + + let fields: Vec = data_part.split(';').map(|f| f.trim().to_string()).collect(); + + if fields.is_empty() { + continue; + } + + let hexcode = fields[0].clone(); + lines.push(ParsedLine { + hexcode, + fields, + comment, + }); + } + + lines +} + +/// Extract an emoji version number from a comment string. +/// +/// Looks for patterns like `E1.0`, `E13.1`, `E17.0`. +pub fn extract_emoji_version(comment: &str) -> Option { + let re = regex::Regex::new(r"E(\d+\.\d)").ok()?; + let caps = re.captures(comment)?; + caps.get(1)?.as_str().parse().ok() +} + +/// Map an emoji version to the corresponding Unicode version. +pub fn emoji_version_to_unicode_version(emoji_version: f32) -> f32 { + match emoji_version as u32 { + 1 => 6.1, + 2 => 6.1, + 3 => 9.0, + 4 => 9.0, + 5 => 10.0, + v if v >= 11 => emoji_version, + _ => emoji_version, + } +} diff --git a/crates/emojibase-generator/src/parser/sequences.rs b/crates/emojibase-generator/src/parser/sequences.rs new file mode 100644 index 00000000..ba9e987e --- /dev/null +++ b/crates/emojibase-generator/src/parser/sequences.rs @@ -0,0 +1,92 @@ +//! Parse `emoji-sequences.txt` and `emoji-zwj-sequences.txt`. +//! +//! Lines look like: +//! ```text +//! 1F468 200D 2764 FE0F 200D 1F468 ; RGI_Emoji_ZWJ_Sequence ; couple with heart: man, man # E2.0 +//! ``` + +use crate::model::emoji::{EmojiData, EmojiDataMap, Property}; +use crate::model::hexcode::format_hexcode; +use crate::parser::{extract_emoji_version, emoji_version_to_unicode_version, parse_unicode_file}; +use emojibase_core::types::Presentation; + +/// Parse an emoji sequence file and return a hexcode → [`EmojiData`] map. +/// +/// `default_property` should be the property to assign when none is specified +/// in the data (e.g. `Property::EmojiZwjSequence` for the ZWJ file). +pub fn parse_sequences(content: &str, default_property: Property) -> EmojiDataMap { + let lines = parse_unicode_file(content); + let mut map = EmojiDataMap::new(); + + for line in &lines { + // Convert space-separated codepoints to hyphen-separated hexcode. + let hexcode = line + .hexcode + .split_whitespace() + .map(|part| format_hexcode(part)) + .collect::>() + .join("-"); + + // Property from field 1, or use default. + let property = line + .fields + .get(1) + .and_then(|s| Property::from_str_name(s)) + .unwrap_or(default_property); + + let emoji_version = line + .comment + .as_deref() + .and_then(extract_emoji_version) + .unwrap_or(0.0); + + let unicode_version = if emoji_version > 0.0 { + Some(emoji_version_to_unicode_version(emoji_version)) + } else { + None + }; + + // Description from field 2 (if present) or comment. + let description = line + .fields + .get(2) + .filter(|s| !s.is_empty()) + .cloned() + .or_else(|| { + line.comment.as_deref().and_then(|c| { + c.rfind(')') + .map(|idx| c[idx + 1..].trim().to_string()) + .or_else(|| Some(c.to_string())) + }) + }) + .unwrap_or_default(); + + let entry = map.entry(hexcode.clone()).or_insert_with(|| EmojiData { + description, + hexcode: hexcode.clone(), + property: Vec::new(), + presentation: Presentation::Emoji, + unicode_version, + version: emoji_version, + gender: None, + }); + + if !entry.property.contains(&property) { + entry.property.push(property); + } + } + + map +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_zwj_sequence() { + let content = "1F468 200D 2764 FE0F 200D 1F468 ; RGI_Emoji_ZWJ_Sequence # E2.0 [1] (👨‍❤️‍👨) couple with heart: man, man\n"; + let map = parse_sequences(content, Property::EmojiZwjSequence); + assert!(map.contains_key("1F468-200D-2764-FE0F-200D-1F468")); + } +} diff --git a/crates/emojibase-generator/src/parser/test_file.rs b/crates/emojibase-generator/src/parser/test_file.rs new file mode 100644 index 00000000..12f72404 --- /dev/null +++ b/crates/emojibase-generator/src/parser/test_file.rs @@ -0,0 +1,117 @@ +//! Parse `emoji-test.txt` for group, subgroup, order, and qualification. +//! +//! ```text +//! # group: Smileys & Emotion +//! # subgroup: face-smiling +//! 1F600 ; fully-qualified # 😀 E1.0 grinning face +//! ``` + +use crate::model::emoji::EmojiMetadataMap; +use crate::model::hexcode::format_hexcode; +use emojibase_core::types::{Qualifier, Subgroup}; + +use super::extract_emoji_version; + +/// Parse `emoji-test.txt` and return a map of hexcode → [`EmojiMetadata`]. +/// +/// Also populates the group hierarchy. +pub fn parse_test_file( + content: &str, +) -> (EmojiMetadataMap, Vec, Vec<(u8, String)>) { + let mut map = EmojiMetadataMap::new(); + let mut group_names: Vec = Vec::new(); + let mut subgroup_entries: Vec<(u8, String)> = Vec::new(); + + let mut current_group: u8 = 0; + let mut current_subgroup: Subgroup = 0; + let mut order: u32 = 0; + + for raw_line in content.lines() { + let trimmed = raw_line.trim(); + + if trimmed.starts_with("# group:") { + let group_name = trimmed.strip_prefix("# group:").unwrap().trim().to_string(); + current_group = group_names.len() as u8; + group_names.push(group_name); + continue; + } + + if trimmed.starts_with("# subgroup:") { + let subgroup_name = trimmed.strip_prefix("# subgroup:").unwrap().trim().to_string(); + current_subgroup = subgroup_entries.len() as Subgroup; + subgroup_entries.push((current_group, subgroup_name)); + continue; + } + + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + + // Data line: "1F600 ; fully-qualified # ..." + let (data_part, comment) = if let Some(idx) = trimmed.find('#') { + (trimmed[..idx].trim(), Some(trimmed[idx + 1..].trim())) + } else { + (trimmed, None) + }; + + let fields: Vec<&str> = data_part.split(';').collect(); + if fields.is_empty() { + continue; + } + + // Build hexcode from space-separated codepoints. + let hexcode = fields[0] + .trim() + .split_whitespace() + .map(|part| format_hexcode(part)) + .collect::>() + .join("-"); + + // Qualifier + let qualifier = fields.get(1).and_then(|q| match q.trim() { + "fully-qualified" => Some(Qualifier::FullyQualified), + "minimally-qualified" => Some(Qualifier::MinimallyQualified), + "unqualified" => Some(Qualifier::Unqualified), + _ => None, + }); + + order += 1; + + map.insert( + hexcode, + crate::model::emoji::EmojiMetadata { + group: current_group, + subgroup: current_subgroup, + order, + qualifier, + }, + ); + } + + (map, group_names, subgroup_entries) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_test_file() { + let content = "\ +# group: Smileys & Emotion + +# subgroup: face-smiling +1F600 ; fully-qualified # 😀 E1.0 grinning face +1F603 ; fully-qualified # 😃 E0.6 grinning face with big eyes +"; + let (map, groups, subgroups) = parse_test_file(content); + assert_eq!(groups, vec!["Smileys & Emotion"]); + assert_eq!(subgroups, vec![(0, "face-smiling".to_string())]); + assert!(map.contains_key("1F600")); + assert!(map.contains_key("1F603")); + let entry = &map["1F600"]; + assert_eq!(entry.group, 0); + assert_eq!(entry.order, 1); + assert_eq!(entry.qualifier, Some(Qualifier::FullyQualified)); + } +} diff --git a/crates/emojibase-generator/src/parser/unicode_data.rs b/crates/emojibase-generator/src/parser/unicode_data.rs new file mode 100644 index 00000000..7211898b --- /dev/null +++ b/crates/emojibase-generator/src/parser/unicode_data.rs @@ -0,0 +1,52 @@ +//! Parse `UnicodeData.txt` for character names. +//! +//! Each line looks like: +//! ```text +//! 0023;NUMBER SIGN;Po;0;ON;;;;;N;;;;; +//! ``` +//! Field 0 = hexcode, field 1 = character name. + +use crate::model::emoji::UnicodeNamesMap; +use crate::model::hexcode::format_hexcode; + +/// Parse `UnicodeData.txt` and return a hexcode → name map. +pub fn parse_unicode_names(content: &str) -> UnicodeNamesMap { + let mut map = UnicodeNamesMap::new(); + + for raw_line in content.lines() { + let trimmed = raw_line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + + let fields: Vec<&str> = trimmed.splitn(3, ';').collect(); + if fields.len() < 2 { + continue; + } + + let hexcode = format_hexcode(fields[0].trim()); + let name = fields[1].trim().to_string(); + + if !name.is_empty() && !name.starts_with('<') { + map.insert(hexcode, name); + } + } + + map +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_names() { + let content = "\ +0023;NUMBER SIGN;Po;0;ON;;;;;N;;;;; +0024;DOLLAR SIGN;Sc;0;ET;;;;;N;;;;; +"; + let map = parse_unicode_names(content); + assert_eq!(map.get("0023"), Some(&"NUMBER SIGN".to_string())); + assert_eq!(map.get("0024"), Some(&"DOLLAR SIGN".to_string())); + } +} diff --git a/crates/emojibase-generator/src/parser/variations.rs b/crates/emojibase-generator/src/parser/variations.rs new file mode 100644 index 00000000..36498910 --- /dev/null +++ b/crates/emojibase-generator/src/parser/variations.rs @@ -0,0 +1,64 @@ +//! Parse `emoji-variation-sequences.txt`. +//! +//! Lines look like: +//! ```text +//! 0023 FE0E ; text style; # (1.1) NUMBER SIGN +//! 0023 FE0F ; emoji style; # (6.0) NUMBER SIGN +//! ``` + +use crate::model::emoji::{EmojiVariation, EmojiVariationMap}; +use crate::model::hexcode::format_hexcode; +use crate::parser::parse_unicode_file; + +/// Parse the content of `emoji-variation-sequences.txt`. +pub fn parse_variations(content: &str) -> EmojiVariationMap { + let lines = parse_unicode_file(content); + let mut map = EmojiVariationMap::new(); + + for line in &lines { + // Field 0 = "0023 FE0E" or "0023 FE0F" + let parts: Vec<&str> = line.hexcode.split_whitespace().collect(); + if parts.len() < 2 { + continue; + } + + let base_hex = format_hexcode(parts[0]); + let selector = parts[1].to_uppercase(); + + // Field 1 = "text style" or "emoji style" + let style = line.fields.get(1).map(|s| s.as_str()).unwrap_or(""); + + let entry = map.entry(base_hex.clone()).or_insert_with(|| EmojiVariation { + text: String::new(), + emoji: String::new(), + property: None, + }); + + let full_hex = format!("{}-{}", base_hex, selector); + if style.contains("text") || selector == "FE0E" { + entry.text = full_hex; + } else { + entry.emoji = full_hex; + } + } + + map +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_variations() { + let content = "\ +0023 FE0E ; text style; # (1.1) NUMBER SIGN +0023 FE0F ; emoji style; # (6.0) NUMBER SIGN +"; + let map = parse_variations(content); + assert!(map.contains_key("0023")); + let v = &map["0023"]; + assert_eq!(v.text, "0023-FE0E"); + assert_eq!(v.emoji, "0023-FE0F"); + } +} diff --git a/crates/emojibase-generator/src/util/hexcode.rs b/crates/emojibase-generator/src/util/hexcode.rs new file mode 100644 index 00000000..11926ab6 --- /dev/null +++ b/crates/emojibase-generator/src/util/hexcode.rs @@ -0,0 +1,33 @@ +//! Re-export core hexcode utilities plus generator-specific helpers. + +pub use emojibase_core::unicode::*; + +use crate::model::hexcode::format_hexcode; + +/// Expand a hexcode range `"1F600..1F64F"` into individual hexcodes, +/// or return a single hexcode unchanged. +pub fn spread_hexcode(hexcode: &str) -> Vec { + if let Some((start_str, end_str)) = hexcode.split_once("..") { + let start = u32::from_str_radix(start_str.trim(), 16).unwrap_or(0); + let end = u32::from_str_radix(end_str.trim(), 16).unwrap_or(0); + (start..=end).map(|cp| format!("{cp:X}")).collect() + } else { + vec![format_hexcode(hexcode)] + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_spread_single() { + assert_eq!(spread_hexcode("1F600"), vec!["1F600"]); + } + + #[test] + fn test_spread_range() { + let result = spread_hexcode("1F600..1F603"); + assert_eq!(result, vec!["1F600", "1F601", "1F602", "1F603"]); + } +} diff --git a/crates/emojibase-generator/src/util/log.rs b/crates/emojibase-generator/src/util/log.rs new file mode 100644 index 00000000..0cbf166e --- /dev/null +++ b/crates/emojibase-generator/src/util/log.rs @@ -0,0 +1,51 @@ +//! Logging and progress-reporting utilities. + +use indicatif::{ProgressBar, ProgressStyle}; +use std::time::Duration; + +/// Create a progress bar for processing items. +pub fn create_progress_bar(total: u64, message: &str) -> ProgressBar { + let pb = ProgressBar::new(total); + pb.set_style( + ProgressStyle::default_bar() + .template("{spinner:.cyan} [{bar:40.cyan/blue}] {pos}/{len} {msg}") + .unwrap() + .progress_chars("█▓░"), + ); + pb.set_message(message.to_string()); + pb.enable_steady_tick(Duration::from_millis(100)); + pb +} + +/// Create a spinner for an indeterminate task. +pub fn create_spinner(message: &str) -> ProgressBar { + let pb = ProgressBar::new_spinner(); + pb.set_style( + ProgressStyle::default_spinner() + .template("{spinner:.cyan} {msg}") + .unwrap(), + ); + pb.set_message(message.to_string()); + pb.enable_steady_tick(Duration::from_millis(80)); + pb +} + +/// Log an informational message. +pub fn info(message: &str) { + eprintln!("ℹ {message}"); +} + +/// Log a success message. +pub fn success(message: &str) { + eprintln!("✅ {message}"); +} + +/// Log a warning message. +pub fn warn(message: &str) { + eprintln!("⚠️ {message}"); +} + +/// Log an error message. +pub fn error(message: &str) { + eprintln!("❌ {message}"); +} diff --git a/crates/emojibase-generator/src/util/mod.rs b/crates/emojibase-generator/src/util/mod.rs new file mode 100644 index 00000000..0080f054 --- /dev/null +++ b/crates/emojibase-generator/src/util/mod.rs @@ -0,0 +1,3 @@ +pub mod hexcode; +pub mod log; +pub mod slug; diff --git a/crates/emojibase-generator/src/util/slug.rs b/crates/emojibase-generator/src/util/slug.rs new file mode 100644 index 00000000..f855be01 --- /dev/null +++ b/crates/emojibase-generator/src/util/slug.rs @@ -0,0 +1,42 @@ +//! String slugification for shortcode generation. + +/// Slugify a string for use as an emoji shortcode. +/// +/// - Converts to lowercase ASCII (via [`any_ascii`]). +/// - Replaces whitespace and special chars with underscores. +/// - Strips non-alphanumeric/underscore characters. +/// - Collapses consecutive underscores. +pub fn slugify(label: &str) -> String { + let ascii = any_ascii::any_ascii(label); + let mut slug = String::with_capacity(ascii.len()); + for ch in ascii.chars() { + if ch.is_ascii_alphanumeric() { + slug.push(ch.to_ascii_lowercase()); + } else if ch == ' ' || ch == '-' || ch == ':' || ch == '&' || ch == '\'' { + slug.push('_'); + } + } + // Collapse consecutive underscores and trim. + slug.split('_') + .filter(|s| !s.is_empty()) + .collect::>() + .join("_") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_slugify_basic() { + assert_eq!(slugify("Grinning Face"), "grinning_face"); + assert_eq!(slugify("woman: red hair"), "woman_red_hair"); + } + + #[test] + fn test_slugify_non_latin() { + // any_ascii transliterates non-Latin characters. + let result = slugify("日本語"); + assert!(!result.is_empty()); + } +}