diff --git a/.circleci/cmake_test.sh b/.circleci/cmake_test.sh deleted file mode 100755 index cdf8e4e5d..000000000 --- a/.circleci/cmake_test.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2016-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -# - -RESULTDIR=result -DATADIR=data - -./.circleci/pull_data.sh -mkdir buildc && cd buildc && cmake .. && make && cd .. -cp buildc/fasttext . -./fasttext supervised -input "${DATADIR}/dbpedia.train" -output "${RESULTDIR}/dbpedia" -dim 10 -lr 0.1 -wordNgrams 2 -minCount 1 -bucket 10000000 -epoch 5 -thread 4 -verbose 0 -./fasttext test "${RESULTDIR}/dbpedia.bin" "${DATADIR}/dbpedia.test" -./fasttext predict "${RESULTDIR}/dbpedia.bin" "${DATADIR}/dbpedia.test" > "${RESULTDIR}/dbpedia.test.predict" diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100755 index 40b4eeb87..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,199 +0,0 @@ -# Python CircleCI 2.0 configuration file -# -# Check https://circleci.com/docs/2.0/language-python/ for more details -# -# Copyright (c) 2016-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -# - -# Maybe one day this will work -# "mac": -# macos: -# xcode: "9.0" -# working_directory: ~/repo -# steps: -# - checkout -# - run: -# command: | -# . .circleci/cmake_test.sh - -version: 2 -jobs: - "py368": - docker: - - image: circleci/python:3.6.8 - working_directory: ~/repo - steps: - - checkout - - run: - command: | - . .circleci/setup_circleimg.sh - . .circleci/python_test.sh - - - "py357": - docker: - - image: circleci/python:3.5.7 - working_directory: ~/repo - steps: - - checkout - - run: - command: | - . .circleci/setup_circleimg.sh - . .circleci/python_test.sh - - "py3410": - docker: - - image: circleci/python:3.4.10 - working_directory: ~/repo - steps: - - checkout - - run: - command: | - . .circleci/setup_circleimg.sh - . .circleci/python_test.sh - - "py2715": - docker: - - image: circleci/python:2.7.15 - working_directory: ~/repo - steps: - - checkout - - run: - command: | - . .circleci/setup_circleimg.sh - . .circleci/python_test.sh - - "gcc5": - docker: - - image: gcc:5 - working_directory: ~/repo - steps: - - checkout - - run: - command: | - . .circleci/setup_debian.sh - . .circleci/gcc_test.sh - - "gcc6": - docker: - - image: gcc:6 - working_directory: ~/repo - steps: - - checkout - - run: - command: | - . .circleci/setup_debian.sh - . .circleci/gcc_test.sh - - "gcc7": - docker: - - image: gcc:7 - working_directory: ~/repo - steps: - - checkout - - run: - command: | - . .circleci/setup_debian.sh - . .circleci/gcc_test.sh - - "gcclatest": - docker: - - image: gcc:latest - working_directory: ~/repo - steps: - - checkout - - run: - command: | - . .circleci/setup_debian.sh - . .circleci/gcc_test.sh - - "debian-stretch-gcc": - docker: - - image: debian:stretch - working_directory: ~/repo - steps: - - checkout - - run: - command: | - . .circleci/setup_debian.sh - . .circleci/gcc_test.sh - - "debian-stretch-cmake": - docker: - - image: debian:stretch - working_directory: ~/repo - steps: - - checkout - - run: - command: | - . .circleci/setup_debian.sh - . .circleci/cmake_test.sh - - "debian-stretch-python": - docker: - - image: debian:stretch - working_directory: ~/repo - steps: - - checkout - - run: - command: | - . .circleci/setup_debian.sh - pip install . - python runtests.py -u - - "debian-jessie-gcc": - docker: - - image: debian:jessie - working_directory: ~/repo - steps: - - checkout - - run: - command: | - . .circleci/setup_debian.sh - . .circleci/gcc_test.sh - - "debian-jessie-cmake": - docker: - - image: debian:jessie - working_directory: ~/repo - steps: - - checkout - - run: - command: | - . .circleci/setup_debian.sh - . .circleci/cmake_test.sh - - "website-build": - docker: - - image: node:latest - working_directory: ~/repo - steps: - - checkout - - run: - command: | - git config --global user.email "docusaurus-bot@users.noreply.github.com" - git config --global user.name "Website Deployment Script" - echo "machine github.com login docusaurus-bot password $GITHUB_TOKEN_DOCUSAURUS_BOT" > ~/.netrc - cd website && npm install && GIT_USER=docusaurus-bot npm run publish-gh-pages - -workflows: - version: 2 - build: - jobs: - - "py368" - - "py357" - - "py3410" - - "py2715" - - "gcc5" - - "gcc6" - - "gcc7" - - "gcclatest" - - "debian-stretch-gcc" - - "debian-stretch-cmake" - - "debian-stretch-python" - - "debian-jessie-gcc" - - "debian-jessie-cmake" diff --git a/.circleci/gcc_test.sh b/.circleci/gcc_test.sh deleted file mode 100755 index a58b68507..000000000 --- a/.circleci/gcc_test.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2016-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -# - -RESULTDIR=result -DATADIR=data - -./.circleci/pull_data.sh -make opt -./fasttext supervised -input "${DATADIR}/dbpedia.train" -output "${RESULTDIR}/dbpedia" -dim 10 -lr 0.1 -wordNgrams 2 -minCount 1 -bucket 10000000 -epoch 5 -thread 4 -verbose 0 -./fasttext test "${RESULTDIR}/dbpedia.bin" "${DATADIR}/dbpedia.test" -./fasttext predict "${RESULTDIR}/dbpedia.bin" "${DATADIR}/dbpedia.test" > "${RESULTDIR}/dbpedia.test.predict" - -make clean -make debug -./fasttext supervised -input "${DATADIR}/dbpedia.train" -output "${RESULTDIR}/dbpedia" -dim 10 -lr 0.1 -wordNgrams 2 -minCount 1 -bucket 10000000 -epoch 5 -thread 4 -verbose 0 -./fasttext test "${RESULTDIR}/dbpedia.bin" "${DATADIR}/dbpedia.test" -./fasttext predict "${RESULTDIR}/dbpedia.bin" "${DATADIR}/dbpedia.test" > "${RESULTDIR}/dbpedia.test.predict" - - diff --git a/.circleci/pip_test.sh b/.circleci/pip_test.sh deleted file mode 100644 index 661109662..000000000 --- a/.circleci/pip_test.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2016-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -# - -sudo pip install --index-url https://test.pypi.org/simple/ fasttext -python runtests.py -u diff --git a/.circleci/pull_data.sh b/.circleci/pull_data.sh deleted file mode 100755 index 6cc045a25..000000000 --- a/.circleci/pull_data.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2016-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -# - -myshuf() { - perl -MList::Util=shuffle -e 'print shuffle(<>);' "$@"; -} - -normalize_text() { - tr '[:upper:]' '[:lower:]' | sed -e 's/^/__label__/g' | \ - sed -e "s/'/ ' /g" -e 's/"//g' -e 's/\./ \. /g' -e 's/
/ /g' \ - -e 's/,/ , /g' -e 's/(/ ( /g' -e 's/)/ ) /g' -e 's/\!/ \! /g' \ - -e 's/\?/ \? /g' -e 's/\;/ /g' -e 's/\:/ /g' | tr -s " " | myshuf -} - -RESULTDIR=result -DATADIR=data - -mkdir -p "${RESULTDIR}" -mkdir -p "${DATADIR}" - -if [ ! -f "${DATADIR}/dbpedia.train" ] -then - wget -c "https://github.com/le-scientifique/torchDatasets/raw/master/dbpedia_csv.tar.gz" -O "${DATADIR}/dbpedia_csv.tar.gz" - tar -xzvf "${DATADIR}/dbpedia_csv.tar.gz" -C "${DATADIR}" - cat "${DATADIR}/dbpedia_csv/train.csv" | normalize_text > "${DATADIR}/dbpedia.train" - cat "${DATADIR}/dbpedia_csv/test.csv" | normalize_text > "${DATADIR}/dbpedia.test" -fi diff --git a/.circleci/python_test.sh b/.circleci/python_test.sh deleted file mode 100644 index 3296082b2..000000000 --- a/.circleci/python_test.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2016-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -# - -sudo pip install . -python runtests.py -u diff --git a/.circleci/run_locally.sh b/.circleci/run_locally.sh deleted file mode 100644 index 0ff882de1..000000000 --- a/.circleci/run_locally.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2016-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -# - -# This script illustrates how to run the build tests locally -# This requires docker - -tail -n 15 .circleci/config.yml | sed s/.\\+\"\\\(\.\\+\\\)\"/\\1/g | xargs -P 4 -o -I {} bash -c "circleci build --job {} && (>&2 echo "{}")" > /dev/null diff --git a/.circleci/setup_circleimg.sh b/.circleci/setup_circleimg.sh deleted file mode 100644 index b66589e8b..000000000 --- a/.circleci/setup_circleimg.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2016-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -# - -sudo apt-get update || sudo apt-get --allow-releaseinfo-change-suite update -sudo apt-get install -y cmake python-pip python-dev build-essential libboost-all-dev diff --git a/.circleci/setup_debian.sh b/.circleci/setup_debian.sh deleted file mode 100755 index 99f2fcadb..000000000 --- a/.circleci/setup_debian.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2016-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -# - -apt-get update -apt-get install -y vim g++ make cmake wget git python-pip python-dev build-essential libboost-all-dev diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0f614dab8..8f606aebb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,7 +31,7 @@ jobs: with: # this might remove tools that are actually needed, # if set to "true" but frees about 6 GB - tool-cache: false + tool-cache: true # all of these default to true, but feel free to set to # "false" if necessary for your workflow diff --git a/.gitignore b/.gitignore index 039bab8de..2b8a32250 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +build/ +*.log *.sif *.kdev4 .*.swp diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..7bee3fc62 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,47 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What this repo is + +This is the **aymara fork** of Facebook's [fastText](https://fasttext.cc/) (a C++ library + CLI for word representation learning and text classification, with Python bindings). The fork's distinguishing feature is the ability to **read training/test input directly from compressed archives** (`.xz`/LZMA and `.gz`/gzip) using Boost.Iostreams. Most divergence from upstream lives on the `read_from_xz` branch. + +## Building + +**CMake** is the only build system: `mkdir build && cd build && cmake .. && make`. It builds shared, static, and PIC libraries plus the CLI. Requires CMake ≥ 3.22 and C++11. The `-march=native` flag is on by default. + +(The upstream Makefile, CircleCI config, and the Emscripten/WebAssembly build were removed when this fork dropped upstream compatibility — see commit history.) + +### The READ_FROM_XZ option (fork-specific) +CMake exposes `option(READ_FROM_XZ ... ON)`. When ON (default), the build does `find_package(Boost 1.46 REQUIRED COMPONENTS iostreams)` and links `Boost::iostreams`. Turn it OFF (`cmake -DREAD_FROM_XZ=OFF ..`) to build without the Boost dependency. **Boost iostreams (with liblzma/zlib) is a hard build dependency unless this option is disabled.** On Debian/Ubuntu: `apt install libboost-all-dev`. + +### Python bindings +`pip install .` from the repo root (uses `setup.py`, requires pybind11, NumPy, SciPy). Source under `python/fasttext_module/`. + +### Docker / release artifacts +`Dockerfile` builds against `quay.io/pypa/manylinux_2_28_x86_64` for portable Linux/Python wheels. `.github/workflows/build.yml` (GitHub Actions) builds and pushes `aymara/fasttext_manylinux_2_28` to ghcr.io on push to `read_from_xz`. This is the only CI. + +## Testing + +- Python tests: `python runtests.py -u` (unit) or `python runtests.py -i --data-dir ` (integration). Integration tests need data fetched via `tests/fetch_test_data.sh` first. +- There is no C++ unit test suite; the shell scripts in the repo root (`word-vector-example.sh`, `classification-example.sh`, `classification-results.sh`, `quantization-example.sh`) double as end-to-end smoke tests that download data and exercise the CLI. + +## Architecture + +All C++ lives in `src/`. Core flow, top-down: + +- **`main.cc`** — CLI dispatch: maps subcommands (`skipgram`, `cbow`, `supervised`, `test`, `predict`, `predict-prob`, `quantize`, `print-word-vectors`, `print-sentence-vectors`, etc.) onto `FastText` methods. +- **`fasttext.{h,cc}`** — the `FastText` orchestrator. Owns training loop, model save/load (`.bin`/`.vec`/`.ftz`), inference, and quantization. This is where input is opened: training (`train`) and dictionary building read through `impl::ArchiveReader` (see below). +- **`dictionary.{h,cc}`** — vocabulary, subword (char n-gram) hashing, and line reading. `getLine(...)` and `reset(...)` take an `impl::ArchiveReader&` rather than a raw stream — this is the main fork touch point for transparent decompression. +- **`model.{h,cc}`** + **`loss.{h,cc}`** — the training model and pluggable loss functions (negative sampling `ns`, hierarchical softmax `hs`, full `softmax`, one-vs-all `ova`). +- **Matrix layer:** `matrix.h` (abstract) → `densematrix` (training-time weights) and `quantmatrix` (quantized inference). `productquantizer.{h,cc}` implements the PQ compression behind `quantize`/`.ftz`. +- **`args.{h,cc}`** — all hyperparameters and CLI argument parsing/defaults. +- **`vector`, `utils`, `meter`, `real.h`** — math vectors, helpers, evaluation metrics (P@k/R@k), and the `real` (float) typedef. + +### archivereader.h (the fork's core addition) +`src/archivereader.h` defines `fasttext::impl::ArchiveReader`, a header-only RAII wrapper over a `boost::iostreams::filtering_stream`. It auto-detects/pushes gzip and lzma decompression filters so that `FastText` and `Dictionary` consume compressed input files transparently. It is only compiled meaningfully when the Boost iostreams path is enabled. When modifying input-reading code, route through `ArchiveReader` rather than `std::ifstream` to preserve archive support. + +## Conventions + +- C++11 only (original upstream constraint); when adding `.cc`/`.h` files, update the explicit source lists in `CMakeLists.txt` (`SOURCE_FILES`/`HEADER_FILES`). +- Upstream fastText is archived, so changes no longer need to be merge-compatible with upstream `master`. diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d4b8e966..66514a764 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,7 @@ set (fasttext_VERSION_MINOR 1) option(READ_FROM_XZ "Building with libboost::iostreams (allows archive reading)" ON) if (READ_FROM_XZ) - find_package(Boost 1.46 REQUIRED COMPONENTS iostreams) + find_package(Boost 1.46 REQUIRED CONFIG COMPONENTS iostreams) endif() include_directories(fasttext) diff --git a/Makefile b/Makefile deleted file mode 100644 index 79bcf800b..000000000 --- a/Makefile +++ /dev/null @@ -1,125 +0,0 @@ -# -# Copyright (c) 2016-present, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -# - -CXX = c++ -CXXFLAGS = -pthread -std=c++11 -march=native -OBJS = args.o autotune.o matrix.o dictionary.o loss.o productquantizer.o densematrix.o quantmatrix.o vector.o model.o utils.o meter.o fasttext.o -INCLUDES = -I. - -opt: CXXFLAGS += -O3 -funroll-loops -DNDEBUG -opt: fasttext - -coverage: CXXFLAGS += -O0 -fno-inline -fprofile-arcs --coverage -coverage: fasttext - -debug: CXXFLAGS += -g -O0 -fno-inline -debug: fasttext - -wasm: webassembly/fasttext_wasm.js - -wasmdebug: export EMCC_DEBUG=1 -wasmdebug: webassembly/fasttext_wasm.js - - -args.o: src/args.cc src/args.h - $(CXX) $(CXXFLAGS) -c src/args.cc - -autotune.o: src/autotune.cc src/autotune.h - $(CXX) $(CXXFLAGS) -c src/autotune.cc - -matrix.o: src/matrix.cc src/matrix.h - $(CXX) $(CXXFLAGS) -c src/matrix.cc - -dictionary.o: src/dictionary.cc src/dictionary.h src/args.h - $(CXX) $(CXXFLAGS) -c src/dictionary.cc - -loss.o: src/loss.cc src/loss.h src/matrix.h src/real.h - $(CXX) $(CXXFLAGS) -c src/loss.cc - -productquantizer.o: src/productquantizer.cc src/productquantizer.h src/utils.h - $(CXX) $(CXXFLAGS) -c src/productquantizer.cc - -densematrix.o: src/densematrix.cc src/densematrix.h src/utils.h src/matrix.h - $(CXX) $(CXXFLAGS) -c src/densematrix.cc - -quantmatrix.o: src/quantmatrix.cc src/quantmatrix.h src/utils.h src/matrix.h - $(CXX) $(CXXFLAGS) -c src/quantmatrix.cc - -vector.o: src/vector.cc src/vector.h src/utils.h - $(CXX) $(CXXFLAGS) -c src/vector.cc - -model.o: src/model.cc src/model.h src/args.h - $(CXX) $(CXXFLAGS) -c src/model.cc - -utils.o: src/utils.cc src/utils.h - $(CXX) $(CXXFLAGS) -c src/utils.cc - -meter.o: src/meter.cc src/meter.h - $(CXX) $(CXXFLAGS) -c src/meter.cc - -fasttext.o: src/fasttext.cc src/*.h - $(CXX) $(CXXFLAGS) -c src/fasttext.cc - -fasttext: $(OBJS) src/fasttext.cc src/main.cc - $(CXX) $(CXXFLAGS) $(OBJS) -lboost_iostreams src/main.cc -o fasttext - -clean: - rm -rf *.o *.gcno *.gcda fasttext *.bc webassembly/fasttext_wasm.js webassembly/fasttext_wasm.wasm - - -EMCXX = em++ -EMCXXFLAGS = --bind --std=c++11 -s WASM=1 -s ALLOW_MEMORY_GROWTH=1 -s "EXTRA_EXPORTED_RUNTIME_METHODS=['addOnPostRun', 'FS']" -s "DISABLE_EXCEPTION_CATCHING=0" -s "EXCEPTION_DEBUG=1" -s "FORCE_FILESYSTEM=1" -s "MODULARIZE=1" -s "EXPORT_ES6=1" -s 'EXPORT_NAME="FastTextModule"' -Isrc/ -EMOBJS = args.bc autotune.bc matrix.bc dictionary.bc loss.bc productquantizer.bc densematrix.bc quantmatrix.bc vector.bc model.bc utils.bc meter.bc fasttext.bc main.bc - - -main.bc: webassembly/fasttext_wasm.cc - $(EMCXX) $(EMCXXFLAGS) webassembly/fasttext_wasm.cc -o main.bc - -args.bc: src/args.cc src/args.h - $(EMCXX) $(EMCXXFLAGS) src/args.cc -o args.bc - -autotune.bc: src/autotune.cc src/autotune.h - $(EMCXX) $(EMCXXFLAGS) src/autotune.cc -o autotune.bc - -matrix.bc: src/matrix.cc src/matrix.h - $(EMCXX) $(EMCXXFLAGS) src/matrix.cc -o matrix.bc - -dictionary.bc: src/dictionary.cc src/dictionary.h src/args.h - $(EMCXX) $(EMCXXFLAGS) src/dictionary.cc -o dictionary.bc - -loss.bc: src/loss.cc src/loss.h src/matrix.h src/real.h - $(EMCXX) $(EMCXXFLAGS) src/loss.cc -o loss.bc - -productquantizer.bc: src/productquantizer.cc src/productquantizer.h src/utils.h - $(EMCXX) $(EMCXXFLAGS) src/productquantizer.cc -o productquantizer.bc - -densematrix.bc: src/densematrix.cc src/densematrix.h src/utils.h src/matrix.h - $(EMCXX) $(EMCXXFLAGS) src/densematrix.cc -o densematrix.bc - -quantmatrix.bc: src/quantmatrix.cc src/quantmatrix.h src/utils.h src/matrix.h - $(EMCXX) $(EMCXXFLAGS) src/quantmatrix.cc -o quantmatrix.bc - -vector.bc: src/vector.cc src/vector.h src/utils.h - $(EMCXX) $(EMCXXFLAGS) src/vector.cc -o vector.bc - -model.bc: src/model.cc src/model.h src/args.h - $(EMCXX) $(EMCXXFLAGS) src/model.cc -o model.bc - -utils.bc: src/utils.cc src/utils.h - $(EMCXX) $(EMCXXFLAGS) src/utils.cc -o utils.bc - -meter.bc: src/meter.cc src/meter.h - $(EMCXX) $(EMCXXFLAGS) src/meter.cc -o meter.bc - -fasttext.bc: src/fasttext.cc src/*.h - $(EMCXX) $(EMCXXFLAGS) src/fasttext.cc -o fasttext.bc - -webassembly/fasttext_wasm.js: $(EMOBJS) webassembly/fasttext_wasm.cc Makefile - $(EMCXX) $(EMCXXFLAGS) $(EMOBJS) -o webassembly/fasttext_wasm.js - - diff --git a/README.md b/README.md index ce734ed82..74cef9103 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # fastText [fastText](https://fasttext.cc/) is a library for efficient learning of word representations and sentence classification. -[![CircleCI](https://circleci.com/gh/aymara/fastText/tree/master.svg?style=svg)](https://circleci.com/gh/aymara/fastText/tree/master) +[![Build](https://github.com/aymara/fastText/actions/workflows/build.yml/badge.svg)](https://github.com/aymara/fastText/actions/workflows/build.yml) ## Table of contents @@ -13,7 +13,6 @@ * [Requirements](#requirements) * [Building fastText](#building-fasttext) * [Getting the source code](#getting-the-source-code) - * [Building fastText using make (preferred)](#building-fasttext-using-make-preferred) * [Building fastText using cmake](#building-fasttext-using-cmake) * [Building fastText for Python](#building-fasttext-for-python) * [Example use cases](#example-use-cases) @@ -48,7 +47,7 @@ We also provide a [cheatsheet](https://fasttext.cc/docs/en/cheatsheet.html#conte ## Requirements -We are continuously building and testing our library, CLI and Python bindings under various docker images using [circleci](https://circleci.com/). +We are continuously building and testing our library, CLI and Python bindings under a manylinux docker image using [GitHub Actions](https://github.com/aymara/fastText/actions). Generally, **fastText** builds on modern Mac OS and Linux distributions. Since it uses some C++11 features, it requires a compiler with good C++11 support. @@ -56,8 +55,7 @@ These include : * (g++-4.7.2 or newer) or (clang-3.3 or newer) -Compilation is carried out using a Makefile, so you will need to have a working **make**. -If you want to use **cmake** you need at least version 2.8.9. +Compilation is carried out using **cmake** (version 3.22 or newer). One of the oldest distributions we successfully built and tested the CLI under is [Debian jessie](https://www.debian.org/releases/jessie/). @@ -86,24 +84,10 @@ You can find our [latest stable release](https://github.com/facebookresearch/fas There is also the master branch that contains all of our most recent work, but comes along with all the usual caveats of an unstable branch. You might want to use this if you are a developer or power-user. -### Building fastText using make (preferred) - -``` -$ wget https://github.com/facebookresearch/fastText/archive/v0.9.2.zip -$ unzip v0.9.2.zip -$ cd fastText-0.9.2 -$ make -``` - -This will produce object files for all the classes as well as the main binary `fasttext`. -If you do not plan on using the default system-wide compiler, update the two macros defined at the beginning of the Makefile (CC and INCLUDES). - ### Building fastText using cmake -For now this is not part of a release, so you will need to clone the master branch. - ``` -$ git clone https://github.com/facebookresearch/fastText.git +$ git clone https://github.com/aymara/fastText.git $ cd fastText $ mkdir build && cd build && cmake .. $ make && make install diff --git a/classification-example.sh b/classification-example.sh index 7085f7fad..97967cc0a 100755 --- a/classification-example.sh +++ b/classification-example.sh @@ -32,10 +32,10 @@ then cat "${DATADIR}/dbpedia_csv/test.csv" | normalize_text > "${DATADIR}/dbpedia.test" fi -make +mkdir -p build && cd build && cmake .. && make && cd .. -./fasttext supervised -input "${DATADIR}/dbpedia.train" -output "${RESULTDIR}/dbpedia" -dim 10 -lr 0.1 -wordNgrams 2 -minCount 1 -bucket 10000000 -epoch 5 -thread 4 +./build/fasttext supervised -input "${DATADIR}/dbpedia.train" -output "${RESULTDIR}/dbpedia" -dim 10 -lr 0.1 -wordNgrams 2 -minCount 1 -bucket 10000000 -epoch 5 -thread 4 -./fasttext test "${RESULTDIR}/dbpedia.bin" "${DATADIR}/dbpedia.test" +./build/fasttext test "${RESULTDIR}/dbpedia.bin" "${DATADIR}/dbpedia.test" -./fasttext predict "${RESULTDIR}/dbpedia.bin" "${DATADIR}/dbpedia.test" > "${RESULTDIR}/dbpedia.test.predict" +./build/fasttext predict "${RESULTDIR}/dbpedia.bin" "${DATADIR}/dbpedia.test" > "${RESULTDIR}/dbpedia.test.predict" diff --git a/classification-results.sh b/classification-results.sh index 0c945cf68..be8f372a1 100755 --- a/classification-results.sh +++ b/classification-results.sh @@ -86,9 +86,9 @@ make for i in {0..7} do echo "Working on dataset ${DATASET[i]}" - ./fasttext supervised -input "${DATADIR}/${DATASET[i]}.train" \ + ./build/fasttext supervised -input "${DATADIR}/${DATASET[i]}.train" \ -output "${RESULTDIR}/${DATASET[i]}" -dim 10 -lr "${LR[i]}" -wordNgrams 2 \ -minCount 1 -bucket 10000000 -epoch 5 -thread 4 > /dev/null - ./fasttext test "${RESULTDIR}/${DATASET[i]}.bin" \ + ./build/fasttext test "${RESULTDIR}/${DATASET[i]}.bin" \ "${DATADIR}/${DATASET[i]}.test" done diff --git a/docs/support.md b/docs/support.md index 46ca97147..d254bf00f 100644 --- a/docs/support.md +++ b/docs/support.md @@ -15,7 +15,7 @@ These include : * (gcc-4.6.3 or newer) or (clang-3.3 or newer) -Compilation is carried out using a Makefile, so you will need to have a working **make**. +Compilation is carried out using **cmake** (version 3.22 or newer). For the word-similarity evaluation script you will need: * python 2.6 or newer @@ -26,13 +26,13 @@ For the word-similarity evaluation script you will need: In order to build `fastText`, use the following: ```bash -$ git clone https://github.com/facebookresearch/fastText.git +$ git clone https://github.com/aymara/fastText.git $ cd fastText -$ make +$ mkdir build && cd build && cmake .. +$ make && make install ``` -This will produce object files for all the classes as well as the main binary `fasttext`. -If you do not plan on using the default system-wide compiler, update the two macros defined at the beginning of the Makefile (CC and INCLUDES). +This will create the `fasttext` binary and all relevant libraries (shared, static, PIC). ## Building `fasttext` python module diff --git a/docs/webassembly-module.md b/docs/webassembly-module.md deleted file mode 100644 index c1e984de1..000000000 --- a/docs/webassembly-module.md +++ /dev/null @@ -1,338 +0,0 @@ ---- -id: webassembly-module -title: WebAssembly module ---- - -In this document we present how to use fastText in javascript with WebAssembly. - -## Table of contents - -* [Requirements](#requirements) -* [Building WebAssembly binaries](#building-webassembly-binaries) -* [Build a webpage that uses fastText](#build-a-webpage-that-uses-fasttext) -* [Load a model](#load-a-model) -* [Train a model](#train-a-model) - * [Disclaimer](#disclaimer) - * [Text classification](#text-classification) - * [Word representations](#word-representations) -* [Quantized models](#quantized-models) -* [API](#api) - * [`model` object](#model-object) - * [`loadModel`](#loadmodel) - * [`trainSupervised`](#trainsupervised) - * [`trainUnsupervised`](#trainunsupervised) - -# Requirements - -For building [fastText](https://fasttext.cc/) with WebAssembly bindings, we will need: - - a compiler with good C++11 support, since it uses C\++11 features, - - [emscripten](https://emscripten.org/), - - a [browser that supports WebAssembly](https://caniuse.com/#feat=wasm). - - -# Building WebAssembly binaries - -First, download and install emscripten sdk as [described here](https://emscripten.org/docs/getting_started/downloads.html#installation-instructions). - - -We need to make sure we activated the PATH for emscripten: -```bash -$ source /path/to/emsdk/emsdk_env.sh -``` - -Clone [fastText repository](https://github.com/facebookresearch/fastText/): - -```bash -$ git clone git@github.com:facebookresearch/fastText.git -``` - -Build WebAssembly binaries: -```bash -$ cd fastText -$ make wasm -``` - -This will create `fasttext_wasm.wasm` and `fasttext_wasm.js` in the `webassembly` folder. - -- `fasttext_wasm.wasm` is the binary file that will be loaded in the webassembly's virtual machine. -- `fasttext_wasm.js` is a javascript file built by emscripten, that helps to load `fasttext_wasm.wasm` file in the virtual machine and provides some helper functions. -- `fasttext.js` is the wrapper that provides a nice API for fastText. - -As the user of the library, we will interact with classes and methods defined in `fasttext.js`. We won't deal with `fasttext_wasm.*` files, but they are necessary to run fastText in the javascript's VM. - -# Build a webpage that uses fastText - -In this section we are going to build a minimal HTML page that loads fastText WebAssembly module. - -At the root of the repository, create a folder `webassembly-test`, and copy the files mentioned in the previous section: - -```bash -$ mkdir webassembly-test -$ cp webassembly/fasttext_wasm.wasm webassembly-test/ -$ cp webassembly/fasttext_wasm.js webassembly-test/ -$ cp webassembly/fasttext.js webassembly-test/ -``` - -Inside that folder, create `test.html` file containing: -```html - - - - - - - - - - -``` - -It is important to add the attribute `type="module"` to the script tag, because we use ES6 style imports. `addOnPostRun` is a function that helps to provide a handler that is called when fastText is successfully loaded in the virtual machine. Once we are called inside that function, we can create an instance of `FastText`, that we will use to access the api. - - -Let's test it. - -Opening `test.html` directly in the browser won't work since we are dynamically loading webassembly resources. The `test.html` file must be served from a webserver. The easiest way to achieve this is to use python's simple http server module: - -```bash -$ cd webassembly-test -$ python -m SimpleHTTPServer -``` - -Then browse `http://localhost:8000/test.html` in your browser. If everything worked as expected, you should see `FastText {f: FastText}` in the javascript console. - - -# Load a model - -In order to load a fastText model that was already trained, we can use `loadModel` function. In the example below we use `lid.176.ftz` that you can download from [here](/docs/en/language-identification.html). - -Place the model file you want to load inside the same directory than the HTML file, and inside the script part: -```javascript -import {FastText, addOnPostRun} from "./fasttext.js"; - -const printVector = function(predictions) { - for (let i=0; i { - let ft = new FastText(); - - const url = "lid.176.ftz"; - ft.loadModel(url).then(model => { - - console.log("Model loaded.") - - let text = "Bonjour à tous. Ceci est du français"; - console.log(text); - printVector(model.predict(text, 5, 0.0)); - - text = "Hello, world. This is english"; - console.log(text); - printVector(model.predict(text, 5, 0.0)); - - text = "Merhaba dünya. Bu da türkçe" - console.log(text); - printVector(model.predict(text, 5, 0.0)); - }); -}); -``` - -`loadModel` function returns a [Promise](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise) that resolves to a `model` object. We can then use [`model` object](#model-object) to call various methods, such as `predict`. - -We define `printVector` function that loops through a representation of `std::vector` in javascript, and displays the items. Here, we use it to display prediction results. - -You can also refer to `webassembly/doc/examples/predict.html` in the source code. - -# Calling other methods - -Once the model is loaded, you can call any method like `model.getDimension()` or `model.getSubwords(word)`. You can refer to [this](#api) section of the document for a complete API. You can also have a look to `webassembly/doc/examples/misc.html` file in the source code for further examples. - - -# Train a model - -### Disclaimer - -It is also possible to train a model inside the browser with fastText's WebAssembly API. The training can be slow because at the time of writing, it is not possible to use multithreading in WebAssembly (along with dynamic memory growth). So most of the time, we would train a model with the python or command line tool, eventually quantize it, and load it in the WebAssembly module. However, training a model inside the browser can be useful for creating animations or educational tools. - -### Text classification - -Place the `cooking.train` file (as described [here](/docs/en/supervised-tutorial.html)) inside the same directory: - -```javascript -import {FastText, addOnPostRun} from "./fasttext.js"; - -const trainCallback = (progress, loss, wst, lr, eta) => { - console.log([progress, loss, wst, lr, eta]); -}; - -addOnPostRun(() => { - let ft = new FastText(); - - ft.trainSupervised("cooking.train", { - 'lr':1.0, - 'epoch':10, - 'loss':'hs', - 'wordNgrams':2, - 'dim':50, - 'bucket':200000 - }, trainCallback).then(model => { - console.log('Trained.'); - }); -}); -``` - -`trainCallback` function is called by the module to show progress, average training cost, number of words per second (per thread, but there is only one thread), learning rate, estimated remaining time. - - -### Word representations - -Place the `fil9` file (as described [here](/docs/en/unsupervised-tutorial.html)) inside the same directory: - -```javascript -import {FastText, addOnPostRun} from "./fasttext.js"; - -const trainCallback = (progress, loss, wst, lr, eta) => { - console.log([progress, loss, wst, lr, eta]); -}; - -addOnPostRun(() => { - let ft = new FastText(); - - ft.trainUnsupervised("fil9", 'skipgram', { - 'lr':0.1, - 'epoch':1, - 'loss':'ns', - 'wordNgrams':2, - 'dim':50, - 'bucket':200000 - }, trainCallback).then(model => { - console.log('Trained.'); - }); -}); -``` - -# Quantized models - -Quantization is a technique that reduces the size of your models. You can quantize your model as [described here](/docs/en/faqs.html#how-can-i-reduce-the-size-of-my-fasttext-models). - -You can load a quantized model in fastText's WebAssembly module, as we did in ["Load a model" section](#load-a-model). - - -In the context of web, it is particularly useful to have smaller models since they can be downloaded much faster. You can use our autotune feature as [described here](/docs/en/autotune.html#constrain-model-size) in order to find the best trade-off between accuracy and model size that fits your needs. - - -# API - -## `model` object - -`trainSupervised`, `trainUnsupervised` and `loadModel` functions return a Promise that resolves to an instance of `FastTextModel` class, that we generaly name `model` object. - -This object exposes several functions: - -```javascript -isQuant // true if the model is quantized. -getDimension // the dimension (size) of a lookup vector (hidden layer). -getWordVector(word) // the vector representation of `word`. -getSentenceVector(text) // the vector representation of `text`. -getNearestNeighbors(word, k=10) // nearest `k` neighbors of `word`. -getAnalogies(wordA, wordB, wordC, k) // nearest `k` neighbors of the operation `wordA - wordB + wordC`. -getWordId(word) // get the word id within the dictionary. -getSubwordId(subword) // the index (within input matrix) a subword hashes to. -getSubwords(word) // the subwords and their indicies. -getInputVector(ind) // given an index, get the corresponding vector of the Input Matrix. -predict(text, k = 1, threshold = 0.0) // Given a string, get a list of labels and a list of corresponding - // probabilities. k controls the number of returned labels. -getInputMatrix() // get a reference to the full input matrix of a (non-quantized) Model. -getOutputMatrix() // get a reference to the full output matrix of a (non-quantized) Model. -getWords() // get the entire list of words of the dictionary including the frequency - // of the individual words. This does not include any subwords. For that - // please consult the function get_subwords. -getLabels() // get the entire list of labels of the dictionary including the frequency -getLine(text) // split a line of text into words and labels. -saveModel() // saves the model file in WebAssembly's in-memory FS and returns a blob -test(url, k, threshold) // downloads the test file from the specified url, evaluates the supervised model with it. -``` - -You can also have a look to `webassembly/doc/examples/misc.html` file in the source code for further examples. - -## `loadModel` - -You can load a model as follows: - -`ft.loadModel(url);` - -`loadModel` returns a [Promise](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise) that resolves to a [`model` object](#model-object). - - -## `trainSupervised` - -You can train a text classification model with fastText's WebAssembly API as follows: - -`ft.trainSupervised(trainFile, args, trainCallback);` - -- `trainFile`: the url of the input file -- `args`: a dictionary with following keys: -```javascript - lr # learning rate [0.1] - dim # size of word vectors [100] - ws # size of the context window [5] - epoch # number of epochs [5] - minCount # minimal number of word occurences [1] - minCountLabel # minimal number of label occurences [1] - minn # min length of char ngram [0] - maxn # max length of char ngram [0] - neg # number of negatives sampled [5] - wordNgrams # max length of word ngram [1] - loss # loss function {ns, hs, softmax, ova} [softmax] - bucket # number of buckets [2000000] - thread # number of threads [number of cpus] - lrUpdateRate # change the rate of updates for the learning rate [100] - t # sampling threshold [0.0001] - label # label prefix ['__label__'] -``` -- `trainCallback` is the name of the function that will be called during training to provide various information. Set this argument to `null` if you don't need a callback, or provide a function that has the following signature: `function myCallback(progress, loss, wst, lr, eta){ ... }` - -`trainSupervised` returns a [Promise](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise) that resolves to a [`model` object](#model-object). - -## `trainUnsupervised` - -You can train a word representation model with fastText's WebAssembly API as follows: - -`ft.trainUnsupervised(trainFile, modelname, args, trainCallback);` - -- `trainFile`: the url of the input file -- `modelName`: must be `"cbow"` or `"skipgram"` -- `args`: a dictionary with following keys: -```javascript - lr # learning rate [0.05] - dim # size of word vectors [100] - ws # size of the context window [5] - epoch # number of epochs [5] - minCount # minimal number of word occurences [5] - minn # min length of char ngram [3] - maxn # max length of char ngram [6] - neg # number of negatives sampled [5] - wordNgrams # max length of word ngram [1] - loss # loss function {ns, hs, softmax, ova} [ns] - bucket # number of buckets [2000000] - thread # number of threads [number of cpus] - lrUpdateRate # change the rate of updates for the learning rate [100] - t # sampling threshold [0.0001] -``` -- `trainCallback` is the name of the function that will be called during training to provide various information. Set this argument to `null` if you don't need a callback, or provide a function that has the following signature: `function myCallback(progress, loss, wst, lr, eta){ ... }` - -`trainUnsupervised` returns a [Promise](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise) that resolves to a [`model` object](#model-object). - - - diff --git a/python/README.md b/python/README.md index f6dbda228..1bf8083d7 100644 --- a/python/README.md +++ b/python/README.md @@ -1,4 +1,4 @@ -# fastText [![CircleCI](https://circleci.com/gh/facebookresearch/fastText/tree/master.svg?style=svg)](https://circleci.com/gh/facebookresearch/fastText/tree/master) +# fastText [![Build](https://github.com/aymara/fastText/actions/workflows/build.yml/badge.svg)](https://github.com/aymara/fastText/actions/workflows/build.yml) [fastText](https://fasttext.cc/) is a library for efficient learning of word representations and sentence classification. diff --git a/python/README.rst b/python/README.rst index 6e6f38d84..0b4f0c95a 100644 --- a/python/README.rst +++ b/python/README.rst @@ -1,4 +1,4 @@ -fastText |CircleCI| +fastText |Build| =================== `fastText `__ is a library for efficient learning @@ -402,5 +402,5 @@ Join the fastText community group `__ - `GitHub `__ -.. |CircleCI| image:: https://circleci.com/gh/facebookresearch/fastText/tree/master.svg?style=svg - :target: https://circleci.com/gh/facebookresearch/fastText/tree/master +.. |Build| image:: https://github.com/aymara/fastText/actions/workflows/build.yml/badge.svg + :target: https://github.com/aymara/fastText/actions/workflows/build.yml diff --git a/quantization-example.sh b/quantization-example.sh index cd256afaa..99b05aba8 100755 --- a/quantization-example.sh +++ b/quantization-example.sh @@ -23,18 +23,18 @@ then cat "${DATADIR}/dbpedia_csv/test.csv" | normalize_text > "${DATADIR}/dbpedia.test" fi -make +mkdir -p build && cd build && cmake .. && make && cd .. echo "Training..." -./fasttext supervised -input "${DATADIR}/dbpedia.train" -output "${RESULTDIR}/dbpedia" -dim 10 -lr 0.1 -wordNgrams 2 -minCount 1 -bucket 10000000 -epoch 5 -thread 4 +./build/fasttext supervised -input "${DATADIR}/dbpedia.train" -output "${RESULTDIR}/dbpedia" -dim 10 -lr 0.1 -wordNgrams 2 -minCount 1 -bucket 10000000 -epoch 5 -thread 4 echo "Quantizing..." -./fasttext quantize -output "${RESULTDIR}/dbpedia" -input "${DATADIR}/dbpedia.train" -qnorm -retrain -epoch 1 -cutoff 100000 +./build/fasttext quantize -output "${RESULTDIR}/dbpedia" -input "${DATADIR}/dbpedia.train" -qnorm -retrain -epoch 1 -cutoff 100000 echo "Testing original model..." -./fasttext test "${RESULTDIR}/dbpedia.bin" "${DATADIR}/dbpedia.test" +./build/fasttext test "${RESULTDIR}/dbpedia.bin" "${DATADIR}/dbpedia.test" echo "Testing quantized model..." -./fasttext test "${RESULTDIR}/dbpedia.ftz" "${DATADIR}/dbpedia.test" +./build/fasttext test "${RESULTDIR}/dbpedia.ftz" "${DATADIR}/dbpedia.test" wc -c < "${RESULTDIR}/dbpedia.bin" | awk '{print "Size of the original model:\t",$1;}' wc -c < "${RESULTDIR}/dbpedia.ftz" | awk '{print "Size of the quantized model:\t",$1;}' diff --git a/scripts/kbcompletion/fb15k.sh b/scripts/kbcompletion/fb15k.sh index 4c1edc776..bfd10b5e2 100755 --- a/scripts/kbcompletion/fb15k.sh +++ b/scripts/kbcompletion/fb15k.sh @@ -12,9 +12,9 @@ FASTTEXTDIR=../../ # compile pushd $FASTTEXTDIR -make opt +mkdir -p build && cd build && cmake .. && make popd -ft=${FASTTEXTDIR}/fasttext +ft=${FASTTEXTDIR}/build/fasttext g++ -std=c++0x eval.cpp -o eval diff --git a/scripts/kbcompletion/fb15k237.sh b/scripts/kbcompletion/fb15k237.sh index e06f24a33..b09f7d13b 100755 --- a/scripts/kbcompletion/fb15k237.sh +++ b/scripts/kbcompletion/fb15k237.sh @@ -13,9 +13,9 @@ FASTTEXTDIR=../../ # compile pushd $FASTTEXTDIR -make opt +mkdir -p build && cd build && cmake .. && make popd -ft=${FASTTEXTDIR}/fasttext +ft=${FASTTEXTDIR}/build/fasttext g++ -std=c++0x eval.cpp -o eval diff --git a/scripts/kbcompletion/svo.sh b/scripts/kbcompletion/svo.sh index 61e4636ef..41ec531b7 100755 --- a/scripts/kbcompletion/svo.sh +++ b/scripts/kbcompletion/svo.sh @@ -12,9 +12,9 @@ FASTTEXTDIR=../../ # compile pushd $FASTTEXTDIR -make opt +mkdir -p build && cd build && cmake .. && make popd -ft=${FASTTEXTDIR}/fasttext +ft=${FASTTEXTDIR}/build/fasttext ## Train model and test it on validation: diff --git a/scripts/kbcompletion/wn18.sh b/scripts/kbcompletion/wn18.sh index fac7bb660..a161f66bd 100755 --- a/scripts/kbcompletion/wn18.sh +++ b/scripts/kbcompletion/wn18.sh @@ -13,9 +13,9 @@ FASTTEXTDIR=../../ # compile pushd $FASTTEXTDIR -make opt +mkdir -p build && cd build && cmake .. && make popd -ft=${FASTTEXTDIR}/fasttext +ft=${FASTTEXTDIR}/build/fasttext g++ -std=c++0x eval.cpp -o eval diff --git a/src/args.h b/src/args.h index d36fd3479..d328a6b86 100644 --- a/src/args.h +++ b/src/args.h @@ -8,6 +8,7 @@ #pragma once +#include #include #include #include diff --git a/src/autotune.h b/src/autotune.h index 8b300ae5d..f8e141d19 100644 --- a/src/autotune.h +++ b/src/autotune.h @@ -8,6 +8,7 @@ #pragma once +#include #include #include #include diff --git a/src/dictionary.h b/src/dictionary.h index 2dfd73075..f47443611 100644 --- a/src/dictionary.h +++ b/src/dictionary.h @@ -8,6 +8,7 @@ #pragma once +#include #include #include #include diff --git a/src/fasttext.h b/src/fasttext.h index 1e35f6fe3..de5cb3e9c 100644 --- a/src/fasttext.h +++ b/src/fasttext.h @@ -8,6 +8,7 @@ #pragma once +#include #include #include diff --git a/src/loss.h b/src/loss.h index 3aea72f87..f1a9812a1 100644 --- a/src/loss.h +++ b/src/loss.h @@ -8,6 +8,7 @@ #pragma once +#include #include #include #include diff --git a/src/meter.h b/src/meter.h index b6fad1f36..30e4d9e3c 100644 --- a/src/meter.h +++ b/src/meter.h @@ -8,6 +8,7 @@ #pragma once +#include #include #include diff --git a/src/model.h b/src/model.h index 65987d681..3e0ca8212 100644 --- a/src/model.h +++ b/src/model.h @@ -8,6 +8,7 @@ #pragma once +#include #include #include #include diff --git a/src/productquantizer.h b/src/productquantizer.h index 0d71d86e4..ca3346681 100644 --- a/src/productquantizer.h +++ b/src/productquantizer.h @@ -8,6 +8,7 @@ #pragma once +#include #include #include #include diff --git a/src/utils.h b/src/utils.h index 2198dc152..1b4bce7ac 100644 --- a/src/utils.h +++ b/src/utils.h @@ -8,6 +8,7 @@ #pragma once +#include #include "real.h" #include diff --git a/webassembly/README.md b/webassembly/README.md deleted file mode 100644 index 7fd9960d7..000000000 --- a/webassembly/README.md +++ /dev/null @@ -1,37 +0,0 @@ - fastText [![CircleCI](https://circleci.com/gh/facebookresearch/fastText/tree/master.svg?style=svg)](https://circleci.com/gh/facebookresearch/fastText/tree/master) - -[fastText](https://fasttext.cc/) is a library for efficient learning of word representations and sentence classification. - -In this document we present how to use fastText in a browser with WebAssembly. - - -# Requirements - -[fastText](https://fasttext.cc/) builds on modern Mac OS and Linux distributions. -Since it uses C\++11 features, it requires a compiler with good C++11 support. -You will need [emscripten](https://emscripten.org/) and a [browser that supports WebAssembly](https://caniuse.com/#feat=wasm). - - -# Building WebAssembly binaries - -First, download and install emscripten sdk as [described here](https://emscripten.org/docs/getting_started/downloads.html#installation-instructions) - - -Make sure you activated the PATH for emscripten: -```bash -$ source /path/to/emsdk/emsdk_env.sh -``` - -Clone our [repository](https://github.com/facebookresearch/fastText/). - -```bash -$ git clone git@github.com:facebookresearch/fastText.git -``` - -Build WebAssembly binaries: -```bash -$ cd fastText -$ make wasm -``` - - diff --git a/webassembly/doc/examples/misc.html b/webassembly/doc/examples/misc.html deleted file mode 100644 index 811574d1b..000000000 --- a/webassembly/doc/examples/misc.html +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/webassembly/doc/examples/predict.html b/webassembly/doc/examples/predict.html deleted file mode 100644 index 294e8f978..000000000 --- a/webassembly/doc/examples/predict.html +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/webassembly/doc/examples/train_supervised.html b/webassembly/doc/examples/train_supervised.html deleted file mode 100644 index 2c42b3f73..000000000 --- a/webassembly/doc/examples/train_supervised.html +++ /dev/null @@ -1,66 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/webassembly/doc/examples/train_unsupervised.html b/webassembly/doc/examples/train_unsupervised.html deleted file mode 100644 index 12f0c4de9..000000000 --- a/webassembly/doc/examples/train_unsupervised.html +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/webassembly/fasttext.js b/webassembly/fasttext.js deleted file mode 100644 index 86600b9ac..000000000 --- a/webassembly/fasttext.js +++ /dev/null @@ -1,520 +0,0 @@ -/** - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -import fastTextModularized from './fasttext_wasm.js'; -const fastTextModule = fastTextModularized(); - -let postRunFunc = null; -const addOnPostRun = function(func) { - postRunFunc = func; -}; - -fastTextModule.addOnPostRun(() => { - if (postRunFunc) { - postRunFunc(); - } -}); - -const thisModule = this; -const trainFileInWasmFs = 'train.txt'; -const testFileInWasmFs = 'test.txt'; -const modelFileInWasmFs = 'model.bin'; - -const getFloat32ArrayFromHeap = (len) => { - const dataBytes = len * Float32Array.BYTES_PER_ELEMENT; - const dataPtr = fastTextModule._malloc(dataBytes); - const dataHeap = new Uint8Array(fastTextModule.HEAPU8.buffer, - dataPtr, - dataBytes); - return { - 'ptr':dataHeap.byteOffset, - 'size':len, - 'buffer':dataHeap.buffer - }; -}; - -const heapToFloat32 = (r) => new Float32Array(r.buffer, r.ptr, r.size); - -class FastText { - constructor() { - this.f = new fastTextModule.FastText(); - } - - /** - * loadModel - * - * Loads the model file from the specified url, and returns the - * corresponding `FastTextModel` object. - * - * @param {string} url - * the url of the model file. - * - * @return {Promise} promise object that resolves to a `FastTextModel` - * - */ - loadModel(url) { - const fetchFunc = (thisModule && thisModule.fetch) || fetch; - - const fastTextNative = this.f; - return new Promise(function(resolve, reject) { - fetchFunc(url).then(response => { - return response.arrayBuffer(); - }).then(bytes => { - const byteArray = new Uint8Array(bytes); - const FS = fastTextModule.FS; - FS.writeFile(modelFileInWasmFs, byteArray); - }).then(() => { - fastTextNative.loadModel(modelFileInWasmFs); - resolve(new FastTextModel(fastTextNative)); - }).catch(error => { - reject(error); - }); - }); - } - - _train(url, modelName, kwargs = {}, callback = null) { - const fetchFunc = (thisModule && thisModule.fetch) || fetch; - const fastTextNative = this.f; - - return new Promise(function(resolve, reject) { - fetchFunc(url).then(response => { - return response.arrayBuffer(); - }).then(bytes => { - const byteArray = new Uint8Array(bytes); - const FS = fastTextModule.FS; - FS.writeFile(trainFileInWasmFs, byteArray); - }).then(() => { - const argsList = ['lr', 'lrUpdateRate', 'dim', 'ws', 'epoch', - 'minCount', 'minCountLabel', 'neg', 'wordNgrams', 'loss', - 'model', 'bucket', 'minn', 'maxn', 't', 'label', 'verbose', - 'pretrainedVectors', 'saveOutput', 'seed', 'qout', 'retrain', - 'qnorm', 'cutoff', 'dsub', 'qnorm', 'autotuneValidationFile', - 'autotuneMetric', 'autotunePredictions', 'autotuneDuration', - 'autotuneModelSize']; - const args = new fastTextModule.Args(); - argsList.forEach(k => { - if (k in kwargs) { - args[k] = kwargs[k]; - } - }); - args.model = fastTextModule.ModelName[modelName]; - args.loss = ('loss' in kwargs) ? - fastTextModule.LossName[kwargs['loss']] : 'hs'; - args.thread = 1; - args.input = trainFileInWasmFs; - - fastTextNative.train(args, callback); - - resolve(new FastTextModel(fastTextNative)); - }).catch(error => { - reject(error); - }); - }); - } - - /** - * trainSupervised - * - * Downloads the input file from the specified url, trains a supervised - * model and returns a `FastTextModel` object. - * - * @param {string} url - * the url of the input file. - * The input file must must contain at least one label per line. For an - * example consult the example datasets which are part of the fastText - * repository such as the dataset pulled by classification-example.sh. - * - * @param {dict} kwargs - * train parameters. - * For example {'lr': 0.5, 'epoch': 5} - * - * @param {function} callback - * train callback function - * `callback` function is called regularly from the train loop: - * `callback(progress, loss, wordsPerSec, learningRate, eta)` - * - * @return {Promise} promise object that resolves to a `FastTextModel` - * - */ - trainSupervised(url, kwargs = {}, callback) { - const self = this; - return new Promise(function(resolve, reject) { - self._train(url, 'supervised', kwargs, callback).then(model => { - resolve(model); - }).catch(error => { - reject(error); - }); - }); - } - - /** - * trainUnsupervised - * - * Downloads the input file from the specified url, trains an unsupervised - * model and returns a `FastTextModel` object. - * - * @param {string} url - * the url of the input file. - * The input file must not contain any labels or use the specified label - * prefixunless it is ok for those words to be ignored. For an example - * consult the dataset pulled by the example script word-vector-example.sh - * which is part of the fastText repository. - * - * @param {string} modelName - * Model to be used for unsupervised learning. `cbow` or `skipgram`. - * - * @param {dict} kwargs - * train parameters. - * For example {'lr': 0.5, 'epoch': 5} - * - * @param {function} callback - * train callback function - * `callback` function is called regularly from the train loop: - * `callback(progress, loss, wordsPerSec, learningRate, eta)` - * - * @return {Promise} promise object that resolves to a `FastTextModel` - * - */ - trainUnsupervised(url, modelName, kwargs = {}, callback) { - const self = this; - return new Promise(function(resolve, reject) { - self._train(url, modelName, kwargs, callback).then(model => { - resolve(model); - }).catch(error => { - reject(error); - }); - }); - } - -} - - -class FastTextModel { - /** - * `FastTextModel` represents a trained model. - * - * @constructor - * - * @param {object} fastTextNative - * webassembly object that makes the bridge between js and C++ - */ - constructor(fastTextNative) { - this.f = fastTextNative; - } - - /** - * isQuant - * - * @return {bool} true if the model is quantized - * - */ - isQuant() { - return this.f.isQuant; - } - - /** - * getDimension - * - * @return {int} the dimension (size) of a lookup vector (hidden layer) - * - */ - getDimension() { - return this.f.args.dim; - } - - /** - * getWordVector - * - * @param {string} word - * - * @return {Float32Array} the vector representation of `word`. - * - */ - getWordVector(word) { - const b = getFloat32ArrayFromHeap(this.getDimension()); - this.f.getWordVector(b, word); - - return heapToFloat32(b); - } - - /** - * getSentenceVector - * - * @param {string} text - * - * @return {Float32Array} the vector representation of `text`. - * - */ - getSentenceVector(text) { - if (text.indexOf('\n') != -1) { - "sentence vector processes one line at a time (remove '\\n')"; - } - text += '\n'; - const b = getFloat32ArrayFromHeap(this.getDimension()); - this.f.getSentenceVector(b, text); - - return heapToFloat32(b); - } - - /** - * getNearestNeighbors - * - * returns the nearest `k` neighbors of `word`. - * - * @param {string} word - * @param {int} k - * - * @return {Array.>} - * words and their corresponding cosine similarities. - * - */ - getNearestNeighbors(word, k = 10) { - return this.f.getNN(word, k); - } - - /** - * getAnalogies - * - * returns the nearest `k` neighbors of the operation - * `wordA - wordB + wordC`. - * - * @param {string} wordA - * @param {string} wordB - * @param {string} wordC - * @param {int} k - * - * @return {Array.>} - * words and their corresponding cosine similarities - * - */ - getAnalogies(wordA, wordB, wordC, k) { - return this.f.getAnalogies(k, wordA, wordB, wordC); - } - - /** - * getWordId - * - * Given a word, get the word id within the dictionary. - * Returns -1 if word is not in the dictionary. - * - * @return {int} word id - * - */ - getWordId(word) { - return this.f.getWordId(word); - } - - /** - * getSubwordId - * - * Given a subword, return the index (within input matrix) it hashes to. - * - * @return {int} subword id - * - */ - getSubwordId(subword) { - return this.f.getSubwordId(subword); - } - - /** - * getSubwords - * - * returns the subwords and their indicies. - * - * @param {string} word - * - * @return {Pair., Array.>} - * words and their corresponding indicies - * - */ - getSubwords(word) { - return this.f.getSubwords(word); - } - - /** - * getInputVector - * - * Given an index, get the corresponding vector of the Input Matrix. - * - * @param {int} ind - * - * @return {Float32Array} the vector of the `ind`'th index - * - */ - getInputVector(ind) { - const b = getFloat32ArrayFromHeap(this.getDimension()); - this.f.getInputVector(b, ind); - - return heapToFloat32(b); - } - - /** - * predict - * - * Given a string, get a list of labels and a list of corresponding - * probabilities. k controls the number of returned labels. - * - * @param {string} text - * @param {int} k, the number of predictions to be returned - * @param {number} probability threshold - * - * @return {Array.>} - * labels and their probabilities - * - */ - predict(text, k = 1, threshold = 0.0) { - return this.f.predict(text, k, threshold); - } - - /** - * getInputMatrix - * - * Get a reference to the full input matrix of a Model. This only - * works if the model is not quantized. - * - * @return {DenseMatrix} - * densematrix with functions: `rows`, `cols`, `at(i,j)` - * - * example: - * let inputMatrix = model.getInputMatrix(); - * let value = inputMatrix.at(1, 2); - */ - getInputMatrix() { - if (this.isQuant()) { - throw new Error("Can't get quantized Matrix"); - } - return this.f.getInputMatrix(); - } - - /** - * getOutputMatrix - * - * Get a reference to the full input matrix of a Model. This only - * works if the model is not quantized. - * - * @return {DenseMatrix} - * densematrix with functions: `rows`, `cols`, `at(i,j)` - * - * example: - * let outputMatrix = model.getOutputMatrix(); - * let value = outputMatrix.at(1, 2); - */ - getOutputMatrix() { - if (this.isQuant()) { - throw new Error("Can't get quantized Matrix"); - } - return this.f.getOutputMatrix(); - } - - /** - * getWords - * - * Get the entire list of words of the dictionary including the frequency - * of the individual words. This does not include any subwords. For that - * please consult the function get_subwords. - * - * @return {Pair., Array.>} - * words and their corresponding frequencies - * - */ - getWords() { - return this.f.getWords(); - } - - /** - * getLabels - * - * Get the entire list of labels of the dictionary including the frequency - * of the individual labels. - * - * @return {Pair., Array.>} - * labels and their corresponding frequencies - * - */ - getLabels() { - return this.f.getLabels(); - } - - /** - * getLine - * - * Split a line of text into words and labels. Labels must start with - * the prefix used to create the model (__label__ by default). - * - * @param {string} text - * - * @return {Pair., Array.>} - * words and labels - * - */ - getLine(text) { - return this.f.getLine(text); - } - - /** - * saveModel - * - * Saves the model file in web assembly in-memory FS and returns a blob - * - * @return {Blob} blob data of the file saved in web assembly FS - * - */ - saveModel() { - this.f.saveModel(modelFileInWasmFs); - const content = fastTextModule.FS.readFile(modelFileInWasmFs, - { encoding: 'binary' }); - return new Blob( - [new Uint8Array(content, content.byteOffset, content.length)], - { type: ' application/octet-stream' } - ); - } - - /** - * test - * - * Downloads the test file from the specified url, evaluates the supervised - * model with it. - * - * @param {string} url - * @param {int} k, the number of predictions to be returned - * @param {number} probability threshold - * - * @return {Promise} promise object that resolves to a `Meter` object - * - * example: - * model.test("/absolute/url/to/test.txt", 1, 0.0).then((meter) => { - * console.log(meter.precision); - * console.log(meter.recall); - * console.log(meter.f1Score); - * console.log(meter.nexamples()); - * }); - * - */ - test(url, k, threshold) { - const fetchFunc = (thisModule && thisModule.fetch) || fetch; - const fastTextNative = this.f; - - return new Promise(function(resolve, reject) { - fetchFunc(url).then(response => { - return response.arrayBuffer(); - }).then(bytes => { - const byteArray = new Uint8Array(bytes); - const FS = fastTextModule.FS; - FS.writeFile(testFileInWasmFs, byteArray); - }).then(() => { - const meter = fastTextNative.test(testFileInWasmFs, k, threshold); - resolve(meter); - }).catch(error => { - reject(error); - }); - }); - } -} - - -export {FastText, addOnPostRun}; diff --git a/webassembly/fasttext_wasm.cc b/webassembly/fasttext_wasm.cc deleted file mode 100644 index 071e84ff5..000000000 --- a/webassembly/fasttext_wasm.cc +++ /dev/null @@ -1,328 +0,0 @@ -/** - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include -#include -#include -#include -#include -#include -#include - -using namespace emscripten; -using namespace fasttext; - -struct Float32ArrayBridge { - uintptr_t ptr; - int size; -}; - -void fillFloat32ArrayFromVector( - const Float32ArrayBridge& vecFloat, - const Vector& v) { - float* buffer = reinterpret_cast(vecFloat.ptr); - assert(vecFloat.size == v.size()); - for (int i = 0; i < v.size(); i++) { - buffer[i] = v[i]; - } -} - -std::vector> -predict(FastText* fasttext, std::string text, int k, double threshold) { - std::stringstream ioss(text + std::string("\n")); - - std::vector> predictions; - fasttext->predictLine(ioss, predictions, k, threshold); - - return predictions; -} - -void getWordVector( - FastText* fasttext, - const Float32ArrayBridge& vecFloat, - std::string word) { - assert(fasttext); - Vector v(fasttext->getDimension()); - fasttext->getWordVector(v, word); - - fillFloat32ArrayFromVector(vecFloat, v); -} - -void getSentenceVector( - FastText* fasttext, - const Float32ArrayBridge& vecFloat, - std::string text) { - assert(fasttext); - Vector v(fasttext->getDimension()); - std::stringstream ioss(text); - fasttext->getSentenceVector(ioss, v); - - fillFloat32ArrayFromVector(vecFloat, v); -} - -std::pair, std::vector> getSubwords( - FastText* fasttext, - std::string word) { - assert(fasttext); - std::vector subwords; - std::vector ngrams; - std::shared_ptr d = fasttext->getDictionary(); - d->getSubwords(word, ngrams, subwords); - - return std::pair, std::vector>( - subwords, ngrams); -} - -void getInputVector( - FastText* fasttext, - const Float32ArrayBridge& vecFloat, - int32_t ind) { - assert(fasttext); - Vector v(fasttext->getDimension()); - fasttext->getInputVector(v, ind); - - fillFloat32ArrayFromVector(vecFloat, v); -} - -void train(FastText* fasttext, Args* args, emscripten::val jsCallback) { - assert(args); - assert(fasttext); - fasttext->train( - *args, - [=](float progress, float loss, double wst, double lr, int64_t eta) { - jsCallback(progress, loss, wst, lr, static_cast(eta)); - }); -} - -const DenseMatrix* getInputMatrix(FastText* fasttext) { - assert(fasttext); - std::shared_ptr mm = fasttext->getInputMatrix(); - return mm.get(); -} - -const DenseMatrix* getOutputMatrix(FastText* fasttext) { - assert(fasttext); - std::shared_ptr mm = fasttext->getOutputMatrix(); - return mm.get(); -} - -std::pair, std::vector> getTokens( - const FastText& fasttext, - const std::function getter, - entry_type entryType) { - std::vector tokens; - std::vector retVocabFrequencies; - std::shared_ptr d = fasttext.getDictionary(); - std::vector vocabFrequencies = d->getCounts(entryType); - for (int32_t i = 0; i < vocabFrequencies.size(); i++) { - tokens.push_back(getter(*d, i)); - retVocabFrequencies.push_back(vocabFrequencies[i]); - } - return std::pair, std::vector>( - tokens, retVocabFrequencies); -} - -std::pair, std::vector> getWords( - FastText* fasttext) { - assert(fasttext); - return getTokens(*fasttext, &Dictionary::getWord, entry_type::word); -} - -std::pair, std::vector> getLabels( - FastText* fasttext) { - assert(fasttext); - return getTokens(*fasttext, &Dictionary::getLabel, entry_type::label); -} - -std::pair, std::vector> getLine( - FastText* fasttext, - const std::string text) { - assert(fasttext); - std::shared_ptr d = fasttext->getDictionary(); - std::stringstream ioss(text); - std::string token; - std::vector words; - std::vector labels; - while (d->readWord(ioss, token)) { - uint32_t h = d->hash(token); - int32_t wid = d->getId(token, h); - entry_type type = wid < 0 ? d->getType(token) : d->getType(wid); - - if (type == entry_type::word) { - words.push_back(token); - } else if (type == entry_type::label && wid >= 0) { - labels.push_back(token); - } - if (token == Dictionary::EOS) - break; - } - return std::pair, std::vector>( - words, labels); -} - -Meter test( - FastText* fasttext, - const std::string& filename, - int32_t k, - float threshold) { - assert(fasttext); - std::ifstream ifs(filename); - if (!ifs.is_open()) { - throw std::invalid_argument("Test file cannot be opened!"); - } - Meter meter(false); - fasttext->test(ifs, k, threshold, meter); - ifs.close(); - - return meter; -} - -EMSCRIPTEN_BINDINGS(fasttext) { - class_("Args") - .constructor<>() - .property("input", &Args::input) - .property("output", &Args::output) - .property("lr", &Args::lr) - .property("lrUpdateRate", &Args::lrUpdateRate) - .property("dim", &Args::dim) - .property("ws", &Args::ws) - .property("epoch", &Args::epoch) - .property("minCount", &Args::minCount) - .property("minCountLabel", &Args::minCountLabel) - .property("neg", &Args::neg) - .property("wordNgrams", &Args::wordNgrams) - .property("loss", &Args::loss) - .property("model", &Args::model) - .property("bucket", &Args::bucket) - .property("minn", &Args::minn) - .property("maxn", &Args::maxn) - .property("thread", &Args::thread) - .property("t", &Args::t) - .property("label", &Args::label) - .property("verbose", &Args::verbose) - .property("pretrainedVectors", &Args::pretrainedVectors) - .property("saveOutput", &Args::saveOutput) - .property("seed", &Args::seed) - .property("qout", &Args::qout) - .property("retrain", &Args::retrain) - .property("qnorm", &Args::qnorm) - .property("cutoff", &Args::cutoff) - .property("dsub", &Args::dsub) - .property("qnorm", &Args::qnorm) - .property("autotuneValidationFile", &Args::autotuneValidationFile) - .property("autotuneMetric", &Args::autotuneMetric) - .property("autotunePredictions", &Args::autotunePredictions) - .property("autotuneDuration", &Args::autotuneDuration) - .property("autotuneModelSize", &Args::autotuneModelSize); - - class_("FastText") - .constructor<>() - .function( - "loadModel", - select_overload(&FastText::loadModel)) - .function( - "getNN", - select_overload>( - const std::string& word, int32_t k)>(&FastText::getNN)) - .function("getAnalogies", &FastText::getAnalogies) - .function("getWordId", &FastText::getWordId) - .function("getSubwordId", &FastText::getSubwordId) - .function("getInputMatrix", &getInputMatrix, allow_raw_pointers()) - .function("getOutputMatrix", &getOutputMatrix, allow_raw_pointers()) - .function("getWords", &getWords, allow_raw_pointers()) - .function("getLabels", &getLabels, allow_raw_pointers()) - .function("getLine", &getLine, allow_raw_pointers()) - .function("test", &test, allow_raw_pointers()) - .function("predict", &predict, allow_raw_pointers()) - .function("getWordVector", &getWordVector, allow_raw_pointers()) - .function("getSentenceVector", &getSentenceVector, allow_raw_pointers()) - .function("getSubwords", &getSubwords, allow_raw_pointers()) - .function("getInputVector", &getInputVector, allow_raw_pointers()) - .function("train", &train, allow_raw_pointers()) - .function("saveModel", &FastText::saveModel) - .property("isQuant", &FastText::isQuant) - .property("args", &FastText::getArgs); - - class_("DenseMatrix") - .constructor<>() - // we return int32_t because "JS can't represent int64s" - .function( - "rows", - optional_override( - [](const DenseMatrix* self) -> int32_t { return self->rows(); }), - allow_raw_pointers()) - .function( - "cols", - optional_override( - [](const DenseMatrix* self) -> int32_t { return self->cols(); }), - allow_raw_pointers()) - .function( - "at", - optional_override( - [](const DenseMatrix* self, int32_t i, int32_t j) -> const float { - return self->at(i, j); - }), - allow_raw_pointers()); - - class_("Meter") - .constructor() - .property( - "precision", select_overload(&Meter::precision)) - .property("recall", select_overload(&Meter::recall)) - .property("f1Score", select_overload(&Meter::f1Score)) - .function( - "nexamples", - optional_override( - [](const Meter* self) -> int32_t { return self->nexamples(); }), - allow_raw_pointers()); - - enum_("ModelName") - .value("cbow", model_name::cbow) - .value("skipgram", model_name::sg) - .value("supervised", model_name::sup); - - enum_("LossName") - .value("hs", loss_name::hs) - .value("ns", loss_name::ns) - .value("softmax", loss_name::softmax) - .value("ova", loss_name::ova); - - emscripten::value_object("Float32ArrayBridge") - .field("ptr", &Float32ArrayBridge::ptr) - .field("size", &Float32ArrayBridge::size); - - emscripten::value_array>( - "std::pair") - .element(&std::pair::first) - .element(&std::pair::second); - - emscripten::register_vector>( - "std::vector>"); - - emscripten::value_array< - std::pair, std::vector>>( - "std::pair, std::vector>") - .element( - &std::pair, std::vector>::first) - .element( - &std::pair, std::vector>::second); - - emscripten::value_array< - std::pair, std::vector>>( - "std::pair, std::vector>") - .element( - &std::pair, std::vector>::first) - .element(&std::pair, std::vector>:: - second); - - emscripten::register_vector("std::vector"); - - emscripten::register_vector("std::vector"); - - emscripten::register_vector("std::vector"); -} diff --git a/website/sidebars.json b/website/sidebars.json index 9595f7669..4fd204de1 100644 --- a/website/sidebars.json +++ b/website/sidebars.json @@ -2,7 +2,7 @@ "docs": { "Introduction": ["support", "cheatsheet", "options"], "Tutorials": ["supervised-tutorial", "unsupervised-tutorial"], - "Help": ["autotune", "python-module", "webassembly-module", "faqs", "api", "references"] + "Help": ["autotune", "python-module", "faqs", "api", "references"] }, "download": { "Resources": [ diff --git a/word-vector-example.sh b/word-vector-example.sh index 4a450d090..281998630 100755 --- a/word-vector-example.sh +++ b/word-vector-example.sh @@ -26,14 +26,14 @@ then unzip "${DATADIR}/rw.zip" -d "${DATADIR}" fi -make +mkdir -p build && cd build && cmake .. && make && cd .. -./fasttext skipgram -input "${DATADIR}"/fil9 -output "${RESULTDIR}"/fil9 -lr 0.025 -dim 100 \ +./build/fasttext skipgram -input "${DATADIR}"/fil9 -output "${RESULTDIR}"/fil9 -lr 0.025 -dim 100 \ -ws 5 -epoch 1 -minCount 5 -neg 5 -loss ns -bucket 2000000 \ -minn 3 -maxn 6 -thread 4 -t 1e-4 -lrUpdateRate 100 cut -f 1,2 "${DATADIR}"/rw/rw.txt | awk '{print tolower($0)}' | tr '\t' '\n' > "${DATADIR}"/queries.txt -cat "${DATADIR}"/queries.txt | ./fasttext print-word-vectors "${RESULTDIR}"/fil9.bin > "${RESULTDIR}"/vectors.txt +cat "${DATADIR}"/queries.txt | ./build/fasttext print-word-vectors "${RESULTDIR}"/fil9.bin > "${RESULTDIR}"/vectors.txt python eval.py -m "${RESULTDIR}"/vectors.txt -d "${DATADIR}"/rw/rw.txt