llama integration with exception safety

.gitignore

@@ -7,3 +7,5 @@ version.h
 ports/docwire/.disable_binary_cache
 .cache
 compile_commands.json
+.zed
+.clang-format

CMakeLists.txt

@@ -2,6 +2,9 @@ cmake_minimum_required(VERSION 3.17)
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # for VSCode CLangd extension
 
+option(DOCWIRE_LOCAL_CT2 "Enable local AI (Translation / Text Generation / Embedding models)" OFF)
+option(DOCWIRE_LLAMA    "Enable llama.cpp engine" OFF)
+
 # Get version from ChangeLog.md and store it in DOCWIRE_VERSION and SIMPLE_DOCWIRE_VERSION
 function(extract_version)
 	file(READ "${CMAKE_CURRENT_SOURCE_DIR}/doc/ChangeLog.md" changelog_text)

README.md

@@ -365,7 +365,7 @@ std::filesystem::path("test.zip") | content_type::detector{} | archives_parser{}
 Classify file in any format (Office, PDF, mail, etc) to any categories using build-in local AI model:
 
 ```cpp
-std::filesystem::path("...") | ... | local_ai::model_chain_element("Classify to...: agreement, invoice, report...") | out_stream;
+std::filesystem::path("...") | ... | ai::local::task("Classify to...: agreement, invoice, report...") | out_stream;
 ensure(out_stream.str()) == "report";
 ```
 [Full example](https://docwire.readthedocs.io/en/latest/local_ai_classify_8cpp-example.html)
@@ -381,7 +381,7 @@ ensure(out_stream.str()) == "report\n";
 Translate document in any format (Office, PDF, mail, etc) to other language using build-in local AI model:
 
 ```cpp
-std::filesystem::path("...") | ... | local_ai::model_chain_element("Translate to spanish:\n\n") | out_stream;
+std::filesystem::path("...") | ... | ai::local::translate("spanish") | out_stream;
 ensure(fuzzy_match::ratio(out_stream.str(), "La procesación de datos se refiere a las actividades...")) > 80;
 ```
 [Full example](https://docwire.readthedocs.io/en/latest/local_ai_translate_8cpp-example.html)
@@ -397,7 +397,7 @@ ensure(fuzzy_match::ratio(out_stream.str(), "El procesamiento de datos se refier
 Detect sentiment of document in any format (Office, PDF, mail, etc) using build-in local AI model:
 
 ```cpp
-std::filesystem::path("...") | ... | local_ai::model_chain_element("Detect sentiment:\n\n") | out_stream;
+std::filesystem::path("...") | ... | ai::local::task("Detect sentiment:\n\n") | out_stream;
 ensure(out_stream.str()) == "positive";
 ```
 [Full example](https://docwire.readthedocs.io/en/latest/local_ai_sentiment_8cpp-example.html)
@@ -412,7 +412,7 @@ std::filesystem::path("1.doc") | ... | openai::DetectSentiment(...) | std::cout;
 Make a summary of document in any format (Office, PDF, mail, etc) using build-in local AI model:
 
 ```cpp
-std::filesystem::path("...") | ... | local_ai::model_chain_element("Write a short summary...") | out_stream;
+std::filesystem::path("...") | ... | ai::local::summarize() | out_stream;
 ensure(out_stream.str()).is_one_of({ "Data processing is the collection, organization, analysis, and interpretation of data to extract useful insights and support decision-making."...
 ```
 [Full example](https://docwire.readthedocs.io/en/latest/local_ai_summary_8cpp-example.html)
@@ -435,7 +435,7 @@ ensure(fuzzy_match::ratio(out_stream.str(), "Data processing involves converting
 Find phrases, objects and events with smart matching in documents in any format (Office, PDF, mail, etc) using build-in local AI model:
 
 ```cpp
-std::filesystem::path("...") | ... | local_ai::model_chain_element("Find sentence about \"data conversion\"...") | out_stream;
+std::filesystem::path("...") | ... | ai::local::task("Find sentence about \"data conversion\"...") | out_stream;
 ensure(out_stream.str()).is_one_of({ "Data processing refers to the activities performed on raw data to convert it into meaningful information."...
 ```
 [Full example](https://docwire.readthedocs.io/en/latest/local_ai_find_8cpp-example.html)
@@ -461,9 +461,9 @@ ensure(out_msgs[0]->get<ai::embedding>().values.size()) == 1536;
 Create embedding for document in any format (Office, PDF, mail, etc) using build-in local AI model, create embeddings for two queries and calculate similarity:
 
 ```cpp
-std::filesystem::path("data_processing_definition.doc") | ... | local_ai::embed(local_ai::embed::e5_passage_prefix) | passage_msgs;
+std::filesystem::path("data_processing_definition.doc") | ... | ai::local::passage::embedder{} | passage_msgs;
 ...
-docwire::data_source{std::string{"What is data processing?"}, ...} | local_ai::embed(local_ai::embed::e5_query_prefix) | similar_query_msgs;
+docwire::data_source{std::string{"What is data processing?"}, ...} | ai::local::query::embedder{} | similar_query_msgs;
 ...
 double sim = cosine_similarity(passage_embedding.values, similar_query_embedding.values);
 ...

build_demo.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -e  # stop on first error
+
+# ==========================
+# CONFIGURATION — adjust paths
+# ==========================
+VCPKG_TOOLCHAIN=/home/reeshabh/coderepos/docwire/vcpkg/scripts/buildsystems/vcpkg.cmake
+VCPKG_TRIPLET=x64-linux-dynamic
+DOCWIRE_DIR=/home/reeshabh/coderepos/docwire/vcpkg/installed/x64-linux-dynamic/share/docwire
+BUILD_DIR=./build
+DEMO_EXEC=demo
+
+
+# ==========================
+# CLEAN BUILD
+# ==========================
+echo "Cleaning old build folder..."
+rm -rf "$BUILD_DIR"
+mkdir "$BUILD_DIR"
+cd "$BUILD_DIR"
+
+# ==========================
+# CONFIGURE CMAKE
+# ==========================
+echo "Configuring CMake..."
+cmake .. \
+  -DCMAKE_TOOLCHAIN_FILE="$VCPKG_TOOLCHAIN" \
+  -DVCPKG_TARGET_TRIPLET="$VCPKG_TRIPLET" \
+  -Ddocwire_DIR="$DOCWIRE_DIR" \
+  -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
+
+# ==========================
+# BUILD
+# ==========================
+echo "Building project..."
+cmake --build .
+
+# ==========================
+# SET LIBRARY PATH
+# ==========================
+export LD_LIBRARY_PATH=/home/reeshabh/coderepos/docwire/vcpkg/installed/x64-linux-dynamic/lib:$LD_LIBRARY_PATH
+
+# ==========================
+# RUN DEMO
+# ==========================
+# echo "Running demo..."
+# ./$DEMO_EXEC

download_llama_model.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -e
+
+# Configurable defaults
+
+MODEL_NAME="${MODEL_NAME:-qwen2-7b-instruct}"
+MODEL_QUANT="${MODEL_QUANT:-q4_k_m}"
+MODEL_REPO="${MODEL_REPO:-Qwen/Qwen2-7B-Instruct-GGUF}"
+
+# Derived values
+MODEL_FILE="${MODEL_NAME}.${MODEL_QUANT}.gguf"
+OUTPUT_DIR="${OUTPUT_DIR:-models}"
+HF_URL="https://huggingface.co/${MODEL_REPO}/resolve/main/${MODEL_FILE}"
+
+# Checks
+if ! command -v wget &> /dev/null && ! command -v curl &> /dev/null; then
+    echo "Error: Neither wget nor curl is installed."
+    exit 1
+fi
+
+mkdir -p "${OUTPUT_DIR}"
+cd "${OUTPUT_DIR}"
+
+if [ -f "${MODEL_FILE}" ]; then
+    echo "Model already exists: ${OUTPUT_DIR}/${MODEL_FILE}"
+    exit 0
+fi
+
+echo "Downloading model:"
+echo "  Repository : ${MODEL_REPO}"
+echo "  File       : ${MODEL_FILE}"
+echo "  Destination: ${OUTPUT_DIR}"
+echo ""
+
+# Download
+
+if command -v wget &> /dev/null; then
+    wget -c "${HF_URL}"
+else
+    curl -L -C - -o "${MODEL_FILE}" "${HF_URL}"
+fi
+
+echo ""
+echo "Download complete."
+echo "Model saved to: ${OUTPUT_DIR}/${MODEL_FILE}"

ports/docwire/portfile.cmake

@@ -13,6 +13,8 @@ vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS
 		asan ADDRESS_SANITIZER
 		tsan THREAD_SANITIZER
 		helgrind HELGRIND_ENABLED
+		local-ai DOCWIRE_LOCAL_CT2
+        llama-engine DOCWIRE_LLAMA
 )
 
 if(DEFINED ENV{CMAKE_MESSAGE_LOG_LEVEL})

ports/docwire/vcpkg.json

@@ -28,6 +28,32 @@
 		"callgrind":
 		{
 			"description": "Enable valgrind callgrind in automatic tests"
+		},
+		"local-ai":
+		{
+			"description": "Enable local AI runtime",
+			"dependencies": [
+			"ctranslate2",
+			"sentencepiece",
+			"multilingual-e5-small-ct2-int8",
+			"flan-t5-large-ct2-int8"
+			]
+	    },
+		"llama-engine":
+		{
+			"description": "Enable GGUF-based LLM inference (llama.cpp)",
+			"dependencies": [
+			{ "name": "docwire", "features": ["local-ai"] },
+			"llama-cpp"
+			]
+		},
+    	"llama-qwen":
+		{
+			"description": "Install Qwen2 7B GGUF model",
+			"dependencies": [
+			{ "name": "docwire", "features": ["llama-engine"] },
+			"qwen2-7b-instruct-q4-k-m"
+			]
 		}
 	},
 	"dependencies": [
@@ -99,18 +125,6 @@
 		{
 			"name": "tessdata-fast"
 		},
-		{
-			"name": "ctranslate2"
-		},
-		{
-			"name": "sentencepiece"
-		},
-		{
-			"name": "flan-t5-large-ct2-int8"
-		},
-		{
-			"name": "multilingual-e5-small-ct2-int8"
-		},
 		{
 			"name": "rapidfuzz-cpp"
 		},

ports/qwen2-7b-instruct-q4-k-m/portfile.cmake

@@ -0,0 +1,21 @@
+set(MODEL_NAME "qwen2-7b-instruct")
+set(MODEL_QUANT "q4_k_m")
+
+set(MODEL_FILE "${MODEL_NAME}-${MODEL_QUANT}.gguf")
+
+vcpkg_download_distfile(
+    MODEL_ARCHIVE
+    URLS "https://huggingface.co/Qwen/Qwen2-7B-Instruct-GGUF/resolve/main/${MODEL_FILE}"
+    FILENAME "${MODEL_FILE}"
+    SHA512 39c1f9702856cf5faff13b672033c5c99246b5393550ed58ab9ba0eb2d5ce5d50cc2710b2d9f08d51ad6ce7f6b66826f5c916128fa06c3ac2f78865e167146b8
+)
+
+file(INSTALL
+    ${MODEL_ARCHIVE}
+    DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT}
+)
+
+file(WRITE
+    ${CURRENT_PACKAGES_DIR}/share/${PORT}/copyright
+    "Model weights from HuggingFace repository Qwen/Qwen2-7B-Instruct-GGUF."
+)

ports/qwen2-7b-instruct-q4-k-m/vcpkg.json

@@ -0,0 +1,7 @@
+{
+    "name": "qwen2-7b-instruct-q4-k-m",
+    "version": "1.0.0",
+    "description": "Qwen2 7B Instruct GGUF model",
+    "homepage": "https://huggingface.co/Qwen/Qwen2-7B-Instruct-GGUF",
+    "license": "Apache-2.0"
+}

src/CMakeLists.txt

@@ -23,7 +23,19 @@ include(ocr.cmake)
 include(mail.cmake)
 include(archives.cmake)
 include(ai.cmake)
-include(local_ai.cmake)
+
+if(DOCWIRE_LOCAL_CT2)
+    include(ai_ct2.cmake)
+endif()
+
+if(DOCWIRE_LLAMA)
+    include(ai_llama.cmake)
+endif()
+
+if(DOCWIRE_LOCAL_CT2 OR DOCWIRE_LLAMA)
+    include(local_ai.cmake)
+endif()
+
 include(fuzzy_match.cmake)
 include(content_type.cmake)
 

src/ai.cmake

@@ -1,13 +1,7 @@
-set(EMPTY_SOURCE ${CMAKE_CURRENT_BINARY_DIR}/ai_empty.cpp)
-file(GENERATE OUTPUT ${EMPTY_SOURCE} CONTENT "
-	#include \"ai_elements.h\"
-	namespace docwire::ai
-	{
-		// This dummy function is required to ensure that the shared library is created.
-		DOCWIRE_AI_EXPORT void dummy_function_for_docwire_ai() {}
-	}
-")
-add_library(docwire_ai SHARED ${EMPTY_SOURCE})
+
+add_library(docwire_ai SHARED model_chain_element.cpp ai_summarize.cpp ai_translate.cpp ai_embed.cpp ai_task.cpp)
+
+target_link_libraries(docwire_ai PUBLIC docwire_core)
 
 target_compile_features(docwire_ai PUBLIC cxx_std_20)
 if(MSVC)
@@ -27,4 +21,4 @@ endif()
 include(GenerateExportHeader)
 generate_export_header(docwire_ai EXPORT_FILE_NAME ai_export.h)
 target_include_directories(docwire_ai PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>)
-install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ai_export.h DESTINATION include/docwire)
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ai_export.h DESTINATION include/docwire)

src/ai_ct2.cmake

@@ -0,0 +1,51 @@
+
+message(STATUS "DOCWIRE_LOCAL_CT2 enabled: Building CT2 backend.")
+
+add_library(docwire_ai_ct2 SHARED ct2_runner.cpp tokenizer.cpp)
+
+target_compile_definitions(docwire_ai_ct2 PUBLIC DOCWIRE_LOCAL_CT2)
+
+find_package(Boost REQUIRED COMPONENTS filesystem system json)
+find_package(ctranslate2 CONFIG REQUIRED)
+find_library(sentencepiece_LIBRARIES sentencepiece REQUIRED)
+
+if(MSVC)
+    find_package(absl CONFIG REQUIRED)
+    list(APPEND sentencepiece_LIBRARIES
+        absl::strings
+        absl::flags
+        absl::flags_parse
+        absl::log
+        absl::check)
+
+    find_package(protobuf CONFIG REQUIRED)
+    list(APPEND sentencepiece_LIBRARIES protobuf::libprotobuf-lite)
+endif()
+
+target_link_libraries(docwire_ai_ct2 PRIVATE docwire_core docwire_ai Boost::filesystem Boost::system Boost::json CTranslate2::ctranslate2 ${sentencepiece_LIBRARIES})
+
+docwire_find_resource(FLAN_T5_FULL_PATH REL_PATH "flan-t5-large-ct2-int8" REQUIRED)
+docwire_target_resources(docwire_ai_ct2 "flan-t5-large-ct2-int8" SOURCE "${FLAN_T5_FULL_PATH}")
+
+docwire_find_resource(E5_MODEL_FULL_PATH REL_PATH "multilingual-e5-small-ct2-int8" REQUIRED)
+docwire_target_resources(docwire_ai_ct2 "multilingual-e5-small-ct2-int8" SOURCE "${E5_MODEL_FULL_PATH}")
+
+if(MSVC)
+    install(FILES $<TARGET_PDB_FILE:docwire_ai_ct2> DESTINATION bin CONFIGURATIONS Debug)
+endif()
+
+include(GenerateExportHeader)
+
+generate_export_header(docwire_ai_ct2 EXPORT_FILE_NAME ai_ct2_export.h)
+
+target_include_directories(docwire_ai_ct2 PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    $<INSTALL_INTERFACE:include/docwire>
+)
+
+install(TARGETS docwire_ai_ct2 EXPORT docwire_targets)
+
+install(FILES
+    ${CMAKE_CURRENT_BINARY_DIR}/ai_ct2_export.h
+    DESTINATION include/docwire
+)