Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion packages/app-expo/src/components/rag/ExtractorWebView.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,22 @@ import { WebView } from "react-native-webview";

const READER_HTML_ASSET = Asset.fromModule(require("../../../assets/reader/reader.html"));

const EXTRACTOR_EXTENSIONS_BY_MIME: Record<string, string> = {
"application/epub+zip": "epub",
"application/pdf": "pdf",
"application/x-mobipocket-ebook": "mobi",
"application/vnd.amazon.ebook": "azw3",
"application/vnd.comicbook+zip": "cbz",
"application/x-fictionbook+xml": "fb2",
"application/x-zip-compressed-fb2": "fbz",
"text/plain": "txt",
};

function getExtractorFileName(mimeType: string) {
const normalized = mimeType.split(";")[0]?.trim().toLowerCase() || "application/epub+zip";
return `book.${EXTRACTOR_EXTENSIONS_BY_MIME[normalized] || "epub"}`;
}

export interface ExtractorRef {
extractChapters: (base64BookData: string, mimeType?: string) => Promise<ChapterData[]>;
}
Expand Down Expand Up @@ -87,8 +103,8 @@ export const ExtractorWebView = forwardRef<ExtractorRef>((_, ref) => {
const cmd = {
type: "openBook",
base64: base64BookData,
fileName: "book.epub",
mimeType,
fileName: getExtractorFileName(mimeType),
};

webViewRef.current.injectJavaScript(`
Expand Down
105 changes: 67 additions & 38 deletions packages/app-expo/src/screens/library/useVectorizationQueue.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,35 @@
import type { ExtractorRef } from "@/components/rag/ExtractorWebView";
import { triggerVectorizeBook } from "@/lib/rag/vectorize-trigger";
import type { RootStackParamList } from "@/navigation/RootNavigator";
import { useVectorModelStore } from "@/stores/vector-model-store";
import type { NativeStackNavigationProp } from "@react-navigation/native-stack";
import { getPlatformService } from "@readany/core/services";
import type { Book } from "@readany/core/types";
import * as FileSystem from "expo-file-system/legacy";
import { useCallback, useRef, useState } from "react";
import { useTranslation } from "react-i18next";
import { Alert } from "react-native";
import type { NativeStackNavigationProp } from "@react-navigation/native-stack";
import type { RootStackParamList } from "@/navigation/RootNavigator";

type Nav = NativeStackNavigationProp<RootStackParamList>;

const VECTORIZE_MIME_TYPES: Record<string, string> = {
epub: "application/epub+zip",
pdf: "application/pdf",
mobi: "application/x-mobipocket-ebook",
azw: "application/vnd.amazon.ebook",
azw3: "application/vnd.amazon.ebook",
cbz: "application/vnd.comicbook+zip",
cbr: "application/vnd.comicbook+zip",
fb2: "application/x-fictionbook+xml",
fbz: "application/x-zip-compressed-fb2",
txt: "text/plain",
umd: "application/octet-stream",
};

function getVectorizeMimeType(book: Book) {
return VECTORIZE_MIME_TYPES[String(book.format || "").toLowerCase()] || "application/epub+zip";
}

interface UseVectorizationQueueOptions {
extractorRef: React.RefObject<ExtractorRef | null>;
nav: Nav;
Expand All @@ -30,50 +48,61 @@ export function useVectorizationQueue({ extractorRef, nav }: UseVectorizationQue
} | null>(null);
const isProcessingRef = useRef(false);

const processOneBook = useCallback(async (book: Book) => {
setVectorizingBookId(book.id);
setVectorizingBookTitle(book.meta.title);
setVectorProgress({ status: "chunking", processedChunks: 0, totalChunks: 0 });

try {
if (!extractorRef.current) {
throw new Error("Extractor WebView not ready");
}

const platform = getPlatformService();
const appData = await platform.getAppDataDir();
const absPath = await platform.joinPath(appData, book.filePath);

const base64 = await FileSystem.readAsStringAsync(absPath, {
encoding: FileSystem.EncodingType.Base64,
});

const chapters = await extractorRef.current.extractChapters(base64, "application/epub+zip");
if (!chapters || chapters.length === 0) {
throw new Error("No chapters extracted from book");
const processOneBook = useCallback(
async (book: Book) => {
setVectorizingBookId(book.id);
setVectorizingBookTitle(book.meta.title);
setVectorProgress({ status: "chunking", processedChunks: 0, totalChunks: 0 });

try {
if (!extractorRef.current) {
throw new Error("Extractor WebView not ready");
}

const platform = getPlatformService();
const appData = await platform.getAppDataDir();
const absPath = await platform.joinPath(appData, book.filePath);

const base64 = await FileSystem.readAsStringAsync(absPath, {
encoding: FileSystem.EncodingType.Base64,
});

const chapters = await extractorRef.current.extractChapters(
base64,
getVectorizeMimeType(book),
);
if (!chapters || chapters.length === 0) {
throw new Error("No chapters extracted from book");
}

await triggerVectorizeBook(book.id, book.filePath, chapters, (progress) => {
setVectorProgress(progress);
});

setVectorProgress({ status: "completed", processedChunks: 1, totalChunks: 1 });
await new Promise((resolve) => setTimeout(resolve, 800));
} catch (err) {
console.error(
`[useVectorizationQueue] Vectorization failed for "${book.meta.title}":`,
err,
);
setVectorProgress({ status: "error", processedChunks: 0, totalChunks: 0 });
await new Promise((resolve) => setTimeout(resolve, 1500));
}

await triggerVectorizeBook(book.id, book.filePath, chapters, (progress) => {
setVectorProgress(progress);
});

setVectorProgress({ status: "completed", processedChunks: 1, totalChunks: 1 });
await new Promise((resolve) => setTimeout(resolve, 800));
} catch (err) {
console.error(`[useVectorizationQueue] Vectorization failed for "${book.meta.title}":`, err);
setVectorProgress({ status: "error", processedChunks: 0, totalChunks: 0 });
await new Promise((resolve) => setTimeout(resolve, 1500));
}
}, [extractorRef]);
},
[extractorRef],
);

const processQueue = useCallback(async () => {
if (isProcessingRef.current) return;
isProcessingRef.current = true;

try {
while (vectorQueueRef.current.length > 0) {
const nextBook = vectorQueueRef.current[0]!;
vectorQueueRef.current = vectorQueueRef.current.slice(1);
const [nextBook, ...remainingBooks] = vectorQueueRef.current;
if (!nextBook) break;

vectorQueueRef.current = remainingBooks;
setVectorQueue([...vectorQueueRef.current]);
await processOneBook(nextBook);
}
Expand Down