From 2839e9d8a2ec2b9a3307236668782d93663868a1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 16 Sep 2026 07:32:07 +0000 Subject: [PATCH 1/2] Fix Windows knowledge-mount UTF-8 paths and allow mount-first preview. Keep folder picker and mount listing on UTF-8 so Chinese paths and filenames are not shown as Latin-1/CP1252 mojibake. Mount and list preview require only the knowledge feature switch, not a ready embedding model. Co-authored-by: XYAI Labs --- dashboard/src/locales/en.json | 1 + dashboard/src/locales/zh.json | 1 + dashboard/src/utils/desktopFolder.test.ts | 18 +++ dashboard/src/utils/desktopFolder.ts | 7 +- dashboard/src/utils/utf8Mojibake.test.ts | 18 +++ dashboard/src/utils/utf8Mojibake.ts | 55 ++++++++ desktop/portable/templates/start.bat | 2 + desktop/portable/templates/start.sh | 2 + desktop/src/folder_dialog.go | 146 +++++++++++++++++++++- desktop/src/folder_dialog_test.go | 51 ++++++++ desktop/src/process.go | 2 + desktop/src/settings_home_test.go | 3 + src/octop/api/routers/knowledge_bases.py | 29 ++++- src/octop/i18n/en.json | 1 + src/octop/i18n/zh.json | 1 + src/octop/infra/errors.py | 2 + src/octop/infra/knowledge/local_mount.py | 30 +++-- src/octop/infra/utils/host_dirs.py | 17 ++- src/octop/infra/utils/win_utf8.py | 21 ++++ tests/unit/api/test_knowledge_bases.py | 115 +++++++++++++++++ tests/unit/desktop/test_nsis_uninstall.py | 10 ++ tests/unit/infra/utils/test_host_dirs.py | 13 ++ tests/unit/infra/utils/test_win_utf8.py | 13 ++ tests/unit/test_knowledge_mount.py | 26 ++++ 24 files changed, 564 insertions(+), 20 deletions(-) create mode 100644 dashboard/src/utils/utf8Mojibake.test.ts create mode 100644 dashboard/src/utils/utf8Mojibake.ts create mode 100644 desktop/src/folder_dialog_test.go create mode 100644 src/octop/infra/utils/win_utf8.py create mode 100644 tests/unit/infra/utils/test_win_utf8.py diff --git a/dashboard/src/locales/en.json b/dashboard/src/locales/en.json index f00da4e8..15cca0ae 100644 --- a/dashboard/src/locales/en.json +++ b/dashboard/src/locales/en.json @@ -299,6 +299,7 @@ "KNOWLEDGE_UNSUPPORTED_TYPE": "This document type is not supported for knowledge bases.", "KNOWLEDGE_NAME_TAKEN": "You already have a knowledge base with this name.", "KNOWLEDGE_NAME_INVALID": "Invalid name.", + "KNOWLEDGE_MOUNT_INVALID": "The knowledge-base folder path is invalid or the directory does not exist.", "AVATAR_INVALID": "Avatar must be a PNG, JPEG, WebP, or GIF image.", "AVATAR_TOO_LARGE": "Avatar exceeds the 5 MB size limit.", "INVITE_INVALID": "This invite code is invalid.", diff --git a/dashboard/src/locales/zh.json b/dashboard/src/locales/zh.json index c8d589fc..8e30722e 100644 --- a/dashboard/src/locales/zh.json +++ b/dashboard/src/locales/zh.json @@ -299,6 +299,7 @@ "KNOWLEDGE_UNSUPPORTED_TYPE": "知识库不支持此文档类型。", "KNOWLEDGE_NAME_TAKEN": "您已有同名知识库。", "KNOWLEDGE_NAME_INVALID": "名称无效。", + "KNOWLEDGE_MOUNT_INVALID": "知识库文件夹路径无效，或目录不存在。", "AVATAR_INVALID": "头像须为 PNG、JPEG、WebP 或 GIF 图片。", "AVATAR_TOO_LARGE": "头像超过 5 MB 大小限制。", "INVITE_INVALID": "邀请码无效。", diff --git a/dashboard/src/utils/desktopFolder.test.ts b/dashboard/src/utils/desktopFolder.test.ts index a92304c8..00edda90 100644 --- a/dashboard/src/utils/desktopFolder.test.ts +++ b/dashboard/src/utils/desktopFolder.test.ts @@ -28,6 +28,24 @@ describe("desktopFolder", () => { expect(path).toBe("D:\\Projects\\demo"); }); + it("repairs UTF-8 Chinese paths that arrived as Windows-1252 mojibake", async () => { + const bytes = new TextEncoder().encode("D:\\项目"); + const garbled = Array.from(bytes, (byte) => String.fromCharCode(byte)).join( + "", + ); + const invoke = vi.fn(() => { + window.dispatchEvent( + new CustomEvent("freeos-folder-selected", { detail: garbled }), + ); + }); + const win = { + _wails: { invoke }, + addEventListener: window.addEventListener.bind(window), + removeEventListener: window.removeEventListener.bind(window), + } as unknown as Window & { _wails?: { invoke?: (m: string) => void } }; + await expect(pickDesktopFolder(win, 1000)).resolves.toBe("D:\\项目"); + }); + it("returns null when the user cancels", async () => { const invoke = vi.fn(() => { window.dispatchEvent(new CustomEvent("freeos-folder-selected", { detail: "" })); diff --git a/dashboard/src/utils/desktopFolder.ts b/dashboard/src/utils/desktopFolder.ts index 6b7ada04..d447dfa1 100644 --- a/dashboard/src/utils/desktopFolder.ts +++ b/dashboard/src/utils/desktopFolder.ts @@ -1,4 +1,5 @@ import { isDesktopShell } from "./desktopChrome"; +import { repairUtf8Mojibake } from "./utf8Mojibake"; type DesktopWindow = Window & { _wails?: { invoke?: (message: string) => void }; @@ -29,7 +30,11 @@ export function pickDesktopFolder( }; const onSelected = (event: Event) => { const detail = (event as CustomEvent).detail; - finish(typeof detail === "string" && detail.trim() ? detail.trim() : null); + finish( + typeof detail === "string" && detail.trim() + ? repairUtf8Mojibake(detail.trim()) + : null, + ); }; const timer = window.setTimeout(() => finish(null), timeoutMs); win.addEventListener(FOLDER_EVENT, onSelected as EventListener); diff --git a/dashboard/src/utils/utf8Mojibake.test.ts b/dashboard/src/utils/utf8Mojibake.test.ts new file mode 100644 index 00000000..1f3324f6 --- /dev/null +++ b/dashboard/src/utils/utf8Mojibake.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest"; +import { repairUtf8Mojibake } from "./utf8Mojibake"; + +describe("repairUtf8Mojibake", () => { + it("restores UTF-8 Chinese that was decoded as Windows-1252", () => { + const bytes = new TextEncoder().encode("项目结构.png"); + const garbled = Array.from(bytes, (byte) => String.fromCharCode(byte)).join( + "", + ); + expect(garbled).not.toBe("项目结构.png"); + expect(repairUtf8Mojibake(garbled)).toBe("项目结构.png"); + }); + + it("leaves already-valid Unicode and ASCII alone", () => { + expect(repairUtf8Mojibake("项目结构.png")).toBe("项目结构.png"); + expect(repairUtf8Mojibake("D:\\docs\\notes")).toBe("D:\\docs\\notes"); + }); +}); diff --git a/dashboard/src/utils/utf8Mojibake.ts b/dashboard/src/utils/utf8Mojibake.ts new file mode 100644 index 00000000..da51d1f3 --- /dev/null +++ b/dashboard/src/utils/utf8Mojibake.ts @@ -0,0 +1,55 @@ +/** Map Unicode code points that CP1252 uses for bytes 0x80–0x9F back to those bytes. */ +const CP1252_REVERSE: Record = { + 0x20ac: 0x80, + 0x201a: 0x82, + 0x0192: 0x83, + 0x201e: 0x84, + 0x2026: 0x85, + 0x2020: 0x86, + 0x2021: 0x87, + 0x02c6: 0x88, + 0x2030: 0x89, + 0x0160: 0x8a, + 0x2039: 0x8b, + 0x0152: 0x8c, + 0x017d: 0x8e, + 0x2018: 0x91, + 0x2019: 0x92, + 0x201c: 0x93, + 0x201d: 0x94, + 0x2022: 0x95, + 0x2013: 0x96, + 0x2014: 0x97, + 0x02dc: 0x98, + 0x2122: 0x99, + 0x0161: 0x9a, + 0x203a: 0x9b, + 0x0153: 0x9c, + 0x017e: 0x9e, + 0x0178: 0x9f, +}; + +function cp1252Byte(code: number): number | null { + if (code <= 0xff) return code; + return CP1252_REVERSE[code] ?? null; +} + +/** + * Repair UTF-8 text that a Windows bridge exposed as Latin-1 / CP1252. + * Example: ``项目结构.png`` → ``项目结构.png``. + */ +export function repairUtf8Mojibake(text: string): string { + if (!text) return text; + const bytes = new Uint8Array(text.length); + for (let i = 0; i < text.length; i += 1) { + const mapped = cp1252Byte(text.charCodeAt(i)); + if (mapped === null) return text; + bytes[i] = mapped; + } + try { + const decoded = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + return decoded || text; + } catch { + return text; + } +} diff --git a/desktop/portable/templates/start.bat b/desktop/portable/templates/start.bat index 69866d2d..1c6f65e0 100644 --- a/desktop/portable/templates/start.bat +++ b/desktop/portable/templates/start.bat @@ -86,6 +86,8 @@ if not exist "%ROOT%\launch.py" ( rem Prefer launch.py (site.addsitedir + pywin32 DLL path). Do not set PYTHONPATH. set "PYTHONNOUSERSITE=1" +set "PYTHONUTF8=1" +set "PYTHONIOENCODING=utf-8" set "PYTHONPATH=" if exist "%ROOT%\org-sidecar\start-sidecar.bat" ( diff --git a/desktop/portable/templates/start.sh b/desktop/portable/templates/start.sh index 9065516b..9e5ebf13 100755 --- a/desktop/portable/templates/start.sh +++ b/desktop/portable/templates/start.sh @@ -74,6 +74,8 @@ fi # launch.py adds packages/ via site.addsitedir (honours .pth / pywin32). export PYTHONNOUSERSITE=1 +export PYTHONUTF8=1 +export PYTHONIOENCODING=utf-8 unset PYTHONPATH || true if [[ -x "${ROOT}/org-sidecar/start-sidecar.sh" ]]; then diff --git a/desktop/src/folder_dialog.go b/desktop/src/folder_dialog.go index 374fead6..41aef96c 100644 --- a/desktop/src/folder_dialog.go +++ b/desktop/src/folder_dialog.go @@ -1,11 +1,14 @@ package main import ( + "encoding/base64" "fmt" "log" "os/exec" "runtime" "strings" + "unicode/utf16" + "unicode/utf8" ) func (a *App) emitPickedFolder() { @@ -42,6 +45,8 @@ func pickFolderNative() (string, error) { func pickFolderWindows() (string, error) { // FolderBrowserDialog with MyComputer so every drive letter is reachable. + // SelectedPath is a .NET Unicode string; emit UTF-8 as Base64 so the + // hidden-window PowerShell console code page cannot mojibake CJK. script := strings.Join( []string{ "Add-Type -AssemblyName System.Windows.Forms", @@ -50,7 +55,9 @@ func pickFolderWindows() (string, error) { "$d.Description = 'Select a work folder'", "$d.RootFolder = [System.Environment+SpecialFolder]::MyComputer", "$d.ShowNewFolderButton = $true", - "if ($d.ShowDialog() -eq [System.Windows.Forms.DialogResult]::OK) { $d.SelectedPath }", + "if ($d.ShowDialog() -ne [System.Windows.Forms.DialogResult]::OK) { exit 0 }", + "$bytes = [System.Text.Encoding]::UTF8.GetBytes($d.SelectedPath)", + "[Convert]::ToBase64String($bytes)", }, "; ", ) @@ -67,11 +74,142 @@ func pickFolderWindows() (string, error) { if err != nil { return "", fmt.Errorf("folder dialog: %w", err) } - path := strings.TrimSpace(string(out)) - if path == "" { + return decodeFolderPickerOutput(out) +} + +func decodeFolderPickerOutput(out []byte) (string, error) { + raw := strings.TrimSpace(normalizePickerBytes(out)) + if raw == "" { return "", nil } - return path, nil + if i := strings.LastIndex(raw, "\n"); i >= 0 { + if last := strings.TrimSpace(raw[i+1:]); last != "" { + raw = last + } + } + if decoded, err := base64.StdEncoding.DecodeString(raw); err == nil && len(decoded) > 0 && utf8.Valid(decoded) { + return string(decoded), nil + } + return repairUTF8Mojibake(raw), nil +} + +func normalizePickerBytes(out []byte) string { + if len(out) >= 2 && out[0] == 0xFF && out[1] == 0xFE { + return utf16LEToString(out[2:]) + } + if looksLikeUTF16LE(out) { + return utf16LEToString(out) + } + return string(out) +} + +func looksLikeUTF16LE(b []byte) bool { + if len(b) < 4 || len(b)%2 != 0 { + return false + } + zeros := 0 + pairs := 0 + for i := 1; i < len(b); i += 2 { + pairs++ + if b[i] == 0 { + zeros++ + } + } + return pairs > 0 && zeros*2 >= pairs +} + +func utf16LEToString(b []byte) string { + if len(b)%2 == 1 { + b = b[:len(b)-1] + } + u := make([]uint16, 0, len(b)/2) + for i := 0; i+1 < len(b); i += 2 { + u = append(u, uint16(b[i])|uint16(b[i+1])<<8) + } + return string(utf16.Decode(u)) +} + +func repairUTF8Mojibake(text string) string { + raw := make([]byte, 0, len(text)) + for _, r := range text { + if mapped, ok := cp1252Byte(r); ok { + raw = append(raw, mapped) + continue + } + return text + } + if !utf8.Valid(raw) { + return text + } + repaired := string(raw) + if repaired != text { + return repaired + } + return text +} + +func cp1252Byte(r rune) (byte, bool) { + if r <= 0xFF { + return byte(r), true + } + switch r { + case 0x20AC: + return 0x80, true + case 0x201A: + return 0x82, true + case 0x0192: + return 0x83, true + case 0x201E: + return 0x84, true + case 0x2026: + return 0x85, true + case 0x2020: + return 0x86, true + case 0x2021: + return 0x87, true + case 0x02C6: + return 0x88, true + case 0x2030: + return 0x89, true + case 0x0160: + return 0x8A, true + case 0x2039: + return 0x8B, true + case 0x0152: + return 0x8C, true + case 0x017D: + return 0x8E, true + case 0x2018: + return 0x91, true + case 0x2019: + return 0x92, true + case 0x201C: + return 0x93, true + case 0x201D: + return 0x94, true + case 0x2022: + return 0x95, true + case 0x2013: + return 0x96, true + case 0x2014: + return 0x97, true + case 0x02DC: + return 0x98, true + case 0x2122: + return 0x99, true + case 0x0161: + return 0x9A, true + case 0x203A: + return 0x9B, true + case 0x0153: + return 0x9C, true + case 0x017E: + return 0x9E, true + case 0x0178: + return 0x9F, true + default: + return 0, false + } } func pickFolderDarwin() (string, error) { diff --git a/desktop/src/folder_dialog_test.go b/desktop/src/folder_dialog_test.go new file mode 100644 index 00000000..797c9488 --- /dev/null +++ b/desktop/src/folder_dialog_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "encoding/base64" + "testing" +) + +func TestDecodeFolderPickerOutputPrefersUTF8Base64(t *testing.T) { + path := `D:\项目\资料` + encoded := base64.StdEncoding.EncodeToString([]byte(path)) + got, err := decodeFolderPickerOutput([]byte(encoded + "\r\n")) + if err != nil { + t.Fatal(err) + } + if got != path { + t.Fatalf("got %q want %q", got, path) + } +} + +func TestDecodeFolderPickerOutputReadsUTF16LE(t *testing.T) { + path := `D:\项目` + encoded := base64.StdEncoding.EncodeToString([]byte(path)) + utf16 := make([]byte, 0, 2+len(encoded)*2) + utf16 = append(utf16, 0xFF, 0xFE) + for i := 0; i < len(encoded); i++ { + utf16 = append(utf16, encoded[i], 0) + } + got, err := decodeFolderPickerOutput(utf16) + if err != nil { + t.Fatal(err) + } + if got != path { + t.Fatalf("got %q want %q", got, path) + } +} + +func TestRepairUTF8MojibakeChineseFilename(t *testing.T) { + // UTF-8 of 项目结构.png interpreted as Windows-1252. + mojibake := string([]rune{ + 0x00E9, 0x00A1, 0x00B9, + 0x00E7, 0x009B, 0x00AE, + 0x00E7, 0x00BB, 0x201C, + 0x00E6, 0x017E, 0x201E, + }) + ".png" + if got := repairUTF8Mojibake(mojibake); got != "项目结构.png" { + t.Fatalf("got %q", got) + } + if got := repairUTF8Mojibake(`D:\docs`); got != `D:\docs` { + t.Fatalf("ascii changed: %q", got) + } +} diff --git a/desktop/src/process.go b/desktop/src/process.go index 151c7513..5a55f14c 100644 --- a/desktop/src/process.go +++ b/desktop/src/process.go @@ -26,6 +26,8 @@ func hostLaunchEnv(root string, port int) map[string]string { "OCTOP_HOME": home, "OCTOP_GREEN_PACKAGES": filepath.Join(root, "packages"), "PYTHONNOUSERSITE": "1", + "PYTHONUTF8": "1", + "PYTHONIOENCODING": "utf-8", "PYTHONPATH": "", "FREEOS_ORG_ENABLE": "1", "FREEOS_ORG_SIDECAR_URL": sidecarURL(), diff --git a/desktop/src/settings_home_test.go b/desktop/src/settings_home_test.go index 7c4e5992..1b6c57ad 100644 --- a/desktop/src/settings_home_test.go +++ b/desktop/src/settings_home_test.go @@ -54,6 +54,9 @@ func TestHostLaunchEnvSetsFreeosHomeAndOrgEnable(t *testing.T) { if env["OCTOP_DESKTOP"] != "1" || env["FREEOS_DESKTOP"] != "1" { t.Fatalf("desktop flags: %+v", env) } + if env["PYTHONUTF8"] != "1" || env["PYTHONIOENCODING"] != "utf-8" { + t.Fatalf("utf8 env: %+v", env) + } } func TestWithDesktopQueryMarksSpa(t *testing.T) { diff --git a/src/octop/api/routers/knowledge_bases.py b/src/octop/api/routers/knowledge_bases.py index 343a338c..e2e426c2 100644 --- a/src/octop/api/routers/knowledge_bases.py +++ b/src/octop/api/routers/knowledge_bases.py @@ -214,6 +214,19 @@ def _map_knowledge_error( else ErrorCode.KNOWLEDGE_PREREQUISITES_FAILED ) return OctopError.localized(code, locale) + if isinstance(exc, ValueError) and any( + token in text + for token in ( + "source_path", + "distill_path", + "existing directory", + "invalid path", + "cloud_url", + "outside the mounted", + "not a directory", + ) + ): + return OctopError.localized(ErrorCode.KNOWLEDGE_MOUNT_INVALID, locale) if "at most 100" in text: return OctopError.localized(ErrorCode.KNOWLEDGE_DOC_LIMIT, locale) if "document size exceeds" in text: @@ -731,14 +744,18 @@ async def upload_document( max_bytes=_max_upload_bytes(server), code=ErrorCode.KNOWLEDGE_DOC_TOO_LARGE, ) + from octop.infra.utils.win_utf8 import repair_utf8_mojibake + + filename = repair_utf8_mojibake(upload.filename or "") + relative = repair_utf8_mojibake(path or filename) document = _knowledge_service(server).upload_document( kb_id, actor_user_id=user.id, - filename=upload.filename or "", + filename=filename, content_type=upload.content_type or "", content=content, is_admin=_is_admin(user), - path=path or upload.filename or "", + path=relative, ) assert server.services is not None enqueue_index_document(server.services, kb_id, document.id) @@ -990,6 +1007,7 @@ async def get_kb_mount( from octop.infra.knowledge.local_mount import load_mounts, scan_mount try: + _require_enabled(server, request) _knowledge_service(server).get_readable_base( kb_id, actor_user_id=user.id, is_admin=_is_admin(user) ) @@ -1000,7 +1018,7 @@ async def get_kb_mount( return {**mount.to_dict(), "mounted": True, "entries": []} entries = [ {"path": item.path, "name": item.name, "is_dir": item.is_dir, "size": item.size} - for item in scan_mount(mount.source_path) + for item in scan_mount(mount.source_path, preview=True) ] return {**mount.to_dict(), "mounted": True, "entries": entries} except Exception as exc: @@ -1020,6 +1038,7 @@ async def put_kb_mount( from octop.infra.knowledge.local_mount import KnowledgeMount, save_mount, scan_mount try: + _require_enabled(server, request) _knowledge_service(server).get_writable_base( kb_id, actor_user_id=user.id, is_admin=_is_admin(user) ) @@ -1038,7 +1057,7 @@ async def put_kb_mount( return {**stored.to_dict(), "mounted": True, "entries": []} entries = [ {"path": item.path, "name": item.name, "is_dir": item.is_dir, "size": item.size} - for item in scan_mount(stored.source_path) + for item in scan_mount(stored.source_path, preview=True) ] return {**stored.to_dict(), "mounted": True, "entries": entries} except Exception as exc: @@ -1057,6 +1076,7 @@ async def delete_kb_mount( from octop.infra.knowledge.local_mount import clear_mount try: + _require_enabled(server, request) _knowledge_service(server).get_writable_base( kb_id, actor_user_id=user.id, is_admin=_is_admin(user) ) @@ -1087,6 +1107,7 @@ async def distill_kb_mount( ) try: + _require_enabled(server, request) _knowledge_service(server).get_writable_base( kb_id, actor_user_id=user.id, is_admin=_is_admin(user) ) diff --git a/src/octop/i18n/en.json b/src/octop/i18n/en.json index e0fd2274..2eaf74b4 100644 --- a/src/octop/i18n/en.json +++ b/src/octop/i18n/en.json @@ -392,6 +392,7 @@ "KNOWLEDGE_UNSUPPORTED_TYPE": "This document type is not supported for knowledge bases.", "KNOWLEDGE_NAME_TAKEN": "You already have a knowledge base with this name.", "KNOWLEDGE_NAME_INVALID": "Invalid name.", + "KNOWLEDGE_MOUNT_INVALID": "The knowledge-base folder path is invalid or the directory does not exist.", "AVATAR_INVALID": "Avatar must be a PNG, JPEG, WebP, or GIF image.", "AVATAR_TOO_LARGE": "Avatar exceeds the 5 MB size limit.", "INVITE_INVALID": "This invite code is invalid.", diff --git a/src/octop/i18n/zh.json b/src/octop/i18n/zh.json index 272da1ed..6e231f00 100644 --- a/src/octop/i18n/zh.json +++ b/src/octop/i18n/zh.json @@ -392,6 +392,7 @@ "KNOWLEDGE_UNSUPPORTED_TYPE": "知识库不支持此文档类型。", "KNOWLEDGE_NAME_TAKEN": "您已有同名知识库。", "KNOWLEDGE_NAME_INVALID": "名称无效。", + "KNOWLEDGE_MOUNT_INVALID": "知识库文件夹路径无效，或目录不存在。", "AVATAR_INVALID": "头像须为 PNG、JPEG、WebP 或 GIF 图片。", "AVATAR_TOO_LARGE": "头像超过 5 MB 大小限制。", "INVITE_INVALID": "邀请码无效。", diff --git a/src/octop/infra/errors.py b/src/octop/infra/errors.py index 81837027..4ad4349f 100644 --- a/src/octop/infra/errors.py +++ b/src/octop/infra/errors.py @@ -96,6 +96,7 @@ class ErrorCode(StrEnum): KNOWLEDGE_UNSUPPORTED_TYPE = "KNOWLEDGE_UNSUPPORTED_TYPE" KNOWLEDGE_NAME_TAKEN = "KNOWLEDGE_NAME_TAKEN" KNOWLEDGE_NAME_INVALID = "KNOWLEDGE_NAME_INVALID" + KNOWLEDGE_MOUNT_INVALID = "KNOWLEDGE_MOUNT_INVALID" AVATAR_INVALID = "AVATAR_INVALID" AVATAR_TOO_LARGE = "AVATAR_TOO_LARGE" INVITE_INVALID = "INVITE_INVALID" @@ -192,6 +193,7 @@ class ErrorCode(StrEnum): ErrorCode.KNOWLEDGE_UNSUPPORTED_TYPE: 400, ErrorCode.KNOWLEDGE_NAME_TAKEN: 409, ErrorCode.KNOWLEDGE_NAME_INVALID: 400, + ErrorCode.KNOWLEDGE_MOUNT_INVALID: 400, ErrorCode.AVATAR_INVALID: 400, ErrorCode.AVATAR_TOO_LARGE: 413, ErrorCode.INVITE_INVALID: 400, diff --git a/src/octop/infra/knowledge/local_mount.py b/src/octop/infra/knowledge/local_mount.py index 8c7bcae8..3a119b91 100644 --- a/src/octop/infra/knowledge/local_mount.py +++ b/src/octop/infra/knowledge/local_mount.py @@ -14,6 +14,7 @@ from octop.infra.utils.host_dirs import assert_safe_host_path from octop.infra.utils.paths import PathLayout +from octop.infra.utils.win_utf8 import repair_utf8_mojibake _FILE = "knowledge-mounts.json" _PARSEABLE = {".md", ".txt", ".markdown", ".rst", ".csv", ".json", ".html", ".htm"} @@ -131,11 +132,12 @@ def attach_cloud_pointer( def save_mount(mount: KnowledgeMount, home: Path | None = None) -> KnowledgeMount: if mount.kind == "cloud" or mount.cloud_url.strip(): return save_cloud_mount(mount, home) - assert_safe_host_path(mount.source_path) - source = Path(mount.source_path).expanduser().resolve() + source_path = repair_utf8_mojibake(mount.source_path) + assert_safe_host_path(source_path) + source = Path(source_path).expanduser().resolve() if not source.is_dir(): raise ValueError("source_path must be an existing directory") - distill = mount.distill_path.strip() + distill = repair_utf8_mojibake(mount.distill_path).strip() if distill: assert_safe_host_path(distill) dest = Path(distill).expanduser().resolve() @@ -166,8 +168,16 @@ def clear_mount(kb_id: str, home: Path | None = None) -> None: ) -def scan_mount(source_path: str, *, max_docs: int = _MAX_DOCS) -> list[MountEntry]: - """List subdirectories and parseable files. Read-only; does not copy or write.""" +def scan_mount( + source_path: str, *, max_docs: int = _MAX_DOCS, preview: bool = False +) -> list[MountEntry]: + """List subdirectories and files. Read-only; does not copy or write. + + When *preview* is false (distill), only parseable text suffixes are included. + Mount list preview includes every non-hidden entry so Chinese image names + stay visible before embeddings are ready. + """ + source_path = repair_utf8_mojibake(source_path) assert_safe_host_path(source_path) root = Path(source_path).expanduser().resolve() if not root.is_dir(): @@ -176,16 +186,18 @@ def scan_mount(source_path: str, *, max_docs: int = _MAX_DOCS) -> list[MountEntr for child in sorted(root.iterdir(), key=lambda p: (not p.is_dir(), p.name.lower())): if child.name.startswith("."): continue + name = repair_utf8_mojibake(child.name) + path_text = repair_utf8_mojibake(str(child)) if child.is_dir(): - entries.append(MountEntry(path=str(child), name=child.name, is_dir=True, size=0)) + entries.append(MountEntry(path=path_text, name=name, is_dir=True, size=0)) continue - if child.suffix.lower() not in _PARSEABLE: + if not preview and child.suffix.lower() not in _PARSEABLE: continue try: size = child.stat().st_size except OSError: size = 0 - entries.append(MountEntry(path=str(child), name=child.name, is_dir=False, size=size)) + entries.append(MountEntry(path=path_text, name=name, is_dir=False, size=size)) if len(entries) >= max_docs: break return entries @@ -198,6 +210,8 @@ def distill_readonly( max_docs: int = _MAX_DOCS, ) -> dict[str, Any]: """Copy parseable text into a new folder. Never writes back to the source.""" + source_path = repair_utf8_mojibake(source_path) + distill_path = repair_utf8_mojibake(distill_path) assert_safe_host_path(source_path) assert_safe_host_path(distill_path) source = Path(source_path).expanduser().resolve() diff --git a/src/octop/infra/utils/host_dirs.py b/src/octop/infra/utils/host_dirs.py index 0f4deb43..90d08b80 100644 --- a/src/octop/infra/utils/host_dirs.py +++ b/src/octop/infra/utils/host_dirs.py @@ -8,6 +8,8 @@ from pathlib import Path from typing import Any, Literal +from octop.infra.utils.win_utf8 import repair_utf8_mojibake + ProbeCode = Literal[ "not_directory", "permission_denied", @@ -29,7 +31,9 @@ def host_path_text(path: Path) -> str: """Serialize a host path for API/UI (POSIX separators, even on Windows).""" - return Path(os.path.realpath(os.path.expanduser(str(path)))).as_posix() + return repair_utf8_mojibake( + Path(os.path.realpath(os.path.expanduser(str(path)))).as_posix() + ) def host_home_dir() -> Path: @@ -77,7 +81,9 @@ def is_within_host_home(resolved: Path, *, home: Path | None = None) -> bool: def normalize_host_path(path: str) -> Path: """Canonical absolute path via ``os.path.realpath`` (CodeQL-recognized).""" - raw = path.strip() or ("/" if os.name == "posix" else str(Path.home().anchor)) + raw = repair_utf8_mojibake(path.strip()) or ( + "/" if os.name == "posix" else str(Path.home().anchor) + ) return Path(os.path.realpath(os.path.expanduser(raw))) @@ -192,7 +198,12 @@ def list_host_subdirs( continue except OSError: continue - entries.append({"path": host_path_text(resolved), "name": child.name}) + entries.append( + { + "path": host_path_text(resolved), + "name": repair_utf8_mojibake(child.name), + } + ) return entries diff --git a/src/octop/infra/utils/win_utf8.py b/src/octop/infra/utils/win_utf8.py new file mode 100644 index 00000000..50dde9fd --- /dev/null +++ b/src/octop/infra/utils/win_utf8.py @@ -0,0 +1,21 @@ +"""Repair UTF-8 text that Windows bridges exposed as Latin-1 / CP1252.""" + +from __future__ import annotations + + +def repair_utf8_mojibake(text: str) -> str: + """Decode *text* when it is UTF-8 bytes mis-read as Latin-1 or CP1252. + + Typical case: ``项目结构.png`` arrives as ``项目结构.png``. Already-valid + Unicode (including CJK that cannot encode as Latin-1) is returned unchanged. + """ + if not text: + return text + for encoding in ("cp1252", "latin-1"): + try: + repaired = text.encode(encoding).decode("utf-8") + except (UnicodeEncodeError, UnicodeDecodeError): + continue + if repaired != text: + return repaired + return text diff --git a/tests/unit/api/test_knowledge_bases.py b/tests/unit/api/test_knowledge_bases.py index ab4cc4cf..a29bc21b 100644 --- a/tests/unit/api/test_knowledge_bases.py +++ b/tests/unit/api/test_knowledge_bases.py @@ -704,3 +704,118 @@ async def test_update_base_rejects_max_documents_out_of_range() -> None: assert knowledge_bases.UpdateBaseBody(max_documents=0).max_documents == 0 assert knowledge_bases.UpdateBaseBody(max_documents=10_000).max_documents == 10_000 assert knowledge_bases.UpdateBaseBody().max_documents is None + + +@pytest.mark.asyncio +async def test_put_mount_allowed_when_embeddings_not_ready( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from octop.api.routers import knowledge_bases + + source = tmp_path / "项目" + source.mkdir() + (source / "项目结构.png").write_bytes(b"png") + settings = {"knowledge_bases_enabled": "true"} + usable_calls: list[str] = [] + server = SimpleNamespace( + services=_services( + settings_repo=SimpleNamespace(get=settings.get, set=settings.__setitem__), + ), + paths=SimpleNamespace(root=tmp_path), + ) + monkeypatch.setattr( + knowledge_bases, + "_knowledge_service", + lambda _server: SimpleNamespace( + get_writable_base=lambda *_a, **_k: _Base(), + ), + ) + original_usable = knowledge_bases._require_usable + + def deny_usable(*_args: object, **_kwargs: object) -> None: + usable_calls.append("usable") + original_usable(*_args, **_kwargs) + + monkeypatch.setattr(knowledge_bases, "_require_usable", deny_usable) + + response = await knowledge_bases.put_kb_mount( + "kb-1", + knowledge_bases.MountBody(source_path=str(source)), + request=_request(), + server=server, + user=SimpleNamespace(id=1, is_admin=False), + ) + + assert usable_calls == [] + assert response["mounted"] is True + assert {item["name"] for item in response["entries"]} == {"项目结构.png"} + + +@pytest.mark.asyncio +async def test_get_mount_lists_unicode_preview_without_embeddings( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from octop.api.routers import knowledge_bases + from octop.infra.knowledge.local_mount import KnowledgeMount, save_mount + + source = tmp_path / "资料" + source.mkdir() + (source / "项目结构.png").write_bytes(b"png") + save_mount(KnowledgeMount(kb_id="kb-1", source_path=str(source)), tmp_path) + settings = {"knowledge_bases_enabled": "true"} + server = SimpleNamespace( + services=_services( + settings_repo=SimpleNamespace(get=settings.get, set=settings.__setitem__), + ), + paths=SimpleNamespace(root=tmp_path), + ) + monkeypatch.setattr( + knowledge_bases, + "_knowledge_service", + lambda _server: SimpleNamespace( + get_readable_base=lambda *_a, **_k: _Base(), + ), + ) + + response = await knowledge_bases.get_kb_mount( + "kb-1", + request=_request(), + server=server, + user=SimpleNamespace(id=1, is_admin=False), + ) + + assert response["mounted"] is True + assert {item["name"] for item in response["entries"]} == {"项目结构.png"} + + +@pytest.mark.asyncio +async def test_put_mount_missing_dir_is_not_prerequisites( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from octop.api.routers import knowledge_bases + + settings = {"knowledge_bases_enabled": "true"} + server = SimpleNamespace( + services=_services( + settings_repo=SimpleNamespace(get=settings.get, set=settings.__setitem__), + ), + paths=SimpleNamespace(root=tmp_path), + ) + monkeypatch.setattr( + knowledge_bases, + "_knowledge_service", + lambda _server: SimpleNamespace( + get_writable_base=lambda *_a, **_k: _Base(), + ), + ) + + with pytest.raises(OctopError) as raised: + await knowledge_bases.put_kb_mount( + "kb-1", + knowledge_bases.MountBody(source_path=str(tmp_path / "missing")), + request=_request(), + server=server, + user=SimpleNamespace(id=1, is_admin=False), + ) + + assert raised.value.code == ErrorCode.KNOWLEDGE_MOUNT_INVALID diff --git a/tests/unit/desktop/test_nsis_uninstall.py b/tests/unit/desktop/test_nsis_uninstall.py index a60449e2..03371099 100644 --- a/tests/unit/desktop/test_nsis_uninstall.py +++ b/tests/unit/desktop/test_nsis_uninstall.py @@ -212,3 +212,13 @@ def test_organization_source_download_uses_native_folder_picker() -> None: assert "canPickDesktopFolder" in page assert "resolveOpenxyosSourceDest" in page assert "browseSourceDest" in page + + +def test_windows_folder_picker_emits_utf8_base64() -> None: + source = (REPO / "desktop" / "src" / "folder_dialog.go").read_text(encoding="utf-8") + assert "[System.Text.Encoding]::UTF8.GetBytes($d.SelectedPath)" in source + assert "[Convert]::ToBase64String($bytes)" in source + assert "decodeFolderPickerOutput" in source + process = (REPO / "desktop" / "src" / "process.go").read_text(encoding="utf-8") + assert "PYTHONUTF8" in process + assert "utf-8" in process diff --git a/tests/unit/infra/utils/test_host_dirs.py b/tests/unit/infra/utils/test_host_dirs.py index 8c884e60..3a390806 100644 --- a/tests/unit/infra/utils/test_host_dirs.py +++ b/tests/unit/infra/utils/test_host_dirs.py @@ -49,6 +49,19 @@ def test_list_host_subdirs_rejects_missing_path(tmp_path: Path) -> None: list_host_subdirs(str(tmp_path / "missing")) +def test_list_host_subdirs_keeps_unicode_names(tmp_path: Path) -> None: + (tmp_path / "项目资料").mkdir() + entries = list_host_subdirs(str(tmp_path)) + assert [item["name"] for item in entries] == ["项目资料"] + + +def test_normalize_host_path_repairs_cp1252_mojibake(tmp_path: Path) -> None: + real = tmp_path / "项目" + real.mkdir() + garbled = str(real).encode("utf-8").decode("cp1252") + assert normalize_host_path(garbled) == real.resolve() + + @posix_only def test_assert_safe_host_path_rejects_proc() -> None: with pytest.raises(ValueError, match="not allowed"): diff --git a/tests/unit/infra/utils/test_win_utf8.py b/tests/unit/infra/utils/test_win_utf8.py new file mode 100644 index 00000000..90ed1498 --- /dev/null +++ b/tests/unit/infra/utils/test_win_utf8.py @@ -0,0 +1,13 @@ +from octop.infra.utils.win_utf8 import repair_utf8_mojibake + + +def test_repair_utf8_mojibake_restores_chinese_filename() -> None: + garbled = "项目结构.png".encode("utf-8").decode("cp1252") + assert garbled != "项目结构.png" + assert repair_utf8_mojibake(garbled) == "项目结构.png" + + +def test_repair_utf8_mojibake_leaves_real_unicode_and_ascii() -> None: + assert repair_utf8_mojibake("项目结构.png") == "项目结构.png" + assert repair_utf8_mojibake(r"D:\docs\notes") == r"D:\docs\notes" + assert repair_utf8_mojibake("") == "" diff --git a/tests/unit/test_knowledge_mount.py b/tests/unit/test_knowledge_mount.py index 47a8efd5..57f289d0 100644 --- a/tests/unit/test_knowledge_mount.py +++ b/tests/unit/test_knowledge_mount.py @@ -73,3 +73,29 @@ def test_save_mount_persists(tmp_path: Path) -> None: ) assert stored.readonly is True assert stored.source_path.endswith("docs") + + +def test_scan_preview_lists_unicode_image_names(tmp_path: Path) -> None: + source = tmp_path / "资料" + source.mkdir() + (source / "项目结构.png").write_bytes(b"png") + (source / "notes.md").write_text("hi", encoding="utf-8") + preview = {item.name for item in scan_mount(str(source), preview=True)} + distill = {item.name for item in scan_mount(str(source))} + assert preview == {"项目结构.png", "notes.md"} + assert distill == {"notes.md"} + + +def test_save_mount_repairs_cp1252_mojibake_path(tmp_path: Path) -> None: + source = tmp_path / "项目" + source.mkdir() + (source / "说明.md").write_text("ok", encoding="utf-8") + garbled = str(source).encode("utf-8").decode("cp1252") + assert garbled != str(source) + stored = save_mount( + KnowledgeMount(kb_id="kb-zh", source_path=garbled, distill_path=""), + tmp_path, + ) + assert Path(stored.source_path) == source.resolve() + names = {item.name for item in scan_mount(garbled, preview=True)} + assert names == {"说明.md"} From 9b62d52f730485d5d2689e3309ba885627ac4d2b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 16 Sep 2026 07:35:11 +0000 Subject: [PATCH 2/2] Harden mount UTF-8 tests and keep upload path repair string-only. Form defaults from FastAPI must not be treated as filenames. Changelog notes the Windows mojibake and mount-first 409 fix. Co-authored-by: XYAI Labs --- CHANGELOG.md | 4 ++++ dashboard/src/utils/desktopFolder.test.ts | 8 ++++++-- src/octop/api/routers/knowledge_bases.py | 4 ++-- src/octop/infra/utils/host_dirs.py | 4 +--- tests/unit/api/test_knowledge_bases.py | 11 ++++++----- tests/unit/infra/utils/test_host_dirs.py | 2 +- tests/unit/infra/utils/test_win_utf8.py | 2 +- tests/unit/test_knowledge_mount.py | 2 +- 8 files changed, 22 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aac685e8..6c1fb93a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ ## [Unreleased] +### 修复 + +- Windows 知识库本地文件夹挂接：选择器与列表预览按 UTF-8 处理中文路径/文件名，避免 `项目…` 这类乱码；挂接与列表预览不再因向量模型未就绪返回 409。 + ### 变更 - 对话与工作区共用一份列表：侧栏「对话」进入工作区「对话」标签；点开一条才进入聊天画布。工作区侧栏默认打开「项目」标签。 diff --git a/dashboard/src/utils/desktopFolder.test.ts b/dashboard/src/utils/desktopFolder.test.ts index 00edda90..7d84e724 100644 --- a/dashboard/src/utils/desktopFolder.test.ts +++ b/dashboard/src/utils/desktopFolder.test.ts @@ -24,7 +24,9 @@ describe("desktopFolder", () => { removeEventListener: window.removeEventListener.bind(window), } as unknown as Window & { _wails?: { invoke?: (m: string) => void } }; const path = await pickDesktopFolder(win, 1000); - expect(invoke).toHaveBeenCalledWith("wails:event:emit:desktop:select-folder"); + expect(invoke).toHaveBeenCalledWith( + "wails:event:emit:desktop:select-folder", + ); expect(path).toBe("D:\\Projects\\demo"); }); @@ -48,7 +50,9 @@ describe("desktopFolder", () => { it("returns null when the user cancels", async () => { const invoke = vi.fn(() => { - window.dispatchEvent(new CustomEvent("freeos-folder-selected", { detail: "" })); + window.dispatchEvent( + new CustomEvent("freeos-folder-selected", { detail: "" }), + ); }); const win = { _wails: { invoke }, diff --git a/src/octop/api/routers/knowledge_bases.py b/src/octop/api/routers/knowledge_bases.py index e2e426c2..3b6216b5 100644 --- a/src/octop/api/routers/knowledge_bases.py +++ b/src/octop/api/routers/knowledge_bases.py @@ -747,7 +747,7 @@ async def upload_document( from octop.infra.utils.win_utf8 import repair_utf8_mojibake filename = repair_utf8_mojibake(upload.filename or "") - relative = repair_utf8_mojibake(path or filename) + relative = path if isinstance(path, str) else None document = _knowledge_service(server).upload_document( kb_id, actor_user_id=user.id, @@ -755,7 +755,7 @@ async def upload_document( content_type=upload.content_type or "", content=content, is_admin=_is_admin(user), - path=relative, + path=repair_utf8_mojibake(relative or filename), ) assert server.services is not None enqueue_index_document(server.services, kb_id, document.id) diff --git a/src/octop/infra/utils/host_dirs.py b/src/octop/infra/utils/host_dirs.py index 90d08b80..09eee3bc 100644 --- a/src/octop/infra/utils/host_dirs.py +++ b/src/octop/infra/utils/host_dirs.py @@ -31,9 +31,7 @@ def host_path_text(path: Path) -> str: """Serialize a host path for API/UI (POSIX separators, even on Windows).""" - return repair_utf8_mojibake( - Path(os.path.realpath(os.path.expanduser(str(path)))).as_posix() - ) + return repair_utf8_mojibake(Path(os.path.realpath(os.path.expanduser(str(path)))).as_posix()) def host_home_dir() -> Path: diff --git a/tests/unit/api/test_knowledge_bases.py b/tests/unit/api/test_knowledge_bases.py index a29bc21b..0964d165 100644 --- a/tests/unit/api/test_knowledge_bases.py +++ b/tests/unit/api/test_knowledge_bases.py @@ -20,11 +20,12 @@ def _request() -> Request: def _services(**extra: object) -> SimpleNamespace: - return SimpleNamespace( - settings_repo=SimpleNamespace(get=lambda _key: None, set=lambda *_: None), - provider_repo=SimpleNamespace(list_all=lambda: []), - **extra, - ) + payload: dict[str, object] = { + "settings_repo": SimpleNamespace(get=lambda _key: None, set=lambda *_: None), + "provider_repo": SimpleNamespace(list_all=lambda: []), + } + payload.update(extra) + return SimpleNamespace(**payload) @dataclass diff --git a/tests/unit/infra/utils/test_host_dirs.py b/tests/unit/infra/utils/test_host_dirs.py index 3a390806..b3bf10f5 100644 --- a/tests/unit/infra/utils/test_host_dirs.py +++ b/tests/unit/infra/utils/test_host_dirs.py @@ -58,7 +58,7 @@ def test_list_host_subdirs_keeps_unicode_names(tmp_path: Path) -> None: def test_normalize_host_path_repairs_cp1252_mojibake(tmp_path: Path) -> None: real = tmp_path / "项目" real.mkdir() - garbled = str(real).encode("utf-8").decode("cp1252") + garbled = str(real).encode().decode("cp1252") assert normalize_host_path(garbled) == real.resolve() diff --git a/tests/unit/infra/utils/test_win_utf8.py b/tests/unit/infra/utils/test_win_utf8.py index 90ed1498..e9e95e3a 100644 --- a/tests/unit/infra/utils/test_win_utf8.py +++ b/tests/unit/infra/utils/test_win_utf8.py @@ -2,7 +2,7 @@ def test_repair_utf8_mojibake_restores_chinese_filename() -> None: - garbled = "项目结构.png".encode("utf-8").decode("cp1252") + garbled = "项目结构.png".encode().decode("cp1252") assert garbled != "项目结构.png" assert repair_utf8_mojibake(garbled) == "项目结构.png" diff --git a/tests/unit/test_knowledge_mount.py b/tests/unit/test_knowledge_mount.py index 57f289d0..b5267bd8 100644 --- a/tests/unit/test_knowledge_mount.py +++ b/tests/unit/test_knowledge_mount.py @@ -90,7 +90,7 @@ def test_save_mount_repairs_cp1252_mojibake_path(tmp_path: Path) -> None: source = tmp_path / "项目" source.mkdir() (source / "说明.md").write_text("ok", encoding="utf-8") - garbled = str(source).encode("utf-8").decode("cp1252") + garbled = str(source).encode().decode("cp1252") assert garbled != str(source) stored = save_mount( KnowledgeMount(kb_id="kb-zh", source_path=garbled, distill_path=""),