diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a1a1316 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +# Set default behaviour, in case users don't have core.autocrlf set. +* text=auto + +# Try to ensure that po files in the repo does not include +# source code line numbers. +# Every person expected to commit po files should change their personal config file as described here: +# https://mail.gnome.org/archives/kupfer-list/2010-June/msg00002.html +*.po filter=cleanpo diff --git a/.gitignore b/.gitignore index b7faf40..1a2c8a7 100644 --- a/.gitignore +++ b/.gitignore @@ -205,3 +205,21 @@ cython_debug/ marimo/_static/ marimo/_lsp/ __marimo__/ + +# NVDA Add-on development +addon/doc/*.css +addon/doc/en/ +*_docHandler.py +*.html +manifest.ini +*.mo +*.pot +*.py[co] +*.nvda-addon +.sconsign.dblite +/[0-9]*.[0-9]*.[0-9]*.json +__pycache__ + +/**/test/*.png +logs +test \ No newline at end of file diff --git a/README.md b/README.md index ef3c2bd..3f8dafc 100644 --- a/README.md +++ b/README.md @@ -1 +1,168 @@ -TODO: Create a file called LICENSE (not LICENSE.TXT, LICENSE.md, etc.)… \ No newline at end of file +# AskEase - NVDA AI Screen Assistant + +AskEase is a powerful NVDA add-on that integrates AI-driven screen reading assistance. It can capture screen content and analyze it through AI, providing detailed screen descriptions, operation guidance, and intelligent Q&A functionality for visually impaired users. + +## Features + +- **Intelligent Screen Analysis**: Automatically captures screen content and provides detailed descriptions +- **AI Chat Assistant**: Ask questions about screen content and get intelligent responses +- **Action Recording & Playback**: Records user operation history and provides context-aware help +- **Multi-language Support**: Supports both Chinese and English interfaces +- **Keyboard Shortcuts**: Convenient keyboard shortcuts for various operations +- **Customizable Settings**: Configurable dialog behavior and search options + +## System Requirements + +- NVDA 2019.3.0 or later +- Windows Operating System +- Valid OpenAI API key or Azure OpenAI service + +## User Guide + +### Installation + +1. Download the `AskEase-0.1.nvda-addon` file +2. Run the file, and NVDA will automatically install the add-on +3. Restart NVDA to activate the add-on + +### Configuration + +#### NVDA Settings + +1. Open NVDA Settings dialog (NVDA menu > Preferences > Settings) +2. Select "AskEase" category from the left panel +3. Configure the following settings: + - **API Key**: Enter your OpenAI API key + - **Model**: Select the AI model to use (default: gpt-5-chat) + - **API Endpoint**: Configure the API service address + +#### Dialog Settings + +Within the AI assistant dialog, you can access additional settings by expanding the "Settings" section. The available options include: + +- **Hide Dialog After Sending**: When enabled, the dialog automatically hides after you send a message, returning focus to the previous application +- **Use Help Documentation**: When enabled, the system searches relevant help documentation to provide more accurate and context-aware responses +- **Advanced Search**: When enabled, provides enhanced search capabilities that may result in more comprehensive answers, though responses may be slightly slower + +**Note**: All of these options are enabled by default to provide the best user experience. The advanced search feature may make responses slightly slower due to additional processing, but typically provides more thorough and helpful guidance. + +### Keyboard Shortcuts + +#### Global Shortcuts (Available anywhere in Windows) + +| Shortcut | Function | +|----------|----------| +| `NVDA+control+J` | Describe current screen and focus | +| `NVDA+control+O` | Open AI assistant dialog | +| `NVDA+control+D` | Analyze current screen and refine earlier guidance | +| `NVDA+control+Up Arrow` | Go back to the previous guidance step | +| `NVDA+control+Down Arrow` | Go to the next guidance step | + +#### Dialog Shortcuts (Available within AI assistant dialog) + +| Shortcut | Function | +|----------|----------| +| `Enter` | Send message (from input field) | +| `Shift+Enter` | New line (from input field) | +| `Escape` | Close the dialog | +| `Alt+C` | Clear chat history | +| `Alt+P` | Go to previous conversation | +| `Alt+N` | Go to next conversation | +| `Alt+S` | Send message | +| `F1` | Get context-sensitive help | + +### How to Use + +1. **Get Screen Description**: Press `NVDA+control+J` to get detailed description of current screen + +2. **AI Chat**: Press `NVDA+control+O` to open dialog and ask questions about screen content + - Type your question and press Enter to send + - Use Alt+C to clear chat history when needed + - Use Alt+H to hide dialog and return focus to previous application + - Navigate through conversation history with Alt+P (previous) and Alt+N (next) + - Access the Settings section to customize dialog behavior + +3. **Smart Analysis**: Press `NVDA+control+D` to let AI analyze current situation and provide operation suggestions + +4. **Navigate Guidance Steps**: Use `NVDA+control+Up/Down Arrow` to navigate through operation steps + +5. **Get Help**: Press F1 within the dialog for context-sensitive help information + +6. **Customize Settings**: + - Expand the Settings section within the dialog to access configuration options + - Toggle "Hide Dialog After Sending" based on your workflow preference + - Enable/disable help documentation search based on your needs + - Adjust advanced search settings for optimal performance vs. thoroughness balance + +## Developer Guide + +### Development Environment Setup + +#### Required Software + +* Python 3.11 ([Download](https://www.python.org)) +* SCons 4.8.1 or later (`pip install scons`) +* GNU Gettext tools (for localization support) +* Markdown 3.7 or later (`pip install markdown`) + +### Development Workflow + +#### 1. Clone and Setup + +```powershell +git clone +cd nvda-addon +``` + +#### 2. Development Build + +```powershell +# Build normal version (uses default version from buildVars.py) +scons + +# Build development version (auto-generates date-based version like 20250820.0.0) +scons dev=True +``` + +#### 3. Localization + +```powershell +# Generate POT file +scons pot + +# Update translation files +# Edit addon/locale/zh_CN/LC_MESSAGES/nvda.po +``` + +#### 4. Testing + +- Install the generated .nvda-addon file to NVDA + +### Core Modules + +#### `__init__.py` +- Main add-on entry point +- Handles global gesture interception +- Manages screen recording and AI interaction + +#### `openai_service.py` +- Wraps OpenAI API calls +- Handles image and text AI analysis +- Manages API configuration and error handling + +#### `addonConfig.py` +- Add-on configuration management +- Stores API keys, model settings, etc. + +#### `helpDialog.py` +- AI chat interface +- Handles user Q&A interactions + +## License + +This project is licensed under the GNU General Public License v2 or v3. See the LICENSE file for details. + +## Support + +For questions or suggestions, please contact us through: +- Create a GitHub Issue \ No newline at end of file diff --git a/addon/doc/zh/readme.md b/addon/doc/zh/readme.md new file mode 100644 index 0000000..0495029 --- /dev/null +++ b/addon/doc/zh/readme.md @@ -0,0 +1,160 @@ +# NVDA Add-on Scons Template + +This package contains a basic template structure for NVDA add-on development, building, distribution and localization. +For details about NVDA add-on development, please see the [NVDA Add-on Development Guide](https://github.com/nvdaaddons/DevGuide/wiki/NVDA-Add-on-Development-Guide). +The NVDA add-on development/discussion list [is here](https://nvda-addons.groups.io/g/nvda-addons) +Information specific to NV Access add-on store [can be found here](https://github.com/nvaccess/addon-datastore). + +Copyright (C) 2012-2025 NVDA Add-on team contributors. + +This package is distributed under the terms of the GNU General Public License, version 2 or later. Please see the file COPYING.txt for further details. + +[alekssamos](https://github.com/alekssamos/) added automatic package of add-ons through Github Actions. + +For details about Github Actions, see the [Workflow syntax for GitHub Actions](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions). + +Copyright (C) 2022 alekssamos + +## Features + +This template provides the following features you can use during NVDA add-on development and packaging: + +* Automatic add-on package creation, with naming and version loaded from a centralized build variables file (buildVars.py) or command-line interface. + * See packaging section for details on using command-line switches when packaging add-ons with custom version information. + * This process will happen automatically when receiving a pull request, and there is also the possibility of manual launch. + * To let the workflow run automatically when pushing to main or master (development) branch, remove the comment for branches line in GitHub Actions (`.github/workflows/build_addon.yml`). + * If you have created a tag (E.G.: `git tag v1.0 && git push --tag`), then a release will be automatically created and the add-on file will be uploaded as an asset. + * Otherwise, with normal commits or with manual startup, you can download the artifacts from the Actions page of your repository. +* Manifest file creation using a template (manifest.ini.tpl). Build variables are replaced on this template. See below for add-on manifest specification. +* Compilation of gettext mo files before distribution, when needed. + * To generate a gettext pot file, please run `scons pot`. An `addon-name.pot` file will be created with all gettext messages for your add-on. You need to check the `buildVars.i18nSources` variable to comply with your requirements. +* Automatic generation of manifest localization files directly from gettext po files. Please make sure buildVars.py is included in i18nFiles. +* Automatic generation of HTML documents from markdown (.md) files, to manage documentation in different languages. + +In addition, this template includes configuration files for the following tools for use in add-on development and testing (see "additional tools" section for details): + +* Ruff (pyproject.toml/tool.ruff sections): a Python linter written in Rust. Sections starting with tool.ruff house configuration options for Ruff. +* Configuration for VS Code. It requires NVDA's repo at the same level as the add-on folder containing your actual source files, with prepared source code (`scons source`). preparing the source code is a step in the instructions for building NVDA itself, see [The NVDA Repository](https://github.com/nvaccess/nvda) for details. + * Place the .vscode in this repo within the addon folder, where your add-on source files (will) reside. The settings file within this folder assumes the NVDA repository is within the parent folder of this folder. If your addon folder is within the addonTemplate folder, then your NVDA repository folder needs to also be within the addonTemplate folder, or the source will not be found. + * Open the addon folder in VS Code. This should initialize VS Code with the correct settings and provide you with code completion and other VS Code features. + * Press `control+shift+m` after saving a file to search for problems. + * Use arrow and tab keys for the autocompletion feature. + * Press `control+shift+p` to open the commands palette and search for recommended extensions to install or check if they are installed. +* Pyright (pyproject.toml/tool.pyright sections): a Python static type checker. Sections starting with tool.pyright house configuration options for Pyright. + +## Requirements + +You need the following software to use this code for your NVDA add-on development and packaging: + +* a Python distribution (3.11 or later is recommended). Check the [Python Website](https://www.python.org) for Windows Installers. Please note that at present, preparing the NVDA source code requires the 32-bit version of Python 3.11 with 64-bit version planned. +* Scons - [Website](https://www.scons.org/) - version 4.8.1 or later. You can install it via PIP. +* GNU Gettext tools, if you want to have localization support for your add-on - Recommended. Any Linux distro or cygwin have those installed. You can find windows builds [here](https://gnuwin32.sourceforge.net/downlinks/gettext.php). +* Markdown 3.7 or later, if you want to convert documentation files to HTML documents. You can install it via PIP. +* Optional: additional tools such as linters and type checkers defined in pyproject.toml file. + +Note, that you may not need these tools in a local build environment, if you are using [Appveyor](https://appveyor.com/) or [GitHub Actions](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions), to build and package your add-ons. + +## Usage + +### To create a new NVDA add-on using this template: + +1. Create an empty folder to hold the files for your add-on. +2. Copy the folder: +``` +site_scons +``` +and the following files, into your new empty folder: +``` +buildVars.py +manifest.ini.tpl +manifest-translated.ini.tpl +sconstruct +.gitignore +.gitattributes +``` +3. If you intend to use the provided GitHub workflow, also copy the folder: +``` +.github +``` +and file: +``` +.pre-commit-config.yaml +``` +4. Create an `addon` folder inside your new folder. You will put your code in the usual folders for NVDA extensions, under the `addon` folder. For instance: `globalPlugins`, `synthDrivers`, etc. +5. In the `buildVars.py` file, change variable `addon_info` with your add-on's information (name, summary, description, version, author, url, source url, license, and license URL). Also, be sure to carefully set the paths contained in the other variables in that file. If you need to use custom Markdown extensions, original add-on interface language is not English, or include custom braille translations tables, be sure to fil out markdown list, base language variable, and braille tables dictioanry, respectively. +6. Gettext translations must be placed into `addon\locale\/LC_MESSAGES\nvda.po`. + +#### Add-on manifest specification + +An add-on manifest generated manually or via `buildVars.py` must include the following information: + +* Name (string): a unique internal identifier for the add-on. It must use camel case (e.g. someModule). This is also used as part of add-on store to identify the add-on uniquely. +* Summary (string): name as shown on NVDA's Add-on store. +* Description (string): a short detailed description about the add-on. +* Version (string), ideally number.number with an optional third number, denoting major.minor.patch. +* Author (string and an email address): one or more add-on author contact information in the form "name ". +* URL (string): a web address where the add-on information can be found such as add-on repository. +* docFileName (string): name of the documentation file. +* minimumNVDAVersion (year.major or year.major.minor): the earliest version of NVDA the add-on is compatible with (e.g. 2019.3). Add-ons are expected to use features introduced in this version of NVDA or declare compatibility with it. +* lastTestedNVDAVersion (year.major or year.major.minor): the latest or last tested version of NVDA the add-on is said to be compatible with (e.g. 2020.3). Add-on authors are expected to declare this value after testing add-ons with the version of NVDA specified. +* addon_updateChannel (string or None): the update channel for the add-on release. + +In addition, the following information must be filled out (not used in the manifest but used elsewhere such as add-on store) in buildVars: + +* sourceURL (string): repository URL for the add-on source code. +* license (string): the license of the add-on and its source code. +* licenseURL: the URL for the license file. + +##### Custom add-on information + +In addition to the core manifest data, custom add-on information can be specified. + +###### Braille translation tables + +Information on custom braille tables must be specified in buildVars under `brailleTables` dictionary as follows: + +* Table name (string key for a nested dictionary): each `brailleTables` entry is a filename for the included custom braille table placed in `brailleTables` folder inside `addon` folder. This nested dictionary should specify: + * displayName (string): the name of the table shown to users and is translatable. + * contracted (True/False): is this a contracted braille table (True) or uncontracted (False). + * output (True/False): the table can be listed in output table list in NVDA's braille settings. + * input (True/False): braille can be entered using this table and listed in input table list in NVDA's braille settings. + +Note: you must fill out this dictionary if at least one custom braille table is included in the add-on. If not, leave the dictionary empty. + +###### Speech symbol dictionaries + +Information on custom symbol dictionaries must be specified in buildVars under `symbolDictionaries` dictionary as follows: + +* Dictionary name (string key for a nested dictionary): each `symbolDictionaries` entry is a name for the included custom symbol dictionary placed in `locale\` folder inside `addon` folder. The file is named `symbols-.dic`. This nested dictionary should specify: + * displayName (string): the name of the dictionary shown to users and is translatable. + * mandatory (True/False): Always enabled (True) or optional and visible in the GUI (False) + +Note: you must fill out this dictionary if at least one custom symbol dictionary is included in the add-on. If not, leave the dictionary empty. + +### To manage documentation files for your addon: + +1. Copy the `readme.md` file for your add-on to the first created folder, where you copied `buildVars.py`. You can also copy `style.css` to improve the presentation of HTML documents. +2. Documentation files (named `readme.md`) must be placed into `addon\doc\/`. + +### To package the add-on for distribution: + +1. Open a command line, change to the folder that has the `sconstruct` file (usually the root of your add-on development folder) and run the `scons` command. The created add-on, if there were no errors, is placed in the current directory. +2. You can further customize variables in the `buildVars.py` file. +3. You can also customize version and update channel information from command line by passing the following switches when running scons: + * version: add-on version string. + * versionNumber: add-on version number of the form major.minor.patch (all integers) + * channel: update channel (do not use this switch unless you know what you are doing). + * dev: suitable for development builds, names the add-on according to current date (yyyymmdd) and sets update channel to "dev". + +### Additional tools + +The template includes configuration files for use with additional tools such as linters. These include: + +* Ruff: a Python linter written in Rust (0.4.10 or later, can be installed with PIP). +* Pyright: a Python static type checker (1.1.402 or later, can be installed with PIP). + +Read the documentation for the tools you wish to use when building and developing add-ons. + +Note that this template only provides a basic add-on structure and build infrastructure. You may need to adapt it for your specific needs such as using additional tools. + +If you have any issues please use the NVDA addon list mentioned above. diff --git a/addon/globalPlugins/screenAssistant/.env.example b/addon/globalPlugins/screenAssistant/.env.example new file mode 100644 index 0000000..ceb3d9d --- /dev/null +++ b/addon/globalPlugins/screenAssistant/.env.example @@ -0,0 +1,6 @@ +# OpenAI API Configuration +# Copy this file to .env and fill in your actual values + +OPENAI_API_KEY=your_openai_api_key_here +OPENAI_MODEL=your_openai_model_here +OPENAI_ENDPOINT=your_openai_endpoint_here \ No newline at end of file diff --git a/addon/globalPlugins/screenAssistant/__init__.py b/addon/globalPlugins/screenAssistant/__init__.py new file mode 100644 index 0000000..c70c3e6 --- /dev/null +++ b/addon/globalPlugins/screenAssistant/__init__.py @@ -0,0 +1,313 @@ +from typing import Callable +import threading + +import globalPluginHandler +import scriptHandler +import speech +from speech.priorities import SpeechPriority +import speechViewer +from queueHandler import queueFunction, eventQueue +import wx +import gui +import inputCore +from keyboardHandler import KeyboardInputGesture +import os +import sys +import addonHandler +import api + +plugin_dir = os.path.dirname(__file__) +sys.path.insert(0, plugin_dir) + +from helpDialog import HelpDialog +import addonConfig as cfg +from settingsPanel import ScreenAssistantSettingsPanel +from screenReader import ScreenReader +from screenReader.views import ActionHistoryItem +from desktop import Desktop +from desktop.views import DesktopState +from log import logging_to_desktop + +addonHandler.initTranslation() + + +class GlobalPlugin(globalPluginHandler.GlobalPlugin): + scriptCategory = _("AskEase") + + def __init__(self): + super(GlobalPlugin, self).__init__() + + cfg.initialize() + # add settings panel + gui.settingsDialogs.NVDASettingsDialog.categoryClasses.append( + ScreenAssistantSettingsPanel + ) + + self.screenReader = ScreenReader() + # Start recording by default + self.screenReader.handle_recording() + self.desktop = Desktop() + self._helpDialog = None + + # speech interception + self._oldSpeak = speech.speech.speak + speech.speech.speak = self._mySpeak + + # gesture interception + self._oldExecuteGesture = inputCore.manager.executeGesture + inputCore.manager.executeGesture = self._myExecuteGesture + + def clear_help_dialog(self): + if hasattr(self, "_helpDialog"): + self._helpDialog = None + + def _myExecuteGesture(self, gesture): + # record key gestures + if isinstance(gesture, KeyboardInputGesture) and self.screenReader.recording: + try: + # check if current focus is in HelpDialog + isInHelp = self._isInHelpDialog() + if isInHelp: + pass # do not record, but continue to execute gesture + # check if the gesture is a modifier or has identifiers + elif hasattr(gesture, "isModifier") and gesture.isModifier: + pass # do not record, but continue to execute gesture + else: + # # Check if this is one of our internal shortcuts + is_internal_shortcut = False + + try: + script = scriptHandler.findScript(gesture) + if script and hasattr(script, "__name__"): + script_name = script.__name__ + # check if script name matches any internal script + if ( + script_name.startswith("script_") + and hasattr(self, script_name) + and getattr(self, script_name) == script + ): + is_internal_shortcut = True + logging_to_desktop( + f"Shortcut executed: {script_name}", + ) + except Exception: + pass + + # Only record if it's not an internal shortcut + if not is_internal_shortcut: + keyName = None + if hasattr(gesture, "displayName") and gesture.displayName: + keyName = gesture.displayName + elif hasattr(gesture, "identifiers") and gesture.identifiers: + keyName = gesture.identifiers[0].split(":")[-1] + + if keyName: + # record the key gesture + queueFunction( + eventQueue, + self.screenReader.add_history, + ActionHistoryItem("key", keyName), + ) + except Exception as e: + print(f"Error recording key: {e}") + + # do not modify the original gesture execution + return self._oldExecuteGesture(gesture) + + def _isInHelpDialog(self): + """Check if current focus is within the HelpDialog""" + if not hasattr(self, "_helpDialog") or not self._helpDialog: + return False + + try: + # Check if dialog is shown + is_shown = self._helpDialog.IsShown() + if not is_shown: + return False + + # Get current focus object and check if it belongs to HelpDialog + try: + focus_obj = api.getFocusObject() + + # Get the window handle of the focused object + focus_window_handle = None + if hasattr(focus_obj, "windowHandle"): + focus_window_handle = focus_obj.windowHandle + + # Get HelpDialog window handle + dialog_handle = None + if hasattr(self._helpDialog, "GetHandle"): + dialog_handle = self._helpDialog.GetHandle() + + # Check if focus window is the dialog or a child of the dialog + if focus_window_handle and dialog_handle: + # Direct match + if focus_window_handle == dialog_handle: + return True + + # Check if focus window is a child of dialog window + try: + import ctypes + + user32 = ctypes.windll.user32 + parent_handle = focus_window_handle + + # Walk up the parent chain to find if dialog is an ancestor + for _ in range(10): # Limit to prevent infinite loops + parent_handle = user32.GetParent(parent_handle) + if not parent_handle: + break + if parent_handle == dialog_handle: + return True + except Exception: + pass + + except Exception: + pass + + return False + + except Exception: + return False + + def _mySpeak(self, sequence, *args, **kwargs): + # do not modify the original speak function + self._oldSpeak(sequence, *args, **kwargs) + if not self.screenReader.recording: + return + + # do not record speech when in HelpDialog + if self._isInHelpDialog(): + return + + text = speechViewer.SPEECH_ITEM_SEPARATOR.join( + [x for x in sequence if isinstance(x, str)] + ) + if text.strip(): + # record the speech output + queueFunction( + eventQueue, + self.screenReader.add_history, + ActionHistoryItem("speech", text), + ) + + def announce(self, message: str, priority=SpeechPriority.NOW): + """Announce a message to the user.""" + # Delay message presentation to avoid interruption by UI changes + # Check if we're in the main thread + if wx.IsMainThread(): + wx.CallLater(1, self._oldSpeak, [str(message)], priority=priority) + else: + self._oldSpeak([str(message)], priority=priority) + + def get_desktop_state(self, on_success: Callable): + self.announce(_("Getting screen information")) + + # get focus and foreground information immediately + try: + focus = self.desktop.get_focus_info() + foreground = self.desktop.get_foreground_info() + except Exception as e: + self.announce( + _("Failed to get screen information: {error}").format(error=str(e)) + ) + return + + def worker(): + try: + # async + screenshot = self.desktop.get_screenshot(focus_rect=focus.get("rect")) + state = DesktopState( + screenshot=screenshot, foreground=foreground, focus=focus + ) + + wx.CallAfter(self.announce, _("Screen information obtained")) + wx.CallAfter(on_success, state) + except Exception as e: + error_msg = _("Failed to get screen information: {error}").format( + error=str(e) + ) + wx.CallAfter(self.announce, error_msg) + + threading.Thread(target=worker, daemon=True).start() + + def describe(self, desktop_state): + # if not has history, ask for description + if not self._helpDialog: + self._helpDialog = HelpDialog(None, self) + self._helpDialog.describe_current_state(desktop_state) + self._helpDialog.init_ui(False) + else: + self._helpDialog.describe_current_state(desktop_state) + + def diagnosis(self, desktop_state): + # if not has history, ask for description + if not self._helpDialog: + self._helpDialog = HelpDialog(None, self) + self._helpDialog.describe_current_state(desktop_state) + self._helpDialog.init_ui(False) + else: + if self._helpDialog.has_history(): + screen_reader_state = self.screenReader.get_state() + self._helpDialog.check_progress(desktop_state, screen_reader_state) + else: + self._helpDialog.describe_current_state(desktop_state) + + def help(self, state): + if self._helpDialog: + self._helpDialog.current_desktop_state = state + self._helpDialog.focus_panel() + return + + self._helpDialog = HelpDialog(None, self) + self._helpDialog.current_desktop_state = state + self._helpDialog.init_ui() + + def terminate(self): + # remove setting panel + gui.settingsDialogs.NVDASettingsDialog.categoryClasses.remove( + ScreenAssistantSettingsPanel + ) + # Stop recording + self.screenReader.handle_recording() + # restore original functions + speech.speech.speak = self._oldSpeak + inputCore.manager.executeGesture = self._oldExecuteGesture + + super(GlobalPlugin, self).terminate() + + def go_to_step(self, step_offset: int): + if self._helpDialog: + self._helpDialog.go_to_step(step_offset) + + @scriptHandler.script( + description=_("Analyze the current screen and refine earlier guidance."), + # if you're unsure, reflecting your earlier request. + gesture="kb:NVDA+control+d", + ) + def script_diagnosis(self, gesture): + self.get_desktop_state(self.diagnosis) + + @scriptHandler.script( + description=_("Describe the screen and current focus."), + gesture="kb:NVDA+control+j", + ) + def script_describeCurrentScreen(self, gesture): + self.get_desktop_state(self.describe) + + @scriptHandler.script(description=_("Ask for help"), gesture="kb:NVDA+control+o") + def script_askForHelp(self, gesture): + self.get_desktop_state(self.help) + + @scriptHandler.script( + description=_("Go back to the previous guidance step"), + gesture="kb:NVDA+control+upArrow", + ) + def script_goToPreviousStep(self, gesture): + self.go_to_step(-1) + + @scriptHandler.script( + description=_("Go to the next guidance step"), gesture="kb:NVDA+control+downArrow" + ) + def script_goToNextStep(self, gesture): + self.go_to_step(+1) diff --git a/addon/globalPlugins/screenAssistant/addonConfig.py b/addon/globalPlugins/screenAssistant/addonConfig.py new file mode 100644 index 0000000..4942061 --- /dev/null +++ b/addon/globalPlugins/screenAssistant/addonConfig.py @@ -0,0 +1,41 @@ +import config +import os +conf = config.conf + +# Load environment variables from .env file if it exists +def load_env_file(): + """Load environment variables from .env file if it exists.""" + env_path = os.path.join(os.path.dirname(__file__), '.env') + if os.path.exists(env_path): + with open(env_path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if line and not line.startswith('#') and '=' in line: + key, value = line.split('=', 1) + os.environ[key.strip()] = value.strip() + + +def initialize(): + """Initialize the Screen Assistant configuration.""" + # Load .env file on module import + load_env_file() + + # Configuration for the Screen Assistant addon + conf_spec = { + "openai": { + "apiKey": f"string(default='{os.environ.get('OPENAI_API_KEY', '')}')", + "model": f"string(default='{os.environ.get('OPENAI_MODEL', '')}')", + "endpoint": f"string(default='{os.environ.get('OPENAI_ENDPOINT', '')}')" + } + } + config.conf.spec["screenAssistant"] = conf_spec + + +def get_config(section, key): + """Get a configuration value for the Screen Assistant addon.""" + return config.conf["screenAssistant"][section][key] + + +def set_config(section, key, value): + """Set a configuration value for the Screen Assistant addon.""" + config.conf["screenAssistant"][section][key] = value \ No newline at end of file diff --git a/addon/globalPlugins/screenAssistant/desktop/__init__.py b/addon/globalPlugins/screenAssistant/desktop/__init__.py new file mode 100644 index 0000000..5df12b6 --- /dev/null +++ b/addon/globalPlugins/screenAssistant/desktop/__init__.py @@ -0,0 +1,134 @@ +import io +import base64 +import api +import os +import sys +import controlTypes + +current_dir = os.path.dirname(__file__) +globalPlugins_dir = os.path.dirname(current_dir) + +if globalPlugins_dir not in sys.path: + sys.path.insert(0, globalPlugins_dir) + +from lib.PIL import ImageGrab, ImageDraw, Image +from desktop.views import DesktopState + + +# TODO: mode and navigator information can be added to DesktopState +class Desktop: + def _getNVDAObjectInfo(self, nvda_object): + info = {} + if nvda_object: + for attr in [ + "name", + "value", + "description", + "help", + "keyboardShortcut", + ]: + value = getattr(nvda_object, attr, None) + if value: + info[attr] = value + + role = getattr(nvda_object, "role", None) + if role: + info["role"] = controlTypes.roleLabels.get( + nvda_object.role, f"role_{nvda_object.role}" + ) + + states = getattr(nvda_object, "states", None) + if states: + if isinstance(states, (list, set)): + states = ", ".join( + controlTypes.stateLabels.get(state.value if hasattr(state, 'value') else state, f"state_{state}") + for state in states + ) + info["states"] = states + + if nvda_object.location: + location = nvda_object.location + if ( + hasattr(location, "left") + and hasattr(location, "top") + and hasattr(location, "right") + and hasattr(location, "bottom") + ): + info["rect"] = ( + location.left, + location.top, + location.right, + location.bottom, + ) + return info + + def is_focus_mode(self): + focus = api.getFocusObject() + if hasattr(focus, 'treeInterceptor') and focus.treeInterceptor: + return bool(focus.treeInterceptor.passThrough) + # If no treeInterceptor, usually means not in a virtual buffer, + # which is typically focus mode + return True + + def get_focus_info(self) -> dict: + focus = api.getFocusObject() + return self._getNVDAObjectInfo(focus) + + def get_navigator_info(self) -> dict: + nav = api.getNavigatorObject() + return self._getNVDAObjectInfo(nav) + + def get_foreground_info(self) -> dict: + foreground_got = api.getForegroundObject() + foreground = {} + if foreground_got: + appModule = getattr(foreground_got, "appModule", None) + if appModule: + appName = getattr(appModule, "appName", None) + if appName: + foreground["appName"] = appName + for attr in ["version", "productVersion", "productName"]: + value = getattr(appModule, attr, None) + if value: + foreground[attr] = value + for attr in ["name", "description", "value", "windowText"]: + value = getattr(foreground_got, attr, None) + if value: + foreground[f"{attr}"] = value + return foreground + + def get_state(self, use_vision: bool = False) -> DesktopState: + focus = self.get_focus_info() + foreground = self.get_foreground_info() + + if use_vision: + screenshot = self.get_screenshot(focus_rect=focus.get("rect")) + else: + screenshot = None + return DesktopState(screenshot=screenshot, foreground=foreground, focus=focus) + + def get_screenshot(self, focus_rect=None) -> str: + # may take a while + screenshot = ImageGrab.grab() + + if focus_rect: + # highlight the focus area + draw = ImageDraw.Draw(screenshot) + # border + draw.rectangle(focus_rect, outline="red", width=3) + # translucent overlay + overlay = Image.new("RGBA", screenshot.size, (0, 0, 0, 0)) + overlay_draw = ImageDraw.Draw(overlay) + overlay_draw.rectangle( + focus_rect, fill=(255, 0, 0, 0) + ) + screenshot = screenshot.convert("RGBA") + screenshot = Image.alpha_composite(screenshot, overlay) + screenshot = screenshot.convert("RGB") + + buffer = io.BytesIO() + screenshot.save(buffer, format="PNG") + img_bytes = buffer.getvalue() + img_base64 = base64.b64encode(img_bytes).decode("utf-8") + + return img_base64 diff --git a/addon/globalPlugins/screenAssistant/desktop/views.py b/addon/globalPlugins/screenAssistant/desktop/views.py new file mode 100644 index 0000000..4428b8f --- /dev/null +++ b/addon/globalPlugins/screenAssistant/desktop/views.py @@ -0,0 +1,54 @@ +from dataclasses import dataclass +from textwrap import dedent +from typing import Optional +import os +import sys + +current_dir = os.path.dirname(__file__) +globalPlugins_dir = os.path.dirname(current_dir) + +if globalPlugins_dir not in sys.path: + sys.path.insert(0, globalPlugins_dir) + +from string_utils import truncate_string + + +@dataclass +class DesktopState: + screenshot: Optional[str] = None + focus: Optional[dict] = None + foreground: Optional[dict] = None + + def foreground_to_string(self) -> str: + if self.foreground: + return ", ".join( + f"{k}: {truncate_string(v)}" for k, v in self.foreground.items() + ) + else: + return "Unknown" + + def focus_to_string(self) -> str: + if self.focus: + important_attrs = ["name", "role", "value", "description", "states"] + attr_value_pairs = [] + + for attr in important_attrs: + if attr in self.focus: + attr_value_pairs.append( + f"{attr}: {truncate_string(self.focus[attr])}" + ) + + for k, v in self.focus.items(): + if k not in important_attrs and k != "rect": + attr_value_pairs.append(f"{k}: {truncate_string(v)}") + + return ", ".join(attr_value_pairs) + else: + return "Unknown" + + def to_string(self) -> str: + return dedent( + f""" + **Current window**: {self.foreground_to_string()} + **Current focus**: {self.focus_to_string()}""" + ).strip() diff --git a/addon/globalPlugins/screenAssistant/helpDialog.py b/addon/globalPlugins/screenAssistant/helpDialog.py new file mode 100644 index 0000000..cc81603 --- /dev/null +++ b/addon/globalPlugins/screenAssistant/helpDialog.py @@ -0,0 +1,964 @@ +import time +import tones +import wx +import threading +import base64 +import json +import io +import os +import sys +import requests +import re +from speech.priorities import SpeechPriority + + +plugin_dir = os.path.dirname(__file__) +sys.path.insert(0, plugin_dir) + +# Load environment variables using the existing function +import addonConfig as cfg +cfg.load_env_file() + +from openai_service import request_openai_model +from lib.PIL import Image +from message import MessageList +from prompt import ( + SYSTEM_PROMPT, + NEXT_STEP_PROMPT, + NEXT_STEP_WITH_HELP_INFO_PROMPT, + PROGRESS_CHECK_PROMPT, +) +from log import logging_to_desktop +import addonHandler + +addonHandler.initTranslation() + +PREVIEW_WIDTH = 160 +PREVIEW_HEIGHT = 120 +FULL_SCREENSHOT_SIZE = (800, 600) +BEEP_INTERVAL = 4 + +RAG_SERVER = os.environ.get("RAG_SERVER", "") +API_PATH = "/api/v1/retrieval" + +DEBUG = False + + +def find_rag_dataset(product_name, app_name, window_text): + text = f"{product_name} {app_name} {window_text}".lower() + if "word" in text: + return "word_split_aug" + if "excel" in text: + return "excel_split_aug" + if "libreoffice" in text: + if "writer" in text: + return "writer_aug" + if "calc" in text: + return "calc_aug" + if "msedge" in text or "microsoft edge" in text: + return "edge_aug" + if "notepad" in text: + return "notepad_aug" + if "mspaint" in text: + return "paint_aug" + if "systemsettings" in text or "settings" in text: + return "setting_aug" + if "visual studio code" in text: + return "vs code_aug" + if "explorer" in text: + return "file explorer_aug" + if "calculator" in text: + return "ms calc_aug" + if "vlc" in text: + return "VLC_aug" + if "windowsalarms" in text: + return "clock_aug" + if "google chrome" in text: + return "chrome_aug" + return None + +def format_guide_steps(json_string): + """Format the JSON string to a readable guide format""" + try: + if json_string.startswith('```json'): + json_string = re.sub(r'^```json\s*\n?', '', json_string) + json_string = re.sub(r'\n?```\s*$', '', json_string) + json_string = json_string.replace('\\n', '\n') + data = json.loads(json_string) + steps = data.get("answer", []) + if not steps: + return json_string + formatted_steps = "\n".join(f"{i + 1}. {step}" for i, step in enumerate(steps)) + return formatted_steps + except json.JSONDecodeError: + return json_string + + +def create_screenshot_bitmap( + img_base64, target_width=PREVIEW_WIDTH, target_height=PREVIEW_HEIGHT +): + try: + img_data = base64.b64decode(img_base64) + img_stream = io.BytesIO(img_data) + pil_img = Image.open(img_stream) + + if target_width and target_height: + original_width, original_height = pil_img.size + width_ratio = target_width / original_width + height_ratio = target_height / original_height + scale_ratio = min(width_ratio, height_ratio) + + new_width = int(original_width * scale_ratio) + new_height = int(original_height * scale_ratio) + + pil_img = pil_img.resize((new_width, new_height), Image.LANCZOS) + + canvas = Image.new("RGB", (target_width, target_height), (255, 255, 255)) + paste_x = (target_width - new_width) // 2 + paste_y = (target_height - new_height) // 2 + canvas.paste(pil_img, (paste_x, paste_y)) + pil_img = canvas + + # Convert PIL image to wx.Bitmap + wx_img = wx.Image(pil_img.size[0], pil_img.size[1]) + wx_img.SetData(pil_img.convert("RGB").tobytes()) + return wx.Bitmap(wx_img) + except Exception: + return None + + +class HelpDialog(wx.Dialog): + def __init__(self, parent, plugin): + # import inside to make sure translation is initialized + import addonHandler + + addonHandler.initTranslation() + super().__init__(parent, title=_("AskEase"), size=(900, 1000)) + self.plugin = plugin + self.message_list = MessageList() + # index of user and AI messages pair + self.current_history_index = -1 + self.current_step = -1 + # store the current state for reuse + self.current_desktop_state = None + self.last_request = None + + def on_close_window(self, event): + self.plugin.clear_help_dialog() + + self.Destroy() + + def focus_panel(self, focus_element=None): + if not self.IsActive(): + self.Show() + self.Raise() + self.SetFocus() + self.plugin.announce(_("AskEase dialog opened. Press ESC to close.")) + if focus_element: + focus_element.SetFocus() + + def init_ui(self, shown=True): + import wx + + panel = wx.Panel(self) + main_vbox = wx.BoxSizer(wx.VERTICAL) + + """create a history navigation area""" + nav_box = wx.StaticBoxSizer(wx.HORIZONTAL, panel, _("Chat History Navigation")) + nav_container = nav_box.GetStaticBox() + self.prev_btn = wx.Button(nav_container, label=_("Previous (&P)")) + self.next_btn = wx.Button(nav_container, label=_("Next (&N)")) + self.current_label = wx.StaticText( + nav_container, label=_("No conversations yet") + ) + nav_box.Add(self.current_label, 1, wx.ALIGN_CENTER_VERTICAL | wx.ALL, 5) + nav_box.Add(self.prev_btn, 0, wx.ALL, 5) + nav_box.Add(self.next_btn, 0, wx.ALL, 5) + main_vbox.Add(nav_box, 0, wx.EXPAND | wx.ALL, 5) + + if DEBUG: + self.add_screenshot_preview(panel, main_vbox) + + """create a message display area""" + # display user request + user_request_static_box = wx.StaticBoxSizer( + wx.VERTICAL, panel, _("Your Previous Question") + ) + self.user_request_text = wx.TextCtrl( + user_request_static_box.GetStaticBox(), + style=wx.TE_MULTILINE | wx.TE_READONLY | wx.BORDER_NONE, + ) + self.user_request_text.SetMinSize((800, 80)) + user_request_static_box.Add(self.user_request_text, 1, wx.EXPAND | wx.ALL, 5) + main_vbox.Add(user_request_static_box, 0, wx.EXPAND | wx.ALL, 5) + + # display AI reply + ai_reply_static_box = wx.StaticBoxSizer(wx.VERTICAL, panel, _("AI Reply")) + self.ai_reply_text = wx.TextCtrl( + ai_reply_static_box.GetStaticBox(), + style=wx.TE_MULTILINE | wx.TE_READONLY | wx.TE_WORDWRAP | wx.BORDER_NONE, + ) + self.ai_reply_text.SetMinSize((800, 180)) + self.ai_reply_text.SetCanFocus(True) + ai_reply_static_box.Add(self.ai_reply_text, 1, wx.EXPAND | wx.ALL, 5) + main_vbox.Add(ai_reply_static_box, 1, wx.EXPAND | wx.ALL, 5) + + """create a user input area""" + input_box = wx.StaticBoxSizer(wx.VERTICAL, panel, _("Chat Input")) + input_container = input_box.GetStaticBox() + input_tips = wx.StaticText( + input_container, + label=_("Ask a question."), + ) + input_box.Add(input_tips, 0, wx.ALL, 5) + self.input_text = wx.TextCtrl(input_container, style=wx.TE_MULTILINE) + self.input_text.SetMinSize((800, 100)) + input_box.Add(self.input_text, 0, wx.EXPAND | wx.ALL, 5) + input_tips = wx.StaticText( + input_container, + label=_("AI-generated content may be incorrect."), + ) + input_box.Add(input_tips, 0, wx.ALL, 5) + + # a collapsible settings area + self.settings_pane = wx.CollapsiblePane(input_container, label=_("Settings")) + self.settings_pane.Collapse() + input_box.Add(self.settings_pane, 0, wx.EXPAND | wx.ALL, 5) + # Add settings content + settings_panel = self.settings_pane.GetPane() + settings_sizer = wx.BoxSizer(wx.VERTICAL) + self.auto_hide_chk = wx.CheckBox( + settings_panel, label=_("Hide Dialog After Sending") + ) + self.auto_hide_chk.SetValue(True) + self.enable_cross_languages_chk = wx.CheckBox( + settings_panel, label=_("Advanced Search") + ) + self.enable_cross_languages_chk.SetValue(False) + self.enable_retrieval_chk = wx.CheckBox( + settings_panel, label=_("Use Help Documentation") + ) + self.enable_retrieval_chk.SetValue(True) + + settings_sizer.Add(self.auto_hide_chk, 0, wx.ALL, 5) + settings_sizer.Add(self.enable_cross_languages_chk, 0, wx.ALL, 5) + settings_sizer.Add(self.enable_retrieval_chk, 0, wx.ALL, 5) + + settings_panel.SetSizer(settings_sizer) + self.additional_settings = [ + self.auto_hide_chk, + self.enable_cross_languages_chk, + self.enable_retrieval_chk, + ] + # Add buttons + btn_box = wx.BoxSizer(wx.HORIZONTAL) + self.clear_btn = wx.Button(input_container, label=_("Clear History (&C)")) + self.send_btn = wx.Button(input_container, label=_("Send (&S)")) + + btn_box.AddStretchSpacer() + btn_box.Add(self.clear_btn, 0, wx.ALL, 8) + btn_box.Add(self.send_btn, 0, wx.ALL, 8) + + input_box.Add(btn_box, 0, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.BOTTOM, 10) + main_vbox.Add(input_box, 0, wx.EXPAND | wx.ALL, 5) + + self.send_btn.Bind(wx.EVT_BUTTON, self.on_send) + self.clear_btn.Bind(wx.EVT_BUTTON, self.on_clear) + self.prev_btn.Bind(wx.EVT_BUTTON, self.on_previous) + self.next_btn.Bind(wx.EVT_BUTTON, self.on_next) + + self.prev_btn.Bind(wx.EVT_SET_FOCUS, self.on_navigation_button_focus) + self.next_btn.Bind(wx.EVT_SET_FOCUS, self.on_navigation_button_focus) + self.input_text.Bind(wx.EVT_KEY_DOWN, self.on_key_down) + + self.Bind(wx.EVT_CHAR_HOOK, self.on_char_hook) + self.Bind(wx.EVT_CLOSE, self.on_close_window) + + panel.SetSizer(main_vbox) + + self.update_display_area() + self.update_navigation_buttons() + + self.Centre() + + if shown: + self.focus_panel(focus_element=self.input_text) + else: + self.Hide() + + def on_navigation_button_focus(self, event): + current_status = self.current_label.GetLabel() + self.plugin.announce(current_status, priority=SpeechPriority.NORMAL) + event.Skip() + + def on_send(self, event): + question = self.input_text.GetValue().strip() + if not question: + self.plugin.announce(_("Please enter your question")) + return + + self.input_text.Clear() + self.send_btn.Enable(False) + self.last_request = question + self.process_request(question) + + def process_request(self, question, desktop_state=None, use_rag=True): + """Process user question: use provided state or current stored state""" + if desktop_state is not None: + self.current_desktop_state = desktop_state + + if self.current_desktop_state is None: + self.plugin.announce(_("Unable to capture current screen information")) + self.send_btn.Enable(True) + return + + self.add_user_message(question, self.current_desktop_state) + + # handle dialog visibility based on user preference + if hasattr(self, "auto_hide_chk") and self.auto_hide_chk.IsChecked(): + self.Hide() + + if ( + hasattr(self, "enable_retrieval_chk") + and self.enable_retrieval_chk.IsChecked() + and use_rag + ): + + def on_help_retrieved(help_info): + AIRequestManager.make_ai_request( + self.plugin, + self.message_list, + help_info, + lambda response: self.add_ai_message(response), + ) + + AIRequestManager.retrieve_help_info( + self.plugin, question, self.current_desktop_state, on_help_retrieved + ) + else: + AIRequestManager.make_ai_request( + self.plugin, + self.message_list, + None, + lambda response: self.add_ai_message(response), + ) + + def check_progress(self, desktop_state, screen_reader_state): + """Check user progress and provide guidance based on their current state""" + # Get the current AI message (the guidance steps) + ai_message = self.get_ai_message() + guideline = ai_message.content if ai_message else "" + + # Store the current state for reuse + self.current_desktop_state = desktop_state + step_number = max(1, self.current_step + 1) + + self.add_user_message( + _( + "It seems I encountered difficulty at step {step_number}. What should I do?" + ).format(step_number=step_number), + self.current_desktop_state, + screen_reader_state, + ) + # Create a progress check request using the PROGRESS_CHECK_PROMPT + AIRequestManager.make_progress_check_request( + self.plugin, + self.message_list, + self.last_request, + guideline, + step_number, + lambda response: self.add_ai_message(response), + ) + + def describe_current_state(self, desktop_state): + description_request = _( + "Describe the current screen, specifying what the current focus is and its function. If any unexpected elements such as pop-ups, error messages, or interface changes are present, describe them as well." + ) + self.process_request(description_request, desktop_state, False) + + def add_user_message(self, question, desktop_state, screen_reader_state=None): + logging_to_desktop(f"user request: {question}") + + self.message_list.add_message( + role="user", + content=question, + desktop_state=desktop_state, + screen_reader_state=screen_reader_state, + ) + + self.current_history_index = self.message_list.len() - 1 + self.update_display_area() + self.update_navigation_buttons() + + def add_ai_message(self, response): + logging_to_desktop(f"AI answer: {response}") + + # announce the response + self.plugin.announce(format_guide_steps(response)) + + self.message_list.add_message(role="AI", content=response) + self.current_history_index = self.message_list.len() - 2 + self.current_step = -1 + self.update_display_area() + self.update_navigation_buttons() + self.send_btn.Enable(True) + + def on_key_down(self, event): + # enter: send message + if event.GetKeyCode() == wx.WXK_RETURN and not event.ShiftDown(): + self.on_send(None) + # Shift+enter: allow new line + elif event.GetKeyCode() == wx.WXK_RETURN and event.ShiftDown(): + event.Skip() + else: + event.Skip() + + def on_clear(self, event): + self.message_list.clear() + self.current_history_index = -1 + self.current_step = -1 + self.last_request = None + self.update_display_area() + self.update_navigation_buttons() + self.plugin.announce(_("Chat history cleared")) + + def on_char_hook(self, event): + # handle global key events + key_code = event.GetKeyCode() + current_focus = wx.Window.FindFocus() + + if key_code == wx.WXK_F1: + if current_focus == self.input_text: + self.plugin.announce(_("Input field: Type your question here. Press Enter to send, Shift+Enter for new line.")) + elif current_focus == self.ai_reply_text: + self.plugin.announce(_("AI response area: Contains step-by-step guidance. Use arrow keys to read.")) + elif current_focus == self.user_request_text: + self.plugin.announce(_("Your previous question area: Shows your previously asked question.")) + elif current_focus in [self.prev_btn, self.next_btn]: + self.plugin.announce(_("""History navigation: +Use the Previous button, or press Alt and P, to go to the previous conversation. +Use the Next button, or press Alt and N, to go to the next conversation.""")) + else: + self.plugin.announce(_("""AskEase Help: +Press F1 to open context help. +Press Escape to close the window. +Press Alt and C to clear chat history. + +In Settings, you can choose whether the dialog is hidden automatically, +whether to use help documentation, and whether to enable advanced search. +Please note, enabling advanced search may make responses slightly slower. +All of these options are turned on by default. +""")) + return + + if current_focus == self.settings_pane or ( + current_focus and self.settings_pane.IsDescendant(current_focus) + ): + if current_focus not in self.additional_settings: + # if focus is in settings pane (e.g., title) but not on checkboxes + if key_code == wx.WXK_RETURN or key_code == wx.WXK_SPACE: + if self.settings_pane.IsExpanded(): + self.settings_pane.Collapse() + else: + self.settings_pane.Expand() + wx.CallLater(50, self.auto_hide_chk.SetFocus) + self.Layout() + return + else: + if ( + key_code == wx.WXK_TAB + and current_focus == self.additional_settings[-1] + ): + self.settings_pane.Collapse() + self.Layout() + self.clear_btn.SetFocus() + return + elif ( + key_code == wx.WXK_TAB + and event.ShiftDown() + and current_focus == self.additional_settings[0] + ): + self.settings_pane.Collapse() + self.Layout() + event.Skip() + return + elif key_code == wx.WXK_DOWN: + # Handle down arrow for all elements + current_index = self.additional_settings.index(current_focus) + next_index = (current_index + 1) % len(self.additional_settings) + self.additional_settings[next_index].SetFocus() + return + elif key_code == wx.WXK_UP: + # Handle up arrow for all elements + current_index = self.additional_settings.index(current_focus) + prev_index = (current_index - 1) % len(self.additional_settings) + self.additional_settings[prev_index].SetFocus() + return + + if key_code == wx.WXK_ESCAPE: + self.Close() + else: + event.Skip() + + def on_previous(self, event): + # show previous message + prev_index = self.current_history_index - 2 + if prev_index >= 0: + self.current_history_index = prev_index + self.current_step = -1 + self.update_display_area() + self.update_navigation_buttons() + + total_pairs = (self.message_list.len() + 1) // 2 + current_pair = (self.current_history_index // 2) + 1 + self.plugin.announce( + _("Conversation {current} of {total}").format( + current=current_pair, total=total_pairs + ) + ) + else: + self.plugin.announce(_("Already at the first conversation")) + + def on_next(self, event): + # show next message + next_index = self.current_history_index + 2 + if next_index < self.message_list.len(): + self.current_history_index = next_index + self.current_step = -1 + self.update_display_area() + self.update_navigation_buttons() + total_pairs = (self.message_list.len() + 1) // 2 + current_pair = (self.current_history_index // 2) + 1 + self.plugin.announce( + _("Conversation {current} of {total}").format( + current=current_pair, total=total_pairs + ) + ) + else: + self.plugin.announce(_("Already at the last conversation")) + + def update_navigation_buttons(self): + if ( + not hasattr(self, "current_label") + or not hasattr(self, "prev_btn") + or not hasattr(self, "next_btn") + ): + return + # update navigation buttons and current label + if self.current_label: + total_pairs = (self.message_list.len() + 1) // 2 + current_pair = (self.current_history_index // 2) + 1 + + if total_pairs == 0: + self.current_label.SetLabel(_("No conversations yet")) + else: + self.current_label.SetLabel( + _("Conversation {current} of {total}").format( + current=current_pair, total=total_pairs + ) + ) + + def update_display_area(self): + if not hasattr(self, "user_request_text") or not hasattr(self, "ai_reply_text"): + return + + if not self.has_history(): + self.user_request_text.SetValue("") + self.ai_reply_text.SetValue("") + return + + userMessage = self.get_user_message() + user_msg = userMessage.content + self.user_request_text.SetValue(user_msg) + + img_base64 = userMessage.desktop_state.screenshot + if DEBUG and img_base64 and hasattr(self, "screenshot_preview"): + bitmap = create_screenshot_bitmap(img_base64, PREVIEW_WIDTH, PREVIEW_HEIGHT) + if bitmap: + self.screenshot_preview.SetBitmap(bitmap) + self.view_full_btn.Enable(True) + + ai_message = self.get_ai_message() + if ai_message: + ai_msg = ai_message.content + else: + ai_msg = "" + self.ai_reply_text.SetValue(format_guide_steps(ai_msg)) + + def add_screenshot_preview(self, panel, main_vbox): + preview_box = wx.StaticBox( + panel, + label=_( + "Screenshot Preview (visually impaired users can ignore this area)" + ), + ) + preview_sizer = wx.StaticBoxSizer(preview_box, wx.HORIZONTAL) + + placeholder = wx.Bitmap(PREVIEW_WIDTH, PREVIEW_HEIGHT) + dc = wx.MemoryDC(placeholder) + dc.SetBackground(wx.Brush(wx.WHITE)) + dc.Clear() + dc.DrawRectangle(0, 0, PREVIEW_WIDTH, PREVIEW_HEIGHT) + dc = None + + self.screenshot_preview = wx.StaticBitmap(preview_box, bitmap=placeholder) + self.screenshot_preview.SetMinSize((PREVIEW_WIDTH, PREVIEW_HEIGHT)) + preview_sizer.Add(self.screenshot_preview, 0, wx.ALL, 5) + + # Add button to view full screenshot + btn_sizer = wx.BoxSizer(wx.VERTICAL) + self.view_full_btn = wx.Button(preview_box, label=_("View Full Image (&V)")) + self.view_full_btn.Bind(wx.EVT_BUTTON, self.on_view_full_screenshot) + self.view_full_btn.Enable(False) + btn_sizer.Add(self.view_full_btn, 0, wx.EXPAND | wx.ALL, 5) + + preview_sizer.Add(btn_sizer, 0, wx.ALIGN_CENTER_VERTICAL | wx.ALL, 5) + main_vbox.Add(preview_sizer, 0, wx.EXPAND | wx.ALL, 5) + + def on_view_full_screenshot(self, event): + try: + message = self.get_user_message() + desktop_state = getattr(message, "desktop_state", None) + screen_reader_state = getattr(message, "screen_reader_state", None) + + if desktop_state: + screenshot = desktop_state.screenshot + status = desktop_state.to_string() + if screen_reader_state: + status += ( + f"\nScreen Reader history: {screen_reader_state.to_string()}" + ) + else: + self.plugin.announce(_("No corresponding screenshot data found")) + return + + frame = wx.Frame( + None, title=_("Full Screenshot"), size=FULL_SCREENSHOT_SIZE + ) + panel = wx.Panel(frame) + main_sizer = wx.BoxSizer(wx.VERTICAL) + + try: + bitmap = create_screenshot_bitmap(screenshot, None, None) + if bitmap: + original_width = bitmap.GetWidth() + original_height = bitmap.GetHeight() + + scroll_window = wx.ScrolledWindow( + panel, style=wx.HSCROLL | wx.VSCROLL + ) + scroll_window.SetScrollRate(10, 10) + + scroll_sizer = wx.BoxSizer(wx.VERTICAL) + img_ctrl = wx.StaticBitmap(scroll_window, bitmap=bitmap) + scroll_sizer.Add(img_ctrl, 0, wx.ALL, 0) + + scroll_window.SetSizer(scroll_sizer) + scroll_window.SetVirtualSize((original_width, original_height)) + + main_sizer.Add(scroll_window, 1, wx.EXPAND | wx.ALL, 5) + + status_bar = frame.CreateStatusBar() + status_bar.SetStatusText(status) + else: + self.plugin.announce( + _("Error displaying screenshot: unable to create bitmap") + ) + + except Exception as e: + error_text = wx.StaticText( + panel, label=_("Failed to load image: {error}").format(error=str(e)) + ) + main_sizer.Add(error_text, 0, wx.ALL, 20) + + close_btn = wx.Button(panel, label=_("Close")) + close_btn.Bind(wx.EVT_BUTTON, lambda evt: frame.Close()) + main_sizer.Add(close_btn, 0, wx.ALIGN_CENTER | wx.BOTTOM, 10) + + panel.SetSizer(main_sizer) + frame.Centre() + frame.Show() + + except Exception as e: + self.plugin.announce( + _("Error displaying screenshot: {error}").format(error=str(e)) + ) + + def get_user_message(self): + return self.message_list.get_message(self.current_history_index) + + def get_ai_message(self): + return self.message_list.get_message(self.current_history_index + 1) + + def go_to_step(self, step_offset: int): + message = self.get_ai_message() + if message: + try: + step_string = format_guide_steps(message.content) + steps = step_string.split("\n") + except json.JSONDecodeError: + steps = [message.content] + + if self.current_step + step_offset < 0: + self.plugin.announce( + _("This is the first step: {step}").format(step=steps[0]) + ) + elif self.current_step + step_offset >= len(steps): + self.plugin.announce( + _("This is the final step: {step}").format(step=steps[-1]) + ) + else: + self.current_step += step_offset + step_text = steps[self.current_step] + self.plugin.announce(f"{step_text}") + else: + self.plugin.announce(_("There is no step-by-step guidance available. Press NVDA+Ctrl+O to open the dialog and ask your question.")) + + def has_history(self): + return self.message_list.len() > 0 + + +class AIRequestManager: + @staticmethod + def create_notification_thread(stop_event): + def notify_loop(): + while not stop_event.is_set(): + time.sleep(BEEP_INTERVAL) + if not stop_event.is_set(): + # Mature, steady processing pattern + # F note - deeper, concluding tone + tones.beep(174, 100) + time.sleep(0.4) + tones.beep(174, 100) + time.sleep(0.4) + tones.beep(174, 100) + time.sleep(1.5) # Longer, contemplative pause + + return threading.Thread(target=notify_loop, daemon=True) + + @staticmethod + def format_message_history(message_list): + # Format messages for LLM by converting them to OpenAI message format. + formatted_messages = [{"role": "system", "content": SYSTEM_PROMPT}] + + for message in message_list.messages[-7:-1]: + if message.role == "user": + prompt = NEXT_STEP_PROMPT.format( + question=message.content, + desktop_state=message.desktop_state.to_string(), + ) + if DEBUG: + print(f"Formatted user prompt: {prompt}") + + formatted_messages.append( + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image_url", + "image_url": { + "url": ( + f"data:image/jpeg;base64," + f"{message.desktop_state.screenshot}" + ) + }, + }, + ], + } + ) + elif message.role == "AI": + formatted_messages.append( + {"role": "assistant", "content": message.content} + ) + + return formatted_messages + + @staticmethod + def make_ai_request(plugin, message_list, help_info, on_finished): + plugin.announce(_("Getting AI response")) + stop_event = threading.Event() + + notify_thread = AIRequestManager.create_notification_thread(stop_event) + notify_thread.start() + + def worker(): + try: + messages = AIRequestManager.format_message_history(message_list) + # last user message + last_message = message_list.messages[-1] + assert ( + last_message.role == "user" + ), "Last message must be a user message" + if help_info: + formatted = [] + for j, res in enumerate(help_info, 1): + text = res.get("text", "") + text = re.sub(r"(?m)^###(?!#)", "#####", text) + text = re.sub(r"(?m)^##(?!#)", "####", text) + text = text.strip() + + formatted.append( + f"### Document{j}\n{text}\n\nScore: {res.get('score', 0):.4f}" + ) + help_info_formatted = "\n\n".join(formatted) + + prompt = NEXT_STEP_WITH_HELP_INFO_PROMPT.format( + question=last_message.content, + desktop_state=last_message.desktop_state.to_string(), + help_info=help_info_formatted, + ) + if DEBUG: + print(f"Formatted user prompt with help info: {prompt}") + else: + prompt = NEXT_STEP_PROMPT.format( + question=last_message.content, + desktop_state=last_message.desktop_state.to_string(), + ) + if DEBUG: + print(f"Formatted user prompt without help info: {prompt}") + messages.append( + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image_url", + "image_url": { + "url": ( + f"data:image/jpeg;base64," + f"{last_message.desktop_state.screenshot}" + ) + }, + }, + ], + } + ) + answer = request_openai_model(messages) + stop_event.set() + wx.CallAfter(on_finished, answer) + except Exception as e: + stop_event.set() + error_msg = str(_("Unable to get AI assistance: {error}")).format( + error=str(e) + ) + wx.CallAfter(on_finished, error_msg) + + threading.Thread(target=worker, daemon=True).start() + + @staticmethod + def make_progress_check_request( + plugin, message_list, last_request, guideline, step_number, on_finished + ): + """Make a progress check request to AI""" + plugin.announce(_("Getting AI response")) + stop_event = threading.Event() + + notify_thread = AIRequestManager.create_notification_thread(stop_event) + notify_thread.start() + + def worker(): + try: + messages = [{"role": "system", "content": SYSTEM_PROMPT}] + # last user message + last_message = message_list.messages[-1] + assert ( + last_message.role == "user" + ), "Last message must be a user message" + # Format the progress check prompt + prompt = PROGRESS_CHECK_PROMPT.format( + desktop_state=last_message.desktop_state.to_string(), + screen_reader_state=(last_message.screen_reader_state.to_string()), + question=last_request, + guideline=format_guide_steps(guideline), + step_number=step_number, + ) + if DEBUG: + print(f"Progress check prompt: {prompt}") + messages.append( + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image_url", + "image_url": { + "url": ( + f"data:image/jpeg;base64," + f"{last_message.desktop_state.screenshot}" + ) + }, + }, + ], + } + ) + + answer = request_openai_model(messages) + stop_event.set() + wx.CallAfter(on_finished, answer) + except Exception as e: + stop_event.set() + error_msg = str(_("Unable to get AI assistance: {error}")).format( + error=str(e) + ) + wx.CallAfter(on_finished, error_msg) + + threading.Thread(target=worker, daemon=True).start() + + @staticmethod + def retrieve_help_info(plugin, query, desktopState, on_finished, top_k=5): + # Check if RAG server is configured first + if not RAG_SERVER: + if DEBUG: + print("[-] RAG_SERVER is not configured, skipping retrieval") + wx.CallAfter(on_finished, []) + return + + plugin.announce(_("Fetching relevant help articles...")) + + stop_event = threading.Event() + notify_thread = AIRequestManager.create_notification_thread(stop_event) + notify_thread.start() + + def worker(): + try: + if desktopState.foreground: + productName = desktopState.foreground.get("productName", "") + appName = desktopState.foreground.get("appName", "") + windowText = desktopState.foreground.get("windowText", "") + dataset = find_rag_dataset(productName, appName, windowText) + if not dataset: + if DEBUG: + print( + f"[-] No RAG dataset found for: {productName} {appName} {windowText}" + ) + results = [] + else: + payload = { + "question": query, + "dataset_ids": [dataset], + "top_k": top_k, + } + # Add cross_languages if enabled + if ( + hasattr(plugin._helpDialog, "enable_cross_languages_chk") + and plugin._helpDialog.enable_cross_languages_chk.IsChecked() + ): + payload["cross_languages"] = ["hyde"] + resp = requests.post(f"{RAG_SERVER}{API_PATH}", json=payload) + if resp.status_code != 200: + print("Retrieval failed:", resp.text) + results = [] + else: + results = resp.json().get("hits", []) + else: + results = [] + + stop_event.set() + wx.CallAfter(on_finished, results) + except Exception as e: + stop_event.set() + print(f"Error retrieving help info: {e}") + wx.CallAfter(on_finished, []) + + threading.Thread(target=worker, daemon=True).start() diff --git a/addon/globalPlugins/screenAssistant/log.py b/addon/globalPlugins/screenAssistant/log.py new file mode 100644 index 0000000..39c56f8 --- /dev/null +++ b/addon/globalPlugins/screenAssistant/log.py @@ -0,0 +1,33 @@ +import ctypes +import os + +CSIDL_DESKTOPDIRECTORY = 0x0010 # Desktop folder +SHGFP_TYPE_CURRENT = 0 +# logging chat history to desktop +LOGGING_ENABLED = False + + +def get_user_desktop_dir() -> str: + try: + buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH) + ctypes.windll.shell32.SHGetFolderPathW( + 0, CSIDL_DESKTOPDIRECTORY, 0, SHGFP_TYPE_CURRENT, buf + ) + return buf.value + except Exception: + return os.path.join(os.path.expanduser("~"), "Desktop") + + +def logging_to_desktop(content: str, withTime=True, filename: str = "study_log.txt") -> str: + if LOGGING_ENABLED: + desktop = get_user_desktop_dir() + path = os.path.join(desktop, filename) + with open(path, "a", encoding="utf-8") as f: + # get current time + from datetime import datetime + current_time = datetime.now().isoformat(timespec="milliseconds").split("T")[1] + if withTime: + from time import time + f.write(f"{current_time}\n{time():.3f}\n{content}\n\n") + else: + f.write(f"{content}\n") \ No newline at end of file diff --git a/addon/globalPlugins/screenAssistant/message/__init__.py b/addon/globalPlugins/screenAssistant/message/__init__.py new file mode 100644 index 0000000..9fbd665 --- /dev/null +++ b/addon/globalPlugins/screenAssistant/message/__init__.py @@ -0,0 +1,35 @@ +import os +import sys + +current_dir = os.path.dirname(__file__) +globalPlugins_dir = os.path.dirname(current_dir) + +if globalPlugins_dir not in sys.path: + sys.path.insert(0, globalPlugins_dir) + +from message.views import Message + + +class MessageList: + def __init__(self): + self.messages = [] + + def add_message(self, role, content, desktop_state=None, screen_reader_state=None): + message = Message( + role=role, + content=content, + desktop_state=desktop_state, + screen_reader_state=screen_reader_state, + ) + self.messages.append(message) + + def clear(self): + self.messages.clear() + + def len(self): + return len(self.messages) + + def get_message(self, index): + if index >= 0 and index < self.len(): + return self.messages[index] + return None diff --git a/addon/globalPlugins/screenAssistant/message/views.py b/addon/globalPlugins/screenAssistant/message/views.py new file mode 100644 index 0000000..7fc6c5b --- /dev/null +++ b/addon/globalPlugins/screenAssistant/message/views.py @@ -0,0 +1,20 @@ +from dataclasses import dataclass +import os +import sys + +current_dir = os.path.dirname(__file__) +globalPlugins_dir = os.path.dirname(current_dir) + +if globalPlugins_dir not in sys.path: + sys.path.insert(0, globalPlugins_dir) + +from desktop.views import DesktopState +from screenReader.views import ScreenReaderState + + +@dataclass +class Message: + role: str + content: str + desktop_state: DesktopState | None = None + screen_reader_state: ScreenReaderState | None = None \ No newline at end of file diff --git a/addon/globalPlugins/screenAssistant/openai_service.py b/addon/globalPlugins/screenAssistant/openai_service.py new file mode 100644 index 0000000..243dd4a --- /dev/null +++ b/addon/globalPlugins/screenAssistant/openai_service.py @@ -0,0 +1,114 @@ +import requests +import addonConfig as cfg +import languageHandler +import addonHandler +import os +import sys +import json +import time + +addonHandler.initTranslation() + +plugin_dir = os.path.dirname(__file__) +sys.path.insert(0, plugin_dir) + +from log import logging_to_desktop + + +def language_code_to_string(lang_code): + """Convert language code to a human-readable string.""" + if lang_code == "zh_CN": + return "Chinese (Simplified)" + elif lang_code == "zh_TW" or lang_code == "zh_HK": + return "Chinese (Traditional)" + elif lang_code == "en": + return "English" + else: + return lang_code # Fallback to the code itself if unknown + + +effective_lang = language_code_to_string(languageHandler.getLanguage()) + +OUTPUT_FORMAT_PROMPT = """Return EXACTLY one JSON object in the format below. Answer in {language}. +{{ + "thoughts": "string", + "answer": ["string", "..."], +}} +""" + + +def get_api_settings(): + """Get API Setting""" + return { + "api_key": cfg.get_config("openai", "apiKey"), + "model": cfg.get_config("openai", "model"), + "endpoint": cfg.get_config("openai", "endpoint"), + } + + +def _append_format_prompt_to_last_message(messages): + """Append OUTPUT_FORMAT_PROMPT into the last message's content.""" + if not messages: + return messages + msgs = list(messages) # avoid mutating caller's list + last = dict(msgs[-1]) + + content = last.get("content") + last["content"] = content + [ + {"type": "text", "text": OUTPUT_FORMAT_PROMPT.format(language=effective_lang)} + ] + + msgs[-1] = last + return msgs + + +def request_openai_model(messages): + settings = get_api_settings() + + if not settings["api_key"] or not settings["endpoint"]: + raise ValueError(_("API key or endpoint is not set in the configuration.")) + + messages = _append_format_prompt_to_last_message(messages) + + payload = { + "messages": messages, + "max_tokens": 8192, + "temperature": 0.1, + } + + headers = {"api-key": settings["api_key"], "Content-Type": "application/json"} + start_time = time.time() + response = requests.post(settings["endpoint"], headers=headers, json=payload) + result = response.json() + time_cost = time.time() - start_time + usage = result.get("usage", {}) + usage["time_cost"] = time_cost + logging_to_desktop(f"{json.dumps(usage)}", False) + + return result["choices"][0]["message"]["content"] + + +def check_api_settings(): + """ + Checks if the current API key and endpoint are valid by making a minimal request. + Returns (True, None) if OK, or (False, error_message) if not. + """ + settings = get_api_settings() + headers = {"api-key": settings["api_key"], "Content-Type": "application/json"} + # Minimal payload for a chat/completion request + payload = {"messages": [{"role": "user", "content": "Hello"}], "max_tokens": 1} + try: + response = requests.post( + settings["endpoint"], headers=headers, json=payload, timeout=10 + ) + if response.status_code == 200: + return True, None + else: + # Try to extract error message from response + try: + err = response.json().get("error", {}).get("message", response.text) + except Exception: + err = response.text + return False, err + except Exception as e: + return False, str(e) diff --git a/addon/globalPlugins/screenAssistant/prompt.py b/addon/globalPlugins/screenAssistant/prompt.py new file mode 100644 index 0000000..e1f1065 --- /dev/null +++ b/addon/globalPlugins/screenAssistant/prompt.py @@ -0,0 +1,60 @@ +SYSTEM_PROMPT = """You are an assistant for the NVDA screen reader. Your role is to help the user understand the current interface and guide them to operate applications using NVDA. + +## STRICT RULES +Be an approachable and helpful teacher who helps users learn to master applications with NVDA. + - Be concise and direct. Structure your response as individual sentences, with each sentence being a complete, standalone statement. Do not add any numbers or bullet symbols before sentences. + - Always analyze the provided screenshot to identify the interface structure, current focus, and visible controls. Use this to provide extra context that a screen reader user might not detect directly, and integrate it into the keyboard navigation guidance. + - Prioritize keyboard interaction (e.g., shortcuts, Tab, arrow keys, Enter, Esc, Space). Avoid mentioning the mouse. + - Avoid purely visual descriptions (e.g., "red button"). Instead, use structural or functional descriptions (e.g., "the first tab", "the Settings item in the main menu"). + - When a technical term is needed, give a short clarification and what will happen after the action. + - If you are not certain, explicitly say you are unsure and suggest the user try an exploratory action or consult documentation rather than guessing. + +## THINGS YOU SHOULD DO + - If the user needs **step-by-step guidance to accomplish a task**, provide clear operational steps. + - If the user is **asking about current state** or **seeking information**, provide a direct answer. + - If the user's question is **unclear**, ask for clarification. +""" + +NEXT_STEP_PROMPT = ( + "## Current Screen State\n{desktop_state}\n" + "**Desktop screenshot**: The focus is highlighted with a red rectangle.\n\n" + "## User Query\n{question}\n\n" + "## Your Task\n" + "**Note**: Please refer to the desktop screenshot\n" + "1. **Current State Analysis**: Analyze the current screen state, including the active window, current focus element, and visible UI components.\n" + "2. **Response Strategy**: Provide a concise response." +) + +NEXT_STEP_WITH_HELP_INFO_PROMPT = ( + "## Current Screen State\n{desktop_state}\n" + "**Desktop screenshot**: The focus is highlighted with a red rectangle.\n\n" + "## Most Recent Relevant Help Content\n" + "Below are related entries with their similarity scores:\n\n{help_info}\n\n" + "## User Query\n{question}\n\n" + "## Your Task\n" + "**Note**: Please refer to the desktop screenshot\n" + "1. **Current State Analysis**: Analyze the current screen state, including the active window, current focus element, and visible UI components.\n" + "2. **Knowledge Integration Analysis**: Examine the **Most Recent Relevant Help Content** above and identify which specific information is relevant to the user's query.\n" + "3. **Response Strategy**: \n" + " - **MANDATORY**: The **Most Recent Relevant Help Content** contains current and authoritative information that should be prioritized when relevant.\n" + " - If the help content contains specific keyboard shortcuts or procedures for the user's task, use those EXACTLY as provided.\n" + " - Only supplement with your knowledge if the help content doesn't fully address the question.\n" + " - Provide a concise response." +) + +PROGRESS_CHECK_PROMPT = ( + "## Current Screen State\n{desktop_state}\n" + "**Desktop screenshot**: The focus is highlighted with a red rectangle.\n\n" + "## Previous guideline\n{guideline}\n" + "**Note**: They appear to be stuck at step {step_number} because the guideline is *unclear* or *incorrect*.\n\n" + "### Recent user actions (keystrokes) and auditory feedback\n{screen_reader_state}\n\n" + "## User Intent\n{question}\n\n" + "## Your Task\n" + "**Note**: Please refer to the desktop screenshot\n" + "1. **Current State Analysis**: Analyze the current screen state, including the active window, current focus element, and visible UI components.\n" + "2. **Identify User Confusion**: Based on the user's recent actions, auditory feedback received, and previous guideline, determine specifically why the user is experiencing confusion or feeling stuck.\n" + "3. **Response Strategy**: To resolve their confusion and guide them toward their goal:\n" + " - Summary current state.\n" + " - Provide actionable next steps.\n" + " - Avoid confusing descriptions or incorrect as Previous guideline." +) diff --git a/addon/globalPlugins/screenAssistant/screenReader/__init__.py b/addon/globalPlugins/screenAssistant/screenReader/__init__.py new file mode 100644 index 0000000..e44065e --- /dev/null +++ b/addon/globalPlugins/screenAssistant/screenReader/__init__.py @@ -0,0 +1,38 @@ +from collections import deque +import os +import sys + +current_dir = os.path.dirname(__file__) +globalPlugins_dir = os.path.dirname(current_dir) + +if globalPlugins_dir not in sys.path: + sys.path.insert(0, globalPlugins_dir) + +from screenReader.views import ActionHistoryItem, ScreenReaderState + +# Constants +MAX_HISTORY_LEN = 16 + + +class ScreenReader: + def __init__(self): + self.history = deque(maxlen=MAX_HISTORY_LEN) + self.recording = False + + def get_state(self) -> ScreenReaderState: + return ScreenReaderState( + history=list(self.history) + ) + + def add_history(self, item: ActionHistoryItem): + if not self.recording: + return + self.history.append(item) + + def handle_recording(self): + if not self.recording: + self.recording = True + else: + # Stop recording + self.recording = False + self.history.clear() \ No newline at end of file diff --git a/addon/globalPlugins/screenAssistant/screenReader/views.py b/addon/globalPlugins/screenAssistant/screenReader/views.py new file mode 100644 index 0000000..fc70f4b --- /dev/null +++ b/addon/globalPlugins/screenAssistant/screenReader/views.py @@ -0,0 +1,29 @@ +from dataclasses import dataclass +import os +import sys + +current_dir = os.path.dirname(__file__) +globalPlugins_dir = os.path.dirname(current_dir) + +if globalPlugins_dir not in sys.path: + sys.path.insert(0, globalPlugins_dir) + +from string_utils import truncate_string + + +@dataclass +class ActionHistoryItem: + type: str # 'key', 'speech' + value: str # the actual value of the item + + +@dataclass +class ScreenReaderState: + history: list[ActionHistoryItem] + + def to_string(self) -> str: + trans = {"key": "key input", "speech": "speech output"} + return "\n".join( + f"[{trans[item.type]}] {truncate_string(item.value)}" + for item in self.history + ) diff --git a/addon/globalPlugins/screenAssistant/settingsPanel.py b/addon/globalPlugins/screenAssistant/settingsPanel.py new file mode 100644 index 0000000..b4e1174 --- /dev/null +++ b/addon/globalPlugins/screenAssistant/settingsPanel.py @@ -0,0 +1,90 @@ +import wx +import gui +import config +import addonHandler +import addonConfig as cfg +import openai_service + +addonHandler.initTranslation() + +class ScreenAssistantSettingsPanel(gui.settingsDialogs.SettingsPanel): + # Translators: Settings panel title + title = _("AskEase") + + def makeSettings(self, sizer): + helper = gui.guiHelper.BoxSizerHelper(self, sizer=sizer) + + # OpenAI Settings group + openaiGroupSizer = wx.StaticBoxSizer(wx.VERTICAL, self, _("OpenAI Settings")) + openaiGroupBox = openaiGroupSizer.GetStaticBox() + openaiGroup = gui.guiHelper.BoxSizerHelper(openaiGroupBox, sizer=openaiGroupSizer) + helper.addItem(openaiGroup) + + # API Key + Verify Button in one line + apiKeyLine = wx.BoxSizer(wx.HORIZONTAL) + apiKeyLabel = wx.StaticText(openaiGroup.sizer.GetStaticBox(), label=_("API Key")) + self.apiKeyCtrl = wx.TextCtrl( + openaiGroup.sizer.GetStaticBox(), + value=cfg.get_config("openai", "apiKey"), + size=(400, -1) + ) + self.verifyBtn = wx.Button(openaiGroup.sizer.GetStaticBox(), label=_("Verify")) + self.verifyBtn.Bind(wx.EVT_BUTTON, self.onVerify) + apiKeyLine.Add(apiKeyLabel, flag=wx.ALIGN_CENTER_VERTICAL) + apiKeyLine.Add(self.apiKeyCtrl, flag=wx.LEFT, border=5) + apiKeyLine.Add(self.verifyBtn, flag=wx.LEFT, border=5) + openaiGroup.sizer.Add(apiKeyLine, flag=wx.TOP | wx.BOTTOM, border=5) + + # Endpoint setting + self.endpointCtrl = openaiGroup.addLabeledControl( + _("Endpoint"), + wx.TextCtrl, + value=cfg.get_config("openai", "endpoint"), + size=(600, -1) + ) + + # Model setting + self.modelCtrl = openaiGroup.addLabeledControl( + _("Model"), + wx.ComboBox, + choices=["gpt-4.1", "gpt-4o", "gpt-5-chat"], + style=wx.CB_READONLY + ) + + # Set default selection based on config + current_model = cfg.get_config("openai", "model") + model_index = self.modelCtrl.FindString(current_model) + if model_index != wx.NOT_FOUND: + self.modelCtrl.SetSelection(model_index) + + def onSave(self): + apiKey = self.apiKeyCtrl.GetValue() + endpoint = self.endpointCtrl.GetValue() + model = self.modelCtrl.GetStringSelection() + # first check with current input + cfg.set_config("openai", "apiKey", apiKey) + cfg.set_config("openai", "model", model) + cfg.set_config("openai", "endpoint", endpoint) + ok, err = openai_service.check_api_settings() + if not ok: + wx.MessageBox(_("API check failed: %s") % err, _(u"Error"), wx.OK | wx.ICON_ERROR) + return False # prevent saving + # save if check passed + config.setConfig("openai", "apiKey", apiKey) + config.setConfig("openai", "model", model) + config.setConfig("openai", "endpoint", endpoint) + return True + + def onVerify(self, event): + apiKey = self.apiKeyCtrl.GetValue() + endpoint = self.endpointCtrl.GetValue() + model = self.modelCtrl.GetStringSelection() + # temporary settings + cfg.set_config("openai", "apiKey", apiKey) + cfg.set_config("openai", "model", model) + cfg.set_config("openai", "endpoint", endpoint) + ok, err = openai_service.check_api_settings() + if ok: + wx.MessageBox(_("API is valid!"), _("Success"), wx.OK | wx.ICON_INFORMATION) + else: + wx.MessageBox(_("API check failed: %s") % err, _("Error"), wx.OK | wx.ICON_ERROR) \ No newline at end of file diff --git a/addon/globalPlugins/screenAssistant/string_utils.py b/addon/globalPlugins/screenAssistant/string_utils.py new file mode 100644 index 0000000..317657b --- /dev/null +++ b/addon/globalPlugins/screenAssistant/string_utils.py @@ -0,0 +1,3 @@ +def truncate_string(s, max_length=200): + s = s.strip().replace('\n', ' ').replace('\r', ' ') + return s[:max_length] + ("..." if len(s) > max_length else "") \ No newline at end of file diff --git a/addon/locale/zh_CN/LC_MESSAGES/nvda.po b/addon/locale/zh_CN/LC_MESSAGES/nvda.po new file mode 100644 index 0000000..0fb497e --- /dev/null +++ b/addon/locale/zh_CN/LC_MESSAGES/nvda.po @@ -0,0 +1,336 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the 'AskEase' package. +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: 'AskEase' '0.1'\n" +"Report-Msgid-Bugs-To: 'nvda-translations@groups.io'\n" +"POT-Creation-Date: 2025-08-21 16:46+0800\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: zh_CN\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#. Translators: Settings panel title +#. Add-on summary/title, usually the user visible name of the add-on +#. Translators: Summary/title for this add-on +#. to be shown on installation and add-on information found in add-on store +#: addon\globalPlugins\screenAssistant\__init__.py:35 +#: addon\globalPlugins\screenAssistant\helpDialog.py:106 +#: addon\globalPlugins\screenAssistant\settingsPanel.py:12 buildVars.py:23 +msgid "AskEase" +msgstr "AskEase" + +#: addon\globalPlugins\screenAssistant\__init__.py:205 +msgid "Getting screen information" +msgstr "正在获取屏幕信息" + +#: addon\globalPlugins\screenAssistant\__init__.py:213 +#: addon\globalPlugins\screenAssistant\__init__.py:228 +#, python-brace-format +msgid "Failed to get screen information: {error}" +msgstr "获取屏幕信息失败: {error}" + +#: addon\globalPlugins\screenAssistant\__init__.py:225 +msgid "Screen information obtained" +msgstr "已获取屏幕信息" + +#: addon\globalPlugins\screenAssistant\__init__.py:285 +msgid "Analyze the current screen and refine earlier guidance." +msgstr "分析当前屏幕并优化先前的指导。" + +#: addon\globalPlugins\screenAssistant\__init__.py:293 +msgid "Describe the screen and current focus." +msgstr "描述当前屏幕内容和焦点位置。" + +#: addon\globalPlugins\screenAssistant\__init__.py:299 +msgid "Ask for help" +msgstr "获取帮助" + +#: addon\globalPlugins\screenAssistant\__init__.py:304 +msgid "Go back to the previous guidance step" +msgstr "返回上一指导步骤" + +#: addon\globalPlugins\screenAssistant\__init__.py:311 +msgid "Go to the next guidance step" +msgstr "前往下一指导步骤" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:126 +msgid "AskEase dialog opened. Press ESC to close." +msgstr "AskEase 对话框已打开。按 ESC 键可关闭。" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:137 +msgid "Chat History Navigation" +msgstr "对话历史导航" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:139 +msgid "Previous (&P)" +msgstr "上一个 (&P)" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:140 +msgid "Next (&N)" +msgstr "下一个 (&N)" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:142 +#: addon\globalPlugins\screenAssistant\helpDialog.py:504 +msgid "No conversations yet" +msgstr "暂无对话" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:155 +msgid "Your Previous Question" +msgstr "你之前的提问" + +#. display AI reply +#: addon\globalPlugins\screenAssistant\helpDialog.py:166 +msgid "AI Reply" +msgstr "AI回复" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:177 +msgid "Chat Input" +msgstr "聊天输入" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:181 +msgid "Ask a question." +msgstr "请输入您的问题。" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:211 +msgid "AI-generated content may be incorrect." +msgstr "AI生成的内容可能存在错误。" + +#. a collapsible settings area +#: addon\globalPlugins\screenAssistant\helpDialog.py:189 +msgid "Settings" +msgstr "设置" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:196 +msgid "Hide Dialog After Sending" +msgstr "发送后自动隐藏对话框" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:204 +msgid "Advanced Search" +msgstr "高级搜索" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:209 +msgid "Use Help Documentation" +msgstr "使用帮助文档" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:220 +msgid "Hide Window (&H)" +msgstr "隐藏窗口 (&H)" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:221 +msgid "Clear History (&C)" +msgstr "清空历史 (&C)" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:222 +msgid "Send (&S)" +msgstr "发送 (&S)" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:255 +msgid "Please enter your question" +msgstr "请输入您的问题" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:269 +msgid "Unable to capture current screen information" +msgstr "无法获取当前屏幕信息" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:316 +#, python-brace-format +msgid "" +"It seems I encountered difficulty at step {step_number}. What should I do?" +msgstr "在第 {step_number} 步遇到困难，我该怎么办？" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:333 +msgid "" +"Describe the current screen, specifying what the current focus is and its " +"function. If any unexpected elements such as pop-ups, error messages, or " +"interface changes are present, describe them as well." +msgstr "请描述当前屏幕，说明当前焦点是什么以及它的功能。如果出现意外元素（如弹窗、错误提示或界面变化），也请一并描述。" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:376 +msgid "Chat window hidden, use NVDA+Control+H to reopen" +msgstr "对话窗口已隐藏，使用 NVDA+Ctrl+H 重新打开" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:385 +msgid "Chat history cleared" +msgstr "对话历史已清空" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:394 +msgid "" +"Input field: Type your question here. Press Enter to send, Shift+Enter for " +"new line." +msgstr "输入框：在此输入您的问题。回车键发送，Shift+回车键换行。" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:396 +msgid "" +"AI response area: Contains step-by-step guidance. Use arrow keys to read." +msgstr "AI 回复区域：包含分步指导。使用方向键阅读。" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:398 +msgid "Your previous question area: Shows your previously asked question." +msgstr "你之前的提问区域：显示您之前提出的问题。" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:400 +msgid "" +"History navigation:\n" +"Use the Previous button, or press Alt and P, to go to the previous " +"conversation.\n" +"Use the Next button, or press Alt and N, to go to the next conversation." +msgstr "历史导航：\n" +"使用上一个按钮，或按 Alt 加 P，查看上一条对话。\n" +"使用下一个按钮，或按 Alt 加 N，查看下一条对话。" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:415 +msgid "" +"AskEase Help:\n" +"Press F1 to open context help.\n" +"Press Escape to close the window.\n" +"Press Alt and C to clear chat history.\n" +"In Settings, you can choose whether the dialog is hidden automatically,\n" +"whether to use help documentation, and whether to enable advanced search.\n" +"Please note, enabling advanced search may make responses slightly slower.\n" +"All of these options are turned on by default.\n" +msgstr "" +"AskEase 帮助：\n" +"按 F1 打开上下文帮助。\n" +"按 Escape 关闭窗口。\n" +"按 Alt 加 C 清除聊天记录。\n" +"\n" +"在设置中，您可以选择是否自动隐藏对话框，\n" +"是否使用帮助文档，以及是否启用高级搜索。\n" +"请注意，启用高级搜索可能会让回复稍微变慢。\n" +"这些选项默认都是开启的。\n" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:466 +#: addon\globalPlugins\screenAssistant\helpDialog.py:484 +#: addon\globalPlugins\screenAssistant\helpDialog.py:507 +#, python-brace-format +msgid "Conversation {current} of {total}" +msgstr "第 {current} 个对话，共 {total} 个" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:471 +msgid "Already at the first conversation" +msgstr "已经是第一个对话" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:489 +msgid "Already at the last conversation" +msgstr "已经是最后一个对话" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:543 +msgid "Screenshot Preview (visually impaired users can ignore this area)" +msgstr "屏幕截图预览（视障用户可忽略此区域）" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:561 +msgid "View Full Image (&V)" +msgstr "查看大图 (&V)" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:587 +msgid "No corresponding screenshot data found" +msgstr "没有找到对应的截图数据" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:591 +msgid "Full Screenshot" +msgstr "完整截图" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:620 +msgid "Error displaying screenshot: unable to create bitmap" +msgstr "显示截图时出错：无法创建位图" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:625 +#, python-brace-format +msgid "Failed to load image: {error}" +msgstr "加载图像失败：{error}" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:629 +msgid "Close" +msgstr "关闭" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:639 +#, python-brace-format +msgid "Error displaying screenshot: {error}" +msgstr "显示截图时出错：{error}" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:661 +#, python-brace-format +msgid "This is the first step: {step}" +msgstr "这是第一步：{step}" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:665 +#, python-brace-format +msgid "This is the final step: {step}" +msgstr "这是最后一步：{step}" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:672 +msgid "" +"There is no step-by-step guidance available. Press NVDA+Ctrl+O to open the dialog and ask your question." +msgstr "暂无分步指导。请按 NVDA+Ctrl+O 打开对话框并提问。" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:737 +#: addon\globalPlugins\screenAssistant\helpDialog.py:811 +msgid "Getting AI response" +msgstr "正在获取AI回答" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:799 +#: addon\globalPlugins\screenAssistant\helpDialog.py:857 +#, python-brace-format +msgid "Unable to get AI assistance: {error}" +msgstr "无法获取 AI 帮助：{error}" + +#: addon\globalPlugins\screenAssistant\helpDialog.py:866 +msgid "Fetching relevant help articles..." +msgstr "正在获取相关帮助文档..." + +#: addon\globalPlugins\screenAssistant\openai_service.py:60 +msgid "API key or endpoint is not set in the configuration." +msgstr "API 密钥或接口地址未在配置中设置。请在 NVDA 设置中进行配置。" + +#. OpenAI Settings group +#: addon\globalPlugins\screenAssistant\settingsPanel.py:18 +msgid "OpenAI Settings" +msgstr "OpenAI设置" + +#: addon\globalPlugins\screenAssistant\settingsPanel.py:25 +msgid "API Key" +msgstr "API密钥" + +#: addon\globalPlugins\screenAssistant\settingsPanel.py:27 +msgid "Verify" +msgstr "检查" + +#: addon\globalPlugins\screenAssistant\settingsPanel.py:36 +msgid "Endpoint" +msgstr "接口地址" + +#: addon\globalPlugins\screenAssistant\settingsPanel.py:42 +msgid "Model" +msgstr "模型" + +#: addon\globalPlugins\screenAssistant\settingsPanel.py:64 +#: addon\globalPlugins\screenAssistant\settingsPanel.py:84 +#, python-format +msgid "API check failed: %s" +msgstr "API检查失败：%s" + +#: addon\globalPlugins\screenAssistant\settingsPanel.py:64 +#: addon\globalPlugins\screenAssistant\settingsPanel.py:84 +msgid "Error" +msgstr "错误" + +#: addon\globalPlugins\screenAssistant\settingsPanel.py:82 +msgid "API is valid!" +msgstr "API有效" + +#: addon\globalPlugins\screenAssistant\settingsPanel.py:82 +msgid "Success" +msgstr "成功" + +#. Add-on description +#. Translators: Long description to be shown for this add-on on add-on information from add-on store +#: buildVars.py:26 +msgid "Take screenshots and ask questions about them" +msgstr "智能屏幕截图分析与问答助手" diff --git a/buildVars.py b/buildVars.py new file mode 100644 index 0000000..29c7315 --- /dev/null +++ b/buildVars.py @@ -0,0 +1,104 @@ +# -*- coding: UTF-8 -*- + +# Build customizations +# Change this file instead of sconstruct or manifest files, whenever possible. + + +# Since some strings in `addon_info` are translatable, +# we need to include them in the .po files. +# Gettext recognizes only strings given as parameters to the `_` function. +# To avoid initializing translations in this module we simply roll our own "fake" `_` function +# which returns whatever is given to it as an argument. +def _(arg): + return arg + + +# Add-on information variables +addon_info = { + # add-on Name/identifier, internal for NVDA + "addon_name": "AskEase", + # Add-on summary/title, usually the user visible name of the add-on + # Translators: Summary/title for this add-on + # to be shown on installation and add-on information found in add-on store + "addon_summary": _("AskEase"), + # Add-on description + # Translators: Long description to be shown for this add-on on add-on information from add-on store + "addon_description": _("""Take screenshots and ask questions about them"""), + # version + "addon_version": "0.1", + # Author(s) + "addon_author": "name ", + # URL for the add-on documentation support + "addon_url": None, + # URL for the add-on repository where the source code can be found + "addon_sourceURL": None, + # Documentation file name + "addon_docFileName": "readme.html", + # Minimum NVDA version supported (e.g. "2019.3.0", minor version is optional) + "addon_minimumNVDAVersion": "2019.3.0", + # Last NVDA version supported/tested (e.g. "2024.4.0", ideally more recent than minimum version) + "addon_lastTestedNVDAVersion": "2025.1.2", + # Add-on update channel (default is None, denoting stable releases, + # and for development releases, use "dev".) + # Do not change unless you know what you are doing! + "addon_updateChannel": None, + # Add-on license such as GPL 2 + "addon_license": "MIT", + # URL for the license document the add-on is licensed under + "addon_licenseURL": None, +} + +# Define the python files that are the sources of your add-on. +# You can either list every file (using ""/") as a path separator, +# or use glob expressions. +# For example to include all files with a ".py" extension from the "globalPlugins" dir of your add-on +# the list can be written as follows: +# pythonSources = ["addon/globalPlugins/*.py"] +# For more information on SCons Glob expressions please take a look at: +# https://scons.org/doc/production/HTML/scons-user/apd.html +pythonSources = [ + "addon/globalPlugins/*.py", + "addon/globalPlugins/*/*.py", + "addon/globalPlugins/*/*/*.py", + "addon/globalPlugins/*/*/*.pyd", + "addon/globalPlugins/*/*/*.pyi", +] + +# Files that contain strings for translation. Usually your python sources +i18nSources = pythonSources + ["buildVars.py"] + +# Files that will be ignored when building the nvda-addon file +# Paths are relative to the addon directory, not to the root directory of your addon sources. +excludedFiles = [] + +# Base language for the NVDA add-on +# If your add-on is written in a language other than english, modify this variable. +# For example, set baseLanguage to "es" if your add-on is primarily written in spanish. +# You must also edit .gitignore file to specify base language files to be ignored. +baseLanguage = "en" + +# Markdown extensions for add-on documentation +# Most add-ons do not require additional Markdown extensions. +# If you need to add support for markup such as tables, fill out the below list. +# Extensions string must be of the form "markdown.extensions.extensionName" +# e.g. "markdown.extensions.tables" to add tables. +markdownExtensions = [] + +# Custom braille translation tables +# If your add-on includes custom braille tables (most will not), fill out this dictionary. +# Each key is a dictionary named according to braille table file name, +# with keys inside recording the following attributes: +# displayName (name of the table shown to users and translatable), +# contracted (contracted (True) or uncontracted (False) braille code), +# output (shown in output table list), +# input (shown in input table list). +brailleTables = {} + +# Custom speech symbol dictionaries +# Symbol dictionary files reside in the locale folder, e.g. `locale\en`, and are named `symbols-.dic`. +# If your add-on includes custom speech symbol dictionaries (most will not), fill out this dictionary. +# Each key is the name of the dictionary, +# with keys inside recording the following attributes: +# displayName (name of the speech dictionary shown to users and translatable), +# mandatory (True when always enabled, False when not. +symbolDictionaries = {} \ No newline at end of file diff --git a/manifest-translated.ini.tpl b/manifest-translated.ini.tpl new file mode 100644 index 0000000..c06aa84 --- /dev/null +++ b/manifest-translated.ini.tpl @@ -0,0 +1,2 @@ +summary = "{addon_summary}" +description = """{addon_description}""" diff --git a/manifest.ini.tpl b/manifest.ini.tpl new file mode 100644 index 0000000..d44355d --- /dev/null +++ b/manifest.ini.tpl @@ -0,0 +1,10 @@ +name = {addon_name} +summary = "{addon_summary}" +description = """{addon_description}""" +author = "{addon_author}" +url = {addon_url} +version = {addon_version} +docFileName = {addon_docFileName} +minimumNVDAVersion = {addon_minimumNVDAVersion} +lastTestedNVDAVersion = {addon_lastTestedNVDAVersion} +updateChannel = {addon_updateChannel} diff --git a/sconstruct b/sconstruct new file mode 100644 index 0000000..0ca3acc --- /dev/null +++ b/sconstruct @@ -0,0 +1,303 @@ +# NVDA add-on template SCONSTRUCT file +# Copyright (C) 2012-2025 Rui Batista, Noelia Martinez, Joseph Lee +# This file is covered by the GNU General Public License. +# See the file COPYING.txt for more details. + +import codecs +import gettext +import os +import os.path +import zipfile +import sys + +# Add-on localization exchange facility and the template requires Python 3.10. +# For best practice, use Python 3.11 or later to align with NVDA development. +EnsurePythonVersion(3, 10) +sys.dont_write_bytecode = True + +# Bytecode should not be written for build vars module to keep the repository root folder clean. +import buildVars # NOQA: E402 + + +def md2html(source, dest): + import markdown + + # Use extensions if defined. + mdExtensions = buildVars.markdownExtensions + lang = os.path.basename(os.path.dirname(source)).replace("_", "-") + localeLang = os.path.basename(os.path.dirname(source)) + try: + _ = gettext.translation( + "nvda", localedir=os.path.join("addon", "locale"), languages=[localeLang] + ).gettext + summary = _(buildVars.addon_info["addon_summary"]) + except Exception: + summary = buildVars.addon_info["addon_summary"] + title = "{addonSummary} {addonVersion}".format( + addonSummary=summary, addonVersion=buildVars.addon_info["addon_version"] + ) + headerDic = { + '[[!meta title="': "# ", + '"]]': " #", + } + with codecs.open(source, "r", "utf-8") as f: + mdText = f.read() + for k, v in headerDic.items(): + mdText = mdText.replace(k, v, 1) + htmlText = markdown.markdown(mdText, extensions=mdExtensions) + # Optimization: build resulting HTML text in one go instead of writing parts separately. + docText = "\n".join( + [ + "", + f'', + "", + '', + '', + '', + f"{title}", + "\n", + htmlText, + "\n", + ] + ) + with codecs.open(dest, "w", "utf-8") as f: + f.write(docText) + + +def mdTool(env): + mdAction = env.Action( + lambda target, source, env: md2html(source[0].path, target[0].path), + lambda target, source, env: f"Generating {target[0]}", + ) + mdBuilder = env.Builder( + action=mdAction, + suffix=".html", + src_suffix=".md", + ) + env["BUILDERS"]["markdown"] = mdBuilder + + +def validateVersionNumber(key, val, env): + # Used to make sure version major.minor.patch are integers to comply with NV Access add-on store. + # Ignore all this if version number is not specified. + if val == "0.0.0": + return + versionNumber = val.split(".") + if len(versionNumber) < 3: + raise ValueError("versionNumber must have three parts (major.minor.patch)") + if not all([part.isnumeric() for part in versionNumber]): + raise ValueError("versionNumber (major.minor.patch) must be integers") + + +vars = Variables() +vars.Add("version", "The version of this build", buildVars.addon_info["addon_version"]) +vars.Add("versionNumber", "Version number of the form major.minor.patch", "0.0.0", validateVersionNumber) +vars.Add(BoolVariable("dev", "Whether this is a daily development version", False)) +vars.Add("channel", "Update channel for this build", buildVars.addon_info["addon_updateChannel"]) + +env = Environment(variables=vars, ENV=os.environ, tools=["gettexttool", mdTool]) +env.Append(**buildVars.addon_info) + +if env["dev"]: + import datetime + + buildDate = datetime.datetime.now() + year, month, day = str(buildDate.year), str(buildDate.month), str(buildDate.day) + versionTimestamp = "".join([year, month.zfill(2), day.zfill(2)]) + env["addon_version"] = f"{versionTimestamp}.0.0" + env["versionNumber"] = f"{versionTimestamp}.0.0" + env["channel"] = "dev" +elif env["version"] is not None: + env["addon_version"] = env["version"] +if "channel" in env and env["channel"] is not None: + env["addon_updateChannel"] = env["channel"] + +buildVars.addon_info["addon_version"] = env["addon_version"] +buildVars.addon_info["addon_updateChannel"] = env["addon_updateChannel"] + +addonFile = env.File("${addon_name}-${addon_version}.nvda-addon") + + +def addonGenerator(target, source, env, for_signature): + action = env.Action( + lambda target, source, env: createAddonBundleFromPath(source[0].abspath, target[0].abspath) and None, + lambda target, source, env: f"Generating Addon {target[0]}", + ) + return action + + +def manifestGenerator(target, source, env, for_signature): + action = env.Action( + lambda target, source, env: generateManifest(source[0].abspath, target[0].abspath) and None, + lambda target, source, env: f"Generating manifest {target[0]}", + ) + return action + + +def translatedManifestGenerator(target, source, env, for_signature): + dir = os.path.abspath(os.path.join(os.path.dirname(str(source[0])), "..")) + lang = os.path.basename(dir) + action = env.Action( + lambda target, source, env: generateTranslatedManifest(source[1].abspath, lang, target[0].abspath) + and None, + lambda target, source, env: f"Generating translated manifest {target[0]}", + ) + return action + + +env["BUILDERS"]["NVDAAddon"] = Builder(generator=addonGenerator) +env["BUILDERS"]["NVDAManifest"] = Builder(generator=manifestGenerator) +env["BUILDERS"]["NVDATranslatedManifest"] = Builder(generator=translatedManifestGenerator) + + +def createAddonHelp(dir): + docsDir = os.path.join(dir, "doc") + if os.path.isfile("style.css"): + cssPath = os.path.join(docsDir, "style.css") + cssTarget = env.Command(cssPath, "style.css", Copy("$TARGET", "$SOURCE")) + env.Depends(addon, cssTarget) + if os.path.isfile("readme.md"): + readmePath = os.path.join(docsDir, buildVars.baseLanguage, "readme.md") + readmeTarget = env.Command(readmePath, "readme.md", Copy("$TARGET", "$SOURCE")) + env.Depends(addon, readmeTarget) + + +def createAddonBundleFromPath(path, dest): + """Creates a bundle from a directory that contains an addon manifest file.""" + basedir = os.path.abspath(path) + with zipfile.ZipFile(dest, "w", zipfile.ZIP_DEFLATED) as z: + # FIXME: the include/exclude feature may or may not be useful. Also python files can be pre-compiled. + for dir, dirnames, filenames in os.walk(basedir): + relativePath = os.path.relpath(dir, basedir) + for filename in filenames: + pathInBundle = os.path.join(relativePath, filename) + absPath = os.path.join(dir, filename) + if pathInBundle not in buildVars.excludedFiles: + z.write(absPath, pathInBundle) + return dest + + +def generateManifest(source, dest): + # Prepare the root manifest section + addon_info = buildVars.addon_info + with codecs.open(source, "r", "utf-8") as f: + manifest_template = f.read() + manifest = manifest_template.format(**addon_info) + # Add additional manifest sections such as custom braille tables + # Custom braille translation tables + if getattr(buildVars, "brailleTables", {}): + manifest_brailleTables = ["\n[brailleTables]"] + for table in buildVars.brailleTables.keys(): + manifest_brailleTables.append(f"[[{table}]]") + for key, val in buildVars.brailleTables[table].items(): + manifest_brailleTables.append(f"{key} = {val}") + manifest += "\n".join(manifest_brailleTables) + "\n" + + # Custom speech symbol dictionaries + if getattr(buildVars, "symbolDictionaries", {}): + manifest_symbolDictionaries = ["\n[symbolDictionaries]"] + for dictionary in buildVars.symbolDictionaries.keys(): + manifest_symbolDictionaries.append(f"[[{dictionary}]]") + for key, val in buildVars.symbolDictionaries[dictionary].items(): + manifest_symbolDictionaries.append(f"{key} = {val}") + manifest += "\n".join(manifest_symbolDictionaries) + "\n" + + with codecs.open(dest, "w", "utf-8") as f: + f.write(manifest) + + +def generateTranslatedManifest(source, language, out): + _ = gettext.translation("nvda", localedir=os.path.join("addon", "locale"), languages=[language]).gettext + vars = {} + for var in ("addon_summary", "addon_description"): + vars[var] = _(buildVars.addon_info[var]) + with codecs.open(source, "r", "utf-8") as f: + manifest_template = f.read() + result = manifest_template.format(**vars) + # Add additional manifest sections such as custom braille tables + # Custom braille translation tables + if getattr(buildVars, "brailleTables", {}): + result_brailleTables = ["\n[brailleTables]"] + for table in buildVars.brailleTables.keys(): + result_brailleTables.append(f"[[{table}]]") + # Fetch display name only. + result_brailleTables.append(f"displayName = {_(buildVars.brailleTables[table]['displayName'])}") + result += "\n".join(result_brailleTables) + "\n" + + # Custom speech symbol dictionaries + if getattr(buildVars, "symbolDictionaries", {}): + result_symbolDictionaries = ["\n[symbolDictionaries]"] + for dictionary in buildVars.symbolDictionaries.keys(): + result_symbolDictionaries.append(f"[[{dictionary}]]") + # Fetch display name only. + result_symbolDictionaries.append( + f"displayName = {_(buildVars.symbolDictionaries[dictionary]['displayName'])}" + ) + result += "\n".join(result_symbolDictionaries) + "\n" + + with codecs.open(out, "w", "utf-8") as f: + f.write(result) + + +def expandGlobs(files): + return [f for pattern in files for f in env.Glob(pattern)] + + +addon = env.NVDAAddon(addonFile, env.Dir("addon")) + +langDirs = [f for f in env.Glob(os.path.join("addon", "locale", "*", "LC_MESSAGES", "nvda.po"))] + +# Allow all NVDA's gettext po files to be compiled in source/locale, and manifest files to be generated +moByLang = {} +for dir in langDirs: + poFile = dir + moFile = env.gettextMoFile(poFile) + moByLang[dir] = moFile + env.Depends(moFile, poFile) + translatedManifest = env.NVDATranslatedManifest( + dir.File("manifest.ini"), [moFile, os.path.join("manifest-translated.ini.tpl")] + ) + env.Depends(translatedManifest, ["buildVars.py"]) + env.Depends(addon, [translatedManifest, moFile]) + +pythonFiles = expandGlobs(buildVars.pythonSources) +for file in pythonFiles: + env.Depends(addon, file) + +# Convert markdown files to html +# We need at least doc in English and should enable the Help button for the add-on in Add-ons Manager +createAddonHelp("addon") +for mdFile in env.Glob(os.path.join("addon", "doc", "*", "*.md")): + # the title of the html file is translated based on the contents of something in the moFile for a language. + # Thus, we find the moFile for this language and depend on it if it exists. + lang = os.path.basename(os.path.dirname(mdFile.get_abspath())) + moFile = moByLang.get(lang) + htmlFile = env.markdown(mdFile) + env.Depends(htmlFile, mdFile) + if moFile: + env.Depends(htmlFile, moFile) + env.Depends(addon, htmlFile) + +# Pot target +i18nFiles = expandGlobs(buildVars.i18nSources) +gettextvars = { + "gettext_package_bugs_address": "nvda-translations@groups.io", + "gettext_package_name": buildVars.addon_info["addon_name"], + "gettext_package_version": buildVars.addon_info["addon_version"], +} + +pot = env.gettextPotFile("${addon_name}.pot", i18nFiles, **gettextvars) +env.Alias("pot", pot) +env.Depends(pot, i18nFiles) +mergePot = env.gettextMergePotFile("${addon_name}-merge.pot", i18nFiles, **gettextvars) +env.Alias("mergePot", mergePot) +env.Depends(mergePot, i18nFiles) + +# Generate Manifest path +manifest = env.NVDAManifest(os.path.join("addon", "manifest.ini"), os.path.join("manifest.ini.tpl")) +# Ensure manifest is rebuilt if buildVars is updated. +env.Depends(manifest, "buildVars.py") + +env.Depends(addon, manifest) +env.Default(addon) +env.Clean(addon, [".sconsign.dblite", "addon/doc/" + buildVars.baseLanguage + "/"]) diff --git a/site_scons/site_tools/gettexttool/__init__.py b/site_scons/site_tools/gettexttool/__init__.py new file mode 100644 index 0000000..900f8dc --- /dev/null +++ b/site_scons/site_tools/gettexttool/__init__.py @@ -0,0 +1,55 @@ +"""This tool allows generation of gettext .mo compiled files, pot files from source code files +and pot files for merging. + +Three new builders are added into the constructed environment: + +- gettextMoFile: generates .mo file from .pot file using msgfmt. +- gettextPotFile: Generates .pot file from source code files. +- gettextMergePotFile: Creates a .pot file appropriate for merging into existing .po files. + +To properly configure get text, define the following variables: + +- gettext_package_bugs_address +- gettext_package_name +- gettext_package_version + + +""" + +from SCons.Action import Action + + +def exists(env): + return True + + +XGETTEXT_COMMON_ARGS = ( + "--msgid-bugs-address='$gettext_package_bugs_address' " + "--package-name='$gettext_package_name' " + "--package-version='$gettext_package_version' " + "--keyword=pgettext:1c,2 " + "-c -o $TARGET $SOURCES" +) + + +def generate(env): + env.SetDefault(gettext_package_bugs_address="example@example.com") + env.SetDefault(gettext_package_name="") + env.SetDefault(gettext_package_version="") + + env["BUILDERS"]["gettextMoFile"] = env.Builder( + action=Action("msgfmt -o $TARGET $SOURCE", "Compiling translation $SOURCE"), + suffix=".mo", + src_suffix=".po", + ) + + env["BUILDERS"]["gettextPotFile"] = env.Builder( + action=Action("xgettext " + XGETTEXT_COMMON_ARGS, "Generating pot file $TARGET"), suffix=".pot" + ) + + env["BUILDERS"]["gettextMergePotFile"] = env.Builder( + action=Action( + "xgettext " + "--omit-header --no-location " + XGETTEXT_COMMON_ARGS, "Generating pot file $TARGET" + ), + suffix=".pot", + )