[pull] dev from ArchiveBox:dev

.claude/settings.json

@@ -0,0 +1,5 @@
+{
+  "enabledPlugins": {
+    "pyright-lsp@claude-plugins-official": true
+  }
+}

.claude/settings.local.json

@@ -0,0 +1,51 @@
+{
+  "permissions": {
+    "allow": [
+      "Read(**)",
+      "Glob(**)",
+      "Grep(**)",
+      "Bash(python -m archivebox:*)",
+      "Bash(ls:*)",
+      "Bash(xargs:*)",
+      "Bash(python -c:*)",
+      "Bash(printf:*)",
+      "Bash(pkill:*)",
+      "Bash(python3:*)",
+      "Bash(sqlite3:*)",
+      "WebFetch(domain:github.com)",
+      "Bash(uv add:*)",
+      "Bash(mkdir:*)",
+      "Bash(chmod:*)",
+      "Bash(python -m forum_dl:*)",
+      "Bash(archivebox manage migrate:*)",
+      "Bash(cat:*)",
+      "Bash(python archivebox/plugins/pip/on_Dependency__install_using_pip_provider.py:*)",
+      "Bash(forum-dl:*)",
+      "Bash(pip uninstall:*)",
+      "Bash(python:*)",
+      "Bash(source .venv/bin/activate)",
+      "Bash(mv:*)",
+      "Bash(echo:*)",
+      "Bash(grep:*)",
+      "WebFetch(domain:python-statemachine.readthedocs.io)",
+      "Bash(./bin/run_plugin_tests.sh:*)",
+      "Bash(done)",
+      "Bash(coverage erase:*)",
+      "Bash(gh api:*)"
+    ]
+  },
+  "hooks": {
+    "PreToolUse": [
+      {
+        "matcher": "Bash",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null); if [ -n \"$REPO_ROOT\" ] && [ \"$PWD\" != \"$REPO_ROOT\" ]; then echo \"ERROR: Not in repo root ($REPO_ROOT). Current dir: $PWD\" >&2; exit 1; fi",
+            "statusMessage": "Checking working directory..."
+          }
+        ]
+      }
+    ]
+  }
+}

.dockerignore

@@ -1,6 +1,45 @@
-output
-__pycache__
 .DS_Store
-venv
-.venv
-data
+._*
+*.pyc
+__pycache__/
+.mypy_cache/
+.pytest_cache/
+.github/
+.pdm-build/
+.pdm-python
+.eggs/
+.git/
+.vscode/
+!.git/HEAD
+!.git/refs/heads/*
+
+venv/
+.venv/
+.venv-old/
+.docker_venv/
+.docker-venv/
+node_modules/
+chrome/
+chromeprofile/
+chrome_profile/
+
+pdm.dev.lock
+pdm.lock
+
+docs/
+build/
+dist/
+brew_dist/
+deb_dist/
+pip_dist/
+assets/
+docker/
+website/
+typings/
+
+tmp/
+data/
+data*/
+output/
+index.sqlite3
+index.sqlite3-wal

.github/.readthedocs.yaml

@@ -0,0 +1,26 @@
+# Read the Docs config for https://docs.archivebox.io
+# https://docs.readthedocs.io/en/stable/config-file/v2.html
+
+version: 2
+
+submodules:
+  include: all
+  recursive: true
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    #nodejs: "20"     # not needed unless we need the full archivebox to run while building docs for some reason
+
+sphinx:
+  configuration: docs/conf.py
+
+formats:
+  - pdf
+  - epub
+
+# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: docs/requirements.txt

.github/CONTRIBUTING.md

@@ -1 +1,38 @@
-Make sure check in with me first or confirm your desired features line up with our roadmap: https://github.com/pirate/ArchiveBox#roadmap
+# Contribution Process
+
+1. Confirm your desired features fit into our bigger project goals [Roadmap](https://github.com/pirate/ArchiveBox/wiki/Roadmap).
+2. Open an issue with your planned implementation to discuss
+3. Check in with me before starting development to make sure your work wont conflict with or duplicate existing work
+4. Setup your dev environment, make some changes, and test using the test input files
+5. Commit, push, and submit a PR and wait for review feedback
+6. Have patience, don't abandon your PR! We love contributors but we all have day jobs and don't always have time to respond to notifications instantly. If you want a faster response, ping @theSquashSH on twitter or Patreon.
+
+**Useful links:**
+
+- https://github.com/ArchiveBox/ArchiveBox/issues
+- https://github.com/ArchiveBox/ArchiveBox/pulls
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#manual-setup
+
+### Development Setup
+
+```bash
+git clone https://github.com/ArchiveBox/ArchiveBox
+cd ArchiveBox
+# Ideally do this in a virtualenv
+pip install -e '.[dev]'  # or use: pipenv install --dev
+```
+
+### Running Tests
+
+```bash
+./bin/lint.sh
+./bin/test.sh
+./bin/build.sh
+```
+
+For more common tasks see the `Development` section at the bottom of the README.
+
+### Getting Help
+
+Open issues on Github or message me https://sweeting.me/#contact.

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+github: ["ArchiveBox", "pirate"]
+custom: ["https://donate.archivebox.io", "https://swag.archivebox.io"]

.github/ISSUE_TEMPLATE/1-bug_report.yml

@@ -0,0 +1,198 @@
+name: 🐞 Bug report
+description: Report a bug or error you encountered in ArchiveBox
+title: "Bug: ..."
+assignees:
+  - pirate
+type: 'Bug'
+body:
+  - type: markdown
+    attributes:
+      value: |
+        *Please note:* it is normal to see errors occasionally for some extractors on some URLs (not every extractor will work on every type of page).  
+        Please report archiving errors if you are seeing them *consistently across many URLs* or if they are *preventing you from using ArchiveBox*.
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Provide a screenshot and describe the bug
+      description: |
+        Attach a screenshot and describe what the issue is, what you expected to happen, and if relevant, the *URLs you were trying to archive*.  
+      placeholder: |
+        Got a bunch of 'singlefile was unable to archive this page' errors when trying to archive URLs from this site: https://example.com/xyz ...
+        I also tried to archive the same URLs using `singlefile` directly and some of them worked but not all of them. etc. ...
+    validations:
+      required: true
+
+  - type: textarea
+    id: steps_to_reproduce
+    attributes:
+      label: Steps to reproduce
+      description: Please provide the exact steps you took to trigger the issue (including any shell commands run, URLs visited, buttons clicked, etc.).
+      render: markdown
+      placeholder: |
+        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox` in iTerm2
+        2. Went to the https://127.0.0.1:8000/add/ page in Google Chrome
+        3. Typed 'https://example.com/xyz' into the 'Add URL' input field
+        4. Clicked the 'Add+' button
+        5. Got a 500 error and saw the errors below in terminal
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs or errors
+      description: "Paste any terminal output, logs, or errors (check `data/logs/errors.log` as well)."
+      placeholder: |
+        ╭─────────────────────────────────────────────────────────────────────────────────────────────────────────╮
+        │ [2024-11-02 19:54:28] ArchiveBox v0.8.6rc0: archivebox add https://example.com#1234567                  │
+        ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+
+        [+] [2024-11-02 19:54:29] Adding 1 links to index (crawl depth=0)...
+            > Saved verbatim input to sources/1730577269-import.txt
+            > Parsed 1 URLs from input (Generic TXT)
+            ...
+      render: shell
+    validations:
+      required: false
+
+  - type: textarea
+    id: version
+    attributes:
+      label: ArchiveBox Version
+      description: |
+        **REQUIRED:** Run the `archivebox version` command inside your collection dir and paste the *full output* here (*not just the version number*).  
+        For Docker Compose run: `docker compose run archivebox version`  
+        For plain Docker run:   `docker run -v $PWD:/data archivebox/archivebox version`  
+      render: shell
+      placeholder: |
+        0.8.6
+        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
+        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
+        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
+
+        Binary Dependencies:
+        √  python     3.14.0     venv_pip   ~/.venv/bin/python
+        √  django     6.0        venv_pip   ~/.venv/lib/python3.14/site-packages/django/__init__.py
+        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
+        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
+        ...
+    validations:
+      required: true
+
+  - type: dropdown
+    id: install_method
+    validations:
+      required: true
+    attributes:
+      label: How did you install the version of ArchiveBox you are using?
+      multiple: false
+      options:
+        - pip
+        - apt
+        - brew
+        - nix
+        - Docker (or Podman/LXC/K8s/TrueNAS/Proxmox/etc)
+        - Other
+
+  - type: dropdown
+    id: operating_system
+    validations:
+      required: true
+    attributes:
+      label: What operating system are you running on?
+      description: |
+        Please note we are *unable to provide support for Windows users* unless you are using [Docker on Windows](https://github.com/ArchiveBox/archivebox#:~:text=windows%20without%20docker).
+      multiple: false
+      options:
+        - Linux (Ubuntu/Debian/Arch/Alpine/etc.)
+        - macOS (including Docker on macOS)
+        - BSD (FreeBSD/OpenBSD/NetBSD/etc.)
+        - Windows (including WSL, WSL2, Docker Desktop on Windows)
+        - Other
+
+  - type: checkboxes
+    id: filesystem
+    attributes:
+      label: What type of drive are you using to store your ArchiveBox data?
+      description: Are you using a [remote filesystem](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#supported-remote-filesystems) or FUSE mount for `data/` OR `data/archive`?
+      options:
+        - label: "some of `data/` is on a local SSD or NVMe drive"
+          required: false
+        - label: "some of `data/` is on a spinning hard drive or external USB drive"
+          required: false
+        - label: "some of `data/` is on a network mount (e.g. NFS/SMB/Ceph/GlusterFS/etc.)"
+          required: false
+        - label: "some of `data/` is on a FUSE mount (e.g. SSHFS/RClone/S3/B2/Google Drive/Dropbox/etc.)"
+          required: false
+
+
+  - type: textarea
+    id: docker_compose_yml
+    attributes:
+      label: Docker Compose Configuration
+      description: "If using Docker Compose, please share your full `docker-compose.yml` file. If using plain Docker, paste the `docker run ...` command you use."
+      placeholder: |
+        services:
+            archivebox:
+                image: archivebox/archivebox:latest
+                ports:
+                    - 8000:8000
+                volumes:
+                    - ./data:/data
+                environment:
+                    - ADMIN_USERNAME=admin
+                    - ADMIN_PASSWORD=****<redact any passwords>****
+                    - ALLOWED_HOSTS=*
+                    - CSRF_TRUSTED_ORIGINS=https://archivebox.example.com
+                    - PUBLIC_INDEX=True
+                    - PUBLIC_SNAPSHOTS=True
+                    - PUBLIC_ADD_VIEW=False
+                    ...
+
+            archivebox_scheduler:
+                image: archivebox/archivebox:latest
+                command: schedule --foreground --update --every=day
+                environment:
+                    ...
+
+          ...
+      render: shell
+    validations:
+      required: false
+
+  - type: textarea
+    id: configuration
+    attributes:
+      label: ArchiveBox Configuration
+      description: "Please share your full `data/ArchiveBox.conf` file here."
+      render: shell
+      placeholder: |
+        [SERVER_CONFIG]
+        SECRET_KEY = "*********<redact any secrets/passwords>************"
+
+        WGET_RESTRICT_FILE_NAMES=windows
+        USE_SYSTEM_WGET=true
+        CHECK_SSL_VALIDITY=false
+        ...
+    validations:
+      required: false
+
+
+  - type: markdown
+    attributes:
+      value: |
+        ---
+
+        We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.  
+        Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying anything suggested here, **we are not responsible for data loss**.  
+
+        In the meantime please consider:  
+
+          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
+          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
+          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
+          - 📚 Reading the [Troubleshooting Guide](https://github.com/ArchiveBox/ArchiveBox/wiki)
+          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
+