diff --git a/CLAUDE.md b/CLAUDE.md index 3b88d9ec4..6d9023439 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,6 +8,14 @@ through a responsive web UI. A background librarian daemon watches the filesystem for changes and manages metadata import, cover generation, and search indexing. +## Hard Rules + +@.claude/rules/telemetry-privacy.md — **No personally identifying information, +comic/series/folder/imprint/publisher names, tag values, or admin-flag strings +may ever be collected by or transmitted from `codex/librarian/telemeter/`.** +Telemetry is counts, booleans, and closed-enum keys only. This rule is not +negotiable and not waivable for debugging. + ## Commands Commands are from @\~/.claude/rules/devenv.md diff --git a/NEWS.md b/NEWS.md index d0e6ed1ff..96adfa131 100644 --- a/NEWS.md +++ b/NEWS.md @@ -6,6 +6,33 @@ width: 128px; border-radius: 128px; " /> +## v2.2.3 + +- Features + - Alternate series names: the localized and variant series titles comics + carry in MetronInfo AlternativeNames and Reprints tags (and that Metron + and Comic Vine online tagging now record) are imported, shown in the + metadata panel, editable in the tag editor, and browsable — sort, table + column, filter, and `alternate_series:` search (comicbox 4.6.0). Comics + imported or tagged online from now on pick them up automatically; + already-imported comics need a Force Update Tags. + - Online tagging matches a comic filed under a localized or variant series + title to the right volume, and the match prompt shows each candidate's + other known names (comicbox 4.6.0). + +- Fixes + - Identifier links point at the right page. Hand-tagged keys with a type + prefix like `series:178012`, Comic Vine long codes like `4050-160294`, and + `source:type:key` strings made broken issue-shaped URLs for every source + (comicbox 4.6.1). + - Every identifier URN in a comic's notes is imported; only the first was + read before (comicbox 4.6.1). + - Story arcs tagged online from Metron or Comic Vine get web links (comicbox + 4.6.1). These identifier fixes reach already-imported comics on their next + re-import, or use Force Update Tags. + - User data backups no longer skip every saved filter set after an upgrade + that adds a new filter. + ## v2.2.2 - Fixes diff --git a/bin/fix-django.sh b/bin/fix-django.sh new file mode 100755 index 000000000..828426977 --- /dev/null +++ b/bin/fix-django.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Fix django template lint errors +set -euxo pipefail + +mapfile -t templates < <(find . -mindepth 1 -name '.*' -prune -o -path '*/templates/*' -name '*.html' -print) +if [ ${#templates[@]} -eq 0 ]; then + echo "No django template files found. Nothing fixed." + exit 0 +fi +uv run --group lint djlint --reformat "${templates[@]}" diff --git a/bin/lint-django.sh b/bin/lint-django.sh new file mode 100755 index 000000000..32cefb014 --- /dev/null +++ b/bin/lint-django.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Lint django templates +set -euxo pipefail + +mapfile -t templates < <(find . -mindepth 1 -name '.*' -prune -o -path '*/templates/*' -name '*.html' -print) +if [ ${#templates[@]} -eq 0 ]; then + echo "No django template files found. Nothing linted." + exit 0 +fi +uv run --group lint djlint --lint "${templates[@]}" diff --git a/bin/manage.py b/bin/manage.py index ecd4f2a24..c9dfd2a2e 100755 --- a/bin/manage.py +++ b/bin/manage.py @@ -3,11 +3,52 @@ import os import sys +import tomllib +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +PRUNE = frozenset({"build", "dist", "docs", "node_modules", "test", "tests"}) + + +def _from_pyproject() -> str | None: + """Read an explicitly configured settings module.""" + pyproject = ROOT / "pyproject.toml" + if not pyproject.is_file(): + return None + with pyproject.open("rb") as f: + config = tomllib.load(f) + return config.get("tool", {}).get("devenv", {}).get("django-settings-module") + + +def _is_settings_package(pkg: Path) -> bool: + """Test if a dir is a package that holds a settings module or package.""" + if ( + not pkg.is_dir() + or pkg.name.startswith((".", "_")) + or pkg.name in PRUNE + or not (pkg / "__init__.py").is_file() + ): + return False + return (pkg / "settings.py").is_file() or ( + pkg / "settings" / "__init__.py" + ).is_file() + + +def _discover() -> str: + """Find the sole /settings module or package in the project root.""" + candidates = sorted(pkg.name for pkg in ROOT.iterdir() if _is_settings_package(pkg)) + if len(candidates) == 1: + return f"{candidates[0]}.settings" + reason = ( + f"Found {len(candidates)} django settings modules in {ROOT}: {candidates}. " + "Set [tool.devenv] django-settings-module in pyproject.toml." + ) + raise RuntimeError(reason) def main(): """Run the server.""" - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "codex.settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", _from_pyproject() or _discover()) try: from django.core.management import ( execute_from_command_line, diff --git a/bun.lock b/bun.lock index df499dce2..5ef28b452 100644 --- a/bun.lock +++ b/bun.lock @@ -7,11 +7,11 @@ "@eslint-community/eslint-plugin-eslint-comments": "^4.7.2", "@eslint/js": "^10.0.1", "@eslint/json": "^2.0.1", - "@fsouza/prettierd": "^0.28.0", + "@fsouza/prettierd": "^0.29.0", "@prettier/plugin-xml": "^3.4.2", "@stylistic/eslint-plugin": "^5.10.0", "@vitest/eslint-plugin": "^1.6.24", - "eslint": "^10.7.0", + "eslint": "^10.8.0", "eslint-config-prettier": "^10.1.8", "eslint-plugin-array-func": "^5.1.1", "eslint-plugin-compat": "^7.0.2", @@ -24,14 +24,14 @@ "eslint-plugin-no-secrets": "^2.3.3", "eslint-plugin-no-unsanitized": "^4.1.5", "eslint-plugin-no-use-extend-native": "^0.7.3", - "eslint-plugin-package-json": "^1.6.0", + "eslint-plugin-package-json": "^1.6.2", "eslint-plugin-perfectionist": "^5.10.0", "eslint-plugin-prettier": "^5.5.6", "eslint-plugin-promise": "^7.3.0", "eslint-plugin-regexp": "^3.1.1", "eslint-plugin-security": "^4.0.1", "eslint-plugin-sonarjs": "^4.2.0", - "eslint-plugin-toml": "^1.4.0", + "eslint-plugin-toml": "^1.5.0", "eslint-plugin-unicorn": "^72.0.0", "eslint-plugin-vue": "^10.10.0", "eslint-plugin-vue-scoped-css": "^3.1.1", @@ -60,13 +60,13 @@ "@babel/code-frame": ["@babel/code-frame@7.29.7", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.29.7", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" } }, "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw=="], - "@babel/helper-string-parser": ["@babel/helper-string-parser@7.29.7", "", {}, "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw=="], + "@babel/helper-string-parser": ["@babel/helper-string-parser@8.0.0", "", {}, "sha512-6mJgmFFFIIO82vvoLt9XtRC7/TkzXfts1t/SpRX4IHSzMgqoPYCWesVu1udUPUWioAE/2fcG6WuI8zrkE1gwrg=="], - "@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.29.7", "", {}, "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg=="], + "@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@8.0.4", "", {}, "sha512-4wFaiLd0bVo4cIoTXI3zKI038NIWE/cr3jvBjejOVYVxV/m8Ltav1USiGzG1fmS5J2RhgEOgXNNK46cRPnRsrg=="], - "@babel/parser": ["@babel/parser@7.29.7", "", { "dependencies": { "@babel/types": "^7.29.7" }, "bin": "./bin/babel-parser.js" }, "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg=="], + "@babel/parser": ["@babel/parser@8.0.4", "", { "dependencies": { "@babel/types": "^8.0.4" }, "bin": "./bin/babel-parser.js" }, "sha512-srpptsAkEbbNIC/q8nT7o+m6CQe8CJUTV/t7MYc9NnWlgYVtHOb7JH6SorxMhN0kuRJjVqXbKClG6xSbPtzz+g=="], - "@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="], + "@babel/types": ["@babel/types@8.0.4", "", { "dependencies": { "@babel/helper-string-parser": "^8.0.0", "@babel/helper-validator-identifier": "^8.0.4" } }, "sha512-eY+Yn3dCqTGmyiq2QRU66lA5FL8lqqqvecHt0fF3uHONIa7ToYsaCiWV8lOKqAs0Rb2SjixiKFROngnulPtt2g=="], "@emnapi/core": ["@emnapi/core@1.10.0", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.1", "tslib": "^2.4.0" } }, "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw=="], @@ -96,7 +96,7 @@ "@eslint/plugin-kit": ["@eslint/plugin-kit@0.7.2", "", { "dependencies": { "@eslint/core": "^1.2.1", "levn": "^0.4.1" } }, "sha512-+CNAzxglkrpNf/kKywqQfk74QjtceuOE7Qm+AF8miRvPF/wmmK5+OJOgVh3AVTT3RP2mH3+FOaxlE5v72owk0A=="], - "@fsouza/prettierd": ["@fsouza/prettierd@0.28.0", "", { "dependencies": { "core_d": "^6.1.1", "prettier": "^3.8.3" }, "optionalDependencies": { "@babel/parser": "^7.29.7", "@typescript-eslint/typescript-estree": "^8.60.0" }, "bin": { "prettierd": "bin/prettierd" } }, "sha512-XkRkIp2en2Lcm7Qc+zqNm8ClI3RArLCBNG2A4M2xzTw43S/vYSEDQ4NcfW+6ZsUyQeoVqrFcMU5H1yAKCVkv6A=="], + "@fsouza/prettierd": ["@fsouza/prettierd@0.29.0", "", { "dependencies": { "core_d": "^6.1.1", "prettier": "^3.9.5" }, "optionalDependencies": { "@babel/parser": "^8.0.4", "@typescript-eslint/typescript-estree": "^8.64.0" }, "bin": { "prettierd": "bin/prettierd" } }, "sha512-UMBNFAVAP2Vfd8RrsH4bzchtyvqfzFp3COBYSTYmtrrU6fwK+JQLz9s6g9JNRVY6MDBWkUWNpPO63vPD/6KTFg=="], "@humanfs/core": ["@humanfs/core@0.19.2", "", { "dependencies": { "@humanfs/types": "^0.15.0" } }, "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA=="], @@ -172,19 +172,19 @@ "@types/unist": ["@types/unist@3.0.3", "", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="], - "@typescript-eslint/project-service": ["@typescript-eslint/project-service@8.63.0", "", { "dependencies": { "@typescript-eslint/tsconfig-utils": "^8.63.0", "@typescript-eslint/types": "^8.63.0", "debug": "^4.4.3" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-e5dh0/UI0ok53AlZ5wRkXCB32z/f2jUZqPR/ygAw5WYaSw8j9EoJWlS7wQjr/dmOaqWjnPIn2m+HhVPCMWGZVQ=="], + "@typescript-eslint/project-service": ["@typescript-eslint/project-service@8.65.0", "", { "dependencies": { "@typescript-eslint/tsconfig-utils": "^8.65.0", "@typescript-eslint/types": "^8.65.0", "debug": "^4.4.3" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-SxnPhbTsGahizDgbu7oqFH/xVtzIqMd/s+WtnSxNxJZJpLbdT5IPdzg8EZxO3+PoKahXmwJLeNQOpKJb3/bi7Q=="], "@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.63.0", "", { "dependencies": { "@typescript-eslint/types": "8.63.0", "@typescript-eslint/visitor-keys": "8.63.0" } }, "sha512-uUyfMWCnDSN8bCpcrY8nGP2BLkQ9Xn0GsipcONcpIDWhwhO4ZSyHvyS14U3X75mzxWxL3I2UZIrenTzdzcJO8A=="], - "@typescript-eslint/tsconfig-utils": ["@typescript-eslint/tsconfig-utils@8.63.0", "", { "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-sUAbkulqBAsncKnbRP3+7CtQFRKicexnj7ZwNC6ddCR7EmrXvjvdCYMJbUIqMd6lwoEriZjwLo08aS5tSjVMHg=="], + "@typescript-eslint/tsconfig-utils": ["@typescript-eslint/tsconfig-utils@8.65.0", "", { "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-j6GzGqCiRdA7Qhur2VVmKZAkBLfnHFQfx4TaJGL9RMveZqCo48jSHHO0DTgizEnGhtWnqmbtCUSrqSkdiY/0Hg=="], "@typescript-eslint/types": ["@typescript-eslint/types@8.63.0", "", {}, "sha512-xyLtl9DUBBFrcJS4x2pIqGLH68/tC2uOa4Z7pUteW09D3bXnnXUom4dyPikzWgB7llmIc1zoeI3aoUdC4rPK/Q=="], - "@typescript-eslint/typescript-estree": ["@typescript-eslint/typescript-estree@8.63.0", "", { "dependencies": { "@typescript-eslint/project-service": "8.63.0", "@typescript-eslint/tsconfig-utils": "8.63.0", "@typescript-eslint/types": "8.63.0", "@typescript-eslint/visitor-keys": "8.63.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-ygBkU+B7ex5UI/gKhaqexWev79uISfIv7XQCRNYO/jmD8rGLPyWLAb3KMRT6nd8Gt9bmUBi9+iX6tBdYfOY81Q=="], + "@typescript-eslint/typescript-estree": ["@typescript-eslint/typescript-estree@8.65.0", "", { "dependencies": { "@typescript-eslint/project-service": "8.65.0", "@typescript-eslint/tsconfig-utils": "8.65.0", "@typescript-eslint/types": "8.65.0", "@typescript-eslint/visitor-keys": "8.65.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-JboAE2swaYt4tb1fHhHTABE2K+OLy09XfcTbhnk4Pw96f9dd2e9iYsJ28gBggHlo5z5x1rkyWvcPoTuNTd4oGg=="], "@typescript-eslint/utils": ["@typescript-eslint/utils@8.63.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.63.0", "@typescript-eslint/types": "8.63.0", "@typescript-eslint/typescript-estree": "8.63.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-fUKaeAvrTuQg/Tgt3nliAUSZHJM6DlCcfyEmxCvlX8kieWSStBX+5O5Fnidtc3i2JrH+9c/GL4RY2iasd/GPTA=="], - "@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.63.0", "", { "dependencies": { "@typescript-eslint/types": "8.63.0", "eslint-visitor-keys": "^5.0.0" } }, "sha512-UexrHGnGTpbuQHct2ExOc2ZcFbGUS9FOesCxxqdBGcpI1BxYu/LZ6U8Aq6/72XtF/qRBk9nhuGHFJIXXMhPMdw=="], + "@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.65.0", "", { "dependencies": { "@typescript-eslint/types": "8.65.0", "eslint-visitor-keys": "^5.0.0" } }, "sha512-8C71BQkGjiMmXtop7pHVJu1l2NNShFdkCyD6a2ezzs5vU/L3LRtb69EtcteFwz0mYMPzIgOw0n6OV4VBUWZd7A=="], "@unrs/resolver-binding-android-arm-eabi": ["@unrs/resolver-binding-android-arm-eabi@1.12.2", "", { "os": "android", "cpu": "arm" }, "sha512-g5T90pqg1bo/7mytQx6F4iBNC0Wsh9cu+z9veDbFjc7HjpesJFWD7QMS0NGStXM075+7dJPPVvBbpZlnrdpi/w=="], @@ -404,7 +404,7 @@ "eslint-plugin-no-use-extend-native": ["eslint-plugin-no-use-extend-native@0.7.3", "", { "dependencies": { "is-get-set-prop": "^2.0.0", "is-js-type": "^3.0.0", "is-obj-prop": "^2.0.0", "is-proto-prop": "^3.0.1" }, "peerDependencies": { "eslint": "^9.3.0 || ^10.0.0" } }, "sha512-kYJhgZkiZIavu/wIwrO+n4GemQcMX53kWCNZNr7nGMkRD1aBFLkDpBivEYP7nIJINCo9fzPbFjrpeX5kr2Qbww=="], - "eslint-plugin-package-json": ["eslint-plugin-package-json@1.6.0", "", { "dependencies": { "@altano/repository-tools": "^2.0.1", "change-case": "^5.4.4", "detect-indent": "^7.0.2", "detect-newline": "^4.0.1", "eslint-fix-utils": "~0.4.3", "eslint-json-compat-utils": "^0.2.3", "jsonc-eslint-parser": "^3.1.0", "package-json-validator": "^1.5.0", "semver": "^7.7.3", "sort-object-keys": "^2.0.0", "sort-package-json": "^4.0.0" }, "peerDependencies": { "@eslint/json": ">=1.0.0", "eslint": ">=9.0.0" }, "optionalPeers": ["@eslint/json"] }, "sha512-pROaDdg6qRV7RF6qebqoXIuPBagHI1jNklZBKP0IPikfL2hW25JZnBp4Q0tmdNJZx2LIh3rgdmcqqz4CIkSuCg=="], + "eslint-plugin-package-json": ["eslint-plugin-package-json@1.6.2", "", { "dependencies": { "@altano/repository-tools": "^2.0.1", "@types/estree": "^1.0.0", "change-case": "^5.4.4", "detect-indent": "^7.0.2", "detect-newline": "^4.0.1", "eslint-fix-utils": "~0.4.3", "eslint-json-compat-utils": "^0.2.3", "jsonc-eslint-parser": "^3.1.0", "package-json-validator": "^1.5.0", "semver": "^7.7.3", "sort-object-keys": "^2.0.0", "sort-package-json": "^4.0.0" }, "peerDependencies": { "@eslint/json": ">=1.0.0", "eslint": ">=9.0.0" }, "optionalPeers": ["@eslint/json"] }, "sha512-Zp0CdqXKQqB9luUYa/TOwnGq3VlLotx1UDflZOzEt3D5A5kxu3Lj1VFJ087jKudEhc+K3kF20XV/aKZAlDuslw=="], "eslint-plugin-perfectionist": ["eslint-plugin-perfectionist@5.10.0", "", { "dependencies": { "@typescript-eslint/utils": "^8.62.1", "natural-orderby": "^5.0.0" }, "peerDependencies": { "eslint": "^8.45.0 || ^9.0.0 || ^10.0.0" } }, "sha512-HiqpDrUDbGrMC6iHQbemgDyHJ0366Vyz/qRWmxQcSAkmG25cXr8BdRgx8yAhOKhEfBXn8Rnf/mTCsV4EqUJSxg=="], @@ -418,7 +418,7 @@ "eslint-plugin-sonarjs": ["eslint-plugin-sonarjs@4.2.0", "", { "dependencies": { "@eslint-community/regexpp": "^4.12.2", "builtin-modules": "^3.3.0", "bytes": "^3.1.2", "functional-red-black-tree": "^1.0.1", "globals": "^17.7.0", "jsx-ast-utils-x": "^0.1.0", "lodash.merge": "^4.6.2", "minimatch": "^10.2.5", "scslre": "^0.3.0", "semver": "^7.8.5", "ts-api-utils": "^2.5.0", "typescript": ">=5 <6.1.0", "yaml": "^2.9.0" }, "peerDependencies": { "eslint": "^8.0.0 || ^9.0.0 || ^10.0.0" } }, "sha512-bqADfuNtTL7VK6RU29eoiFTtaaBKIpVPuX3bOl+rBpWSBa0zIBVZlqZNZQjfP6s4iXkAJokv5IsD8OsACkwApg=="], - "eslint-plugin-toml": ["eslint-plugin-toml@1.4.0", "", { "dependencies": { "@eslint/core": "^1.0.1", "@eslint/plugin-kit": "^0.7.0", "@ota-meshi/ast-token-store": "^0.3.0", "debug": "^4.1.1", "toml-eslint-parser": "^1.0.1" }, "peerDependencies": { "eslint": ">=9.38.0" } }, "sha512-3ErTnfUjXq/23f72XeyRcE0Y4Sd/ME1lsZeezczqpn2R4tE7+Sgco/NUKDXm0xAMz15tzcRz/9RfJRm6AqRO+A=="], + "eslint-plugin-toml": ["eslint-plugin-toml@1.5.0", "", { "dependencies": { "@eslint/core": "^1.0.1", "@eslint/plugin-kit": "^0.7.0", "@ota-meshi/ast-token-store": "^0.3.0", "debug": "^4.1.1", "toml-eslint-parser": "^1.0.1" }, "peerDependencies": { "eslint": ">=9.38.0" } }, "sha512-qBjRywEkKxO2uYOjus//6GVF1r+Hg5QDkRO8RTY6XcaXgWfBU0DhwpFmJa2Ljf0Sz49r7DdZlpKwwHmJ4nmH1Q=="], "eslint-plugin-unicorn": ["eslint-plugin-unicorn@72.0.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@eslint/css-tree": "^4.0.4", "browserslist": "^4.28.4", "change-case": "^5.4.4", "ci-info": "^4.4.0", "core-js-compat": "^3.49.0", "detect-indent": "^7.0.2", "entities": "^4.5.0", "find-up-simple": "^1.0.1", "globals": "^17.7.0", "indent-string": "^5.0.0", "is-builtin-module": "^5.0.0", "is-identifier": "^1.1.0", "pluralize": "^8.0.0", "quote-js-string": "^0.1.0", "regjsparser": "^0.13.2", "reserved-identifiers": "^1.2.0", "semver": "^7.8.5", "strip-indent": "^4.1.1", "yaml": "^2.9.0" }, "peerDependencies": { "eslint": ">=10.4" } }, "sha512-hqO6ksoOHO+ZhdseTuKRVQbx9U7PRO/cv8qAR1mctwzdVO2hYud8uS9luAhp43RJgziYgHAph8eHyipT8GL0ng=="], @@ -1110,9 +1110,9 @@ "zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="], - "@eslint-community/eslint-utils/eslint-visitor-keys": ["eslint-visitor-keys@3.4.3", "", {}, "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag=="], + "@babel/code-frame/@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.29.7", "", {}, "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg=="], - "@fsouza/prettierd/prettier": ["prettier@3.9.5", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-/FVl766LpUfB5vXgCYOYa0MeV/441Ia99AeICQIQFTY/Nw0roZwULcXpku5i1/m5kt/baz+s4Zogspd839HSMg=="], + "@eslint-community/eslint-utils/eslint-visitor-keys": ["eslint-visitor-keys@3.4.3", "", {}, "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag=="], "@isaacs/cliui/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="], @@ -1132,6 +1132,16 @@ "@npmcli/promise-spawn/which": ["which@4.0.0", "", { "dependencies": { "isexe": "^3.1.1" }, "bin": { "node-which": "bin/which.js" } }, "sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg=="], + "@typescript-eslint/project-service/@typescript-eslint/types": ["@typescript-eslint/types@8.65.0", "", {}, "sha512-JSSwWNy+H0E/01jJEM+hrX6N0OFDzFzeIhHFSAS01tlVaevpG8cFyYRPhS5yjGOvBUx3sqQHVMjCL1CAZZMxBg=="], + + "@typescript-eslint/scope-manager/@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.63.0", "", { "dependencies": { "@typescript-eslint/types": "8.63.0", "eslint-visitor-keys": "^5.0.0" } }, "sha512-UexrHGnGTpbuQHct2ExOc2ZcFbGUS9FOesCxxqdBGcpI1BxYu/LZ6U8Aq6/72XtF/qRBk9nhuGHFJIXXMhPMdw=="], + + "@typescript-eslint/typescript-estree/@typescript-eslint/types": ["@typescript-eslint/types@8.65.0", "", {}, "sha512-JSSwWNy+H0E/01jJEM+hrX6N0OFDzFzeIhHFSAS01tlVaevpG8cFyYRPhS5yjGOvBUx3sqQHVMjCL1CAZZMxBg=="], + + "@typescript-eslint/utils/@typescript-eslint/typescript-estree": ["@typescript-eslint/typescript-estree@8.63.0", "", { "dependencies": { "@typescript-eslint/project-service": "8.63.0", "@typescript-eslint/tsconfig-utils": "8.63.0", "@typescript-eslint/types": "8.63.0", "@typescript-eslint/visitor-keys": "8.63.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-ygBkU+B7ex5UI/gKhaqexWev79uISfIv7XQCRNYO/jmD8rGLPyWLAb3KMRT6nd8Gt9bmUBi9+iX6tBdYfOY81Q=="], + + "@typescript-eslint/visitor-keys/@typescript-eslint/types": ["@typescript-eslint/types@8.65.0", "", {}, "sha512-JSSwWNy+H0E/01jJEM+hrX6N0OFDzFzeIhHFSAS01tlVaevpG8cFyYRPhS5yjGOvBUx3sqQHVMjCL1CAZZMxBg=="], + "@typescript-eslint/visitor-keys/eslint-visitor-keys": ["eslint-visitor-keys@5.0.1", "", {}, "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA=="], "anymatch/picomatch": ["picomatch@2.3.2", "", {}, "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA=="], @@ -1222,6 +1232,14 @@ "@npmcli/promise-spawn/which/isexe": ["isexe@3.1.5", "", {}, "sha512-6B3tLtFqtQS4ekarvLVMZ+X+VlvQekbe4taUkf/rhVO3d/h0M2rfARm/pXLcPEsjjMsFgrFgSrhQIxcSVrBz8w=="], + "@typescript-eslint/scope-manager/@typescript-eslint/visitor-keys/eslint-visitor-keys": ["eslint-visitor-keys@5.0.1", "", {}, "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA=="], + + "@typescript-eslint/utils/@typescript-eslint/typescript-estree/@typescript-eslint/project-service": ["@typescript-eslint/project-service@8.63.0", "", { "dependencies": { "@typescript-eslint/tsconfig-utils": "^8.63.0", "@typescript-eslint/types": "^8.63.0", "debug": "^4.4.3" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-e5dh0/UI0ok53AlZ5wRkXCB32z/f2jUZqPR/ygAw5WYaSw8j9EoJWlS7wQjr/dmOaqWjnPIn2m+HhVPCMWGZVQ=="], + + "@typescript-eslint/utils/@typescript-eslint/typescript-estree/@typescript-eslint/tsconfig-utils": ["@typescript-eslint/tsconfig-utils@8.63.0", "", { "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-sUAbkulqBAsncKnbRP3+7CtQFRKicexnj7ZwNC6ddCR7EmrXvjvdCYMJbUIqMd6lwoEriZjwLo08aS5tSjVMHg=="], + + "@typescript-eslint/utils/@typescript-eslint/typescript-estree/@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.63.0", "", { "dependencies": { "@typescript-eslint/types": "8.63.0", "eslint-visitor-keys": "^5.0.0" } }, "sha512-UexrHGnGTpbuQHct2ExOc2ZcFbGUS9FOesCxxqdBGcpI1BxYu/LZ6U8Aq6/72XtF/qRBk9nhuGHFJIXXMhPMdw=="], + "csso/css-tree/mdn-data": ["mdn-data@2.0.28", "", {}, "sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g=="], "eslint-mdx/espree/eslint-visitor-keys": ["eslint-visitor-keys@5.0.1", "", {}, "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA=="], @@ -1254,6 +1272,8 @@ "@npmcli/map-workspaces/minimatch/brace-expansion/balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], + "@typescript-eslint/utils/@typescript-eslint/typescript-estree/@typescript-eslint/visitor-keys/eslint-visitor-keys": ["eslint-visitor-keys@5.0.1", "", {}, "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA=="], + "glob/minimatch/brace-expansion/balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], "npm-pick-manifest/npm-package-arg/hosted-git-info/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="], diff --git a/cfg/django.mk b/cfg/django.mk index e5605d5fd..2652dc38b 100644 --- a/cfg/django.mk +++ b/cfg/django.mk @@ -5,13 +5,13 @@ export DEVENV_DJANGO ## Fix django lint errors in templates ## @category Fix fix:: - uv run --group lint djlint --reformat **/templates/**/*.html + bin/fix-django.sh .PHONY: lint ## Lint django templates ## @category Lint lint:: - uv run --group lint djlint --lint **/templates/**/*.html + bin/lint-django.sh .PHONY: django-check ## Django check diff --git a/codex/choices/browser.py b/codex/choices/browser.py index df216e949..d570642c8 100644 --- a/codex/choices/browser.py +++ b/codex/choices/browser.py @@ -18,6 +18,7 @@ { "created_at": "Added Time", "age_rating": "Age Rating", + "reprints": "Alternate Series", "characters": "Characters", "child_count": "Child Count", "community_rating": "Community Rating", @@ -528,6 +529,17 @@ def admin_default_route_for(top_collection: str) -> dict: "editable": False, "edit_widget": None, }, + "reprints": { + # Alternate & localized series names (comicbox ``reprints``). + # "Reprints" reads as reprint editions to users, so the + # column, order-by entry and filter all say "Alternate + # Series"; only the ORM key stays ``reprints``. + "label": "Alternate Series", + "sort_key": "reprints", + "m2m": True, + "editable": False, + "edit_widget": None, + }, "series_groups": { "label": "Series Groups", "sort_key": "series_groups", @@ -602,7 +614,7 @@ def admin_default_route_for(top_collection: str) -> dict: # shape). Display is one query for the whole page; sort scales # with the filtered collection count when the user clicks the header. # - ``high``: composite-M2M columns (``credits`` / ``identifiers`` -# / ``universes`` / ``story_arcs``). Display issues its own +# / ``reprints`` / ``universes`` / ``story_arcs``). Display issues its own # per-column query (composite display strings can't share the # simple-M2M union shape); sort runs a per-outer-row correlated # subquery that JOINs the bespoke composite expression. Both @@ -624,6 +636,7 @@ def admin_default_route_for(top_collection: str) -> dict: # Composite M2M — display + sort both per-column / per-row. "credits": "high", "identifiers": "high", + "reprints": "high", "story_arcs": "high", "universes": "high", } diff --git a/codex/choices/search.py b/codex/choices/search.py index ff1489e30..cd1f1ceee 100644 --- a/codex/choices/search.py +++ b/codex/choices/search.py @@ -62,6 +62,9 @@ def _get_fieldmap_values(*args) -> tuple: "age", "age_rating", "age_rating_metron" ), "age_rating_tagged": (), + "alternate_series": _get_fieldmap_values( + "alternate_series", "alt_series", "reprints" + ), "characters": _get_fieldmap_values("category", "categories", "characters"), "collection_title": ("collection",), "country": (), diff --git a/codex/choices/tagging.py b/codex/choices/tagging.py index fcee03559..1e6413db7 100644 --- a/codex/choices/tagging.py +++ b/codex/choices/tagging.py @@ -78,8 +78,8 @@ def _vuetify_choices(pairs: Iterable[tuple[str, str]]) -> tuple[MappingProxyType # most fields; the splits/renames are codex's relational model (issue split into # number + suffix so issues sort numerically, comicbox "arcs" stored as # story_arcs, ComicInfo's "manga"/"title" surfaced as reading_direction/stories). -# Canonical keys absent here (bookmark, date, pages, reprints, page_count, -# prices, updated_at, ...) are not tag-editor fields and are dropped. +# Canonical keys absent here (bookmark, date, pages, page_count, prices, +# updated_at, ...) are not tag-editor fields and are dropped. _CANONICAL_TO_EDITOR: MappingProxyType[str, tuple[str, ...]] = MappingProxyType( { "publisher": ("publisher",), @@ -113,6 +113,9 @@ def _vuetify_choices(pairs: Iterable[tuple[str, str]]) -> tuple[MappingProxyType "community_rating": ("community_rating",), "protagonist": ("protagonist",), "identifiers": ("identifiers",), + # The series name and issue only; the volume number and language are + # MetronInfo-exclusive — see _EXTRA_FORMAT_FIELDS. + "reprints": ("reprints",), "country": ("country",), "universes": ("universes",), } @@ -121,10 +124,18 @@ def _vuetify_choices(pairs: Iterable[tuple[str, str]]) -> tuple[MappingProxyType # Editor sub-fields comicbox stores only for some formats and which its # transform specs don't surface at the SPECS_TO top level (the # hand-maintained support data): MetronInfo's Series carries a volume -# count, and only MetronInfo's CommunityRating persists a rating count. +# count, only MetronInfo's CommunityRating persists a rating count, and +# only MetronInfo encodes a reprint's volume number (in Reprints) and +# language (in Series/AlternativeNames) — ComicInfo's AlternateSeries / +# AlternateNumber / AlternateCount carry neither. _EXTRA_FORMAT_FIELDS: MappingProxyType[str, tuple[str, ...]] = MappingProxyType( { - "METRON_INFO": ("volume_count", "community_rating_count"), + "METRON_INFO": ( + "volume_count", + "community_rating_count", + "reprint_volume", + "reprint_language", + ), } ) diff --git a/codex/librarian/cron/crond.py b/codex/librarian/cron/crond.py index b00838e56..35c466be7 100644 --- a/codex/librarian/cron/crond.py +++ b/codex/librarian/cron/crond.py @@ -1,23 +1,42 @@ """Perform maintenance tasks.""" +from collections.abc import Callable +from datetime import datetime from threading import Condition, Event -from time import sleep from types import MappingProxyType -from typing import override +from typing import NamedTuple, override from django.db import connections from django.utils import timezone as django_timezone +from loguru._logger import Logger from codex.librarian.scribe.janitor.scheduled_time import get_janitor_time from codex.librarian.scribe.janitor.tasks import JanitorNightlyTask -from codex.librarian.telemeter.scheduled_time import get_telemeter_time +from codex.librarian.telemeter.scheduled_time import ( + get_telemeter_time, + mark_telemeter_attempt, +) from codex.librarian.telemeter.tasks import TelemeterTask from codex.librarian.threads import NamedThread -_TASK_TIME_FUNCTION_MAP = MappingProxyType( + +class _CronJob(NamedTuple): + """When a recurring task runs next, and how to spend the slot it runs in.""" + + get_time: Callable[[Logger], datetime | None] + # Called just before the task is queued, for jobs whose next time is + # read back out of state the job itself writes. ``None`` for jobs + # whose schedule is pure clock arithmetic and so advances on its own. + claim: Callable[[], None] | None = None + + +_CRON_JOBS: MappingProxyType[type, _CronJob] = MappingProxyType( { - JanitorNightlyTask: get_janitor_time, - TelemeterTask: get_telemeter_time, + # Always the next midnight, so queueing it moves the schedule. + JanitorNightlyTask: _CronJob(get_janitor_time), + # Read out of the telemeter Timestamp, which the send writes on a + # thread this one never joins. See ``mark_telemeter_attempt``. + TelemeterTask: _CronJob(get_telemeter_time, mark_telemeter_attempt), } ) @@ -34,8 +53,8 @@ def __init__(self, *args, **kwargs) -> None: def _create_task_times(self) -> None: task_times = {} - for task_class, func in _TASK_TIME_FUNCTION_MAP.items(): - if dttm := func(self.log): + for task_class, job in _CRON_JOBS.items(): + if dttm := job.get_time(self.log): task_times[dttm] = task_class self._task_times = tuple(sorted(task_times.items())) @@ -51,14 +70,34 @@ def _get_timeout(self) -> int: self.log.debug(f"Next scheduled job at {next_time} in {delta}.") return max(0, int(delta.total_seconds())) - def _run_expired_jobs(self) -> None: + def _enqueue_job(self, task_class: type) -> None: + """Spend the job's slot before queueing it, never after.""" + claim = _CRON_JOBS[task_class].claim + if claim is not None: + claim() + self.librarian_queue.put(task_class()) + + def _run_expired_jobs(self, *, timed_out: bool) -> None: + """ + Queue every job whose scheduled time has arrived. + + ``timed_out`` means the wait ran its full course instead of being + cut short by ``end_timeout``, so the job at the head of the + schedule is due even if the clock reads a hair short of its time + — ``_get_timeout`` truncates the delta to whole seconds, so the + wait always ends slightly early. Missing that job cost more than + a late run: ``_create_task_times`` would immediately push the + nightly janitor out to the *following* midnight and skip a night. + That is what ``sleep(2) # fix time rounding problems`` was for. + """ now = django_timezone.now() + if timed_out and self._task_times: + now = max(now, self._task_times[0][0]) for dttm, task_class in self._task_times: - if dttm < now: - self.librarian_queue.put(task_class()) - else: + if dttm > now: # Times are always ordered so stop checking at the first future job. break + self._enqueue_job(task_class) @override def run(self) -> None: @@ -66,10 +105,8 @@ def run(self) -> None: try: self.run_start() with self._cond: + self._create_task_times() while not self._stop_event.is_set(): - self._run_expired_jobs() - self._create_task_times() - sleep(2) # try to fix double jobs timeout = self._get_timeout() # Idle gaps between scheduled tasks are typically # hours-to-days. Release the conn so the next @@ -78,10 +115,18 @@ def run(self) -> None: # whole window. Reopen on the next query is # ~5-20 ms, invisible against the wait. connections.close_all() - self._cond.wait(timeout=timeout) + # ``Condition.wait`` returns False only when the whole + # timeout elapsed; True means ``end_timeout`` notified. + timed_out = not self._cond.wait(timeout=timeout) if self._stop_event.is_set(): break - sleep(2) # fix time rounding problems + self._run_expired_jobs(timed_out=timed_out) + # Recompute *after* queueing, so the new schedule sees + # the slots those jobs just claimed. Recomputing first + # re-read the telemeter's unchanged send time and put + # the same task on the queue again, once per pass, for + # as long as the offloaded send took to finish. + self._create_task_times() except Exception: self.log.exception(f"In {self.__class__.__name__}") self.log.debug(f"Stopped {self.__class__.__name__}.") diff --git a/codex/librarian/onlinetag/session_manager.py b/codex/librarian/onlinetag/session_manager.py index 12d600856..a8a8c92ed 100644 --- a/codex/librarian/onlinetag/session_manager.py +++ b/codex/librarian/onlinetag/session_manager.py @@ -606,7 +606,6 @@ def run_session(self, task: BulkOnlineTagTask) -> None: "sources": list(task.sources), "mode": task.mode, "prompts_mode": task.prompts_mode, - "auto_threshold": task.auto_threshold, "delete_original": task.delete_original, "merge_all_sources": task.merge_all_sources, "rename": task.rename, diff --git a/codex/librarian/onlinetag/session_state.py b/codex/librarian/onlinetag/session_state.py index 5e4a94370..e55093202 100644 --- a/codex/librarian/onlinetag/session_state.py +++ b/codex/librarian/onlinetag/session_state.py @@ -79,7 +79,13 @@ def serialize_candidate(c) -> dict[str, Any]: "year": getattr(summary, "year", None), "publisher": getattr(summary, "publisher", ""), "cover_url": getattr(summary, "cover_url", ""), + # Alternative series names comicbox scored this candidate on. + # Empty for sources whose search results don't carry them. + "alt_series": list(getattr(summary, "alt_series", ())), }, "score": c.score, "url": getattr(c, "url", ""), + # The candidate's parent container id (CV volume, Metron series). + # None for sources that don't expose it. + "volume_id": getattr(c, "volume_id", None), } diff --git a/codex/librarian/onlinetag/tasks.py b/codex/librarian/onlinetag/tasks.py index 4e916fcb8..4a4cab4b4 100644 --- a/codex/librarian/onlinetag/tasks.py +++ b/codex/librarian/onlinetag/tasks.py @@ -21,7 +21,6 @@ class BulkOnlineTagTask(OnlineTagTask): sources: tuple[str, ...] = SOURCE_NAMES mode: str = "auto" prompts_mode: str = "ask" - auto_threshold: float = 0.85 delete_original: bool = False dry_run: bool = False # Query every source per comic and merge (comicbox first_wins=False) diff --git a/codex/librarian/scribe/importer/const.py b/codex/librarian/scribe/importer/const.py index 069e81ab7..a5fa4c7cc 100644 --- a/codex/librarian/scribe/importer/const.py +++ b/codex/librarian/scribe/importer/const.py @@ -32,6 +32,7 @@ Language, Location, OriginalFormat, + Reprint, ScanInfo, SeriesGroup, Story, @@ -172,12 +173,12 @@ def _metron_fts_values(values: tuple) -> tuple: ), ) COMIC_M2M_FIELD_NAMES: tuple[str, ...] = tuple(field.name for field in COMIC_M2M_FIELDS) -COMPLEX_M2M_MODELS = (Credit, Identifier, StoryArcNumber) +COMPLEX_M2M_MODELS = (Credit, Identifier, Reprint, StoryArcNumber) ######################## # COMPLEX M2M METADATA # ######################## -DictModelType = Credit | Identifier | StoryArcNumber +DictModelType = Credit | Identifier | Reprint | StoryArcNumber CREDITS_FIELD_NAME = "credits" CREDIT_PERSON_FIELD_NAME = "person" CREDIT_ROLE_FIELD_NAME = "role" @@ -188,6 +189,13 @@ def _metron_fts_values(values: tuple) -> tuple: IDENTIFIER_TYPE_FIELD_NAME = "id_type" IDENTIFIER_ID_KEY_FIELD_NAME = "key" IDENTIFIER_URL_FIELD_NAME = "url" +REPRINTS_FIELD_NAME = "reprints" +REPRINT_SERIES_NAME_FIELD_NAME = "series_name" +REPRINT_VOLUME_NUMBER_FIELD_NAME = "volume_number" +REPRINT_ISSUE_FIELD_NAME = "issue" +REPRINT_LANGUAGE_FIELD_NAME = "language" +# Reprints index only their series names, under their own ComicFTS column. +ALTERNATE_SERIES_FTS_FIELD_NAME = "alternate_series" UNIVERSES_FIELD_NAME = "universes" NAME_FIELD_NAME = "name" NUMBER_TO_FIELD_NAME = "number_to" @@ -293,6 +301,15 @@ def _metron_fts_values(values: tuple) -> tuple: ), "", ), + Reprint: ( + ( + REPRINT_SERIES_NAME_FIELD_NAME, + REPRINT_VOLUME_NUMBER_FIELD_NAME, + REPRINT_ISSUE_FIELD_NAME, + REPRINT_LANGUAGE_FIELD_NAME, + ), + _IDENTIFIER_RELS, + ), StoryArcNumber: ( (f"{STORY_ARC_FIELD_NAME}__name", NUMBER_FIELD_NAME), "", @@ -311,6 +328,7 @@ def _metron_fts_values(values: tuple) -> tuple: Series: ("publisher", "imprint"), Volume: ("publisher", "imprint", "series"), Credit: (CREDIT_PERSON_FIELD_NAME, CREDIT_ROLE_FIELD_NAME), + Reprint: _IDENTIFIED_SELECT_RELATED, StoryArcNumber: (STORY_ARC_FIELD_NAME,), Universe: _IDENTIFIED_SELECT_RELATED, } @@ -350,6 +368,12 @@ def _metron_fts_values(values: tuple) -> tuple: *_NAMED_MODEL_ATTRS, ), CREDITS_FIELD_NAME: (CREDIT_PERSON_FIELD_NAME, CREDIT_ROLE_FIELD_NAME), + REPRINTS_FIELD_NAME: ( + REPRINT_SERIES_NAME_FIELD_NAME, + REPRINT_VOLUME_NUMBER_FIELD_NAME, + REPRINT_ISSUE_FIELD_NAME, + REPRINT_LANGUAGE_FIELD_NAME, + ), STORY_ARC_NUMBERS_FIELD_NAME: (STORY_ARC_FIELD_NAME, NUMBER_FIELD_NAME), } ) @@ -368,6 +392,7 @@ def _metron_fts_values(values: tuple) -> tuple: Identifier: IDENTIFIER_ID_KEY_FIELD_NAME, Imprint: NAME_FIELD_NAME, Publisher: NAME_FIELD_NAME, + Reprint: REPRINT_SERIES_NAME_FIELD_NAME, Series: NAME_FIELD_NAME, StoryArc: NAME_FIELD_NAME, StoryArcNumber: f"{STORY_ARC_FIELD_NAME}__name", diff --git a/codex/librarian/scribe/importer/create/const.py b/codex/librarian/scribe/importer/create/const.py index 50215d0fb..437a60e1f 100644 --- a/codex/librarian/scribe/importer/create/const.py +++ b/codex/librarian/scribe/importer/create/const.py @@ -20,12 +20,17 @@ NUMBER_FIELD_NAME, NUMBER_TO_FIELD_NAME, PUBLISHER_FIELD_NAME, + REPRINT_ISSUE_FIELD_NAME, + REPRINT_LANGUAGE_FIELD_NAME, + REPRINT_SERIES_NAME_FIELD_NAME, + REPRINT_VOLUME_NUMBER_FIELD_NAME, SERIES_FIELD_NAME, STORY_ARC_FIELD_NAME, VOLUME_COUNT_FIELD_NAME, ) from codex.models import ( Credit, + Reprint, StoryArc, StoryArcNumber, Volume, @@ -117,6 +122,15 @@ }, {}, ), + Reprint: ( + { + REPRINT_SERIES_NAME_FIELD_NAME: None, + REPRINT_VOLUME_NUMBER_FIELD_NAME: None, + REPRINT_ISSUE_FIELD_NAME: None, + REPRINT_LANGUAGE_FIELD_NAME: None, + }, + {IDENTIFIER_FIELD_NAME: Identifier}, + ), StoryArcNumber: ({STORY_ARC_FIELD_NAME: StoryArc, NUMBER_FIELD_NAME: None}, {}), Universe: ( { diff --git a/codex/librarian/scribe/importer/link/const.py b/codex/librarian/scribe/importer/link/const.py index 49ed000b7..ed2391191 100644 --- a/codex/librarian/scribe/importer/link/const.py +++ b/codex/librarian/scribe/importer/link/const.py @@ -5,6 +5,7 @@ from codex.librarian.scribe.importer.const import ( CREDITS_FIELD_NAME, IDENTIFIERS_FIELD_NAME, + REPRINTS_FIELD_NAME, STORY_ARC_NUMBERS_FIELD_NAME, ) from codex.models.base import BaseModel @@ -12,6 +13,7 @@ COMPLEX_MODEL_FIELD_NAMES = ( CREDITS_FIELD_NAME, + REPRINTS_FIELD_NAME, STORY_ARC_NUMBERS_FIELD_NAME, IDENTIFIERS_FIELD_NAME, ) diff --git a/codex/librarian/scribe/importer/link/delete.py b/codex/librarian/scribe/importer/link/delete.py index f525954cc..40f54c95f 100644 --- a/codex/librarian/scribe/importer/link/delete.py +++ b/codex/librarian/scribe/importer/link/delete.py @@ -38,9 +38,7 @@ def _delete_m2m_field_batch( return count def _delete_m2m_fts_entries(self, field_name: str, comic_ids: set[int]) -> None: - fts_field_name = ( - "story_arcs" if field_name == "story_arc_numbers" else field_name - ) + fts_field_name, _ = self.minify_complex_link_to_fts_tuple(field_name, ()) for comic_id in comic_ids: if not self.metadata.get(FTS_UPDATE, {}).get(comic_id, {}).get( fts_field_name diff --git a/codex/librarian/scribe/importer/read/many_to_many.py b/codex/librarian/scribe/importer/read/many_to_many.py index b1f432dea..668e5c0f4 100644 --- a/codex/librarian/scribe/importer/read/many_to_many.py +++ b/codex/librarian/scribe/importer/read/many_to_many.py @@ -1,10 +1,20 @@ """Aggregate ManyToMany Metadata.""" -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from types import MappingProxyType from typing import TYPE_CHECKING, cast -from comicbox.formats.comicbox.schema import IDENTIFIERS_KEY, NUMBER_KEY, ROLES_KEY +from comicbox.formats.comicbox.schema import ( + IDENTIFIERS_KEY, + ISSUE_KEY, + LANGUAGE_KEY, + NAME_KEY, + NUMBER_KEY, + ROLES_KEY, + SERIES_KEY, + SERIES_SORT_NAME_KEY, + VOLUME_KEY, +) from django.db.models import CharField, Field from django.db.models.fields.related import ManyToManyField @@ -13,6 +23,11 @@ CREDITS_FIELD_NAME, IDENTIFIERS_FIELD_NAME, LINK_M2MS, + REPRINT_ISSUE_FIELD_NAME, + REPRINT_LANGUAGE_FIELD_NAME, + REPRINT_SERIES_NAME_FIELD_NAME, + REPRINT_VOLUME_NUMBER_FIELD_NAME, + REPRINTS_FIELD_NAME, STORY_ARC_NUMBERS_FIELD_NAME, get_key_index, ) @@ -27,11 +42,16 @@ from codex.models.collections import Folder from codex.models.comic import Comic from codex.models.identifier import IdentifierSource -from codex.models.named import CreditRole +from codex.models.named import CreditRole, Reprint if TYPE_CHECKING: from codex.models.base import BaseModel +_REPRINT_SERIES_NAME_FIELD = Reprint._meta.get_field(REPRINT_SERIES_NAME_FIELD_NAME) +_REPRINT_VOLUME_NUMBER_FIELD = Reprint._meta.get_field(REPRINT_VOLUME_NUMBER_FIELD_NAME) +_REPRINT_ISSUE_FIELD = Reprint._meta.get_field(REPRINT_ISSUE_FIELD_NAME) +_REPRINT_LANGUAGE_FIELD = Reprint._meta.get_field(REPRINT_LANGUAGE_FIELD_NAME) + class AggregateManyToManyMetadataImporter(AggregateForeignKeyMetadataImporter): """Aggregate ManyToMany Metadata.""" @@ -222,7 +242,44 @@ def _get_m2m_metadata_dict_model_aggregate_sub_values( field, roles_or_numbers, clean_sub_key, clean_sub_values ) - def _get_m2m_metadata_dict_model( + @staticmethod + def _clean_reprint_key(reprint: Mapping) -> tuple | None: + """ + Flatten one comicbox reprint tree into a Reprint key tuple. + + MetronInfo AlternativeNames supply only a series ``sort_name``, + so that stands in when there's no ``series.name``. A reprint + with neither names nothing, so it's dropped. + """ + series = reprint.get(SERIES_KEY) or {} + series_name = _REPRINT_SERIES_NAME_FIELD.get_prep_value( + series.get(NAME_KEY) or series.get(SERIES_SORT_NAME_KEY) + ) + if not series_name: + return None + volume = reprint.get(VOLUME_KEY) or {} + return ( + series_name, + _REPRINT_VOLUME_NUMBER_FIELD.get_prep_value(volume.get(NUMBER_KEY)), + _REPRINT_ISSUE_FIELD.get_prep_value(reprint.get(ISSUE_KEY) or ""), + _REPRINT_LANGUAGE_FIELD.get_prep_value(reprint.get(LANGUAGE_KEY) or ""), + ) + + def _get_m2m_metadata_reprints(self, values: Iterable[Mapping]) -> dict: + """ + Aggregate reprints, the only m2m keyed by a nested comicbox tree. + + Every other complex m2m arrives as a mapping keyed by a name, a + shape that cannot express the four part reprint key. + """ + clean_values_map: dict[tuple, frozenset[tuple]] = {} + for reprint in values: + if clean_key := self._clean_reprint_key(reprint): + identifier_tuple = self.get_identifier_tuple(Reprint, reprint) + clean_values_map[clean_key] = frozenset({(identifier_tuple,)}) + return clean_values_map + + def _get_m2m_metadata_named_dict_model( self, md_key: str, field: ManyToManyField, @@ -248,7 +305,21 @@ def _get_m2m_metadata_dict_model( sub_value, ) clean_values_map.update(clean_sub_map) + return clean_values_map + def _get_m2m_metadata_dict_model( + self, + md_key: str, + field: ManyToManyField, + values: Mapping[str, Mapping | None] | list | tuple | set | frozenset, + ) -> dict: + # ``values`` is typed as the union every m2m field can present; + # comicbox always hands reprints over as a list of trees. + clean_values_map = ( + self._get_m2m_metadata_reprints(cast("Iterable[Mapping]", values)) + if field.name == REPRINTS_FIELD_NAME + else self._get_m2m_metadata_named_dict_model(md_key, field, values) + ) related_model: type[BaseModel] = field.related_model if related_model != Folder: for key, value in clean_values_map.items(): diff --git a/codex/librarian/scribe/importer/search/prepare.py b/codex/librarian/scribe/importer/search/prepare.py index 8f6388434..4c69e5502 100644 --- a/codex/librarian/scribe/importer/search/prepare.py +++ b/codex/librarian/scribe/importer/search/prepare.py @@ -1,6 +1,7 @@ """Prepare FTS update methods used in earlier import steps.""" from codex.librarian.scribe.importer.const import ( + ALTERNATE_SERIES_FTS_FIELD_NAME, CREDITS_FIELD_NAME, FTS_CREATE, FTS_CREATED_M2MS, @@ -8,6 +9,7 @@ FTS_FIELD_TARGETS, FTS_UPDATE, NON_FTS_FIELDS, + REPRINTS_FIELD_NAME, STORY_ARC_FIELD_NAME, STORY_ARC_NUMBERS_FIELD_NAME, ) @@ -27,6 +29,9 @@ def minify_complex_link_to_fts_tuple( """Only store the fts relevant parts of complex links.""" if field_name == CREDITS_FIELD_NAME: values = tuple(subvalues[0] for subvalues in values) + elif field_name == REPRINTS_FIELD_NAME: + field_name = ALTERNATE_SERIES_FTS_FIELD_NAME + values = tuple(subvalues[0] for subvalues in values) elif field_name == STORY_ARC_NUMBERS_FIELD_NAME: field_name = STORY_ARC_FIELD_NAME + "s" return field_name, tuple(values) diff --git a/codex/librarian/scribe/janitor/cleanup.py b/codex/librarian/scribe/janitor/cleanup.py index e61b6537c..60da1eb33 100644 --- a/codex/librarian/scribe/janitor/cleanup.py +++ b/codex/librarian/scribe/janitor/cleanup.py @@ -50,6 +50,7 @@ Location, OriginalFormat, Publisher, + Reprint, ScanInfo, Series, SeriesGroup, @@ -88,6 +89,7 @@ Imprint, OriginalFormat, Publisher, + Reprint, Series, SeriesGroup, ScanInfo, @@ -222,7 +224,7 @@ def cleanup_fks(self) -> None: Wrapped in ``transaction.atomic`` so the multi-pass convergence is all-or-nothing: an exception mid-pass rolls back to pre- cleanup state rather than leaving a partially-cleaned graph. - The 25 per-model deletes also coalesce into one fsync. + The per-model deletes also coalesce into one fsync. """ self.abort_event.clear() status = JanitorCleanupTagsStatus(0) diff --git a/codex/librarian/scribe/search/const.py b/codex/librarian/scribe/search/const.py index 950552366..812d40e84 100644 --- a/codex/librarian/scribe/search/const.py +++ b/codex/librarian/scribe/search/const.py @@ -20,6 +20,7 @@ "tagger", ) _COMICFTS_M2MS = ( + "alternate_series", "characters", "credits", "genres", diff --git a/codex/librarian/scribe/search/prepare.py b/codex/librarian/scribe/search/prepare.py index f7a10b16a..449007b72 100644 --- a/codex/librarian/scribe/search/prepare.py +++ b/codex/librarian/scribe/search/prepare.py @@ -29,6 +29,7 @@ "fts_original_format", "fts_scan_info", "fts_tagger", + "fts_alternate_series", "fts_characters", "fts_credits", "fts_country", diff --git a/codex/librarian/scribe/search/sync.py b/codex/librarian/scribe/search/sync.py index 857568b21..c5d6817be 100644 --- a/codex/librarian/scribe/search/sync.py +++ b/codex/librarian/scribe/search/sync.py @@ -69,6 +69,7 @@ # story_arcs) ended up empty for sync-built entries. _M2M_FTS_REL_MAP = MappingProxyType( { + "alternate_series": "reprints__series_name", "characters": "characters__name", "credits": "credits__person__name", "genres": "genres__name", diff --git a/codex/librarian/scribe/tag_writer.py b/codex/librarian/scribe/tag_writer.py index ec27063f4..46f24ca0e 100644 --- a/codex/librarian/scribe/tag_writer.py +++ b/codex/librarian/scribe/tag_writer.py @@ -121,7 +121,7 @@ def _build_base_config(task: BulkTagWriteTask): Never pass codex's read-side ``COMICBOX_CONFIG`` here. Comicbox unions a write's ``delete_keys`` with the base config's, and that config carries the big parse-skip set of schema fields codex - doesn't consume (pages, reprints, cover_image, ...). Using it as a + doesn't consume (pages, cover_image, prices, ...). Using it as a write base would strip every one of those from the user's archive. """ if not task.delete_original: diff --git a/codex/librarian/telemeter/admin_stats.py b/codex/librarian/telemeter/admin_stats.py new file mode 100644 index 000000000..51482fd2e --- /dev/null +++ b/codex/librarian/telemeter/admin_stats.py @@ -0,0 +1,229 @@ +""" +Anonymous stats for admin configuration. + +Everything here is a count, a boolean or a value from a closed vocabulary. +Nothing an administrator typed may leave the process: no credentials (the +encrypted fields), no urls or hostnames, no filesystem paths, no group, +provider or account names, no banner text and no api key. Where the useful +signal is "did they configure this at all", report a ``*_set`` boolean +instead of the value. +""" + +from types import MappingProxyType +from typing import Any, Final + +from comicbox.formats.base.online import SOURCE_NAMES +from django.conf import settings + +from codex.choices.admin import AdminFlagChoices +from codex.collection import Collection +from codex.models.admin import ( + AdminFlag, + ComicboxTaggingDefaults, + EmailSettings, + OIDCSettings, + ThrottleSettings, +) +from codex.models.auth import GroupAuth, UserAuth +from codex.settings.db import ( + get_email_settings, + get_oidc_settings, + get_throttle_settings, + oidc_enabled, +) + +_OTHER: Final = "other" + +# Flags whose signal is the on/off toggle. +_FLAG_BOOLS: Final = MappingProxyType( + { + AdminFlagChoices.AUTO_UPDATE.value: "auto_update", + AdminFlagChoices.FOLDER_VIEW.value: "folder_view", + AdminFlagChoices.IMPORT_METADATA.value: "import_metadata", + AdminFlagChoices.LAZY_IMPORT_METADATA.value: "lazy_import_metadata", + AdminFlagChoices.NON_USERS.value: "non_users", + AdminFlagChoices.REGISTRATION.value: "registration", + AdminFlagChoices.REGISTER_VERIFICATION.value: "register_verification", + AdminFlagChoices.SEND_TELEMETRY.value: "send_telemetry", + } +) +# Flags carrying a secret or admin-authored string. Only "is it set" ships. +_FLAG_SET_ONLY: Final = MappingProxyType( + { + AdminFlagChoices.API_KEY.value: "api_key_set", + AdminFlagChoices.BANNER_TEXT.value: "banner_text_set", + } +) +# Flags whose value is a number. +_FLAG_INTS: Final = MappingProxyType( + { + AdminFlagChoices.BROWSER_MAX_OBJ_PER_PAGE.value: "browser_max_obj_per_page", + AdminFlagChoices.CUSTOM_COVER_MAX_UPLOAD_MB.value: "custom_cover_max_upload_mb", + } +) +# Flags whose value is a collection name. +_FLAG_COLLECTIONS: Final = MappingProxyType( + { + AdminFlagChoices.BROWSER_DEFAULT_COLLECTION.value: ( + "browser_default_collection" + ), + } +) +# Flags whose signal is their AgeRatingMetron row. That table is a fixed +# lookup seeded by migration, so its names are a closed vocabulary. +_FLAG_AGE_RATINGS: Final = MappingProxyType( + { + AdminFlagChoices.AGE_RATING_DEFAULT.value: "age_rating_default", + AdminFlagChoices.ANONYMOUS_USER_AGE_RATING.value: "anonymous_user_age_rating", + } +) +_COLLECTION_VALUES: Final = frozenset(member.value for member in Collection) +# The closed set of OIDC client authentication methods. Anything else is an +# admin typo or a provider we don't know about; report it as "other". +_TOKEN_AUTH_METHODS: Final = frozenset( + { + "", + "client_secret_basic", + "client_secret_post", + "client_secret_jwt", + "private_key_jwt", + "none", + } +) +_OIDC_BOOLS: Final = ( + "create_users", + "link_by_email", + "sync_groups", + "pkce", + "fetch_userinfo", + "rp_initiated_logout", +) +# Claim & scope settings are admin-authored strings. Report only whether they +# differ from the shipped default. +_OIDC_CUSTOM_STRINGS: Final = ("scope", "username_claim", "groups_claim") +_THROTTLE_SCOPES: Final = ("anon", "user", "opds", "opensearch", "reset_password") + + +def _safe_int(value: str) -> int | None: + """Parse an admin flag's numeric value.""" + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _add_flag(stats: dict[str, Any], flag: AdminFlag) -> None: + """Add one admin flag to the stats, by category.""" + if name := _FLAG_BOOLS.get(flag.key): + stats[name] = flag.on + elif name := _FLAG_SET_ONLY.get(flag.key): + stats[name] = bool(flag.value) + elif name := _FLAG_INTS.get(flag.key): + stats[name] = _safe_int(flag.value) + elif name := _FLAG_COLLECTIONS.get(flag.key): + stats[name] = flag.value if flag.value in _COLLECTION_VALUES else _OTHER + elif name := _FLAG_AGE_RATINGS.get(flag.key): + stats[name] = flag.age_rating_metron.name if flag.age_rating_metron else "" + + +def get_admin_flag_stats() -> dict[str, Any]: + """Report every admin flag, in one query.""" + stats: dict[str, Any] = {} + for flag in AdminFlag.objects.select_related("age_rating_metron"): + _add_flag(stats, flag) + return stats + + +def _default_sources(defaults: ComicboxTaggingDefaults) -> dict[str, int]: + """Report which online tagging sources are enabled, as a 1/absent bucket.""" + sources = defaults.default_sources + if not isinstance(sources, list): + return {} + return {source: 1 for source in SOURCE_NAMES if source in sources} + + +def get_tagging_stats() -> dict[str, Any]: + """Report the online tagging defaults. Never the credentials or urls.""" + defaults = ComicboxTaggingDefaults.objects.first() + if not defaults: + return {} + formats = defaults.default_formats + return { + "default_match_mode": defaults.default_match_mode, + "default_prompts_mode": defaults.default_prompts_mode, + "merge_all_sources": defaults.merge_all_sources, + "delete_original": defaults.delete_original, + "rename_files": defaults.rename_files, + "default_sources": _default_sources(defaults), + "default_format_count": len(formats) if isinstance(formats, list) else 0, + "has_metron_credentials": bool( + defaults.metron_user and defaults.metron_password + ), + "has_comicvine_credentials": bool(defaults.comicvine_key), + "metron_url_set": bool(defaults.metron_url), + "comicvine_url_set": bool(defaults.comicvine_url), + } + + +def _is_custom(oidc: OIDCSettings, field: str) -> bool: + """Compare a string setting against the shipped default.""" + default = OIDCSettings._meta.get_field(field).get_default() + return getattr(oidc, field) != default + + +def _add_oidc_stats(stats: dict[str, Any], oidc: OIDCSettings) -> None: + """Report the OIDC client configuration, minus everything identifying.""" + stats["oidc_enabled"] = oidc_enabled(oidc) + for field in _OIDC_BOOLS: + stats[f"oidc_{field}"] = getattr(oidc, field) + for field in _OIDC_CUSTOM_STRINGS: + stats[f"oidc_{field}_custom"] = _is_custom(oidc, field) + method = oidc.token_auth_method + stats["oidc_token_auth_method"] = ( + method if method in _TOKEN_AUTH_METHODS else _OTHER + ) + stats["oidc_admin_group_set"] = bool(oidc.admin_group) + + +def get_auth_stats() -> dict[str, Any]: + """Report single sign on config and the age rating restrictions in use.""" + stats: dict[str, Any] = {} + if oidc := get_oidc_settings(): + _add_oidc_stats(stats, oidc) + stats["user_age_ceiling_count"] = UserAuth.objects.exclude( + age_rating_metron=None + ).count() + stats["auth_group_exclude_count"] = GroupAuth.objects.filter(exclude=True).count() + return stats + + +def get_email_stats() -> dict[str, Any]: + """Report whether outbound email works. Never the host or addresses.""" + email: EmailSettings | None = get_email_settings() + if not email: + return {} + return { + "smtp_configured": bool(email.host and email.from_address), + "smtp_use_tls": email.use_tls, + "smtp_use_ssl": email.use_ssl, + } + + +def get_throttle_stats() -> dict[str, Any]: + """Report the rate limits. 0 means the scope is unthrottled.""" + throttle: ThrottleSettings | None = get_throttle_settings() + if not throttle: + return {} + return {f"throttle_{scope}": getattr(throttle, scope) for scope in _THROTTLE_SCOPES} + + +def get_deployment_stats() -> dict[str, Any]: + """Report deployment shape from codex.toml. Never a path or a prefix.""" + return { + "remote_user_auth": bool(settings.AUTH_REMOTE_USER), + "failed_login_log": bool(settings.AUTH_FAILED_LOGIN_LOG), + "failed_login_log_trust_forwarded_for": bool( + settings.AUTH_FAILED_LOGIN_LOG_TRUST_FORWARDED_FOR + ), + "url_path_prefix_set": bool(settings.GRANIAN_URL_PATH_PREFIX), + } diff --git a/codex/librarian/telemeter/count_stats.py b/codex/librarian/telemeter/count_stats.py new file mode 100644 index 000000000..a7166feca --- /dev/null +++ b/codex/librarian/telemeter/count_stats.py @@ -0,0 +1,110 @@ +""" +Anonymous stats for library contents and reader usage. + +Counts only, plus buckets keyed by closed vocabularies. Identifier source +names come from comic files, so they are mapped through comicbox's known +sources and anything unrecognized is collapsed to "other" before it can +leave the process. +""" + +from types import MappingProxyType +from typing import Any, Final + +from comicbox.enums.maps.identifiers import ID_SOURCE_NAME_MAP +from django.db.models import Count + +from codex.models.bookmark import Bookmark +from codex.models.comic import Comic +from codex.models.favorite import Favorite +from codex.models.identifier import Identifier, IdentifierType +from codex.models.library import Library +from codex.models.paths import CustomCover, FailedImport +from codex.models.settings import SettingsBrowser + +OTHER_IDENTIFIER_SOURCE: Final = "other" +# comicbox's closed source vocabulary. A source name read from a comic file +# that isn't in here is collapsed to "other" before it leaves the process. +IDENTIFIER_SOURCES: Final = frozenset(source.value for source in ID_SOURCE_NAME_MAP) +_ID_TYPES: Final = frozenset(member.value for member in IdentifierType) + +# Comic columns whose signal is "how many comics have this filled in". +_COMIC_POPULATED_FIELDS: Final = MappingProxyType( + { + "comic_community_rating_count": "community_rating", + "comic_community_rating_vote_count": "community_rating_count", + "comic_alternative_issue_number_count": "alternative_issue_number", + "comic_metadata_imported_count": "metadata_imported_at", + } +) + + +def get_library_stats() -> dict[str, int]: + """Report how libraries and custom covers are configured.""" + covers = CustomCover.objects + return { + "library_read_only_count": Library.objects.filter(read_only=True).count(), + "library_poll_count": Library.objects.filter(poll=True).count(), + "library_events_count": Library.objects.filter(events=True).count(), + "library_group_acl_count": Library.objects.filter(groups__isnull=False) + .distinct() + .count(), + "custom_cover_count": covers.count(), + # A null library means the cover was uploaded through the browser + # rather than discovered in a covers directory. + "custom_cover_uploaded_count": covers.filter(library__isnull=True).count(), + "custom_cover_dir_count": covers.filter(library__isnull=False).count(), + "failed_import_count": FailedImport.objects.count(), + } + + +def get_comic_populated_stats() -> dict[str, int]: + """Count comics carrying each of the newer comic columns.""" + return { + name: Comic.objects.exclude(**{f"{field}__isnull": True}).count() + for name, field in _COMIC_POPULATED_FIELDS.items() + } + + +def get_usage_stats() -> dict[str, Any]: + """Report reader engagement: bookmarks and favorites.""" + favorites = Favorite.objects + rows = favorites.values("collection").annotate(count=Count("pk")).order_by() + return { + "bookmark_count": Bookmark.objects.count(), + "favorite_count": favorites.count(), + "favorite_user_count": favorites.values("user").distinct().count(), + "favorites": {row["collection"]: row["count"] for row in rows}, + } + + +def _identifier_bucket_key(source: str | None, id_type: str | None) -> str: + """Build a "source:id_type" key from closed vocabularies only.""" + name = (source or "").lower() + if name not in IDENTIFIER_SOURCES: + name = OTHER_IDENTIFIER_SOURCE + kind = id_type if id_type in _ID_TYPES else OTHER_IDENTIFIER_SOURCE + return f"{name}:{kind}" + + +def get_identifier_stats() -> dict[str, int]: + """ + Count identifiers by source and type. + + The durable proxy for online tagging use: an issue identifier from + metron or comicvine means that comic was matched against that service. + """ + rows = ( + Identifier.objects.values("source__name", "id_type") + .annotate(count=Count("pk")) + .order_by() + ) + buckets: dict[str, int] = {} + for row in rows: + key = _identifier_bucket_key(row["source__name"], row["id_type"]) + buckets[key] = buckets.get(key, 0) + row["count"] + return dict(sorted(buckets.items())) + + +def get_multi_sort_count() -> int: + """Count users who have added a secondary sort column in table view.""" + return SettingsBrowser.objects.exclude(order_extra_keys=[]).count() diff --git a/codex/librarian/telemeter/scheduled_time.py b/codex/librarian/telemeter/scheduled_time.py index 5721f3882..97924a959 100644 --- a/codex/librarian/telemeter/scheduled_time.py +++ b/codex/librarian/telemeter/scheduled_time.py @@ -49,6 +49,24 @@ def _get_scheduled_time(ts) -> datetime | None: return telemeter_time +def mark_telemeter_attempt() -> None: + """ + Spend this week's send slot. + + :func:`_get_scheduled_time` decides whether to schedule by comparing + the week's slot against ``updated_at``, so ``updated_at`` has to move + the moment the task is *queued*. The send itself records its own + attempt, but it runs on a throwaway thread ``BookmarkThread`` spawns + and never joins, and it only gets to that write after roughly thirty + count queries plus up to a five second network timeout. Waiting for + it left the slot unspent, so every pass of the cron loop recomputed + the same overdue time and queued the task again. + """ + ts = get_telemeter_timestamp() + # ``updated_at`` is ``auto_now``, so a plain save is the touch. + ts.save() + + def get_telemeter_time(log: Logger) -> datetime | None: """Get the time to send telemetry.""" # Should we schedule telemeter at all? diff --git a/codex/librarian/telemeter/stats.py b/codex/librarian/telemeter/stats.py index b03526d1e..140df6a0a 100644 --- a/codex/librarian/telemeter/stats.py +++ b/codex/librarian/telemeter/stats.py @@ -4,18 +4,38 @@ from pathlib import Path from platform import machine, python_version, release, system from types import MappingProxyType -from typing import Final +from typing import Any, Final from caseconverter import snakecase from django.contrib.sessions.models import Session from django.db.models import Count +from codex.librarian.telemeter.admin_stats import ( + get_admin_flag_stats, + get_auth_stats, + get_deployment_stats, + get_email_stats, + get_tagging_stats, + get_throttle_stats, +) +from codex.librarian.telemeter.count_stats import ( + get_comic_populated_stats, + get_identifier_stats, + get_library_stats, + get_multi_sort_count, + get_usage_stats, +) from codex.models import ( Comic, ) from codex.models.settings import SettingsBrowser, SettingsReader from codex.version import VERSION -from codex.views.const import CONFIG_MODELS, METADATA_MODELS, STATS_COLLECTION_MODELS +from codex.views.const import ( + CONFIG_MODELS, + METADATA_MODELS, + STATS_COLLECTION_MODELS, + STATS_USAGE_MODELS, +) # Cap on per-call session decodes for the anonymous-session estimate. # ``Session.get_decoded()`` runs HMAC + JSON parse per row; on installs @@ -28,14 +48,35 @@ "config": CONFIG_MODELS, "collections": STATS_COLLECTION_MODELS, "metadata": METADATA_MODELS, + "usage": STATS_USAGE_MODELS, } ) _DOCKERENV_PATH = Path("/.dockerenv") _CGROUP_PATH = Path("/proc/self/cgroup") _USER_STATS: Final = ( - (SettingsBrowser, ("top_collection", "order_by", "dynamic_covers")), + ( + SettingsBrowser, + ( + "top_collection", + "order_by", + "dynamic_covers", + "view_mode", + "table_cover_size", + "custom_covers", + ), + ), (SettingsReader, ("finish_on_last_page", "fit_to", "reading_direction")), ) +# Sections built by a single collector function, in payload order. +_SIMPLE_SECTIONS: Final = ( + ("identifiers", get_identifier_stats), + ("admin_flags", get_admin_flag_stats), + ("tagging", get_tagging_stats), + ("auth", get_auth_stats), + ("email", get_email_stats), + ("throttle", get_throttle_stats), + ("deployment", get_deployment_stats), +) class CodexStats: @@ -118,12 +159,13 @@ def _aggregate_settings_field(model, field) -> dict: @classmethod def _get_session_stats(cls) -> tuple[dict, int]: """Return per-field user-settings buckets and anon session count.""" - user_stats: dict[str, dict] = {} + user_stats: dict[str, Any] = {} for model, fields in _USER_STATS: for field in fields: bucket = cls._aggregate_settings_field(model, field) if bucket: user_stats[field] = bucket + user_stats["multi_sort_count"] = get_multi_sort_count() return user_stats, cls._estimate_anon_session_count() def _add_platform(self, obj) -> None: @@ -156,6 +198,7 @@ def _add_config(self, obj) -> None: # so both fields silently fell back to the default ``0``. config["user_registered_count"] = config.pop("user_count", 0) config["auth_group_count"] = config.pop("group_count", 0) + config.update(get_library_stats()) obj["config"] = config obj["sessions"] = sessions @@ -189,8 +232,22 @@ def _add_metadata(self, obj) -> None: if self.params and "metadata" not in self.params: return metadata = self._get_model_counts("metadata") + metadata.update(get_comic_populated_stats()) obj["metadata"] = metadata + def _add_usage(self, obj) -> None: + """Add dict of reader engagement counts to object.""" + if self.params and "usage" not in self.params: + return + obj["usage"] = get_usage_stats() + + def _add_simple_sections(self, obj) -> None: + """Add every section that one collector builds on its own.""" + for key, collector in _SIMPLE_SECTIONS: + if self.params and key not in self.params: + continue + obj[key] = collector() + def get(self) -> dict: """Construct the stats object.""" obj = {} @@ -199,4 +256,6 @@ def get(self) -> dict: self._add_collections(obj) self._add_file_types(obj) self._add_metadata(obj) + self._add_usage(obj) + self._add_simple_sections(obj) return obj diff --git a/codex/librarian/telemeter/telemeter.py b/codex/librarian/telemeter/telemeter.py index b3916bca4..94094905e 100644 --- a/codex/librarian/telemeter/telemeter.py +++ b/codex/librarian/telemeter/telemeter.py @@ -1,9 +1,11 @@ """Telemeter job.""" import json -from base64 import a85decode +from base64 import a85decode, b64encode from lzma import compress +from os import environ from typing import Final +from urllib.parse import urlsplit, urlunsplit from urllib.request import Request, urlopen from uuid import uuid4 @@ -13,26 +15,58 @@ # Version _APP_NAME: Final = "codex" -_VERSION: Final = "1" +_VERSION: Final = "2" # Sending # this isn't meant to fool you. it's meant to discourage lazy scraper bots. -_BASE: Final = "".join( +_BASE: Final = environ.get("CODEX_TELEMETER_URL") or "".join( ( a85decode(b"BQS?8F#ks-@:XCm@;\\+").decode(), a85decode(b"Ea`frF)to6Bk]hRFCB94/c").decode(), a85decode(b"@rGmhGV*rI@:Wqi/n&^<").decode(), ) ) -# urllib's ``Request`` mutates ``headers`` (it lowercases keys), so this -# can't be a ``MappingProxyType`` — it has to be a real ``MutableMapping``. -# Construct fresh per-call to keep the per-call state isolated. -_POST: Final = _BASE + f"/stats/{_APP_NAME}/{_VERSION}" + + +def _split_credentials(url: str) -> tuple[str, str]: + """ + Separate basic-auth credentials from the url. + + ``requests`` used to pull userinfo out of the url and turn it into an + Authorization header. ``urllib`` does not: it hands the whole netloc to + ``http.client``, which reads everything after the colon as a port and + raises ``InvalidURL``. So do the split ourselves. + """ + parts = urlsplit(url) + if not parts.username: + return url, "" + netloc = parts.hostname or "" + if parts.port: + netloc += f":{parts.port}" + bare_url = urlunsplit( + (parts.scheme, netloc, parts.path, parts.query, parts.fragment) + ) + userinfo = f"{parts.username}:{parts.password or ''}".encode() + return bare_url, "Basic " + b64encode(userinfo).decode() + + +_BARE_BASE, _AUTHORIZATION = _split_credentials(_BASE) +_POST: Final = _BARE_BASE + f"/stats/{_APP_NAME}/{_VERSION}" _TIMEOUT: Final = 5 def _new_headers() -> dict[str, str]: - return {"Content-Type": "application/xz"} + """ + Build request headers. + + urllib's ``Request`` mutates ``headers`` (it lowercases keys), so this + can't be a ``MappingProxyType`` — it has to be a real ``MutableMapping``. + Construct fresh per-call to keep the per-call state isolated. + """ + headers = {"Content-Type": "application/xz"} + if _AUTHORIZATION: + headers["Authorization"] = _AUTHORIZATION + return headers def get_telemeter_timestamp(): @@ -54,8 +88,12 @@ def _post_stats(data) -> None: request = Request( # noqa: S310 _POST, data=compressed_data, headers=_new_headers(), method="POST" ) - response = urlopen(request, timeout=_TIMEOUT) # noqa: S310 - response.raise_for_status() + # urlopen raises HTTPError on any status outside 2xx, and follows 3xx, so + # returning from here is a success. Checking response.status ourselves + # would be dead code; http.client's response has no raise_for_status(), + # that was requests'. The server answers 202: queued, not yet stored. + with urlopen(request, timeout=_TIMEOUT): # noqa: S310 + pass def _send_telemetry(uuid) -> None: @@ -79,8 +117,22 @@ def send_telemetry(log) -> None: try: _send_telemetry(ts.value) except Exception as exc: - log.debug(f"Failed to send anonymous stats: {exc}") - # update updated_at, even on failure to prevent rapid rescheudling. + # This was briefly a warning, because a transport bug went + # unnoticed for months while the stats server answered 200 to + # everything. The server reports honest statuses now, so its + # operator sees failures in the access log and this install has + # no reason to say anything: debug is invisible at the default + # log level, and a stats server is never the user's problem. + # repr, not str: str(TimeoutError()) is empty, and an HTTPError's + # repr names the status code. Neither can carry a response body. + log.debug(f"Failed to send anonymous stats: {exc!r}") + # Record the attempt whether or not it succeeded: retrying a + # failed send would turn a stats outage into a stampede. The cron + # thread already claimed this week's slot before queueing the + # task (``mark_telemeter_attempt``) — it cannot wait for this + # write, which happens on a thread it never joins — so this is a + # backstop for any other caller, and it leaves the recorded time + # honest about when the send actually finished. ts.save() except Exception as exc: log.debug(f"Failed to get or set telemeter timestamp: {exc}") diff --git a/codex/migrations/0049_reprints.py b/codex/migrations/0049_reprints.py new file mode 100644 index 000000000..a35110b46 --- /dev/null +++ b/codex/migrations/0049_reprints.py @@ -0,0 +1,202 @@ +# Generated by Django 6.0.7 on 2026-07-26 05:53 +""" +comicbox 4.6.0: reprints (alternate & localized series names). + +comicbox 4.6.0 emits a ``reprints`` list from MetronInfo +``Series/AlternativeNames`` + ``Reprints`` and from Metron / ComicVine +online tagging. Codex stores them in a denormalized :class:`Reprint` +table rather than reusing Series/Volume — those are browse collections +and alternate names would surface as phantom browser rows. + +``Reprint`` rows carry an ``Identifier``, so ``IdentifierType`` gains +its long-reserved ``reprint`` member. Reprints are also a browser order +key. Both are choices-only churn on existing columns, hence the two +``AlterField`` operations below. + +The FTS5 rebuild adds the ``alternate_series`` column. ``ComicFTS`` is +unmanaged, so makemigrations never sees it — the column list lives in +SQL and has to be dropped and recreated by hand, exactly as 0039 did +for the age-rating columns. The table comes back empty; the librarian's +regular ``SearchIndexSyncTask`` repopulates it. +""" + +import django.db.models.deletion +from django.db import migrations, models + +import codex.models.fields + +# 0039's column list plus ``alternate_series``. +_NEW_FTS_SQL = ( + "CREATE VIRTUAL TABLE codex_comicfts USING fts5(" + "comic_id UNINDEXED, created_at UNINDEXED, updated_at UNINDEXED, " + "publisher, imprint, series, name, collection_title, " + "age_rating_tagged, age_rating_metron, country, language, " + "original_format, review, scan_info, summary, tagger, " + "alternate_series, characters, " + "credits, genres, locations, roles, series_groups, stories, " + "story_arcs, tags, teams, universes, sources" + ")" +) + + +class Migration(migrations.Migration): + """Reprint model, comic m2m, filter column, and FTS alternate_series.""" + + dependencies = [ + ("codex", "0048_community_rating_alternative_issue"), + ] + + operations = [ + migrations.AddField( + model_name="settingsbrowserfilters", + name="reprints", + field=models.JSONField(default=list), + ), + migrations.AlterField( + model_name="identifier", + name="id_type", + field=models.CharField( + choices=[ + ("storyarc", "Arc"), + ("character", "Character"), + ("genre", "Genre"), + ("imprint", "Imprint"), + ("comic", "Issue"), + ("location", "Location"), + ("publisher", "Publisher"), + ("reprint", "Reprint"), + ("series", "Series"), + ("story", "Story"), + ("tag", "Tag"), + ("team", "Team"), + ("universe", "Universe"), + ("creditrole", "Role"), + ("creditperson", "Creator"), + ], + db_index=True, + max_length=16, + ), + ), + migrations.CreateModel( + name="Reprint", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "series_name", + codex.models.fields.CleaningCharField( + db_index=True, max_length=128 + ), + ), + ( + "volume_number", + codex.models.fields.CoercingPositiveSmallIntegerField( + default=None, null=True + ), + ), + ( + "issue", + codex.models.fields.CleaningCharField(default="", max_length=32), + ), + ( + "language", + codex.models.fields.CleaningCharField(default="", max_length=32), + ), + ( + "identifier", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="codex.identifier", + ), + ), + ], + options={ + "get_latest_by": "updated_at", + "abstract": False, + "unique_together": { + ("series_name", "volume_number", "issue", "language") + }, + }, + ), + migrations.AddField( + model_name="comic", + name="reprints", + field=models.ManyToManyField(to="codex.reprint"), + ), + migrations.AlterField( + model_name="settingsbrowser", + name="order_by", + field=models.CharField( + choices=[ + ("created_at", "Added Time"), + ("age_rating", "Age Rating"), + ("reprints", "Alternate Series"), + ("characters", "Characters"), + ("child_count", "Child Count"), + ("community_rating", "Community Rating"), + ("country", "Country"), + ("credits", "Credits"), + ("day", "Day"), + ("favorite", "Favorite"), + ("filename", "Filename"), + ("size", "File Size"), + ("file_type", "File Type"), + ("original_format", "Format"), + ("genres", "Genres"), + ("identifiers", "Identifiers"), + ("imprint_name", "Imprint"), + ("issue", "Issue"), + ("language", "Language"), + ("bookmark_updated_at", "Last Read"), + ("locations", "Locations"), + ("main_character", "Main Character"), + ("main_team", "Main Team"), + ("metadata_mtime", "Tags Updated"), + ("month", "Month"), + ("monochrome", "Monochrome"), + ("sort_name", "Name"), + ("page_count", "Page Count"), + ("publisher_name", "Publisher"), + ("date", "Publish Date"), + ("reading_direction", "Reading Direction"), + ("scan_info", "Scan Info"), + ("search_score", "Search Score"), + ("series_name", "Series"), + ("series_groups", "Series Groups"), + ("stories", "Stories"), + ("story_arc_number", "Story Arc Number"), + ("story_arcs", "Story Arcs"), + ("tags", "Tags"), + ("tagger", "Tagger"), + ("teams", "Teams"), + ("universes", "Universes"), + ("updated_at", "Updated Time"), + ("volume_name", "Volume"), + ("year", "Year"), + ], + default="", + max_length=32, + ), + ), + # ComicFTS is unmanaged; its Django state tracks only (comic, + # created_at, updated_at). The FTS columns are a SQL-level concern — + # no state_operations are needed, only a raw DROP + CREATE. + migrations.RunSQL( + sql="DROP TABLE IF EXISTS codex_comicfts;", + reverse_sql=migrations.RunSQL.noop, + ), + migrations.RunSQL( + sql=_NEW_FTS_SQL, + reverse_sql="DROP TABLE IF EXISTS codex_comicfts;", + ), + ] diff --git a/codex/models/comic.py b/codex/models/comic.py index ba9f93986..dc7d2e239 100644 --- a/codex/models/comic.py +++ b/codex/models/comic.py @@ -59,6 +59,7 @@ Language, Location, OriginalFormat, + Reprint, ScanInfo, SeriesGroup, Story, @@ -209,6 +210,7 @@ class Comic(WatchedPathBrowserCollection): genres = ManyToManyField(Genre) identifiers = ManyToManyField(Identifier) locations = ManyToManyField(Location) + reprints = ManyToManyField(Reprint) series_groups = ManyToManyField(SeriesGroup) stories = ManyToManyField(Story) story_arc_numbers = ManyToManyField(StoryArcNumber) @@ -218,13 +220,16 @@ class Comic(WatchedPathBrowserCollection): ##################### # Comicbox Ignored: - # alternate_issue - # alternate_volumes + # alternate_images + # bookmark # cover_image - # is_version_of - # last_mark + # credit_primaries + # critical_rating + # identifier_primary_source # manga - # price + # pages + # prices + # remainders # rights # codex only @@ -419,6 +424,7 @@ class ComicFTS(BaseModel): scan_info = CharField(db_collation="nocase", max_length=MAX_NAME_LEN) tagger = CharField(db_collation="nocase", max_length=MAX_NAME_LEN) # M2M + alternate_series = CharField(db_collation="nocase", max_length=MAX_NAME_LEN) characters = CharField(db_collation="nocase", max_length=MAX_NAME_LEN) credits = CharField(db_collation="nocase", max_length=MAX_NAME_LEN) genres = CharField(db_collation="nocase", max_length=MAX_NAME_LEN) diff --git a/codex/models/identifier.py b/codex/models/identifier.py index 0197e1709..48af5dcaa 100644 --- a/codex/models/identifier.py +++ b/codex/models/identifier.py @@ -36,7 +36,7 @@ class IdentifierType(TextChoices): ISSUE = "comic" LOCATION = "location" PUBLISHER = "publisher" - # REPRINT = "reprint" not yet implemented + REPRINT = "reprint" SERIES = "series" STORY = "story" TAG = "tag" diff --git a/codex/models/named.py b/codex/models/named.py index 582f35050..fc7c860c9 100644 --- a/codex/models/named.py +++ b/codex/models/named.py @@ -4,11 +4,12 @@ from django.db.models import ( CASCADE, + SET_NULL, ForeignKey, ) -from codex.models.base import MAX_NAME_LEN, BaseModel, NamedModel -from codex.models.collections import BrowserCollectionModel +from codex.models.base import MAX_FIELD_LEN, MAX_NAME_LEN, BaseModel, NamedModel +from codex.models.collections import BrowserCollectionModel, Volume from codex.models.fields import CleaningCharField, CoercingPositiveSmallIntegerField from codex.models.identifier import Identifier @@ -22,6 +23,7 @@ "Language", "Location", "OriginalFormat", + "Reprint", "ScanInfo", "SeriesGroup", "Story", @@ -111,6 +113,59 @@ class OriginalFormat(NamedModel): """The original published format.""" +class Reprint(BaseModel): + """ + An alternate or localized edition of this issue. + + Denormalized on purpose: alternate series names must not become + Series/Volume rows or they'd appear as phantom browser collections. + ``series_name`` absorbs comicbox's ``series.sort_name`` when the + reprint carries no ``series.name`` (MetronInfo AlternativeNames do + this), so this is the only series string stored. + """ + + series_name = CleaningCharField(db_index=True, max_length=MAX_NAME_LEN) + volume_number = CoercingPositiveSmallIntegerField(null=True, default=None) + issue = CleaningCharField(max_length=MAX_FIELD_LEN, default="") + language = CleaningCharField(max_length=MAX_FIELD_LEN, default="") + identifier = ForeignKey(Identifier, on_delete=SET_NULL, null=True) + + class Meta(BaseModel.Meta): + """Declare constraints and indexes.""" + + unique_together = ("series_name", "volume_number", "issue", "language") + + @staticmethod + def compose_name( + series_name: str, + volume_number: int | None = None, + issue: str = "", + language: str = "", + ) -> str: + """ + Compose a display name to imitate a NamedModel. + + Callers that only have the columns (browser filter choices, the + table-view intersection) format through here so every label + matches the metadata panel's. + """ + parts = [series_name] + if volume_number is not None: + parts.append(Volume.to_str(volume_number, None)) + if issue: + parts.append(f"#{issue}") + if language: + parts.append(f"({language})") + return " ".join(parts) + + @property + def name(self) -> str: + """Compose a display name to imitate a NamedModel.""" + return self.compose_name( + self.series_name, self.volume_number, self.issue, self.language + ) + + class ScanInfo(NamedModel): """Whomever scanned the comic.""" diff --git a/codex/models/settings.py b/codex/models/settings.py index 48922bd3c..2b39a4c1b 100644 --- a/codex/models/settings.py +++ b/codex/models/settings.py @@ -237,6 +237,7 @@ class SettingsBrowserFilters(BaseModel): monochrome = JSONField(default=list) original_format = JSONField(default=list) reading_direction = JSONField(default=list) + reprints = JSONField(default=list) series_groups = JSONField(default=list) stories = JSONField(default=list) story_arcs = JSONField(default=list) @@ -265,6 +266,7 @@ class SettingsBrowserFilters(BaseModel): "monochrome", "original_format", "reading_direction", + "reprints", "series_groups", "stories", "story_arcs", diff --git a/codex/serializers/admin/stats.py b/codex/serializers/admin/stats.py index 733fc6596..44139664b 100644 --- a/codex/serializers/admin/stats.py +++ b/codex/serializers/admin/stats.py @@ -9,13 +9,31 @@ Serializer, ) +from codex.librarian.telemeter.count_stats import ( + IDENTIFIER_SOURCES, + OTHER_IDENTIFIER_SOURCE, +) from codex.serializers.fields import ( CountDictField, SerializerChoicesField, StringListMultipleChoiceField, ) -FILE_TYPES_CHOICES: Final[tuple[str, ...]] = ("CBZ", "CBR", "CBT", "PDF", "UNKNOWN") +FILE_TYPES_CHOICES: Final[tuple[str, ...]] = ( + "CBZ", + "CBR", + "CBT", + "CB7", + "PDF", + "UNKNOWN", +) +# Identifier buckets are keyed ":". Requests select whole +# sources; the source vocabulary is comicbox's, plus the "other" catch-all +# the telemeter collapses unrecognized sources into. +IDENTIFIER_SOURCE_CHOICES: Final[tuple[str, ...]] = ( + *sorted(IDENTIFIER_SOURCES), + OTHER_IDENTIFIER_SOURCE, +) class StatsSystemSerializer(Serializer): @@ -40,9 +58,18 @@ class StatsConfigSerializer(Serializer): """Config Information.""" library_count = IntegerField(required=False, read_only=True) + session_count = IntegerField(required=False, read_only=True) user_anonymous_count = IntegerField(required=False, read_only=True) user_registered_count = IntegerField(required=False, read_only=True) auth_group_count = IntegerField(required=False, read_only=True) + library_read_only_count = IntegerField(required=False, read_only=True) + library_poll_count = IntegerField(required=False, read_only=True) + library_events_count = IntegerField(required=False, read_only=True) + library_group_acl_count = IntegerField(required=False, read_only=True) + custom_cover_count = IntegerField(required=False, read_only=True) + custom_cover_uploaded_count = IntegerField(required=False, read_only=True) + custom_cover_dir_count = IntegerField(required=False, read_only=True) + failed_import_count = IntegerField(required=False, read_only=True) # Only for api api_key = CharField(read_only=True, required=False) @@ -56,6 +83,10 @@ class StatsSessionsSerializer(Serializer): finish_on_last_page = CountDictField(required=False, read_only=True) fit_to = CountDictField(required=False, read_only=True) reading_direction = CountDictField(required=False, read_only=True) + view_mode = CountDictField(required=False, read_only=True) + table_cover_size = CountDictField(required=False, read_only=True) + custom_covers = CountDictField(required=False, read_only=True) + multi_sort_count = IntegerField(required=False, read_only=True) class StatsCollectionSerializer(Serializer): @@ -85,14 +116,125 @@ class StatsComicMetadataSerializer(Serializer): language_count = IntegerField(required=False, read_only=True) location_count = IntegerField(required=False, read_only=True) original_format_count = IntegerField(required=False, read_only=True) + reprint_count = IntegerField(required=False, read_only=True) series_group_count = IntegerField(required=False, read_only=True) scan_info_count = IntegerField(required=False, read_only=True) story_arc_count = IntegerField(required=False) story_arc_number_count = IntegerField(required=False, read_only=True) + story_count = IntegerField(required=False, read_only=True) tag_count = IntegerField(required=False, read_only=True) tagger_count = IntegerField(required=False, read_only=True) team_count = IntegerField(required=False, read_only=True) universe_count = IntegerField(required=False, read_only=True) + comic_community_rating_count = IntegerField(required=False, read_only=True) + comic_community_rating_vote_count = IntegerField(required=False, read_only=True) + comic_alternative_issue_number_count = IntegerField(required=False, read_only=True) + comic_metadata_imported_count = IntegerField(required=False, read_only=True) + + +class StatsUsageSerializer(Serializer): + """Reader Engagement Counts.""" + + bookmark_count = IntegerField(required=False, read_only=True) + favorite_count = IntegerField(required=False, read_only=True) + favorite_user_count = IntegerField(required=False, read_only=True) + favorites = CountDictField(required=False, read_only=True) + + +class StatsAdminFlagsSerializer(Serializer): + """ + Admin Flag Settings. + + Flags holding admin-authored text report only whether they are set. + """ + + auto_update = BooleanField(required=False, read_only=True) + folder_view = BooleanField(required=False, read_only=True) + import_metadata = BooleanField(required=False, read_only=True) + lazy_import_metadata = BooleanField(required=False, read_only=True) + non_users = BooleanField(required=False, read_only=True) + registration = BooleanField(required=False, read_only=True) + register_verification = BooleanField(required=False, read_only=True) + send_telemetry = BooleanField(required=False, read_only=True) + api_key_set = BooleanField(required=False, read_only=True) + banner_text_set = BooleanField(required=False, read_only=True) + browser_default_collection = CharField(required=False, read_only=True) + browser_max_obj_per_page = IntegerField(required=False, read_only=True) + custom_cover_max_upload_mb = IntegerField(required=False, read_only=True) + age_rating_default = CharField(required=False, read_only=True, allow_blank=True) + anonymous_user_age_rating = CharField( + required=False, read_only=True, allow_blank=True + ) + + +class StatsTaggingSerializer(Serializer): + """ + Online Tagging Defaults. + + Credentials and service urls report only whether they are configured. + """ + + default_match_mode = CharField(required=False, read_only=True) + default_prompts_mode = CharField(required=False, read_only=True) + merge_all_sources = BooleanField(required=False, read_only=True) + delete_original = BooleanField(required=False, read_only=True) + rename_files = BooleanField(required=False, read_only=True) + default_sources = CountDictField(required=False, read_only=True) + default_format_count = IntegerField(required=False, read_only=True) + has_metron_credentials = BooleanField(required=False, read_only=True) + has_comicvine_credentials = BooleanField(required=False, read_only=True) + metron_url_set = BooleanField(required=False, read_only=True) + comicvine_url_set = BooleanField(required=False, read_only=True) + + +class StatsAuthSerializer(Serializer): + """ + Single Sign On & Access Restrictions. + + The provider name, urls, client id and secret are never reported. + """ + + oidc_enabled = BooleanField(required=False, read_only=True) + oidc_create_users = BooleanField(required=False, read_only=True) + oidc_link_by_email = BooleanField(required=False, read_only=True) + oidc_sync_groups = BooleanField(required=False, read_only=True) + oidc_pkce = BooleanField(required=False, read_only=True) + oidc_fetch_userinfo = BooleanField(required=False, read_only=True) + oidc_rp_initiated_logout = BooleanField(required=False, read_only=True) + oidc_admin_group_set = BooleanField(required=False, read_only=True) + oidc_scope_custom = BooleanField(required=False, read_only=True) + oidc_username_claim_custom = BooleanField(required=False, read_only=True) + oidc_groups_claim_custom = BooleanField(required=False, read_only=True) + oidc_token_auth_method = CharField(required=False, read_only=True, allow_blank=True) + user_age_ceiling_count = IntegerField(required=False, read_only=True) + auth_group_exclude_count = IntegerField(required=False, read_only=True) + + +class StatsEmailSerializer(Serializer): + """Outbound Email. The host, account and addresses are never reported.""" + + smtp_configured = BooleanField(required=False, read_only=True) + smtp_use_tls = BooleanField(required=False, read_only=True) + smtp_use_ssl = BooleanField(required=False, read_only=True) + + +class StatsThrottleSerializer(Serializer): + """Rate Limits. Zero means the scope is unthrottled.""" + + throttle_anon = IntegerField(required=False, read_only=True) + throttle_user = IntegerField(required=False, read_only=True) + throttle_opds = IntegerField(required=False, read_only=True) + throttle_opensearch = IntegerField(required=False, read_only=True) + throttle_reset_password = IntegerField(required=False, read_only=True) + + +class StatsDeploymentSerializer(Serializer): + """Deployment Shape. No paths, prefixes or hostnames are reported.""" + + remote_user_auth = BooleanField(required=False, read_only=True) + failed_login_log = BooleanField(required=False, read_only=True) + failed_login_log_trust_forwarded_for = BooleanField(required=False, read_only=True) + url_path_prefix_set = BooleanField(required=False, read_only=True) class StatsSerializer(Serializer): @@ -104,6 +246,14 @@ class StatsSerializer(Serializer): collections = StatsCollectionSerializer(required=False) file_types = CountDictField(required=False) metadata = StatsComicMetadataSerializer(required=False) + usage = StatsUsageSerializer(required=False) + identifiers = CountDictField(required=False) + admin_flags = StatsAdminFlagsSerializer(required=False) + tagging = StatsTaggingSerializer(required=False) + auth = StatsAuthSerializer(required=False) + email = StatsEmailSerializer(required=False) + throttle = StatsThrottleSerializer(required=False) + deployment = StatsDeploymentSerializer(required=False) class AdminStatsRequestSerializer(Serializer): @@ -123,6 +273,20 @@ class AdminStatsRequestSerializer(Serializer): metadata = SerializerChoicesField( serializer=StatsComicMetadataSerializer, required=False ) + identifiers = StringListMultipleChoiceField(choices=IDENTIFIER_SOURCE_CHOICES) + usage = SerializerChoicesField(serializer=StatsUsageSerializer, required=False) + admin_flags = SerializerChoicesField( + serializer=StatsAdminFlagsSerializer, required=False + ) + tagging = SerializerChoicesField(serializer=StatsTaggingSerializer, required=False) + auth = SerializerChoicesField(serializer=StatsAuthSerializer, required=False) + email = SerializerChoicesField(serializer=StatsEmailSerializer, required=False) + throttle = SerializerChoicesField( + serializer=StatsThrottleSerializer, required=False + ) + deployment = SerializerChoicesField( + serializer=StatsDeploymentSerializer, required=False + ) class APIKeySerializer(Serializer): diff --git a/codex/serializers/admin/tagging.py b/codex/serializers/admin/tagging.py index e1fe27735..122c9aca6 100644 --- a/codex/serializers/admin/tagging.py +++ b/codex/serializers/admin/tagging.py @@ -61,7 +61,6 @@ class OnlineTagStartSerializer(Serializer): sources = ListField(child=CharField(), required=False, default=list(SOURCE_NAMES)) mode = CharField(required=False, default="auto") prompts_mode = CharField(required=False, default="ask") - auto_threshold = CharField(required=False, default="0.85") dry_run = CharField(required=False, default="false") delete_original = BooleanField(required=False, default=None) # None falls back to the admin ComicboxTaggingDefaults default. diff --git a/codex/serializers/browser/choices.py b/codex/serializers/browser/choices.py index 31c04a320..c7504a739 100644 --- a/codex/serializers/browser/choices.py +++ b/codex/serializers/browser/choices.py @@ -10,6 +10,7 @@ ) from rest_framework.serializers import Serializer +from codex.models.named import Reprint from codex.serializers.fields import ( VuetifyBooleanField, VuetifyCharField, @@ -44,6 +45,7 @@ class BrowserFilterChoicesSerializer(Serializer): locations = BooleanField(read_only=True) original_format = BooleanField(read_only=True) reading_direction = BooleanField(read_only=True) + reprints = BooleanField(read_only=True) series_groups = BooleanField(read_only=True) stories = BooleanField(read_only=True) story_arcs = BooleanField(read_only=True) @@ -78,6 +80,7 @@ class BrowserSettingsFilterSerializer(Serializer): reading_direction = VuetifyReadOnlyListField( child=VuetifyReadingDirectionChoiceField ) + reprints = VuetifyReadOnlyListField() series_groups = VuetifyReadOnlyListField() stories = VuetifyReadOnlyListField() story_arcs = VuetifyReadOnlyListField() @@ -113,6 +116,25 @@ class BrowserChoicesUniversePkSerializer(Serializer): designation = CharField(read_only=True) +# ``Reprint`` columns the label composes from, in +# :meth:`Reprint.compose_name` argument order. The choices view projects +# exactly these instead of the ``name`` every other tag model has. +REPRINT_LABEL_FIELDS = ("series_name", "volume_number", "issue", "language") + + +class BrowserChoicesReprintPkSerializer(BrowserChoicesIntegerPkSerializer): + """Reprints have no ``name`` column; compose the label from their four.""" + + name = SerializerMethodField(read_only=True) + + def get_name(self, obj) -> str: + """Compose the same label the metadata panel shows.""" + # The null sentinel row the view prepends already has a name. + if name := obj.get("name"): + return name + return Reprint.compose_name(*(obj[field] for field in REPRINT_LABEL_FIELDS)) + + class BrowserChoicesCharPkSerializer(BrowserChoicesIntegerPkSerializer): """Named Model Serailizer with pk = char hack for languages & countries.""" @@ -134,6 +156,7 @@ class BrowserChoicesDecimalPkSerializer(BrowserChoicesIntegerPkSerializer): "community_rating": BrowserChoicesDecimalPkSerializer, "file_type": BrowserChoicesCharPkSerializer, "language": LanguageSerializer, + "reprints": BrowserChoicesReprintPkSerializer, "universe": BrowserChoicesUniversePkSerializer, } ) diff --git a/codex/serializers/browser/filters.py b/codex/serializers/browser/filters.py index f5666bf5f..233011d8a 100644 --- a/codex/serializers/browser/filters.py +++ b/codex/serializers/browser/filters.py @@ -38,6 +38,7 @@ class BrowserSettingsFilterInputSerializer(Serializer): monochrome = VuetifyListField(child=VuetifyBooleanField) original_format = VuetifyListField() reading_direction = VuetifyListField(child=VuetifyReadingDirectionChoiceField) + reprints = VuetifyListField() series_groups = VuetifyListField() stories = VuetifyListField() story_arcs = VuetifyListField() diff --git a/codex/serializers/models/comic.py b/codex/serializers/models/comic.py index 0dce27115..fb0006ead 100644 --- a/codex/serializers/models/comic.py +++ b/codex/serializers/models/comic.py @@ -20,6 +20,7 @@ IdentifierSerializer, LocationSerializer, OriginalFormatSerializer, + ReprintSerializer, ScanInfoSerializer, SeriesGroupSerializer, StoryArcNumberSerializer, @@ -66,6 +67,7 @@ class ComicSerializer(BaseModelSerializer): genres = GenreSerializer(many=True, allow_null=True) identifiers = IdentifierSerializer(many=True, allow_null=True) locations = LocationSerializer(many=True, allow_null=True) + reprints = ReprintSerializer(many=True, allow_null=True) series_groups = SeriesGroupSerializer(many=True, allow_null=True) stories = StorySerializer(many=True, allow_null=True) story_arc_numbers = StoryArcNumberSerializer( diff --git a/codex/serializers/models/named.py b/codex/serializers/models/named.py index 6968cc4ec..4d70ccf30 100644 --- a/codex/serializers/models/named.py +++ b/codex/serializers/models/named.py @@ -13,6 +13,7 @@ Genre, Location, OriginalFormat, + Reprint, ScanInfo, SeriesGroup, Story, @@ -146,6 +147,30 @@ class Meta(URLNamedModelSerializer.Meta): model = Location +class ReprintSerializer(BaseModelSerializer): + """Reprint model.""" + + name = CharField(read_only=True) + url = URLField(read_only=True, source="identifier.url") + + class Meta(BaseModelSerializer.Meta): + """Configure model.""" + + model = Reprint + # The columns ride along with the composed name so the tag editor + # can seed its rows without parsing the label back apart. + fields = ( + "pk", + "name", + "series_name", + "volume_number", + "issue", + "language", + "url", + ) + depth = 1 + + class SeriesGroupSerializer(NamedModelSerializer): """SeriesGroup model.""" diff --git a/codex/settings/__init__.py b/codex/settings/__init__.py index 16c7b4336..1b161d39f 100644 --- a/codex/settings/__init__.py +++ b/codex/settings/__init__.py @@ -1125,6 +1125,7 @@ def _get_middleware(features: FeatureFlags) -> tuple[str, ...]: "protagonist", "publisher", "reading_direction", + "reprints", "review", "scan_info", "series", @@ -1141,10 +1142,10 @@ def _get_middleware(features: FeatureFlags) -> tuple[str, ...]: ) # Tell comicbox to skip parse work for every top-level field codex -# does not consume. Pages and reprints are short-circuited inside -# their respective computed actions; the rest are excluded at schema -# parse time and dropped post-merge as a backstop. Derived from the -# live comicbox schema rather than hand-curated so a comicbox release +# does not consume. Pages are short-circuited inside their computed +# action; the rest are excluded at schema parse time and dropped +# post-merge as a backstop. Derived from the live comicbox schema +# rather than hand-curated so a comicbox release # that adds a new top-level field is auto-deleted (codex maintainer # adds it to ``USED_COMICBOX_FIELDS`` if it's needed). Closes the # pre-existing ``cover_image`` gap. diff --git a/codex/user_data/identifiers.py b/codex/user_data/identifiers.py index cdcf419a5..e5315f6a7 100644 --- a/codex/user_data/identifiers.py +++ b/codex/user_data/identifiers.py @@ -56,6 +56,7 @@ "language": "codex.Language", "locations": "codex.Location", "original_format": "codex.OriginalFormat", + "reprints": "codex.Reprint", "series_groups": "codex.SeriesGroup", "stories": "codex.Story", "story_arcs": "codex.StoryArc", @@ -66,6 +67,13 @@ } ) FILTER_TAG_COLUMNS: Final[frozenset[str]] = frozenset(_FILTER_TAG_MODEL_REFS) +# Tag models that don't inherit ``NamedModel.name``. ``Reprint`` keys on +# ``series_name``, which is not unique on its own, so a restore may +# resolve more PKs than the dump held when a series has several +# alternate-name variants. +_FILTER_TAG_NAME_FIELDS: Final[MappingProxyType[str, str]] = MappingProxyType( + {"reprints": "series_name"} +) def tag_model_for_filter(column: str) -> type[Model] | None: @@ -79,6 +87,11 @@ def tag_model_for_filter(column: str) -> type[Model] | None: return apps.get_model(app_label, model_name) +def tag_name_field_for_filter(column: str) -> str: + """Return the name-bearing field of a filter column's tag model.""" + return _FILTER_TAG_NAME_FIELDS.get(column, "name") + + def encode_identifier(collection: str, parts: list[Any]) -> str: """JSON-encode an identifier tuple for use as a sidecar primary-key column.""" # ``separators`` produces compact output so equal tuples encode identically. diff --git a/codex/user_data/restore.py b/codex/user_data/restore.py index ed96e1794..37255575f 100644 --- a/codex/user_data/restore.py +++ b/codex/user_data/restore.py @@ -629,13 +629,21 @@ def _restore_one_settings_browser( def _resolve_filter_tags(column: str, raw: list, browser, report: RestoreReport): """Resolve a tag-name list back to PKs, reporting unmatched names.""" - from codex.user_data.identifiers import tag_model_for_filter + from codex.user_data.identifiers import ( + tag_model_for_filter, + tag_name_field_for_filter, + ) model = tag_model_for_filter(column) if model is None: return [] - pks = list(model.objects.filter(name__in=raw).values_list("pk", flat=True)) - if len(pks) != len(raw): + name_field = tag_name_field_for_filter(column) + pks = list( + model.objects.filter(**{f"{name_field}__in": raw}).values_list("pk", flat=True) + ) + # A non-unique name field (``Reprint.series_name``) can resolve to + # more rows than the dump held; only a shortfall is a real drop. + if len(pks) < len(raw): dropped = len(raw) - len(pks) report.note_skipped( "settings_filters", diff --git a/codex/user_data/schema.sql b/codex/user_data/schema.sql index 07a2916ea..6b47b1348 100644 --- a/codex/user_data/schema.sql +++ b/codex/user_data/schema.sql @@ -126,6 +126,7 @@ CREATE TABLE IF NOT EXISTS settings_filters ( monochrome TEXT NOT NULL DEFAULT '[]', original_format TEXT NOT NULL DEFAULT '[]', reading_direction TEXT NOT NULL DEFAULT '[]', + reprints TEXT NOT NULL DEFAULT '[]', series_groups TEXT NOT NULL DEFAULT '[]', stories TEXT NOT NULL DEFAULT '[]', story_arcs TEXT NOT NULL DEFAULT '[]', diff --git a/codex/user_data/serializers.py b/codex/user_data/serializers.py index 37bec6d4e..2e56b846e 100644 --- a/codex/user_data/serializers.py +++ b/codex/user_data/serializers.py @@ -27,6 +27,7 @@ encode_identifier, identifier_for_browse_collection, tag_model_for_filter, + tag_name_field_for_filter, ) @@ -237,9 +238,9 @@ def _rewrite_filter_value(column: str, value: Any) -> Any: model = tag_model_for_filter(column) if model is None: return value - # Every tag model inherits ``NamedModel.name``; missing PKs drop - # silently rather than raise. - return list(model.objects.filter(pk__in=value).values_list("name", flat=True)) + # Missing PKs drop silently rather than raise. + name_field = tag_name_field_for_filter(column) + return list(model.objects.filter(pk__in=value).values_list(name_field, flat=True)) def serialize_settings_filters( diff --git a/codex/user_data/store.py b/codex/user_data/store.py index 2b3ca83e1..72975fccf 100644 --- a/codex/user_data/store.py +++ b/codex/user_data/store.py @@ -42,6 +42,19 @@ SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name != 'schema_version' """ +_TABLE_NAMES_SQL: Final[str] = """\ +SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' +""" + + +def _add_column_clause(column: sqlite3.Row) -> str: + """Render one ``PRAGMA table_info`` row as an ADD COLUMN clause.""" + clause = f"{column['name']} {column['type']}" + if column["notnull"]: + clause += " NOT NULL" + if column["dflt_value"] is not None: + clause += f" DEFAULT {column['dflt_value']}" + return clause class SidecarStore: @@ -93,6 +106,39 @@ def _open_connection(self) -> sqlite3.Connection: conn.execute(pragma) return conn + @staticmethod + def _reconcile_columns(conn: sqlite3.Connection, ddl: str) -> None: + """ + Add columns a release introduced to a sidecar that predates them. + + ``schema.sql`` is entirely ``CREATE TABLE IF NOT EXISTS``, so + re-running it against an existing file adds nothing and every + later upsert naming a new column fails with "no such column". + The wanted columns are read back out of a throwaway database + built from the current schema so they derive from the schema + file instead of a hand-kept list that would rot the same way. + """ + reference = sqlite3.connect(":memory:") + reference.row_factory = sqlite3.Row + try: + reference.executescript(ddl) + for table_row in reference.execute(_TABLE_NAMES_SQL).fetchall(): + table = table_row["name"] + # PRAGMA takes no bind parameters; the table names come + # from the schema file, not from user input. + have = { + column["name"] + for column in conn.execute(f"PRAGMA table_info({table})") + } + for column in reference.execute(f"PRAGMA table_info({table})"): + if column["name"] in have: + continue + clause = _add_column_clause(column) + conn.execute(f"ALTER TABLE {table} ADD COLUMN {clause}") + logger.info(f"Sidecar schema: added {table}.{column['name']}") + finally: + reference.close() + def _ensure_schema(self, conn: sqlite3.Connection) -> None: """Apply ``schema.sql`` and stamp the version row once per process.""" if self._schema_applied: @@ -103,6 +149,8 @@ def _ensure_schema(self, conn: sqlite3.Connection) -> None: ddl = _SCHEMA_PATH.read_text(encoding="utf-8") with conn: conn.executescript(ddl) + if not self._is_memory: + self._reconcile_columns(conn, ddl) conn.execute( "INSERT OR IGNORE INTO schema_version (version) VALUES (?)", (SCHEMA_VERSION,), diff --git a/codex/views/admin/onlinetag.py b/codex/views/admin/onlinetag.py index c0b0bce32..ad2c84fe3 100644 --- a/codex/views/admin/onlinetag.py +++ b/codex/views/admin/onlinetag.py @@ -100,7 +100,6 @@ def post(self, request): sources=tuple(data["sources"]), mode=data["mode"], prompts_mode=data["prompts_mode"], - auto_threshold=float(data.get("auto_threshold", 0.85)), delete_original=delete_original, merge_all_sources=merge_all_sources, rename=rename, @@ -137,8 +136,8 @@ def _resume_task_params(resume: dict) -> dict: ``sources`` round-trips through JSON as a list; the task wants a tuple. A resume descriptor persists to a file-based cache, so one written by an older Codex may carry keys a task field no longer accepts (e.g. the removed - ``effort`` knob) — drop unknown keys so resuming across an upgrade rebuilds - the task instead of raising ``TypeError``. + ``auto_threshold`` knob) — drop unknown keys so resuming across an upgrade + rebuilds the task instead of raising ``TypeError``. """ params = dict(resume.get("params") or {}) params["sources"] = tuple(params.get("sources") or ()) diff --git a/codex/views/browser/choices.py b/codex/views/browser/choices.py index 6fda131ba..8f5ed6b64 100644 --- a/codex/views/browser/choices.py +++ b/codex/views/browser/choices.py @@ -18,8 +18,9 @@ ) from codex.models.age_rating import AgeRating, AgeRatingMetron from codex.models.identifier import IdentifierSource -from codex.models.named import Universe +from codex.models.named import Reprint, Universe from codex.serializers.browser.choices import ( + REPRINT_LABEL_FIELDS, BrowserChoicesFilterSerializer, BrowserFilterChoicesSerializer, ) @@ -267,7 +268,11 @@ def _get_m2m_field_choices(self, model, comic_qs, rel): # Choices qs = self.get_m2m_field_query(model, comic_qs) values = ["pk", "name"] - if qs.model == Universe: + if qs.model == Reprint: + # No ``name`` column at all — BrowserChoicesReprintPkSerializer + # composes the label from these. + values = ["pk", *REPRINT_LABEL_FIELDS] + elif qs.model == Universe: values.append("designation") elif qs.model == AgeRatingMetron: # AgeRatingMetron.Meta.ordering = ("index",), but ``.distinct()`` diff --git a/codex/views/browser/columns.py b/codex/views/browser/columns.py index b8d5aa1c8..a37a705ec 100644 --- a/codex/views/browser/columns.py +++ b/codex/views/browser/columns.py @@ -19,12 +19,13 @@ When, ) from django.db.models.fields import CharField -from django.db.models.functions import Concat +from django.db.models.functions import Cast, Concat from codex.choices.browser import ( BROWSER_TABLE_COLUMNS, BROWSER_TABLE_DEFAULT_COLUMNS, ) +from codex.models.collections import Volume from codex.models.favorite import FAVORITE_MODEL_COLLECTIONS, Favorite from codex.models.functions import JsonGroupArray @@ -71,6 +72,37 @@ output_field=CharField(), ) +# Volume numbers this wide are rendered as a year rather than a volume. +VOLUME_YEAR_RANGE = (10 ** (Volume.YEAR_LEN - 1), 10**Volume.YEAR_LEN - 1) +_REPRINT_VOLUME_STR = Cast("reprints__volume_number", output_field=CharField()) +_M2M_REPRINT_EXPR = Concat( + # Mirror of ``Reprint.name`` in SQL: only the columns the reprint + # carries contribute. ``volume_number`` renders through + # ``Volume.to_str``'s rule — four digits are a year "(1999)", + # anything else a volume "v3". + F("reprints__series_name"), + Case( + When(reprints__volume_number__isnull=True, then=Value("")), + When( + reprints__volume_number__range=VOLUME_YEAR_RANGE, + then=Concat(Value(" ("), _REPRINT_VOLUME_STR, Value(")")), + ), + default=Concat(Value(" v"), _REPRINT_VOLUME_STR), + output_field=CharField(), + ), + Case( + When(reprints__issue="", then=Value("")), + default=Concat(Value(" #"), F("reprints__issue")), + output_field=CharField(), + ), + Case( + When(reprints__language="", then=Value("")), + default=Concat(Value(" ("), F("reprints__language"), Value(")")), + output_field=CharField(), + ), + output_field=CharField(), +) + _M2M_COLUMN_PATHS = MappingProxyType( { "characters": "characters__name", @@ -78,6 +110,7 @@ "genres": "genres__name", "identifiers": _M2M_IDENTIFIER_EXPR, "locations": "locations__name", + "reprints": _M2M_REPRINT_EXPR, "series_groups": "series_groups__name", "stories": "stories__name", "story_arcs": "story_arc_numbers__story_arc__name", @@ -98,6 +131,7 @@ { "credits": Q(credits__person__name__gt=""), "identifiers": Q(identifiers__id_type__gt="") | Q(identifiers__key__gt=""), + "reprints": Q(reprints__series_name__gt=""), } ) diff --git a/codex/views/browser/const.py b/codex/views/browser/const.py index 0e711d110..29f64ab5d 100644 --- a/codex/views/browser/const.py +++ b/codex/views/browser/const.py @@ -16,6 +16,7 @@ "monochrome", "original_format", "reading_direction", + "reprints", "series_groups", "stories", "story_arcs", diff --git a/codex/views/browser/filters/filter.py b/codex/views/browser/filters/filter.py index 669e5f7ed..67a5fa618 100644 --- a/codex/views/browser/filters/filter.py +++ b/codex/views/browser/filters/filter.py @@ -24,6 +24,7 @@ "genres", "identifier_source", "locations", + "reprints", "series_groups", "stories", "story_arcs", diff --git a/codex/views/browser/filters/search/field/column.py b/codex/views/browser/filters/search/field/column.py index c1d4ddc1b..7f8b47489 100644 --- a/codex/views/browser/filters/search/field/column.py +++ b/codex/views/browser/filters/search/field/column.py @@ -9,6 +9,7 @@ _FIELD_TO_REL_SPAN_MAP = MappingProxyType( { + "alternate_series": "reprints__series_name", "role": "credits__role__name", "credits": "credits__person__name", "identifiers": "identifiers__key", diff --git a/codex/views/browser/intersections.py b/codex/views/browser/intersections.py index 1c1552ac5..c724d6f8e 100644 --- a/codex/views/browser/intersections.py +++ b/codex/views/browser/intersections.py @@ -35,7 +35,12 @@ Series, Volume, ) -from codex.views.browser.columns import fk_name_columns, m2m_columns +from codex.models.named import Reprint +from codex.views.browser.columns import ( + VOLUME_YEAR_RANGE, + fk_name_columns, + m2m_columns, +) from codex.views.const import MODEL_REL_MAP # Comic FK column → collection model. Used to correlate the intersection @@ -88,6 +93,15 @@ def _format_identifier(source_name: str | None, id_type: str, key: str) -> str: return f"{id_type}:{key}" +# Positional order matches :meth:`Reprint.compose_name`'s signature. +_REPRINT_VALUE_RELS = ( + "reprints__series_name", + "reprints__volume_number", + "reprints__issue", + "reprints__language", +) + + # Comic field paths for scalar / FK-name columns that participate in # collection-row intersection. Keys mirror the column registry; values are # Django ORM paths from Comic. The intersection path runs only for @@ -202,8 +216,8 @@ def compute_collection_intersections( ) # Simple-M2M columns share one UNION-ALL query (metadata-view - # pattern). Composite shapes (credits / identifiers / universes / - # story_arcs) keep their per-column helpers. + # pattern). Composite shapes (credits / identifiers / reprints / + # universes / story_arcs) keep their per-column helpers. _compute_simple_m2m_intersections_batched( m2m_cols, collection_pks, comic_to_collection, counts, result ) @@ -347,9 +361,10 @@ def _compute_batched_scalars( # the target model's ``name`` field. Eligible for the through-table # batched union (see ``_compute_simple_m2m_intersections_batched``). # ``universes`` (composite ``name:designation``), ``credits`` -# (``Person (Role)``), ``identifiers`` (``[source:]type:key``), and -# ``story_arcs`` (StoryArcNumber → StoryArc indirection) need bespoke -# SQL and stay on per-column helpers. +# (``Person (Role)``), ``identifiers`` (``[source:]type:key``), +# ``reprints`` (no ``name`` column at all — the label composes from +# four), and ``story_arcs`` (StoryArcNumber → StoryArc indirection) +# need bespoke SQL and stay on per-column helpers. _SIMPLE_M2M_BATCH_FIELDS: frozenset[str] = frozenset( { "characters", @@ -474,7 +489,7 @@ def _compute_m2m_intersection( Simple M2M columns route through ``_compute_simple_m2m_intersections_batched`` instead — this helper covers the bespoke shapes (credits / identifiers / - universes / story_arcs). + reprints / universes / story_arcs). """ if col == "credits": _compute_credits_intersection( @@ -486,6 +501,11 @@ def _compute_m2m_intersection( collection_pks, comic_to_collection, counts, result ) return + if col == "reprints": + _compute_reprints_intersection( + collection_pks, comic_to_collection, counts, result + ) + return rel = _M2M_NAME_RELATIONS.get(col) if rel is None: @@ -741,6 +761,38 @@ def _build_identifiers_intersection_sort_sql( return _IntersectionSortRawSQL(sql, []) +def _build_reprints_intersection_sort_sql( + collection_model: type[BrowserCollectionModel], +) -> RawSQL | None: + """Reprints render the same composed label the table cell shows.""" + correlation = _comic_correlation_sql(collection_model) + if correlation is None: + return None + # Mirror of ``Reprint.compose_name``: only the columns the reprint + # carries contribute, and a four-digit volume number is a year. The + # year bounds are the only bound parameters any of these + # intersection subqueries take. + inner = """ + SELECT + r.id AS target_id, + r.series_name + || CASE + WHEN r.volume_number IS NULL THEN '' + WHEN r.volume_number BETWEEN %s AND %s + THEN ' (' || r.volume_number || ')' + ELSE ' v' || r.volume_number + END + || CASE WHEN r.issue = '' THEN '' ELSE ' #' || r.issue END + || CASE WHEN r.language = '' THEN '' ELSE ' (' || r.language || ')' END + AS display_name + FROM codex_reprint r + INNER JOIN codex_comic_reprints th ON th.reprint_id = r.id + INNER JOIN codex_comic c ON c.id = th.comic_id + """ + sql = _wrap_intersection_sort(inner, correlation) + return _IntersectionSortRawSQL(sql, list(VOLUME_YEAR_RANGE)) + + def _build_story_arcs_intersection_sort_sql( collection_model: type[BrowserCollectionModel], ) -> RawSQL | None: @@ -893,6 +945,18 @@ def scalar_intersection_sort_expr( return _IntersectionSortRawSQL(sql, []) +# Composite-M2M columns whose display string needs its own SELECT. +_COMPOSITE_M2M_SORT_BUILDERS = MappingProxyType( + { + "credits": _build_credits_intersection_sort_sql, + "identifiers": _build_identifiers_intersection_sort_sql, + "reprints": _build_reprints_intersection_sort_sql, + "story_arcs": _build_story_arcs_intersection_sort_sql, + "universes": _build_universes_intersection_sort_sql, + } +) + + def m2m_intersection_sort_expr( collection_model: type[BrowserCollectionModel], column: str ) -> RawSQL | None: @@ -904,15 +968,8 @@ def m2m_intersection_sort_expr( """ if column in _SIMPLE_M2M_FIELDS: return _build_simple_m2m_intersection_sort_sql(collection_model, column) - if column == "universes": - return _build_universes_intersection_sort_sql(collection_model) - if column == "credits": - return _build_credits_intersection_sort_sql(collection_model) - if column == "identifiers": - return _build_identifiers_intersection_sort_sql(collection_model) - if column == "story_arcs": - return _build_story_arcs_intersection_sort_sql(collection_model) - return None + builder = _COMPOSITE_M2M_SORT_BUILDERS.get(column) + return builder(collection_model) if builder else None def _compute_credits_intersection( @@ -995,3 +1052,34 @@ def _compute_identifiers_intersection( if cnt == total ) result[gpk]["identifiers"] = names + + +def _compute_reprints_intersection( + collection_pks: list[int], + comic_to_collection: str, + counts: dict[int, int], + result: dict[int, dict[str, Any]], +) -> None: + """Reprints have no ``name`` column; compose the label from their four.""" + rows = ( + Comic.objects.filter(**{f"{comic_to_collection}__in": collection_pks}) + .filter(reprints__series_name__gt="") + .values(comic_to_collection, *_REPRINT_VALUE_RELS) + .annotate(cnt=Count("pk", distinct=True)) + ) + per_collection: dict[int, list[tuple[tuple, int]]] = defaultdict(list) + for row in rows: + per_collection[row[comic_to_collection]].append( + (tuple(row[rel] for rel in _REPRINT_VALUE_RELS), row["cnt"]) + ) + for gpk in collection_pks: + total = counts.get(gpk, 0) + if not total: + result[gpk]["reprints"] = [] + continue + names = sorted( + Reprint.compose_name(*parts) + for parts, cnt in per_collection.get(gpk, ()) + if cnt == total + ) + result[gpk]["reprints"] = names diff --git a/codex/views/browser/metadata/const.py b/codex/views/browser/metadata/const.py index 3fa20f499..cbda90372 100644 --- a/codex/views/browser/metadata/const.py +++ b/codex/views/browser/metadata/const.py @@ -10,6 +10,7 @@ from codex.models.named import ( Character, Credit, + Reprint, SeriesGroup, StoryArcNumber, Team, @@ -75,6 +76,12 @@ "select": ("source",), "only": ("source", "key", "url"), }, + # Reprint.name is a property composed from all four columns; the + # default ``only=("name", ...)`` raises FieldDoesNotExist. + Reprint: { + "select": ("identifier",), + "only": ("series_name", "volume_number", "issue", "language", "identifier"), + }, Universe: {"only": ("name", "designation", "identifier")}, SeriesGroup: { "select": (), diff --git a/codex/views/browser/saved_settings.py b/codex/views/browser/saved_settings.py index 5bd68e726..a7a56f245 100644 --- a/codex/views/browser/saved_settings.py +++ b/codex/views/browser/saved_settings.py @@ -7,7 +7,7 @@ from rest_framework.serializers import BaseSerializer from codex.models.age_rating import AgeRating, AgeRatingMetron -from codex.models.base import NamedModel +from codex.models.base import BaseModel from codex.models.identifier import IdentifierSource from codex.models.named import ( Character, @@ -17,6 +17,7 @@ Language, Location, OriginalFormat, + Reprint, SeriesGroup, Story, StoryArc, @@ -44,7 +45,8 @@ SettingsBaseView, ) -# Map filter field names to the model whose PKs they store. +# Map filter field names to the model whose PKs they store. Most are +# NamedModels; ``Reprint`` is not, so the validator only leans on the pk. _FILTER_FK_MODEL_MAP = MappingProxyType( { "age_rating_metron": AgeRatingMetron, @@ -57,6 +59,7 @@ "language": Language, "locations": Location, "original_format": OriginalFormat, + "reprints": Reprint, "series_groups": SeriesGroup, "stories": Story, "story_arcs": StoryArc, @@ -69,7 +72,7 @@ def _validate_filter_field( - filters_data: dict, field: str, model: type[NamedModel], warnings: list[str] + filters_data: dict, field: str, model: type[BaseModel], warnings: list[str] ): """Validate one filter field by existing models.""" pk_list = filters_data.get(field) diff --git a/codex/views/const.py b/codex/views/const.py index a14ef240d..55bfbe901 100644 --- a/codex/views/const.py +++ b/codex/views/const.py @@ -12,6 +12,7 @@ from codex.collection import Collection from codex.models import ( AgeRating, + Bookmark, BrowserCollectionModel, Character, Comic, @@ -19,6 +20,7 @@ Credit, CreditPerson, CreditRole, + Favorite, Folder, Genre, Identifier, @@ -29,9 +31,11 @@ Location, OriginalFormat, Publisher, + Reprint, ScanInfo, Series, SeriesGroup, + Story, StoryArc, StoryArcNumber, Tag, @@ -163,15 +167,21 @@ Language, Location, OriginalFormat, + Reprint, SeriesGroup, ScanInfo, StoryArc, StoryArcNumber, + Story, Team, Tag, Tagger, Universe, ) +STATS_USAGE_MODELS = ( + Bookmark, + Favorite, +) CONFIG_MODELS = ( Library, User, diff --git a/frontend/bun.lock b/frontend/bun.lock index 3f139a5fb..cc4b7b983 100644 --- a/frontend/bun.lock +++ b/frontend/bun.lock @@ -29,7 +29,7 @@ "@vue/typescript-plugin": "^3.3.8", "happy-dom": "^20.11.1", "rollup-plugin-visualizer": "^7.0.1", - "sass": "^1.101.7", + "sass": "^1.102.0", "sass-loader": "^17.0.0", "toml": "^5.0.0", "typeface-roboto": "^1.1.13", @@ -754,7 +754,7 @@ "run-applescript": ["run-applescript@7.1.0", "", {}, "sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q=="], - "sass": ["sass@1.101.7", "", { "dependencies": { "chokidar": "^5.0.0", "immutable": "^5.1.5", "source-map-js": ">=0.6.2 <2.0.0" }, "optionalDependencies": { "@parcel/watcher": "^2.4.1" }, "bin": { "sass": "sass.js" } }, "sha512-cDeUYU0dhwKVbpYg/ppsjyuoddxYhWlJOkRoI7+/iZsaSp7iowWDfm+tL2HcUafedWBDvbf/+hx0QRgKG4JSHA=="], + "sass": ["sass@1.102.0", "", { "dependencies": { "chokidar": "^5.0.0", "immutable": "^5.1.5", "source-map-js": ">=0.6.2 <2.0.0" }, "optionalDependencies": { "@parcel/watcher": "^2.4.1" }, "bin": { "sass": "sass.js" } }, "sha512-NSOyTnaQF7rTAEOtI2fwb386vL+akyiQLBZu8Na7hXCb+umJy0GAqlcMIaqACZ6Z1VgTBS4K9PG6B3IdjHGJsw=="], "sass-loader": ["sass-loader@17.0.0", "", { "peerDependencies": { "@rspack/core": "0.x || ^1.0.0 || ^2.0.0-0", "sass": "^1.3.0", "sass-embedded": "*", "webpack": "^5.0.0" }, "optionalPeers": ["@rspack/core", "sass", "sass-embedded", "webpack"] }, "sha512-0Ybm8ohBQ9LcrycVrFQp/KQBNX5a3Wda9/smS0mE/xLffzEnwvV8nykOzrbiSWNzTE3IB/jiXx8O4QmDPG2+Gw=="], diff --git a/frontend/package.json b/frontend/package.json index 027dc42fa..7182b7503 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "codex", - "version": "2.2.2", + "version": "2.2.3", "private": true, "description": "ui for codex api", "type": "module", @@ -40,7 +40,7 @@ "@vue/typescript-plugin": "^3.3.8", "happy-dom": "^20.11.1", "rollup-plugin-visualizer": "^7.0.1", - "sass": "^1.101.7", + "sass": "^1.102.0", "sass-loader": "^17.0.0", "toml": "^5.0.0", "typeface-roboto": "^1.1.13", diff --git a/frontend/src/components/admin/tabs/stats-tab.vue b/frontend/src/components/admin/tabs/stats-tab.vue index fd41e2f88..c146881d3 100644 --- a/frontend/src/components/admin/tabs/stats-tab.vue +++ b/frontend/src/components/admin/tabs/stats-tab.vue @@ -14,6 +14,14 @@ :items="browserCollectionsTable" /> + + + + + + + + @@ -28,17 +36,109 @@ import { useCommonStore } from "@/stores/common"; import { ORDER_BY, TOP_COLLECTION } from "@/choices/browser-map.json"; import { FIT_TO, READING_DIRECTION } from "@/choices/reader-map.json"; +const VIEW_MODE = Object.freeze({ cover: "Cover", table: "Table" }); +const TABLE_COVER_SIZE = Object.freeze({ sm: "Small" }); const LOOKUPS = Object.freeze({ topCollection: TOP_COLLECTION, orderBy: ORDER_BY, fitTo: FIT_TO, readingDirection: READING_DIRECTION, + viewMode: VIEW_MODE, + tableCoverSize: TABLE_COVER_SIZE, }); const CONFIG_LABELS = Object.freeze({ authGroupCount: "Authorization Groups", libraryCount: "Libraries", + libraryReadOnlyCount: "+Read Only", + libraryPollCount: "+Polled", + libraryEventsCount: "+Watched for Events", + libraryGroupAclCount: "+Group Restricted", + sessionCount: "Sessions", userAnonymousCount: "Anonymous Users", userRegisteredCount: "Registered Users", + customCoverCount: "Custom Covers", + customCoverUploadedCount: "+Uploaded", + customCoverDirCount: "+From Covers Folder", + failedImportCount: "Failed Imports", +}); +// Metadata keys whose label isn't just the key plus an "s". +const METADATA_LABELS = Object.freeze({ + countryCount: "Countries", + storyCount: "Stories", + comicCommunityRatingCount: "Comics Rated", + comicCommunityRatingVoteCount: "+With Vote Counts", + comicAlternativeIssueNumberCount: "Comics with Alternate Issue Numbers", + comicMetadataImportedCount: "Comics with Imported Tags", +}); +const USAGE_LABELS = Object.freeze({ + bookmarkCount: "Bookmarks", + favoriteCount: "Favorites", + favoriteUserCount: "Users with Favorites", + favorites: "Favorites by Collection", +}); +const ADMIN_FLAG_LABELS = Object.freeze({ + autoUpdate: "Auto Update", + folderView: "Folder View", + importMetadata: "Import Tags on Library Scan", + lazyImportMetadata: "Import Tags on Demand", + nonUsers: "Non Users", + registration: "Registration", + registerVerification: "Verify New User Email", + sendTelemetry: "Send Stats", + apiKeySet: "API Key", + bannerTextSet: "Banner Text", + browserDefaultCollection: "Default View", + browserMaxObjPerPage: "Browser Page Size", + customCoverMaxUploadMb: "Custom Cover Max Upload (MB)", + ageRatingDefault: "Age Rating Default", + anonymousUserAgeRating: "Anonymous User Age Rating", +}); +const TAGGING_LABELS = Object.freeze({ + defaultMatchMode: "Match Mode", + defaultPromptsMode: "Prompts", + mergeAllSources: "Merge All Sources", + deleteOriginal: "Delete Original", + renameFiles: "Rename Files", + defaultSources: "Enabled Sources", + defaultFormatCount: "Write Formats", + hasMetronCredentials: "Metron Credentials", + hasComicvineCredentials: "Comic Vine Credentials", + metronUrlSet: "Custom Metron URL", + comicvineUrlSet: "Custom Comic Vine URL", +}); +const AUTH_LABELS = Object.freeze({ + oidcEnabled: "Single Sign On", + oidcCreateUsers: "+Create Users", + oidcLinkByEmail: "+Link by Email", + oidcSyncGroups: "+Sync Groups", + oidcPkce: "+PKCE", + oidcFetchUserinfo: "+Fetch Userinfo", + oidcRpInitiatedLogout: "+Logout at Provider", + oidcAdminGroupSet: "+Admin Group", + oidcScopeCustom: "+Custom Scope", + oidcUsernameClaimCustom: "+Custom Username Claim", + oidcGroupsClaimCustom: "+Custom Groups Claim", + oidcTokenAuthMethod: "+Token Auth Method", + userAgeCeilingCount: "Age Restricted Users", + authGroupExcludeCount: "Excluded Groups", +}); +const EMAIL_LABELS = Object.freeze({ + smtpConfigured: "SMTP Configured", + smtpUseTls: "Use TLS", + smtpUseSsl: "Use SSL", +}); +const THROTTLE_LABELS = Object.freeze({ + throttleAnon: "Anonymous", + throttleUser: "User", + throttleOpds: "OPDS", + throttleOpensearch: "OpenSearch", + throttleResetPassword: "Reset Password", +}); +const DEPLOYMENT_LABELS = Object.freeze({ + remoteUserAuth: "Remote User SSO", + failedLoginLog: "Failed Login Log", + failedLoginLogTrustForwardedFor: "+Trust Forwarded For", + urlPathPrefixSet: "Reverse Proxy Subpath", }); const INDENT_KEYS = Object.freeze( new Set([ @@ -48,6 +148,9 @@ const INDENT_KEYS = Object.freeze( "storyArcNumbersCount", ]), ); +// Booleans that answer "have you configured this", not "is this turned on". +const SET_SUFFIXES = ["Set", "Custom", "Configured", "Credentials"]; +const NONE = "None"; export default { name: "AdminStatsTab", @@ -66,7 +169,7 @@ export default { }), platformTable() { const table = {}; - for (const [key, value] of Object.entries(this.stats?.platform)) { + for (const [key, value] of Object.entries(this.stats?.platform ?? {})) { const label = this.keyToLabel(key); const labelValue = key === "system" ? `${value.name} ${value.release}` : value; @@ -75,35 +178,33 @@ export default { return table; }, configTable() { - const table = {}; - for (const [key, value] of Object.entries(this.stats?.config)) { - if (key == "apiKey") { - continue; - } - const label = Reflect.get(CONFIG_LABELS, key); - Reflect.set(table, label, value); - } - return table; + return this.labeledTable(this.stats?.config, CONFIG_LABELS, ["apiKey"]); }, userSettingsTable() { const table = {}; - for (const [key, countObj] of Object.entries(this.stats?.sessions)) { + for (const [key, value] of Object.entries(this.stats?.sessions ?? {})) { + const label = this.keyToLabel(key); + if (typeof value !== "object") { + Reflect.set(table, label, this.displayValue(key, value)); + continue; + } const countTable = {}; const lookup = Reflect.get(LOOKUPS, key); - for (const [typeKey, count] of Object.entries(countObj)) { + for (const [typeKey, count] of Object.entries(value)) { const typeLabel = lookup ? Reflect.get(lookup, snakeCase(typeKey)) : typeKey; Reflect.set(countTable, typeLabel, count); } - const label = this.keyToLabel(key); Reflect.set(table, label, countTable); } return table; }, browserCollectionsTable() { const table = {}; - for (const [key, value] of Object.entries(this.stats?.collections)) { + for (const [key, value] of Object.entries( + this.stats?.collections ?? {}, + )) { let label = this.keyToLabel(key); if (label !== "Series") { label += "s"; @@ -114,7 +215,7 @@ export default { }, fileTypesTable() { const table = {}; - for (const [key, value] of Object.entries(this.stats?.fileTypes)) { + for (const [key, value] of Object.entries(this.stats?.fileTypes ?? {})) { const label = key === "unknown" ? capitalCase(key) @@ -125,9 +226,11 @@ export default { }, metadataTable() { const table = {}; - for (const [key, value] of Object.entries(this.stats?.metadata)) { + for (const [key, value] of Object.entries(this.stats?.metadata ?? {})) { + // Most metadata keys are "Count" and pluralize by adding an s. + // The ones that don't are named outright. let label = - key === "countryCount" ? "Countries" : this.keyToLabel(key) + "s"; + Reflect.get(METADATA_LABELS, key) ?? this.keyToLabel(key) + "s"; if (INDENT_KEYS.has(key)) { label = label.replace(/^\w+ /, "+"); } @@ -135,6 +238,38 @@ export default { } return table; }, + usageTable() { + return this.labeledTable(this.stats?.usage, USAGE_LABELS); + }, + identifiersTable() { + const table = {}; + for (const [key, value] of Object.entries( + this.stats?.identifiers ?? {}, + )) { + const [source, idType] = key.split(":"); + const label = `${capitalCase(source)}: ${capitalCase(idType)}`; + Reflect.set(table, label, value); + } + return table; + }, + adminFlagsTable() { + return this.labeledTable(this.stats?.adminFlags, ADMIN_FLAG_LABELS); + }, + taggingTable() { + return this.labeledTable(this.stats?.tagging, TAGGING_LABELS); + }, + authTable() { + return this.labeledTable(this.stats?.auth, AUTH_LABELS); + }, + emailTable() { + return this.labeledTable(this.stats?.email, EMAIL_LABELS); + }, + throttleTable() { + return this.labeledTable(this.stats?.throttle, THROTTLE_LABELS); + }, + deploymentTable() { + return this.labeledTable(this.stats?.deployment, DEPLOYMENT_LABELS); + }, }, created() { this.loadStats(); @@ -145,6 +280,36 @@ export default { key = key.replace(/Count$/, ""); return capitalCase(key); }, + labeledTable(section, labels, skipKeys = []) { + const table = {}; + for (const [key, value] of Object.entries(section ?? {})) { + if (skipKeys.includes(key)) { + continue; + } + const label = Reflect.get(labels, key) ?? this.keyToLabel(key); + Reflect.set(table, label, this.displayValue(key, value)); + } + return table; + }, + // A boolean means one of two different things depending on the setting, + // and the difference is the point of this page: "Yes" is a toggle that is + // on, "Set" is a value we deliberately do not report. + displayValue(key, value) { + if (typeof value === "boolean") { + const isSetFlag = SET_SUFFIXES.some((suffix) => key.endsWith(suffix)); + if (isSetFlag) { + return value ? "Set" : "Not set"; + } + return value ? "Yes" : "No"; + } + if (value === "" || value === null) { + return NONE; + } + if (typeof value === "object" && Object.keys(value).length === 0) { + return NONE; + } + return value; + }, }, }; diff --git a/frontend/src/components/browser/table/browser-table-cell.vue b/frontend/src/components/browser/table/browser-table-cell.vue index a5e362134..b3dbabd54 100644 --- a/frontend/src/components/browser/table/browser-table-cell.vue +++ b/frontend/src/components/browser/table/browser-table-cell.vue @@ -66,6 +66,7 @@ const M2M_COLUMNS = new Set([ "genres", "identifiers", "locations", + "reprints", "series_groups", "stories", "story_arcs", diff --git a/frontend/src/components/browser/table/browser-table-column-picker.vue b/frontend/src/components/browser/table/browser-table-column-picker.vue index 895de0d79..1ca47b4ef 100644 --- a/frontend/src/components/browser/table/browser-table-column-picker.vue +++ b/frontend/src/components/browser/table/browser-table-column-picker.vue @@ -126,6 +126,7 @@ const _CATEGORIES = Object.freeze([ "publisher_name", "imprint_name", "series_name", + "reprints", "volume_name", "issue", "name", diff --git a/frontend/src/components/browser/toolbars/top/filter-sub-menu.vue b/frontend/src/components/browser/toolbars/top/filter-sub-menu.vue index cce61a184..f36443b60 100644 --- a/frontend/src/components/browser/toolbars/top/filter-sub-menu.vue +++ b/frontend/src/components/browser/toolbars/top/filter-sub-menu.vue @@ -126,6 +126,9 @@ import { useBrowserStore } from "@/stores/browser"; const FILTER_TITLE_OVERRIDES = { ageRatingMetron: "Age Rating", + // "Reprints" reads as reprint editions; these are alternate and + // localized series names. + reprints: "Alternate Series", }; // Tab values are the filter keys; "Standardized" is the default tab. const AGE_RATING_DEFAULT_TAB = "ageRatingMetron"; diff --git a/frontend/src/components/metadata/edit-mode/edit-panel.vue b/frontend/src/components/metadata/edit-mode/edit-panel.vue index d18b3fac6..d1af776da 100644 --- a/frontend/src/components/metadata/edit-mode/edit-panel.vue +++ b/frontend/src/components/metadata/edit-mode/edit-panel.vue @@ -847,6 +847,93 @@ +
Alternate Series
+ + + + + + + + + + + + + + + + + + × + + + + + +
+
+ + + Add Alternate Series + +
+ + + Clear All + +
+
Identifiers
@@ -1150,6 +1237,7 @@ export default { creditsByRole: {}, storyArcNames: [], universes: [], + reprints: [], identifiers: [], patch: { publisher: "", @@ -1270,6 +1358,7 @@ export default { credits: this.creditsSerialized, storyArcs: JSON.stringify(this.storyArcNames), universes: JSON.stringify(this.universes), + reprints: JSON.stringify(this.reprints), identifiers: JSON.stringify(this.identifiers), }; }, @@ -1303,6 +1392,12 @@ export default { ) { changed.add("universes"); } + if ( + this.clearedFields.has("reprints") || + cur.reprints !== orig.reprints + ) { + changed.add("reprints"); + } if ( this.clearedFields.has("identifiers") || cur.identifiers !== orig.identifiers @@ -1593,6 +1688,8 @@ export default { this.storyArcNames = []; } else if (field === "universes") { this.universes = []; + } else if (field === "reprints") { + this.reprints = []; } else if (field === "identifiers") { this.identifiers = []; } @@ -1703,6 +1800,18 @@ export default { })); } + // Alternate series — the panel renders the composed `name`, but the + // editor rebuilds comicbox's nested reprint from the flat columns. + if (this.md.reprints?.length) { + this.reprints = this.md.reprints.map((reprint) => ({ + series_name: reprint.seriesName || "", + volume: + reprint.volumeNumber == null ? "" : String(reprint.volumeNumber), + issue: reprint.issue || "", + language: reprint.language || null, + })); + } + /* Identifiers — shaped as {pk, source, type, code, displayName, url} */ if (this.md.identifiers?.length) { this.identifiers = this.md.identifiers.map((id) => ({ @@ -1732,6 +1841,22 @@ export default { this.origSnapshot = { ...this.currentSnapshot }; }, + buildReprints() { + // comicbox nests a reprint's parts under series/volume; codex stores + // them flat. A row without a series name names nothing, so it's + // dropped rather than written as an empty alternate. + const reprints = []; + for (const row of this.reprints) { + if (!row.series_name) continue; + const reprint = { series: { name: row.series_name } }; + const number = parseInt(row.volume, 10); + if (!isNaN(number)) reprint.volume = { number }; + if (row.issue) reprint.issue = row.issue; + if (row.language) reprint.language = row.language; + reprints.push(reprint); + } + return reprints; + }, buildPatch() { // Returns { patch, deleteKeys }. A merge write can only add or replace // values — comicbox prunes empty patch values on schema load — so @@ -1846,6 +1971,13 @@ export default { } } + // Alternate series — only include if changed + if (changed.has("reprints")) { + const reprints = this.buildReprints(); + if (reprints.length) cbPatch.reprints = reprints; + else deleteKeys.push("reprints"); + } + // Identifiers — only include if changed if (changed.has("identifiers")) { if (cleared.has("identifiers") || this.identifiers.length === 0) { diff --git a/frontend/src/components/online-tag/prompt-popup.vue b/frontend/src/components/online-tag/prompt-popup.vue index 5d425c0bb..6a56cba6b 100644 --- a/frontend/src/components/online-tag/prompt-popup.vue +++ b/frontend/src/components/online-tag/prompt-popup.vue @@ -60,6 +60,15 @@ {{ Math.round(candidate.score * 100) }}% + +
+ a.k.a. {{ candidate.summary.altSeries.join(", ") }} +
({ md: undefined, @@ -107,13 +114,7 @@ export const useMetadataStore = defineStore("metadata", { this.md = undefined; }, getTagName(key) { - var tagName; - if (key === "storyArcNumbers") { - tagName = "Story Arcs"; - } else { - tagName = capitalCase(key); - } - return tagName; + return TAG_NAMES[key] || capitalCase(key); }, mapTag(tagSource, keys, filter = undefined) { const tagMap = {}; diff --git a/frontend/tests/unit/browser-table-cell.test.js b/frontend/tests/unit/browser-table-cell.test.js index 1fc036fc3..6517051dd 100644 --- a/frontend/tests/unit/browser-table-cell.test.js +++ b/frontend/tests/unit/browser-table-cell.test.js @@ -87,6 +87,20 @@ describe("BrowserTableCell — list / M2M", () => { }); expect(wrapper.find(".tableListCell").text()).toBe(""); }); + + it("joins the composed alternate-series labels", () => { + /* + * ``reprints`` has to be listed in ``M2M_COLUMNS`` or the array + * falls through to the text cell and stringifies badly. + */ + const wrapper = mountCell({ + column: "reprints", + row: { reprints: ["Kapitän Wissenschaft (de)", "Capitan Sciencia v1"] }, + }); + expect(wrapper.find(".tableListCell").text()).toBe( + "Kapitän Wissenschaft (de), Capitan Sciencia v1", + ); + }); }); describe("BrowserTableCell — bool", () => { diff --git a/frontend/tests/unit/browser-table-column-picker.test.js b/frontend/tests/unit/browser-table-column-picker.test.js index 8f0f192cf..461d989ce 100644 --- a/frontend/tests/unit/browser-table-column-picker.test.js +++ b/frontend/tests/unit/browser-table-column-picker.test.js @@ -17,9 +17,9 @@ import { smartInsertIndex } from "@/components/browser/table/browser-table-colum * Canonical-rank reference for the cases below — kept here so the * test reads as a story rather than an opaque number puzzle: * cover=0, favorite=1, publisher_name=2, imprint_name=3, - * series_name=4, volume_name=5, issue=6, name=7, child_count=8, - * file_name=9, size=10, page_count=11, file_type=12, - * year=13, … + * series_name=4, reprints=5, volume_name=6, issue=7, name=8, + * child_count=9, file_name=10, size=11, page_count=12, + * file_type=13, year=14, … */ describe("smartInsertIndex — canonical-order drafts", () => { diff --git a/frontend/tests/unit/edit-panel-reprints.test.js b/frontend/tests/unit/edit-panel-reprints.test.js new file mode 100644 index 000000000..ccd582ae3 --- /dev/null +++ b/frontend/tests/unit/edit-panel-reprints.test.js @@ -0,0 +1,228 @@ +/* + * Tests for the tag edit panel's Alternate Series section. + * + * The metadata endpoint ships each reprint's flat codex columns + * ({pk, name, seriesName, volumeNumber, issue, language, url}) alongside the + * composed chip `name`, so the editor seeds its rows from the parts instead + * of parsing the label back apart. On save the parts have to be re-nested + * into comicbox's reprint shape ({series: {name}, volume: {number}, issue, + * language}), and clearing the section has to travel as the "reprints" + * delete key — a patch can only add or replace. + * + * Volume and language are MetronInfo-only: ComicInfo's AlternateSeries / + * AlternateNumber / AlternateCount carry neither. + */ +import { createTestingPinia } from "@pinia/testing"; +import { flushPromises, mount } from "@vue/test-utils"; +import { describe, expect, test } from "vitest"; + +import EditPanel from "@/components/metadata/edit-mode/edit-panel.vue"; +import vuetify from "@/plugins/vuetify"; + +const DISABLED_TIP = "Not supported by selected metadata formats"; + +// Absent parts arrive as the model's own empty defaults: "" for the +// strings, null for the volume number. +const SHAPED_REPRINTS = Object.freeze([ + { + pk: 7, + name: "Kapitän Wissenschaft (de)", + seriesName: "Kapitän Wissenschaft", + volumeNumber: null, + issue: "", + language: "de", + }, + { + pk: 8, + name: "Capitan Sciencia v1 #3", + seriesName: "Capitan Sciencia", + volumeNumber: 1, + issue: "3", + language: "", + }, +]); + +async function mountPanel({ formats = ["METRON_INFO"], md = {} } = {}) { + const pinia = createTestingPinia({ + initialState: { + metadata: { md }, + admin: { taggingDefaults: { defaultFormats: formats } }, + browser: { settings: { twentyFourHourTime: false } }, + }, + }); + const wrapper = mount(EditPanel, { + props: { book: { pk: 1, collection: "comics", ids: [1] } }, + global: { plugins: [pinia, vuetify] }, + }); + await flushPromises(); + return wrapper; +} + +function findButton(wrapper, label) { + return wrapper.findAll("button").find((b) => b.text().includes(label)); +} + +describe("EditPanel alternate series rows", () => { + test("seeds one row per reprint from the flat columns", async () => { + const wrapper = await mountPanel({ md: { reprints: SHAPED_REPRINTS } }); + + expect(wrapper.vm.reprints).toStrictEqual([ + { + series_name: "Kapitän Wissenschaft", + volume: "", + issue: "", + language: "de", + }, + { + series_name: "Capitan Sciencia", + volume: "1", + issue: "3", + language: null, + }, + ]); + expect(wrapper.vm.hasChanges).toBe(false); + }); + + test("leaves the rows empty when the comic has no reprints", async () => { + const wrapper = await mountPanel({ md: { reprints: [] } }); + expect(wrapper.vm.reprints).toStrictEqual([]); + }); + + test("the add button appends a blank row", async () => { + const wrapper = await mountPanel(); + await findButton(wrapper, "Add Alternate Series").trigger("click"); + + expect(wrapper.vm.reprints).toStrictEqual([ + { series_name: "", volume: "", issue: "", language: null }, + ]); + }); +}); + +describe("EditPanel alternate series patch", () => { + test("re-nests the parts into comicbox reprints", async () => { + const wrapper = await mountPanel({ md: { reprints: SHAPED_REPRINTS } }); + wrapper.vm.reprints[0].issue = "1"; + await flushPromises(); + + const { patch, deleteKeys } = wrapper.vm.buildPatch(); + expect(patch).toStrictEqual({ + reprints: [ + { + series: { name: "Kapitän Wissenschaft" }, + issue: "1", + language: "de", + }, + { + series: { name: "Capitan Sciencia" }, + volume: { number: 1 }, + issue: "3", + }, + ], + }); + expect(deleteKeys).toStrictEqual([]); + }); + + test("an untouched section stays out of the patch entirely", async () => { + const wrapper = await mountPanel({ md: { reprints: SHAPED_REPRINTS } }); + const { patch, deleteKeys } = wrapper.vm.buildPatch(); + + expect(patch).not.toHaveProperty("reprints"); + expect(deleteKeys).not.toContain("reprints"); + }); + + test("a newly added row travels with only the parts it carries", async () => { + const wrapper = await mountPanel(); + wrapper.vm.reprints.push({ + series_name: "Capitaine Science", + volume: "", + issue: "", + language: null, + }); + await flushPromises(); + + const { patch } = wrapper.vm.buildPatch(); + expect(patch.reprints).toStrictEqual([ + { series: { name: "Capitaine Science" } }, + ]); + }); + + test("removing a row drops it from the patch", async () => { + const wrapper = await mountPanel({ md: { reprints: SHAPED_REPRINTS } }); + wrapper.vm.reprints.splice(0, 1); + await flushPromises(); + + const { patch } = wrapper.vm.buildPatch(); + expect(patch.reprints).toHaveLength(1); + expect(patch.reprints[0].series.name).toBe("Capitan Sciencia"); + }); + + test("a row with no series name names nothing and is dropped", async () => { + const wrapper = await mountPanel(); + wrapper.vm.reprints.push({ + series_name: "", + volume: "2", + issue: "", + language: "fr", + }); + await flushPromises(); + + const { patch, deleteKeys } = wrapper.vm.buildPatch(); + expect(patch).not.toHaveProperty("reprints"); + expect(deleteKeys).toStrictEqual(["reprints"]); + }); + + test("clearing the section deletes the comicbox reprints key", async () => { + const wrapper = await mountPanel({ md: { reprints: SHAPED_REPRINTS } }); + wrapper.vm.clearField("reprints"); + await flushPromises(); + + const { patch, deleteKeys } = wrapper.vm.buildPatch(); + expect(wrapper.vm.reprints).toStrictEqual([]); + expect(deleteKeys).toStrictEqual(["reprints"]); + expect(patch).not.toHaveProperty("reprints"); + }); + + test("emptying every row also deletes rather than patching nothing", async () => { + const wrapper = await mountPanel({ md: { reprints: SHAPED_REPRINTS } }); + wrapper.vm.reprints = []; + await flushPromises(); + + const { patch, deleteKeys } = wrapper.vm.buildPatch(); + expect(deleteKeys).toStrictEqual(["reprints"]); + expect(patch).not.toHaveProperty("reprints"); + }); +}); + +describe("EditPanel alternate series format support", () => { + test("ComicInfo disables the MetronInfo-only volume and language", async () => { + const wrapper = await mountPanel({ + formats: ["COMIC_INFO"], + md: { reprints: SHAPED_REPRINTS }, + }); + const volume = wrapper + .findAll("input") + .find((i) => i.element.value === "1"); + + expect(volume.element.disabled).toBe(true); + expect(volume.element.closest("td").getAttribute("title")).toBe( + DISABLED_TIP, + ); + expect(wrapper.vm.isFieldDisabled("reprint_language")).toBe(true); + // The series name and issue do persist to ComicInfo's Alternates. + expect(wrapper.vm.isFieldDisabled("reprints")).toBe(false); + }); + + test("MetronInfo enables every part", async () => { + const wrapper = await mountPanel({ + formats: ["METRON_INFO"], + md: { reprints: SHAPED_REPRINTS }, + }); + const btn = findButton(wrapper, "Add Alternate Series"); + + expect(btn.element.disabled).toBe(false); + expect(wrapper.vm.isFieldDisabled("reprint_volume")).toBe(false); + expect(wrapper.vm.isFieldDisabled("reprint_language")).toBe(false); + }); +}); + +export default {}; diff --git a/frontend/tests/unit/metadata-reprints.test.js b/frontend/tests/unit/metadata-reprints.test.js new file mode 100644 index 000000000..3e41be1b9 --- /dev/null +++ b/frontend/tests/unit/metadata-reprints.test.js @@ -0,0 +1,88 @@ +/* + * The metadata pane's alternate-series row. + * + * The backend composes ``Reprint.name`` from whichever of series name, + * volume, issue and language the reprint carries, so the client just + * renders it like any other ``{pk, name, url}`` tag row. Only the row + * label differs from the capital-cased key: "Alternate Series". + * + * Chips route through the ``reprints`` browser filter, the key phase E + * registers. + */ +import { createTestingPinia } from "@pinia/testing"; +import { mount } from "@vue/test-utils"; +import { createPinia, setActivePinia } from "pinia"; +import { beforeEach, describe, expect, test } from "vitest"; + +import MetadataTags from "@/components/metadata/metadata-tags.vue"; +import vuetify from "@/plugins/vuetify"; +import { useBrowserStore } from "@/stores/browser"; +import { useMetadataStore } from "@/stores/metadata"; + +const REPRINTS = Object.freeze([ + { pk: 7, name: "Kapitän Wissenschaft" }, + { pk: 8, name: "Capitan Sciencia v1 (es)" }, +]); + +describe("metadata store alternate series row", () => { + beforeEach(() => { + setActivePinia(createPinia()); + }); + + test("labels the reprints row Alternate Series", () => { + const store = useMetadataStore(); + // Copied: the tags getter sorts the row in place. + store.md = { reprints: [...REPRINTS] }; + + const row = store.tags["Alternate Series"]; + expect(row.filter).toBe("reprints"); + expect(row.tags.map((tag) => tag.name)).toStrictEqual([ + "Capitan Sciencia v1 (es)", + "Kapitän Wissenschaft", + ]); + expect(store.tags.Reprints).toBeUndefined(); + }); + + test("omits the row when there are no reprints", () => { + const store = useMetadataStore(); + store.md = { reprints: [] }; + + expect(store.tags["Alternate Series"]).toBeUndefined(); + }); +}); + +describe("alternate series chips", () => { + function mountTags() { + const pinia = createTestingPinia({ + initialState: { + browser: { + settings: { show: {}, filters: {}, topCollection: "publishers" }, + }, + metadata: { md: { collection: "comics", ids: [1] } }, + }, + }); + const wrapper = mount(MetadataTags, { + global: { plugins: [pinia, vuetify] }, + props: { label: "", filter: "reprints", values: REPRINTS }, + }); + return { wrapper, browserStore: useBrowserStore() }; + } + + test("renders the composed names", () => { + const { wrapper } = mountTags(); + const text = wrapper.text(); + expect(text).toContain("Kapitän Wissenschaft"); + expect(text).toContain("Capitan Sciencia v1 (es)"); + }); + + test("clicking filters the browser by the reprint pk", async () => { + const { wrapper, browserStore } = mountTags(); + + await wrapper.findAll(".v-chip")[0].trigger("click"); + + expect(browserStore.routeWithSettings).toHaveBeenCalledWith( + { filters: { reprints: [7] } }, + { name: "browser", params: { collection: "root", pks: "0", page: 1 } }, + ); + }); +}); diff --git a/frontend/tests/unit/prompt-popup.test.js b/frontend/tests/unit/prompt-popup.test.js new file mode 100644 index 000000000..3e4c1887a --- /dev/null +++ b/frontend/tests/unit/prompt-popup.test.js @@ -0,0 +1,130 @@ +/* + * Tests for the Online Tagging match-review popup. + * + * Comicbox scores candidates against alternate series names, so a comic filed + * under a localized title matches a canonical series that looks nothing like + * its filename. The popup shows those aliases so the match is explicable, and + * a pick carries the candidate's volume id so the apply replay can narrow to + * that volume. + */ +import { createTestingPinia } from "@pinia/testing"; +import { mount } from "@vue/test-utils"; +import { describe, expect, test } from "vitest"; + +import OnlineTagPromptPopup from "@/components/online-tag/prompt-popup.vue"; +import vuetify from "@/plugins/vuetify"; +import { useOnlineTagStore } from "@/stores/online-tag"; + +// Render the dialog body inline instead of through v-dialog's overlay/teleport +// (which need browser APIs happy-dom lacks). +const VDialogStub = { name: "VDialog", template: "
" }; + +function candidate(overrides = {}) { + return { + source: "comicvine", + issueId: 42, + summary: { + series: "Captain Science", + issue: "1", + year: 1950, + publisher: "Youthful", + coverUrl: "", + altSeries: [], + }, + score: 0.91, + url: "", + volumeId: null, + ...overrides, + }; +} + +function mountPopup(candidates) { + const pinia = createTestingPinia(); + const wrapper = mount(OnlineTagPromptPopup, { + global: { + plugins: [vuetify, pinia], + stubs: { VDialog: VDialogStub }, + }, + }); + const store = useOnlineTagStore(); + store.pendingPrompts = [ + { + fingerprint: "fp1", + pk: 7, + path: "/comics/kapitan.cbz", + source: "comicvine", + candidates, + }, + ]; + store.promptDialogOpen = true; + return { wrapper, store }; +} + +describe("OnlineTagPromptPopup", () => { + describe("alternate series names", () => { + test("shows the aliases that explain an off-filename match", async () => { + const { wrapper } = mountPopup([ + candidate({ + summary: { + ...candidate().summary, + altSeries: ["Kapitän Wissenschaft", "Capitan Sciencia"], + }, + }), + ]); + await wrapper.vm.$nextTick(); + + expect(wrapper.text()).toContain( + "a.k.a. Kapitän Wissenschaft, Capitan Sciencia", + ); + }); + + test("omits the line when the source carries no aliases", async () => { + const { wrapper } = mountPopup([candidate()]); + await wrapper.vm.$nextTick(); + + expect(wrapper.text()).not.toContain("a.k.a."); + }); + + test("omits the line for a prompt cached before aliases existed", async () => { + const summary = { ...candidate().summary }; + delete summary.altSeries; + const { wrapper } = mountPopup([candidate({ summary })]); + await wrapper.vm.$nextTick(); + + expect(wrapper.text()).not.toContain("a.k.a."); + }); + }); + + describe("pick", () => { + test("passes the chosen candidate's volume id", () => { + const { wrapper, store } = mountPopup([ + candidate(), + candidate({ volumeId: 9876 }), + ]); + + wrapper.vm.pick(store.pendingPrompts[0], 1); + + expect(store.resolvePrompt).toHaveBeenCalledWith( + "fp1", + "choose", + 1, + 9876, + ); + }); + + test("passes null when the source exposes no volume id", () => { + const { wrapper, store } = mountPopup([candidate()]); + + wrapper.vm.pick(store.pendingPrompts[0], 0); + + expect(store.resolvePrompt).toHaveBeenCalledWith( + "fp1", + "choose", + 0, + null, + ); + }); + }); +}); + +export default {}; diff --git a/frontend/tests/unit/stats-tab.test.js b/frontend/tests/unit/stats-tab.test.js new file mode 100644 index 000000000..79f2434f8 --- /dev/null +++ b/frontend/tests/unit/stats-tab.test.js @@ -0,0 +1,178 @@ +/* + * Tests for the Admin Stats tab. + * + * The tab is what an administrator sees of the anonymous stats report, so + * behavior locked in here: + * - Every section the API returns is rendered under a titled table. + * - Toggle booleans read as Yes/No, and "have you configured this" + * booleans read as Set/Not set, so nobody mistakes one for the other. + * - The API key is never rendered, even though the payload carries it. + */ +import { createTestingPinia } from "@pinia/testing"; +import { mount } from "@vue/test-utils"; +import { describe, expect, test } from "vitest"; + +import StatsTab from "@/components/admin/tabs/stats-tab.vue"; +import vuetify from "@/plugins/vuetify"; + +const STATS = { + platform: { + docker: false, + machine: "arm64", + cores: 10, + system: { name: "Darwin", release: "25.5.0" }, + pythonVersion: "3.14.4", + codexVersion: "2.2.3", + }, + config: { + libraryCount: 2, + sessionCount: 1, + userAnonymousCount: 0, + userRegisteredCount: 2, + authGroupCount: 0, + libraryReadOnlyCount: 1, + libraryPollCount: 2, + libraryEventsCount: 2, + libraryGroupAclCount: 0, + customCoverCount: 3, + customCoverUploadedCount: 2, + customCoverDirCount: 1, + failedImportCount: 0, + apiKey: "s3cr3t-api-key", + }, + sessions: { + topCollection: { publishers: 2 }, + orderBy: { sort_name: 2 }, + dynamicCovers: { true: 2 }, + finishOnLastPage: { true: 1 }, + fitTo: { W: 1 }, + readingDirection: { ltr: 1 }, + viewMode: { cover: 2, table: 1 }, + tableCoverSize: { sm: 1 }, + customCovers: { true: 2 }, + multiSortCount: 0, + }, + collections: { + publisherCount: 17, + imprintCount: 20, + seriesCount: 56, + volumeCount: 60, + issueCount: 154, + folderCount: 70, + storyArcCount: 23, + }, + fileTypes: { cbz: 100, cbr: 50, cb7: 2, pdf: 3, unknown: 1 }, + metadata: { characterCount: 609, storyCount: 151 }, + usage: { + bookmarkCount: 1, + favoriteCount: 4, + favoriteUserCount: 1, + favorites: { comics: 4 }, + }, + identifiers: { "metron:comic": 85, "comicvine:comic": 126 }, + adminFlags: { + autoUpdate: false, + folderView: true, + sendTelemetry: true, + apiKeySet: true, + bannerTextSet: false, + browserDefaultCollection: "publishers", + browserMaxObjPerPage: 100, + ageRatingDefault: "Everyone", + }, + tagging: { + defaultMatchMode: "auto", + mergeAllSources: false, + defaultSources: { metron: 1, comicvine: 1 }, + hasMetronCredentials: true, + hasComicvineCredentials: false, + metronUrlSet: false, + }, + auth: { + oidcEnabled: false, + oidcPkce: true, + oidcScopeCustom: false, + oidcTokenAuthMethod: "", + userAgeCeilingCount: 0, + }, + email: { smtpConfigured: false, smtpUseTls: true, smtpUseSsl: false }, + throttle: { throttleAnon: 0, throttleResetPassword: 5 }, + deployment: { + remoteUserAuth: false, + failedLoginLog: true, + urlPathPrefixSet: true, + }, +}; + +const SECTION_TITLES = [ + "Platform", + "Config", + "File Types", + "User Settings", + "Browser Collections", + "Tags", + "Reading", + "Identifiers", + "Admin Flags", + "Online Tagging", + "Authentication", + "Email", + "Rate Limits", + "Deployment", +]; + +function mountTab(stats = STATS) { + const pinia = createTestingPinia({ initialState: { admin: { stats } } }); + return mount(StatsTab, { + global: { plugins: [pinia, vuetify] }, + }); +} + +describe("AdminStatsTab", () => { + test("renders a table for every stats section", () => { + const text = mountTab().text(); + for (const title of SECTION_TITLES) { + expect(text).toContain(title); + } + }); + + test("never renders the api key", () => { + const text = mountTab().text(); + expect(text).not.toContain("s3cr3t-api-key"); + // Its presence is still reported, without the value. + expect(text).toContain("API Key"); + }); + + test("toggles read as Yes/No", () => { + const text = mountTab().text(); + expect(text).toContain("Folder View"); + expect(text).toContain("Yes"); + expect(text).toContain("No"); + }); + + test("configured-or-not booleans read as Set/Not set", () => { + const text = mountTab().text(); + expect(text).toContain("Set"); + expect(text).toContain("Not set"); + }); + + test("renders new v2 sections with readable labels", () => { + const text = mountTab().text(); + expect(text).toContain("Read Only"); + expect(text).toContain("Bookmarks"); + expect(text).toContain("Single Sign On"); + expect(text).toContain("Reverse Proxy Subpath"); + expect(text).toContain("Metron Credentials"); + }); + + test("renders identifier buckets as source and type", () => { + expect(mountTab().text()).toContain("Metron: Comic"); + }); + + test("survives a params-filtered response with sections missing", () => { + // /admin/stats?platform=... returns only the requested sections. + const wrapper = mountTab({ platform: STATS.platform }); + expect(wrapper.text()).toContain("arm64"); + expect(wrapper.text()).not.toContain("Bookmarks"); + }); +}); diff --git a/package.json b/package.json index fe2e561c8..2462b593f 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "@eslint-community/eslint-plugin-eslint-comments": "^4.7.2", "@eslint/js": "^10.0.1", "@eslint/json": "^2.0.1", - "@fsouza/prettierd": "^0.28.0", + "@fsouza/prettierd": "^0.29.0", "@prettier/plugin-xml": "^3.4.2", "@stylistic/eslint-plugin": "^5.10.0", "@vitest/eslint-plugin": "^1.6.24", @@ -82,14 +82,14 @@ "eslint-plugin-no-secrets": "^2.3.3", "eslint-plugin-no-unsanitized": "^4.1.5", "eslint-plugin-no-use-extend-native": "^0.7.3", - "eslint-plugin-package-json": "^1.6.0", + "eslint-plugin-package-json": "^1.6.2", "eslint-plugin-perfectionist": "^5.10.0", "eslint-plugin-prettier": "^5.5.6", "eslint-plugin-promise": "^7.3.0", "eslint-plugin-regexp": "^3.1.1", "eslint-plugin-security": "^4.0.1", "eslint-plugin-sonarjs": "^4.2.0", - "eslint-plugin-toml": "^1.4.0", + "eslint-plugin-toml": "^1.5.0", "eslint-plugin-unicorn": "^72.0.0", "eslint-plugin-vue": "^10.10.0", "eslint-plugin-vue-scoped-css": "^3.1.1", diff --git a/pyproject.toml b/pyproject.toml index b61e7c8e0..d826bb306 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ "adrf~=0.1.12", "bidict~=0.23", "channels~=4.2", - "comicbox[pdf]~=4.5.1", + "comicbox[pdf]~=4.6.1", "cryptography>=48.0.0", "dateparser~=1.2", "django-allauth[socialaccount]~=65.13", @@ -49,7 +49,7 @@ readme = "README.md" requires-python = ">=3.12" license = "GPL-3.0-only" name = "codex" -version = "2.2.2" +version = "2.2.3" [[project.authors]] name = "AJ Slater" email = "aj@slater.net" @@ -91,7 +91,7 @@ lint = [ "mbake~=1.4.5", "pathspec~=1.1.0", "radon~=6.0", - "ruff~=0.15", + "ruff~=0.16", "types-python-dateutil~=2.9", "ty~=0.0.18", "vulture~=2.3", @@ -297,11 +297,13 @@ extend-ignore = [ "E501", "ISC001", "PERF203", + "PLC0415", "S101", "W191", ] extend-select = [ "A", + "AIR", "ARG", "ASYNC", "B", @@ -318,6 +320,7 @@ extend-select = [ "EXE", "F", "FA", + "FAST", "FBT", "FIX", "FLY", @@ -329,9 +332,12 @@ extend-select = [ "ISC", "LOG", "N", + "NPY", + "PD", "PERF", "PGH", "PIE", + "PLC", "PLE", "PLR", "PLW", diff --git a/tests/importer/test_basic.py b/tests/importer/test_basic.py index b13b924c2..5de1d5d6a 100644 --- a/tests/importer/test_basic.py +++ b/tests/importer/test_basic.py @@ -58,6 +58,7 @@ Location, OriginalFormat, Publisher, + Reprint, ScanInfo, Series, SeriesGroup, @@ -81,7 +82,9 @@ PATH = str(LIBRARY_PATH / "test.cbz") PATH_PARENTS = {(str(Path(PATH).parent),)} PATH_PARENTS_QUERY = {(str(Path(PATH).parent),): set()} -COMPLEX_FIELD_NAMES = frozenset({"credits", "story_arc_numbers", "identifiers"}) +COMPLEX_FIELD_NAMES = frozenset( + {"credits", "reprints", "story_arc_numbers", "identifiers"} +) AGGREGATED = MappingProxyType( { FIS: {}, @@ -118,6 +121,10 @@ }, Tagger: {("comicbox dev",): set()}, Publisher: {("Youthful Adventure Stories",): {(None,)}}, + Reprint: { + ("Capitan Sciencia", 1, "", "es"): {(None,)}, + ("Kapitän Wissenschaft", None, "", "de"): {(None,)}, + }, Imprint: { ( "Youthful Adventure Stories", @@ -254,7 +261,10 @@ ) }, "locations": {("The Moon",)}, - # reprints add + "reprints": { + ("Capitan Sciencia", 1, "", "es"), + ("Kapitän Wissenschaft", None, "", "de"), + }, "series_groups": {("science comics",)}, "stories": {("The Beginning",)}, "story_arc_numbers": { @@ -357,6 +367,10 @@ Universe: {("Young Adult Silly Universe", None, "4242")}, Folder: deepcopy(PATH_PARENTS), Publisher: {("Youthful Adventure Stories", None)}, + Reprint: { + ("Capitan Sciencia", 1, "", "es", None), + ("Kapitän Wissenschaft", None, "", "de", None), + }, Imprint: {("Youthful Adventure Stories", "TestImprint", None)}, Series: { ( @@ -377,7 +391,7 @@ 7, ) }, - TOTAL: 41, + TOTAL: 43, }, UPDATE_FKS: { TOTAL: 0, @@ -415,6 +429,10 @@ "genres": {("Science Fiction",)}, "identifiers": {("comicvine", "comic", "145269")}, "locations": {("The Moon",)}, + "reprints": { + ("Capitan Sciencia", 1, "", "es"), + ("Kapitän Wissenschaft", None, "", "de"), + }, "series_groups": {("science comics",)}, "stories": {("The Beginning",)}, "story_arc_numbers": {("c", None), ("d", 1), ("e", 3), ("f", 5)}, @@ -491,6 +509,10 @@ "genres": {("Science Fiction",)}, "identifiers": {("comicvine", "comic", "145269")}, "locations": {("The Moon",)}, + "reprints": { + ("Capitan Sciencia", 1, "", "es"), + ("Kapitän Wissenschaft", None, "", "de"), + }, "series_groups": {("science comics",)}, "stories": {("The Beginning",)}, "story_arc_numbers": {("c", None), ("d", 1), ("e", 3), ("f", 5)}, @@ -516,6 +538,10 @@ "genres": {("Science Fiction",)}, "identifiers": {("comicvine", "comic", "145269")}, "locations": {("The Moon",)}, + "reprints": { + ("Capitan Sciencia", 1, "", "es"), + ("Kapitän Wissenschaft", None, "", "de"), + }, "series_groups": {("science comics",)}, "stories": {("The Beginning",)}, "story_arc_numbers": {("c", None), ("d", 1), ("e", 3), ("f", 5)}, @@ -553,6 +579,7 @@ FIS: {}, FTS_CREATE: { 1: { + "alternate_series": ("Capitan Sciencia", "Kapitän Wissenschaft"), "characters": ("Boy Empirical", "Captain Science"), "collection_title": ("The Big Omnibus",), "country": ("US",), @@ -593,6 +620,7 @@ { FTS_CREATE: { 1: { + "alternate_series": ("Capitan Sciencia", "Kapitän Wissenschaft"), "characters": ("Boy Empirical", "Captain Science"), "collection_title": ("The Big Omnibus",), "country": ("US",), @@ -632,6 +660,7 @@ { FTS_CREATE: { 1: { + "alternate_series": ("Capitan Sciencia", "Kapitän Wissenschaft"), "characters": ("Boy Empirical", "Captain Science"), "collection_title": ("The Big Omnibus",), "country": ("US",), @@ -673,7 +702,7 @@ "volume": -2, } ) -_COMPLEX_KEYS = frozenset({"credits", "identifiers", "story_arc_numbers"}) +_COMPLEX_KEYS = frozenset({"credits", "identifiers", "reprints", "story_arc_numbers"}) _COMICFTS_IGNORE_KEYS = ("comic_id", "updated_at", "created_at") @@ -763,6 +792,18 @@ def _test_comic_creation_field_complex(field_name: str, value): for subval in value.all() ) ) + elif field_name == "reprints": + value = tuple( + sorted( + ( + subval.series_name, + subval.volume_number, + subval.issue, + subval.language, + ) + for subval in value.all() + ) + ) return value @@ -808,6 +849,28 @@ def export_test_fts_creation(values_const: MappingProxyType, comic: Comic): diff_assert(fts_values, comicfts, "COMIC_FTS") +def run_full_import(importer) -> None: + """ + Run every importer phase in order. + + ``ComicImporter.apply()`` wraps the same sequence in + ``importer_pragmas``, whose exit-time ``PRAGMA wal_checkpoint`` raises + ``database table is locked`` inside a ``TestCase``'s wrapping + transaction. Drive the phases directly instead. + """ + importer.read() + importer.query() + importer.create_all_fks() + importer.update_all_fks() + importer.prepare_fk_link_instance_maps() + importer.update_comics() + importer.create_comics() + importer.link() + importer.fail_imports() + importer.delete() + importer.full_text_search() + + class BaseTestImporter(SerializeMixin, TestCase, ABC): lockfile = __file__ diff --git a/tests/importer/test_move_timestamps.py b/tests/importer/test_move_timestamps.py index 7243d1962..0e023b265 100644 --- a/tests/importer/test_move_timestamps.py +++ b/tests/importer/test_move_timestamps.py @@ -32,27 +32,12 @@ StoryArcNumber, Volume, ) -from tests.importer.test_basic import PATH, BaseTestImporter +from tests.importer.test_basic import PATH, BaseTestImporter, run_full_import _FILE_PUBLISHER = "Youthful Adventure Stories" _SOURCE_PUBLISHER = "Throwaway Source Pub" -def _run_full_import(importer) -> None: - """Run every importer phase in order (apply() without pragmas/finish).""" - importer.read() - importer.query() - importer.create_all_fks() - importer.update_all_fks() - importer.prepare_fk_link_instance_maps() - importer.update_comics() - importer.create_comics() - importer.link() - importer.fail_imports() - importer.delete() - importer.full_text_search() - - class TestImporterMoveRestampsSource(BaseTestImporter): """A publisher change re-stamps the collection the comic left.""" @@ -90,7 +75,7 @@ def test_publisher_move_restamps_source(self) -> None: check_metadata_mtime=False, ) importer = ComicImporter(task, logger, LIBRARIAN_QUEUE, Lock(), Event()) - _run_full_import(importer) + run_full_import(importer) # The comic actually moved to the file's publisher... comic = Comic.objects.get(path=PATH) @@ -142,7 +127,7 @@ def test_storyarc_removal_restamps_source(self) -> None: check_metadata_mtime=False, ) importer = ComicImporter(task, logger, LIBRARIAN_QUEUE, Lock(), Event()) - _run_full_import(importer) + run_full_import(importer) # The comic moved to the file's arcs; "Removed Arc" lost it... comic = Comic.objects.get(path=PATH) diff --git a/tests/importer/test_update_all.py b/tests/importer/test_update_all.py index 1ad9c82e2..27966e4e2 100644 --- a/tests/importer/test_update_all.py +++ b/tests/importer/test_update_all.py @@ -40,6 +40,7 @@ Location, OriginalFormat, Publisher, + Reprint, ScanInfo, Series, SeriesGroup, @@ -130,6 +131,10 @@ ("metron", "comic", "999"), }, "locations": {("Mars",)}, + "reprints": { + ("Capitan Sciencia", 1, "", "es"), + ("Kapitän Wissenschaft", None, "", "de"), + }, "series_groups": {("adult comics",)}, "stories": {("The Beginning",), ("The End",)}, "story_arc_numbers": {("c", None), ("d", 1), ("e", 3), ("g", 5)}, @@ -186,6 +191,10 @@ (("metron", "publisher", "111"),), } }, + Reprint: { + ("Capitan Sciencia", 1, "", "es"): {(None,)}, + ("Kapitän Wissenschaft", None, "", "de"): {(None,)}, + }, Imprint: { ("Youthful Adventure Stories", "TestImprint"): { (("metron", "imprint", "123"),), diff --git a/tests/test_browser_reprints_column.py b/tests/test_browser_reprints_column.py new file mode 100644 index 000000000..62875e296 --- /dev/null +++ b/tests/test_browser_reprints_column.py @@ -0,0 +1,230 @@ +""" +The ``reprints`` browser table column and filter. + +``Reprint`` is the only tag model without a ``name`` column — the label +composes from four — so both the table cell (Comic rows and collection-row +intersections) and the filter drawer need their own label paths. These +tests pin all of them to :meth:`Reprint.compose_name`'s output. +""" + +import json +import shutil +from pathlib import Path +from typing import Final, override + +from django.contrib.auth.models import User +from django.core.cache import cache +from django.test import Client, TestCase + +from codex.choices.browser import DUMMY_NULL_NAME, VUETIFY_NULL_CODE +from codex.models import Comic, Imprint, Library, Publisher, Series, Volume +from codex.models.named import Reprint +from codex.startup import init_admin_flags + +_TEST_PASSWORD: Final = "test-pw-hush-S106" # noqa: S105 +_HTTP_OK: Final = 200 +TMP_DIR = Path("/tmp/codex.tests.browser_reprints_column") # noqa: S108 +_SETTINGS_URL: Final = "/api/v4/browse/publishers/settings" + + +def _v4(response): + """Unwrap the v4 ``{data, meta, errors}`` envelope and return ``data``.""" + body = response.json() + if isinstance(body, dict) and "data" in body and "meta" in body: + return body["data"] + return body + + +class _ReprintsFixtureTestCase(TestCase): + """One publisher / imprint / series / volume and one comic to tag.""" + + @override + def setUp(self) -> None: + cache.clear() + init_admin_flags() + TMP_DIR.mkdir(exist_ok=True, parents=True) + self.library = Library.objects.create(path=str(TMP_DIR)) # pyright: ignore[reportUninitializedInstanceVariable] + self.publisher = Publisher.objects.create(name="Pub") # pyright: ignore[reportUninitializedInstanceVariable] + self.imprint = Imprint.objects.create(name="Imp", publisher=self.publisher) # pyright: ignore[reportUninitializedInstanceVariable] + self.series = self._create_series("Ser") # pyright: ignore[reportUninitializedInstanceVariable] + self.comic = self._create_comic("C1", 1) # pyright: ignore[reportUninitializedInstanceVariable] + user = User.objects.create_user( + username="reprints_column_test", password=_TEST_PASSWORD + ) + self.client = Client() + self.client.force_login(user) + + @override + def tearDown(self) -> None: + shutil.rmtree(TMP_DIR, ignore_errors=True) + + def _create_series(self, name: str) -> Series: + return Series.objects.create( + name=name, imprint=self.imprint, publisher=self.publisher + ) + + def _create_comic( + self, name: str, issue_number: int, series: Series | None = None + ) -> Comic: + series = series or self.series + volume, _ = Volume.objects.get_or_create( + name=2024, series=series, imprint=self.imprint, publisher=self.publisher + ) + path = TMP_DIR / f"{name.lower()}.cbz" + path.touch() + return Comic.objects.create( + library=self.library, + path=path, + issue_number=issue_number, + name=name, + publisher=self.publisher, + imprint=self.imprint, + series=series, + volume=volume, + size=42 + issue_number, + year=2024, + page_count=20, + ) + + def _set_view_mode_table(self) -> None: + response = self.client.patch( + _SETTINGS_URL, + data=json.dumps({"viewMode": "table"}), + content_type="application/json", + ) + assert response.status_code == _HTTP_OK, response.content + + def _browse(self, url: str) -> dict: + response = self.client.get(url) + assert response.status_code == _HTTP_OK, response.content + return _v4(response) + + def _browse_comics(self, columns: str = "cover,name,reprints") -> dict: + return self._browse( + f"/api/v4/browse/series/{self.series.pk}?page=1&columns={columns}" + ) + + def _browse_series_rows(self, columns: str = "cover,name,reprints") -> dict: + return self._browse( + f"/api/v4/browse/publishers/{self.publisher.pk}?page=1&columns={columns}" + ) + + +class BrowserReprintsColumnTestCase(_ReprintsFixtureTestCase): + """Table-cell display and sort for the composed alternate-series label.""" + + def test_comic_row_composes_every_part(self) -> None: + """The SQL expression renders the same label as ``Reprint.compose_name``.""" + reprints = ( + Reprint.objects.create(series_name="Kapitän Wissenschaft"), + Reprint.objects.create(series_name="Capitan Sciencia", volume_number=1), + Reprint.objects.create(series_name="Yearly", volume_number=1999), + Reprint.objects.create( + series_name="Full", volume_number=3, issue="2a", language="es" + ), + ) + self.comic.reprints.set(reprints) + + self._set_view_mode_table() + row = self._browse_comics()["rows"][0] + assert sorted(row["reprints"]) == sorted( + reprint.name for reprint in reprints + ), row + # Spell the expectations out so a change to either the SQL or + # the Python composer fails loudly instead of agreeing wrongly. + # The aggregate sorts its own elements — identical reprint sets + # must render identical JSON for the M2M sort to cluster them. + assert row["reprints"] == [ + "Capitan Sciencia v1", + "Full v3 #2a (es)", + "Kapitän Wissenschaft", + "Yearly (1999)", + ] + + def test_collection_row_shows_only_shared_reprints(self) -> None: + """A series row intersects its children's reprints.""" + shared = Reprint.objects.create(series_name="Shared", language="de") + lonely = Reprint.objects.create(series_name="Lonely") + self.comic.reprints.set((shared, lonely)) + sibling = self._create_comic("C2", 2) + sibling.reprints.set((shared,)) + + self._set_view_mode_table() + row = self._browse_series_rows()["rows"][0] + assert row["reprints"] == ["Shared (de)"], row + + def test_collection_row_without_reprints_is_empty(self) -> None: + """Collections whose children carry no reprints render an empty cell.""" + self._set_view_mode_table() + row = self._browse_series_rows()["rows"][0] + assert row["reprints"] == [], row + + def test_collection_row_sort_by_reprints(self) -> None: + """Sorting collection rows by the column uses its intersection SQL.""" + self.comic.reprints.add( + Reprint.objects.create(series_name="Zulu", volume_number=2) + ) + # "Zzz" has no reprints, so ascending by the column puts it + # first — the opposite of the ``sort_name`` order the sort + # falls back to when the intersection SQL isn't wired. + self._create_comic("C2", 1, series=self._create_series("Zzz")) + + self._set_view_mode_table() + response = self.client.patch( + _SETTINGS_URL, + data=json.dumps({"orderBy": "reprints", "orderReverse": False}), + content_type="application/json", + ) + assert response.status_code == _HTTP_OK, response.content + rows = self._browse_series_rows()["rows"] + assert [(row["name"], row["reprints"]) for row in rows] == [ + ("Zzz", []), + ("Ser", ["Zulu v2"]), + ], rows + + +class BrowserReprintsFilterTestCase(_ReprintsFixtureTestCase): + """The filter drawer's choices and the resulting narrowing.""" + + def test_choices_compose_labels(self) -> None: + """The choices endpoint labels reprints like the metadata panel does.""" + reprint = Reprint.objects.create( + series_name="Kapitän Wissenschaft", language="de" + ) + self.comic.reprints.add(reprint) + # A reprint-less sibling makes the view prepend the null + # sentinel, which already carries a name of its own. + self._create_comic("C2", 2) + + body = self._browse(f"/api/v4/browse/series/{self.series.pk}/choices/reprints") + choices = {choice["pk"]: choice["name"] for choice in body["choices"]} + assert choices[reprint.pk] == "Kapitän Wissenschaft (de)", choices + assert choices[VUETIFY_NULL_CODE] == DUMMY_NULL_NAME, choices + + def test_choices_availability(self) -> None: + """``reprints`` only offers a sub-menu once a value and a null exist.""" + body = self._browse(f"/api/v4/browse/series/{self.series.pk}/choices") + assert body["reprints"] is False, body + + self.comic.reprints.add(Reprint.objects.create(series_name="Alt")) + self._create_comic("C2", 2) + cache.clear() + body = self._browse(f"/api/v4/browse/series/{self.series.pk}/choices") + assert body["reprints"] is True, body + + def test_filter_narrows_to_tagged_comics(self) -> None: + """Selecting a reprint pk filters the browse to comics carrying it.""" + reprint = Reprint.objects.create(series_name="Alt", volume_number=1) + self.comic.reprints.add(reprint) + self._create_comic("C2", 2) + + response = self.client.patch( + _SETTINGS_URL, + data=json.dumps({"filters": {"reprints": [reprint.pk]}}), + content_type="application/json", + ) + assert response.status_code == _HTTP_OK, response.content + cache.clear() + body = self._browse(f"/api/v4/browse/series/{self.series.pk}?page=1") + names = [book["name"] for book in body["books"]] + assert names == ["C1"], body diff --git a/tests/test_browser_table_response.py b/tests/test_browser_table_response.py index a34b30392..871be9d5f 100644 --- a/tests/test_browser_table_response.py +++ b/tests/test_browser_table_response.py @@ -632,8 +632,8 @@ def test_table_view_simple_m2m_intersections_share_one_union_query( metadata-view pattern (``query_intersections``'s ``_query_m2m_intersections``) and unions per-through-table sub-queries into a single SQL transmission. Composite M2M - columns (credits / identifiers / universes / story_arcs) - still issue their own queries. + columns (credits / identifiers / reprints / universes / + story_arcs) still issue their own queries. """ from django.db import connection from django.test.utils import CaptureQueriesContext diff --git a/tests/test_crond.py b/tests/test_crond.py new file mode 100644 index 000000000..cfa7f3aed --- /dev/null +++ b/tests/test_crond.py @@ -0,0 +1,204 @@ +""" +Cron scheduling invariants. + +:class:`CronThread` queues due jobs and *then* recomputes the schedule. +For the telemeter that schedule lives in the database and is written by +the send itself, which ``BookmarkThread`` runs on a daemon thread nobody +joins — so the recompute has to see a slot the scheduler spent, not one +the send has not gotten around to writing yet. +""" + +from __future__ import annotations + +from datetime import timedelta +from queue import SimpleQueue +from threading import Lock +from typing import TYPE_CHECKING +from uuid import UUID + +from django.test import TestCase +from django.utils import timezone +from loguru import logger as _loguru_logger + +from codex.choices.admin import AdminFlagChoices +from codex.librarian.cron.crond import CronThread +from codex.librarian.scribe.janitor.tasks import JanitorNightlyTask +from codex.librarian.telemeter.scheduled_time import ( + get_telemeter_time, + mark_telemeter_attempt, +) +from codex.librarian.telemeter.tasks import TelemeterTask +from codex.librarian.telemeter.telemeter import get_telemeter_timestamp +from codex.models.admin import AdminFlag, Timestamp +from codex.startup import init_admin_flags + +if TYPE_CHECKING: + from loguru._logger import Logger + +# The send time is the install uuid scaled across the week, so int=0 puts +# it at Monday 00:00 UTC — always in the past, so the job is always due. +_START_OF_WEEK_UUID = str(UUID(int=0)) +_LONG_AGO = timedelta(days=30) +# ``loguru.logger`` *is* a ``loguru._logger.Logger`` at runtime; loguru's +# stubs declare a second, unrelated ``loguru.Logger`` for the singleton, +# and that is not the type the librarian annotates its ``log`` params with. +_LOG: Logger = _loguru_logger # pyright: ignore[reportAssignmentType] # ty: ignore[invalid-assignment] + + +def _make_telemeter_overdue() -> Timestamp: + """Make the telemeter look installed long ago and never sent.""" + init_admin_flags() + AdminFlag.objects.filter(key=AdminFlagChoices.SEND_TELEMETRY.value).update(on=True) + ts = get_telemeter_timestamp() + long_ago = timezone.now() - _LONG_AGO + # A queryset update writes the columns auto_now/auto_now_add own. + Timestamp.objects.filter(pk=ts.pk).update( + value=_START_OF_WEEK_UUID, created_at=long_ago, updated_at=long_ago + ) + ts.refresh_from_db() + return ts + + +def _new_cron_thread() -> tuple[CronThread, SimpleQueue]: + """Build a cron thread wired to a queue the test can read, never started.""" + queue = SimpleQueue() + return CronThread(_LOG, queue, Lock()), queue + + +def _drain(queue: SimpleQueue) -> list: + """Everything the loop put on the librarian queue.""" + tasks = [] + while not queue.empty(): + tasks.append(queue.get_nowait()) + return tasks + + +def _cron_pass(thread: CronThread) -> None: + """ + One trip around the cron loop, minus the wait. + + ``timed_out=True`` is the worst case for the job at the head of the + schedule: it is the value the loop uses when it has waited out the + full timeout, so nothing about a short clock reading can excuse a + second enqueue. + """ + thread._run_expired_jobs(timed_out=True) # noqa: SLF001 + thread._create_task_times() # noqa: SLF001 + + +def _scheduled_classes(thread: CronThread) -> set[type]: + """Collect the task classes on the thread's current schedule.""" + return {task_class for _, task_class in thread._task_times} # noqa: SLF001 + + +class CronScheduleTestCase(TestCase): + """The loop must queue an overdue job exactly once.""" + + def _start(self) -> tuple[CronThread, SimpleQueue]: + """Prime a cron thread on an overdue telemeter, the way ``run`` primes it.""" + _make_telemeter_overdue() + thread, queue = _new_cron_thread() + thread._create_task_times() # noqa: SLF001 + return thread, queue + + def test_telemeter_is_queued_once_while_the_send_is_in_flight(self) -> None: + """ + The reported bug: three passes, no send, one task. + + Never calling ``send_telemetry`` is the point — it stands in for + the offloaded send still collecting counts or blocked on its five + second network timeout, having written nothing back yet. + """ + thread, queue = self._start() + for _ in range(3): + _cron_pass(thread) + queued = [task for task in _drain(queue) if isinstance(task, TelemeterTask)] + assert len(queued) == 1, ( + f"Queued {len(queued)} TelemeterTasks; the send never ran, so the " + "loop re-read an unclaimed send time and enqueued it again." + ) + + def test_queueing_the_telemeter_takes_it_off_the_schedule(self) -> None: + """A claimed slot must be gone from the very next recompute.""" + thread, _ = self._start() + assert TelemeterTask in _scheduled_classes(thread) + _cron_pass(thread) + assert TelemeterTask not in _scheduled_classes(thread) + # Not merely absent from the tuple — unschedulable at the source, + # which is what keeps ``_get_timeout`` off zero and the loop from + # spinning. + assert get_telemeter_time(_LOG) is None + + def test_next_week_is_scheduled_again(self) -> None: + """Spending this week's slot must not retire the job.""" + thread, _ = self._start() + _cron_pass(thread) + assert get_telemeter_time(_LOG) is None + # Age the claim past the start of this week: the slot moves with + # the calendar, the claim does not. + Timestamp.objects.filter(key=Timestamp.Choices.TELEMETER_SENT.value).update( + updated_at=timezone.now() - timedelta(days=8) + ) + assert get_telemeter_time(_LOG) is not None + + +class CronWakeupTestCase(TestCase): + """Which jobs a wakeup is allowed to run.""" + + @staticmethod + def _janitor_a_moment_out() -> tuple[CronThread, SimpleQueue]: + """Put the nightly janitor just past the clock, as truncation does.""" + thread, queue = _new_cron_thread() + soon = timezone.now() + timedelta(milliseconds=500) + thread._task_times = ((soon, JanitorNightlyTask),) # noqa: SLF001 + return thread, queue + + def test_a_full_timeout_runs_the_job_it_waited_for(self) -> None: + """ + ``_get_timeout`` truncates to whole seconds, so the wait ends early. + + Declining the job then would let ``_create_task_times`` push the + nightly janitor out to the following midnight and skip a night. + """ + thread, queue = self._janitor_a_moment_out() + thread._run_expired_jobs(timed_out=True) # noqa: SLF001 + assert not queue.empty() + assert isinstance(queue.get_nowait(), JanitorNightlyTask) + + def test_being_woken_early_runs_nothing_early(self) -> None: + """``end_timeout`` is a nudge to reschedule, not a licence to run.""" + thread, queue = self._janitor_a_moment_out() + thread._run_expired_jobs(timed_out=False) # noqa: SLF001 + assert queue.empty() + + +class TelemeterScheduleTestCase(TestCase): + """What ``get_telemeter_time`` will and will not schedule.""" + + def test_an_overdue_send_is_scheduled_in_the_past(self) -> None: + """A missed slot stays due rather than rolling over.""" + _make_telemeter_overdue() + dttm = get_telemeter_time(_LOG) + assert dttm is not None + assert dttm <= timezone.now() + + def test_disabled_telemetry_is_not_scheduled(self) -> None: + """The admin flag is checked before anything else.""" + _make_telemeter_overdue() + AdminFlag.objects.filter(key=AdminFlagChoices.SEND_TELEMETRY.value).update( + on=False + ) + assert get_telemeter_time(_LOG) is None + + def test_a_fresh_install_waits_a_day(self) -> None: + """New installs don't report on their first day.""" + ts = _make_telemeter_overdue() + Timestamp.objects.filter(pk=ts.pk).update(created_at=timezone.now()) + assert get_telemeter_time(_LOG) is None + + def test_claiming_the_slot_unschedules_it(self) -> None: + """The unit the cron thread relies on, independent of the loop.""" + _make_telemeter_overdue() + assert get_telemeter_time(_LOG) is not None + mark_telemeter_attempt() + assert get_telemeter_time(_LOG) is None diff --git a/tests/test_metadata_reprints.py b/tests/test_metadata_reprints.py new file mode 100644 index 000000000..30fbd0a5b --- /dev/null +++ b/tests/test_metadata_reprints.py @@ -0,0 +1,159 @@ +""" +The metadata pane serves alternate series names as ready-to-render chips. + +``Reprint.name`` is a property composed from four columns, so the +metadata endpoint has to hydrate all of them plus the optional +identifier in one query. Without the ``M2M_QUERY_OPTIMIZERS`` entry the +default ``only=("name", "identifier")`` raises FieldDoesNotExist. +""" + +import shutil +from pathlib import Path +from typing import Final, override + +from django.contrib.auth.models import User +from django.db import connection +from django.test import Client, TestCase +from django.test.utils import CaptureQueriesContext + +from codex.models import ( + Comic, + Imprint, + Library, + Publisher, + Reprint, + Series, + Volume, +) +from codex.models.identifier import Identifier, IdentifierSource, IdentifierType +from codex.startup import init_admin_flags + +TMP_DIR = Path("/tmp/codex.tests.metadata_reprints") # noqa: S108 +_TEST_PASSWORD: Final = "test-pw-hush-S106" # noqa: S105 +_HTTP_OK: Final = 200 +_REPRINT_TABLE: Final = 'FROM "codex_reprint"' +_REPRINT_COUNT: Final = 3 + + +def _v4(response): + """Unwrap the v4 ``{data, meta, errors}`` envelope and return ``data``.""" + body = response.json() + if isinstance(body, dict) and "data" in body and "meta" in body: + return body["data"] + return body + + +class MetadataReprintsTestCase(TestCase): + """The reprints payload shape and its query cost.""" + + @override + def setUp(self) -> None: + """Seed one comic with three reprints of varying completeness.""" + init_admin_flags() + TMP_DIR.mkdir(exist_ok=True, parents=True) + library = Library.objects.create(path=str(TMP_DIR)) + publisher = Publisher.objects.create(name="Pub") + imprint = Imprint.objects.create(name="Imp", publisher=publisher) + self.series = Series.objects.create( # pyright: ignore[reportUninitializedInstanceVariable] + name="Ser", imprint=imprint, publisher=publisher + ) + volume = Volume.objects.create( + name="2024", series=self.series, imprint=imprint, publisher=publisher + ) + path = TMP_DIR / "c1.cbz" + path.touch() + self.comic = Comic.objects.create( # pyright: ignore[reportUninitializedInstanceVariable] + library=library, + path=path, + issue_number=1, + name="C1", + publisher=publisher, + imprint=imprint, + series=self.series, + volume=volume, + size=42, + ) + source = IdentifierSource.objects.create(name="metron") + identifier = Identifier.objects.create( + source=source, + id_type=IdentifierType.REPRINT.value, + key="12345", + url="https://metron.cloud/series/12345", + ) + self.comic.reprints.set( + [ + Reprint.objects.create(series_name="Kapitän Wissenschaft"), + Reprint.objects.create( + series_name="Capitan Sciencia", volume_number=1, language="es" + ), + Reprint.objects.create( + series_name="Captain Science", + issue="3", + identifier=identifier, + ), + ] + ) + + user = User.objects.create_user( + username="reprint_reader", password=_TEST_PASSWORD, is_staff=True + ) + self.client = Client() + self.client.force_login(user) + + @override + def tearDown(self) -> None: + """Remove the temp comic tree.""" + shutil.rmtree(TMP_DIR, ignore_errors=True) + + def _get_metadata(self, url: str = ""): + url = url or f"/api/v4/browse/comics/{self.comic.pk}/metadata" + with CaptureQueriesContext(connection) as ctx: + response = self.client.get(url) + assert response.status_code == _HTTP_OK, response.content + return _v4(response), ctx.captured_queries + + def test_reprint_names_compose_present_parts(self) -> None: + """Each chip name composes only the columns the reprint carries.""" + data, _ = self._get_metadata() + names = sorted(reprint["name"] for reprint in data["reprints"]) + assert names == [ + "Capitan Sciencia v1 (es)", + "Captain Science #3", + "Kapitän Wissenschaft", + ], names + + def test_reprint_columns_ship_beside_the_composed_name(self) -> None: + """The tag editor seeds its rows from the parts, not the label.""" + data, _ = self._get_metadata() + parts = { + reprint["seriesName"]: ( + reprint["volumeNumber"], + reprint["issue"], + reprint["language"], + ) + for reprint in data["reprints"] + } + assert parts == { + "Kapitän Wissenschaft": (None, "", ""), + "Capitan Sciencia": (1, "", "es"), + "Captain Science": (None, "3", ""), + }, parts + + def test_reprint_identifier_url_ships_when_set(self) -> None: + """The identifier url rides along for the chip's external link.""" + data, _ = self._get_metadata() + urls = {reprint["name"]: reprint.get("url", "") for reprint in data["reprints"]} + assert urls["Captain Science #3"] == "https://metron.cloud/series/12345" + assert not urls["Kapitän Wissenschaft"] + + def test_collection_selection_serves_the_intersection(self) -> None: + """Collections take the plain-attribute branch, not the prefetch cache.""" + url = f"/api/v4/browse/series/{self.series.pk}/metadata" + data, _ = self._get_metadata(url) + assert len(data["reprints"]) == _REPRINT_COUNT, data["reprints"] + + def test_reprints_hydrate_in_one_query(self) -> None: + """No per-reprint query for the deferred columns or the identifier.""" + _, queries = self._get_metadata() + reprint_queries = [q for q in queries if _REPRINT_TABLE in q["sql"]] + assert len(reprint_queries) == 1, reprint_queries diff --git a/tests/test_onlinetag_resume_endpoints.py b/tests/test_onlinetag_resume_endpoints.py index 265c5acd4..e78ce6f6d 100644 --- a/tests/test_onlinetag_resume_endpoints.py +++ b/tests/test_onlinetag_resume_endpoints.py @@ -26,7 +26,6 @@ "sources": ["metron", "comicvine"], "mode": "auto", "prompts_mode": "ask", - "auto_threshold": 0.85, "delete_original": False, "merge_all_sources": False, "dry_run": False, @@ -87,6 +86,19 @@ def test_resume_enqueues_task_over_remaining_pks(self) -> None: assert task.mode == "auto" assert task.prompts_mode == "ask" + def test_resume_drops_params_from_an_older_codex(self) -> None: + # A descriptor written before a knob was removed (auto_threshold) must + # still resume instead of raising TypeError on the task constructor. + set_resume_state({**_PARAMS, "auto_threshold": 0.85}, [11]) + + with patch(_QUEUE_TARGET) as mocked_queue: + response = self.client.post(_RESUME_URL) + + assert response.status_code == HTTPStatus.ACCEPTED + task = mocked_queue.put.call_args.args[0] + assert isinstance(task, BulkOnlineTagTask) + assert task.comic_pks == frozenset([11]) + def test_resume_conflict_when_scan_active(self) -> None: set_resume_state(_PARAMS, [11]) set_active_scan_id("live-scan") diff --git a/tests/test_reprints.py b/tests/test_reprints.py new file mode 100644 index 000000000..33f0887cf --- /dev/null +++ b/tests/test_reprints.py @@ -0,0 +1,133 @@ +""" +Alternate series names, from the archive to the browser and back out. + +``tests/files/comicbox-2-example.cbz`` carries two MetronInfo alternate +names, one of which supplies only a series ``sort_name``. Importing it +is the only place the whole chain — comicbox whitelist, four-column key +flattening, m2m link, display label and the nightly orphan sweep — +meets end to end. The search half of the chain lives in +``tests/test_search_alternate_series.py``. +""" + +from http import HTTPStatus +from multiprocessing import Event as ProcessEvent +from threading import Lock +from typing import Final, override + +from django.contrib.auth.models import User +from django.test import Client, SimpleTestCase +from loguru import logger + +from codex.librarian.mp_queue import LIBRARIAN_QUEUE +from codex.librarian.scribe.importer.read.many_to_many import ( + AggregateManyToManyMetadataImporter, +) +from codex.librarian.scribe.janitor.janitor import Janitor +from codex.models import Comic, Reprint, Series, Volume +from tests.importer.test_basic import PATH, BaseTestImporter, run_full_import + +_TEST_PASSWORD: Final = "test-pw-hush-S106" # noqa: S105 +# The fixture's two alternate names, as ``Reprint`` natural keys. +_FIXTURE_KEYS: Final = { + ("Capitan Sciencia", 1, "", "es"), + ("Kapitän Wissenschaft", None, "", "de"), +} +_SORT_NAME_ONLY: Final = "Capitan Sciencia" + + +def _reprint_keys() -> set[tuple]: + """Every Reprint row in the database, as natural-key tuples.""" + return set( + Reprint.objects.values_list("series_name", "volume_number", "issue", "language") + ) + + +def _make_janitor() -> Janitor: + """Build a Janitor detached from the librarian's thread graph.""" + return Janitor(logger, LIBRARIAN_QUEUE, Lock(), ProcessEvent()) + + +class ReprintImportTestCase(BaseTestImporter): + """Import the example fixture and follow its reprints downstream.""" + + @override + def setUp(self) -> None: + super().setUp() + run_full_import(self.importer) + self.comic = Comic.objects.get(path=PATH) + + def test_import_creates_rows_keyed_on_all_four_columns(self) -> None: + """Every part comicbox supplied lands in its own column.""" + assert _reprint_keys() == _FIXTURE_KEYS + linked = self.comic.reprints.values_list( + "series_name", "volume_number", "issue", "language" + ) + assert set(linked) == _FIXTURE_KEYS + + def test_sort_name_stands_in_for_a_missing_series_name(self) -> None: + """ + One AlternativeNames entry carries no ``series.name`` at all. + + It must still import, and — the reason ``Reprint`` denormalizes + instead of reusing Series/Volume — it must not become a + browsable collection row. + """ + assert Reprint.objects.filter(series_name=_SORT_NAME_ONLY).exists() + assert not Series.objects.filter(name=_SORT_NAME_ONLY).exists() + assert not Volume.objects.filter(series__name=_SORT_NAME_ONLY).exists() + + def test_metadata_endpoint_serves_imported_reprints(self) -> None: + """Imported rows reach the panel as composed display labels.""" + user = User.objects.create_user( + username="reprint_round_trip", password=_TEST_PASSWORD + ) + client = Client() + client.force_login(user) + response = client.get(f"/api/v4/browse/comics/{self.comic.pk}/metadata") + assert response.status_code == HTTPStatus.OK, response.content + names = sorted( + reprint["name"] for reprint in response.json()["data"]["reprints"] + ) + assert names == ["Capitan Sciencia v1 (es)", "Kapitän Wissenschaft (de)"] + + def test_cleanup_fks_keeps_linked_reprints(self) -> None: + """The nightly sweep leaves reprints a comic still points at.""" + _make_janitor().cleanup_fks() + assert _reprint_keys() == _FIXTURE_KEYS + + def test_cleanup_fks_deletes_orphaned_reprints(self) -> None: + """Deleting the last comic that referenced them sweeps them away.""" + self.comic.delete() + _make_janitor().cleanup_fks() + assert not Reprint.objects.exists() + + +class ReprintKeyFlatteningTestCase(SimpleTestCase): + """The nested comicbox tree collapses to the four-column key.""" + + @staticmethod + def _clean(reprint: dict) -> tuple | None: + return AggregateManyToManyMetadataImporter._clean_reprint_key(reprint) # noqa: SLF001 + + def test_series_name_wins_over_sort_name(self) -> None: + """``sort_name`` only fills in for a reprint that has no name.""" + key = self._clean({"series": {"name": "Real", "sort_name": "Sorted"}}) + assert key == ("Real", None, "", "") + + def test_every_part_flattens_to_its_column(self) -> None: + """Volume number, issue and language come out of their subtrees.""" + key = self._clean( + { + "series": {"name": "Kapitän Wissenschaft"}, + "volume": {"number": 2}, + "issue": "3", + "language": "de", + } + ) + assert key == ("Kapitän Wissenschaft", 2, "3", "de") + + def test_reprint_without_any_series_name_is_dropped(self) -> None: + """A reprint naming nothing can't key a row, so it never becomes one.""" + assert self._clean({"volume": {"number": 1}, "language": "de"}) is None + assert self._clean({"series": {}}) is None + assert self._clean({"series": {"name": " "}}) is None diff --git a/tests/test_search_alternate_series.py b/tests/test_search_alternate_series.py new file mode 100644 index 000000000..91cc53889 --- /dev/null +++ b/tests/test_search_alternate_series.py @@ -0,0 +1,133 @@ +""" +Alternate series names are searchable. + +comicbox ``reprints`` become :class:`codex.models.named.Reprint` rows and +are indexed into a dedicated ``ComicFTS.alternate_series`` column rather +than being folded into ``series`` — a ``series:`` token has to keep +meaning the canonical series. + +Two independent code paths fill that column: the importer builds FTS +entries inline, and :class:`SearchIndexerSync` rebuilds them from the +database. Each keeps its own registry of FTS columns and drops values +*silently* when the registries disagree (the historical credits / +sources / story_arcs bug documented in ``scribe/search/sync.py``), so +both are asserted here end to end. +""" + +from datetime import UTC, datetime +from http import HTTPStatus +from pathlib import Path +from threading import Event, Lock +from typing import Final, override +from urllib.parse import urlencode + +from django.contrib.auth.models import User +from django.test import Client +from loguru import logger + +from codex.librarian.mp_queue import LIBRARIAN_QUEUE +from codex.librarian.scribe.search.sync import SearchIndexerSync +from codex.models import Comic +from codex.models.comic import ComicFTS +from tests.importer.test_basic import PATH, BaseTestImporter, run_full_import + +_TEST_PASSWORD: Final = "test-pw-hush-S106" # noqa: S105 +# ``tests/files/comicbox-2-example.cbz`` carries two MetronInfo +# alternate names. The second has only a ``series.sort_name``. +_ALTERNATE_SERIES: Final = "Capitan Sciencia,Kapitän Wissenschaft" +# Comfortably past any index watermark the run just wrote. +_FUTURE: Final = datetime(2100, 1, 1, tzinfo=UTC) + + +class AlternateSeriesSearchTestCase(BaseTestImporter): + """Import the example fixture, then find it by its alternate names.""" + + @override + def setUp(self) -> None: + super().setUp() + run_full_import(self.importer) + self.comic = Comic.objects.get(path=PATH) + self._create_sibling() + user = User.objects.create_user( + username="alt_series_search_test", password=_TEST_PASSWORD + ) + self.client = Client() + self.client.force_login(user) + + def _create_sibling(self) -> None: + """ + Add a reprint-less comic to the same series. + + Without it every positive assertion below would also pass when + the search filter is dropped entirely — the single imported + comic is the whole result set either way. + """ + path = Path(PATH).with_name("sibling.cbz") + path.touch() + Comic.objects.create( + library=self.comic.library, + path=path, + issue_number=2, + name="Sibling", + publisher=self.comic.publisher, + imprint=self.comic.imprint, + series=self.comic.series, + volume=self.comic.volume, + size=1, + page_count=1, + ) + + @staticmethod + def _sync(*, rebuild: bool) -> None: + indexer = SearchIndexerSync(logger, LIBRARIAN_QUEUE, Lock(), Event()) + indexer.update_search_index(rebuild=rebuild) + + def _alternate_series_column(self) -> str: + return ComicFTS.objects.get(comic_id=self.comic.pk).alternate_series + + def _search_book_pks(self, query: str) -> list[int]: + """Browse the comic's series with a search query and return book pks.""" + params = urlencode({"page": 1, "q": query}) + url = f"/api/v4/browse/series/{self.comic.series.pk}?{params}" + response = self.client.get(url) + assert response.status_code == HTTPStatus.OK, response.content + return [book["pk"] for book in response.json()["data"]["books"]] + + def test_import_indexes_alternate_series(self) -> None: + """The importer writes reprint series names to their own FTS column.""" + assert self._alternate_series_column() == _ALTERNATE_SERIES + + def test_unqualified_search_finds_alternate_series(self) -> None: + """A bare token matches every FTS column, alternate names included.""" + assert self._search_book_pks("Wissenschaft") == [self.comic.pk] + assert self._search_book_pks("Sciencia") == [self.comic.pk] + assert self._search_book_pks("Zzyzx") == [] + + def test_field_alias_search_finds_alternate_series(self) -> None: + """Every registered alias resolves to the alternate_series column.""" + for alias in ("alternate_series", "alt_series", "alternateseries", "reprints"): + assert self._search_book_pks(f"{alias}:Sciencia") == [self.comic.pk], alias + # An unregistered column is silently discarded from the query + # rather than erroring, so the miss has to be asserted too. + assert self._search_book_pks(f"{alias}:Zzyzx") == [], alias + + def test_series_token_excludes_alternate_series(self) -> None: + """The canonical series column stays free of alternate names.""" + assert self._search_book_pks("series:Sciencia") == [] + assert self._search_book_pks("series:Captain") == [self.comic.pk] + + def test_search_index_sync_populates_alternate_series(self) -> None: + """A full index rebuild fills the column the importer filled.""" + self._sync(rebuild=True) + assert self._alternate_series_column() == _ALTERNATE_SERIES + assert self._search_book_pks("alt_series:Wissenschaft") == [self.comic.pk] + + def test_search_index_sync_updates_alternate_series(self) -> None: + """An incremental sync carries the column through ``bulk_update``.""" + self._sync(rebuild=True) + self.comic.reprints.clear() + # ``updated_at`` is auto_now, so ``.update()`` is the only way to + # push the comic past the index watermark without a save(). + Comic.objects.filter(pk=self.comic.pk).update(updated_at=_FUTURE) + self._sync(rebuild=False) + assert self._alternate_series_column() == "" diff --git a/tests/test_telemeter.py b/tests/test_telemeter.py new file mode 100644 index 000000000..37a9dba3e --- /dev/null +++ b/tests/test_telemeter.py @@ -0,0 +1,232 @@ +"""Tests for the anonymous stats payload and how it is transmitted.""" + +import json +from datetime import timedelta +from email.message import Message +from lzma import decompress +from typing import override +from unittest.mock import patch +from urllib.error import HTTPError, URLError +from urllib.parse import urlsplit + +import pytest +from django.http import QueryDict +from django.test import TestCase +from django.utils import timezone + +from codex.librarian.telemeter import telemeter +from codex.librarian.telemeter.stats import CodexStats +from codex.models.admin import Timestamp +from codex.serializers.admin.stats import ( + AdminStatsRequestSerializer, + StatsSerializer, +) + +EXPECTED_SECTIONS = ( + "platform", + "config", + "sessions", + "collections", + "file_types", + "metadata", + "usage", + "identifiers", + "admin_flags", + "tagging", + "auth", + "email", + "throttle", + "deployment", +) + + +class _FakeResponse: + """Stand in for http.client.HTTPResponse.""" + + def __enter__(self): + return self + + def __exit__(self, *_args) -> bool: + return False + + +class _DebugOnlyLog: + """ + A log that can only be called at debug. + + Failing to reach the stats server is never the user's problem, so it is + reported at debug and nowhere else. Exposing only that method makes a + revert to warning() an AttributeError rather than a silent regression. + """ + + def __init__(self) -> None: + self.logged: list[str] = [] + + def debug(self, msg) -> None: + self.logged.append(msg) + + def sent_stats_quietly(self, *fragments: str) -> bool: + """Whether one debug line reports a failed send, mentioning fragments.""" + return any( + "Failed to send anonymous stats" in msg + and all(fragment in msg for fragment in fragments) + for msg in self.logged + ) + + +class TelemeterTransportTestCase(TestCase): + """The url and headers the telemeter actually puts on the wire.""" + + @override + def setUp(self) -> None: + """ + Seed the admin flags the send path reads. + + Without the SEND_TELEMETRY row, _send_telemetry raises DoesNotExist at + the flag lookup and never reaches the mocked urlopen, so a transport + test would pass without exercising any transport. + """ + from codex.startup import init_admin_flags + + init_admin_flags() + + def test_posts_to_wire_version_two(self) -> None: + """Codex 2.2.3 and later report the expanded payload as version 2.""" + assert telemeter._VERSION == "2" # noqa: SLF001 + assert telemeter._POST.endswith("/stats/codex/2") # noqa: SLF001 + + def test_credentials_are_not_in_the_url(self) -> None: + """ + Userinfo in the url makes http.client read the password as a port. + + That raised InvalidURL on every send, silently, for months. + """ + netloc = urlsplit(telemeter._POST).netloc # noqa: SLF001 + assert "@" not in netloc + assert netloc + + def test_credentials_move_to_an_authorization_header(self) -> None: + """The credentials taken out of the url are still sent.""" + headers = telemeter._new_headers() # noqa: SLF001 + assert headers["Content-Type"] == "application/xz" + assert headers.get("Authorization", "").startswith("Basic ") + + def test_posts_compressed_json(self) -> None: + """The body is the lzma-compressed payload, and the url is reachable.""" + payload = {"uuid": "abc", "stats": {"platform": {"codex_version": "2.2.3"}}} + with patch.object(telemeter, "urlopen") as mock_urlopen: + mock_urlopen.return_value = _FakeResponse() + telemeter._post_stats(payload) # noqa: SLF001 + + request = mock_urlopen.call_args.args[0] + assert request.get_method() == "POST" + assert request.host == urlsplit(telemeter._POST).netloc # noqa: SLF001 + assert json.loads(decompress(request.data).decode()) == payload + + def test_http_error_reaches_the_caller(self) -> None: + """A rejected post must not look like a successful one.""" + error = HTTPError(telemeter._POST, 500, "boom", Message(), None) # noqa: SLF001 + with ( + patch.object(telemeter, "urlopen", side_effect=error), + pytest.raises(HTTPError), + ): + telemeter._post_stats({"uuid": "abc"}) # noqa: SLF001 + + def test_a_rejected_post_is_survived_quietly(self) -> None: + """A stats server that refuses the post must not break or alarm.""" + log = _DebugOnlyLog() + error = HTTPError(telemeter._POST, 500, "boom", Message(), None) # noqa: SLF001 + with patch.object(telemeter, "urlopen", side_effect=error): + telemeter.send_telemetry(log) + # The status code, not merely some line: the outer handler logs at + # debug too, so presence alone would pass even if this one regressed + # to warning and blew up on the log above. + assert log.sent_stats_quietly("500") + + def test_an_unreachable_server_is_survived_quietly(self) -> None: + """Nor is a timeout or a dns failure the user's problem.""" + log = _DebugOnlyLog() + with patch.object(telemeter, "urlopen", side_effect=URLError(TimeoutError())): + telemeter.send_telemetry(log) + assert log.sent_stats_quietly("TimeoutError") + + def test_a_failed_send_still_waits_a_week(self) -> None: + """ + The timestamp marks the attempt, not the success. + + Retrying a failed send would turn a stats outage into a stream of + requests from every install at once, so the week's slot is spent + either way. + """ + ts = telemeter.get_telemeter_timestamp() + stale = timezone.now() - timedelta(days=30) + # A queryset update writes the column that auto_now would overwrite. + Timestamp.objects.filter(pk=ts.pk).update(updated_at=stale) + error = HTTPError(telemeter._POST, 503, "down", Message(), None) # noqa: SLF001 + with patch.object(telemeter, "urlopen", side_effect=error): + telemeter.send_telemetry(_DebugOnlyLog()) + ts.refresh_from_db() + assert ts.updated_at > stale + + +class TelemeterStatsTestCase(TestCase): + """The shape of the payload the telemeter builds.""" + + @override + def setUp(self) -> None: + from codex.startup import init_admin_flags, init_timestamps + + init_admin_flags() + init_timestamps() + + def test_every_section_present(self) -> None: + """A full payload carries every section, in payload order.""" + stats = CodexStats().get() + assert tuple(stats) == EXPECTED_SECTIONS + + def test_serializer_reports_every_payload_field(self) -> None: + """ + The admin Stats tab is the visible contract for what is collected. + + Anything the telemeter sends must also be rendered there, or the page + understates what leaves the install. + """ + stats = CodexStats().get() + data = StatsSerializer(stats).data + assert set(stats) == set(data) + for section, values in stats.items(): + if not isinstance(values, dict) or not isinstance(data[section], dict): + continue + assert not set(values) - set(data[section]), ( + f"{section} fields missing from StatsSerializer" + ) + + def test_params_select_sections(self) -> None: + """The admin endpoint can ask for a subset.""" + stats = CodexStats({"platform": {}, "throttle": {}}).get() + assert set(stats) == {"platform", "throttle"} + + def test_default_admin_request_returns_every_section(self) -> None: + """ + A section missing from the request serializer vanishes from the tab. + + AdminStatsRequestSerializer fills in every declared section, so params + is always truthy and the section gate drops anything undeclared. Adding + a section to StatsSerializer without adding it here renders an empty + table with no error, which is how the identifiers section was lost. + """ + request_serializer = AdminStatsRequestSerializer(data=QueryDict("ts=1")) + assert request_serializer.is_valid(), request_serializer.errors + params = dict(request_serializer.validated_data) + assert tuple(CodexStats(params).get()) == EXPECTED_SECTIONS + + def test_new_sections_have_content(self) -> None: + """The seeded singletons and flags produce real values.""" + stats = CodexStats().get() + assert stats["admin_flags"]["send_telemetry"] in (True, False) + assert "url_path_prefix_set" in stats["deployment"] + assert "throttle_anon" in stats["throttle"] + assert "bookmark_count" in stats["usage"] + assert "library_read_only_count" in stats["config"] + assert "multi_sort_count" in stats["sessions"] + assert "comic_community_rating_count" in stats["metadata"] diff --git a/tests/test_telemeter_privacy.py b/tests/test_telemeter_privacy.py new file mode 100644 index 000000000..6a775b696 --- /dev/null +++ b/tests/test_telemeter_privacy.py @@ -0,0 +1,281 @@ +""" +The anonymous stats payload must never carry private information. + +Seeds a sentinel into every admin-authored string codex stores -- credentials, +urls, claim names, group names, banner text, api key, library paths -- and +asserts none of them reach the wire. Then walks the payload and requires every +leaf to be a number, a boolean, or a string drawn from a closed vocabulary. + +If a new stat legitimately needs a new kind of string, add its vocabulary to +``_vocabularies`` deliberately. Do not widen the walk. +""" + +import json +from typing import Final, override +from uuid import uuid4 + +from django.test import TestCase + +from codex.choices.admin import AdminFlagChoices +from codex.collection import Collection +from codex.librarian.telemeter.stats import CodexStats +from codex.models.admin import ( + AdminFlag, + ComicboxTaggingDefaults, + EmailSettings, + OIDCSettings, +) +from codex.models.identifier import Identifier, IdentifierSource, IdentifierType +from codex.models.library import Library + +# Anything containing this must never appear in the payload. +SENTINEL: Final = "XXSENTINELXX" + +# Keys whose values are free-form by nature and safe: version strings and the +# platform description, which carry no user or install identity. +_PLATFORM_KEYS: Final = frozenset( + {"machine", "name", "release", "python_version", "codex_version"} +) + +# The only payload dicts whose *keys* come from data rather than from code. +# Everywhere else a key is a field name chosen in codex's source, so checking +# it proves nothing. These are the ones that must stay closed-vocabulary. +_BUCKET_PATHS: Final = frozenset( + { + "stats.file_types", + "stats.identifiers", + "stats.usage.favorites", + "stats.tagging.default_sources", + "stats.sessions.top_collection", + "stats.sessions.order_by", + "stats.sessions.dynamic_covers", + "stats.sessions.view_mode", + "stats.sessions.table_cover_size", + "stats.sessions.custom_covers", + "stats.sessions.finish_on_last_page", + "stats.sessions.fit_to", + "stats.sessions.reading_direction", + } +) + + +def _vocabularies() -> frozenset[str]: + """Every closed vocabulary a payload string may be drawn from.""" + from codex.choices.browser import ( + BROWSER_ORDER_BY_CHOICES, + BROWSER_TABLE_COVER_SIZE_CHOICES, + BROWSER_VIEW_MODE_CHOICES, + ) + from codex.choices.reader import READER_CHOICES + from codex.models.age_rating import AgeRatingMetron + from codex.models.choices import FileTypeChoices + + values: set[str] = {"", "other", "true", "false", "unknown"} + values |= {member.value for member in Collection} + values |= {member.value.lower() for member in FileTypeChoices} + values |= {member.value for member in IdentifierType} + values |= set(BROWSER_ORDER_BY_CHOICES) + values |= set(BROWSER_VIEW_MODE_CHOICES) + values |= set(BROWSER_TABLE_COVER_SIZE_CHOICES) + for reader_choices in READER_CHOICES.values(): + values |= {value.lower() for value in reader_choices} + values |= {mode.value for mode in ComicboxTaggingDefaults.MatchModeChoices} + values |= {mode.value for mode in ComicboxTaggingDefaults.PromptsModeChoices} + values |= set(AgeRatingMetron.objects.values_list("name", flat=True)) + # Identifier buckets are ":", both closed vocabularies. + from comicbox.enums.maps.identifiers import ID_SOURCE_NAME_MAP + + sources = {source.value for source in ID_SOURCE_NAME_MAP} | {"other"} + kinds = {member.value for member in IdentifierType} | {"other"} + values |= {f"{source}:{kind}" for source in sources for kind in kinds} + # comicbox online source names, used as tagging bucket keys. + from comicbox.formats.base.online import SOURCE_NAMES + + values |= set(SOURCE_NAMES) + # OIDC client authentication methods. + values |= { + "client_secret_basic", + "client_secret_post", + "client_secret_jwt", + "private_key_jwt", + "none", + } + return frozenset(values) + + +class TelemeterPrivacyTestCase(TestCase): + """Assert the stats payload leaks nothing an administrator typed.""" + + @override + def setUp(self) -> None: + from codex.startup import init_admin_flags, init_timestamps + + init_admin_flags() + init_timestamps() + self._seed_secrets() + + @staticmethod + def _set_singleton(model, **fields) -> None: + """Update the singleton row, which migrations may already have made.""" + row = model.objects.first() or model() + for name, value in fields.items(): + setattr(row, name, value) + row.save() + + @classmethod + def _seed_secrets(cls) -> None: + """Put a sentinel in every admin-authored string codex stores.""" + cls._set_singleton( + ComicboxTaggingDefaults, + metron_user=f"{SENTINEL}-metron-user", + metron_password=f"{SENTINEL}-metron-password", + metron_url=f"https://{SENTINEL}-metron.example.com", + comicvine_key=f"{SENTINEL}-comicvine-key", + comicvine_url=f"https://{SENTINEL}-comicvine.example.com", + default_sources=["metron", f"{SENTINEL}-source"], + ) + cls._set_singleton( + OIDCSettings, + enabled=True, + provider_name=f"{SENTINEL}-provider", + server_url=f"https://{SENTINEL}-idp.example.com", + client_id=f"{SENTINEL}-client-id", + client_secret=f"{SENTINEL}-client-secret", + scope=f"openid {SENTINEL}-scope", + username_claim=f"{SENTINEL}-claim", + groups_claim=f"{SENTINEL}-groups", + admin_group=f"{SENTINEL}-admins", + token_auth_method=f"{SENTINEL}-auth-method", + ) + cls._set_singleton( + EmailSettings, + host=f"smtp.{SENTINEL}.example.com", + user=f"{SENTINEL}-smtp-user", + password=f"{SENTINEL}-smtp-password", + from_address=f"{SENTINEL}@example.com", + subject_prefix=f"[{SENTINEL}] ", + ) + AdminFlag.objects.filter(key=AdminFlagChoices.BANNER_TEXT.value).update( + value=f"{SENTINEL}-banner" + ) + AdminFlag.objects.filter(key=AdminFlagChoices.API_KEY.value).update( + value=f"{SENTINEL}-api-key" + ) + Library.objects.create(path=f"/{SENTINEL}-library", read_only=True) + source = IdentifierSource.objects.create(name=f"{SENTINEL}-source") + Identifier.objects.create( + source=source, + id_type=IdentifierType.ISSUE.value, + key=f"{SENTINEL}-id", + url=f"https://{SENTINEL}.example.com/1", + ) + + @staticmethod + def _walk(node, path: str, leaves: list[tuple[str, object]]) -> None: + """Collect every value, plus the keys of data-derived count buckets.""" + if isinstance(node, dict): + in_bucket = path in _BUCKET_PATHS + for key, value in node.items(): + if in_bucket: + leaves.append((f"{path}[key]", key)) + TelemeterPrivacyTestCase._walk(value, f"{path}.{key}", leaves) + elif isinstance(node, list): + for index, value in enumerate(node): + TelemeterPrivacyTestCase._walk(value, f"{path}[{index}]", leaves) + else: + leaves.append((path, node)) + + def test_no_sentinel_in_payload(self) -> None: + """No admin-authored string reaches the serialized payload.""" + payload = json.dumps( + {"stats": CodexStats().get(), "uuid": str(uuid4())}, default=str + ) + assert SENTINEL not in payload + + def test_unknown_identifier_source_collapsed(self) -> None: + """A source name from a comic file is replaced by "other".""" + identifiers = CodexStats().get()["identifiers"] + assert identifiers + for key in identifiers: + assert SENTINEL not in key + assert f"other:{IdentifierType.ISSUE.value}" in identifiers + + def test_every_leaf_is_a_number_bool_or_closed_vocabulary(self) -> None: + """Walk the payload; no value may be an open-ended string.""" + vocabularies = {value.lower() for value in _vocabularies()} + leaves: list[tuple[str, object]] = [] + self._walk(CodexStats().get(), "stats", leaves) + assert leaves + + unexpected = [] + for path, value in leaves: + if isinstance(value, bool | int | float) or value is None: + continue + if not isinstance(value, str): + unexpected.append((path, value)) + continue + if path.rsplit(".", 1)[-1] in _PLATFORM_KEYS: + continue + if value.lower() not in vocabularies: + unexpected.append((path, value)) + assert not unexpected, f"open-ended strings in payload: {unexpected}" + + @staticmethod + def _declared_field_names() -> frozenset[str]: + """Every key name StatsSerializer declares, at any nesting depth.""" + from rest_framework.serializers import Serializer + + from codex.serializers.admin.stats import StatsSerializer + + names: set[str] = set() + + def walk(serializer) -> None: + for name, field in serializer.get_fields().items(): + names.add(name) + if isinstance(field, Serializer): + walk(field) + + walk(StatsSerializer()) + return frozenset(names) + + @classmethod + def _undeclared_keys(cls, node, path: str, declared, found: list) -> None: + """Collect dict keys that are neither declared fields nor bucket keys.""" + if not isinstance(node, dict): + return + if path in _BUCKET_PATHS: + # Registered bucket: its keys are data, and the vocabulary walk + # checks them. + return + for key, value in node.items(): + if key not in declared: + found.append(f"{path}.{key}") + cls._undeclared_keys(value, f"{path}.{key}", declared, found) + + def test_no_unregistered_data_keyed_bucket(self) -> None: + """ + A count bucket missing from _BUCKET_PATHS is checked by nothing. + + Its keys never become leaf values, so the vocabulary walk skips them, + and the sentinel test only seeds admin config -- not publisher, series + or user rows. So instead of trusting _BUCKET_PATHS to describe itself, + require every key in the payload to be either a field name declared in + StatsSerializer or a key inside a registered bucket. A new bucket of + library-sourced names satisfies neither. + """ + declared = self._declared_field_names() + found: list[str] = [] + self._undeclared_keys(CodexStats().get(), "stats", declared, found) + assert not found, ( + "payload keys that are neither declared serializer fields nor " + f"registered bucket keys: {sorted(found)}" + ) + + def test_the_bucket_check_catches_an_unregistered_bucket(self) -> None: + """Prove the check above can actually fail.""" + payload = CodexStats().get() + payload["usage"]["top_publishers"] = {"Marvel Comics": 10} + found: list[str] = [] + self._undeclared_keys(payload, "stats", self._declared_field_names(), found) + assert "stats.usage.top_publishers" in found + assert "stats.usage.top_publishers.Marvel Comics" in found diff --git a/tests/test_user_data_restore.py b/tests/test_user_data_restore.py index e9fd7bf51..ce2abb4f6 100644 --- a/tests/test_user_data_restore.py +++ b/tests/test_user_data_restore.py @@ -242,6 +242,43 @@ def test_round_trip_restores_settings_browser_show(self) -> None: assert browser.show.series is False assert browser.show.volumes is True + def test_round_trip_restores_reprints_filter(self) -> None: + """ + A reprints filter survives a rebuild that renumbers its PKs. + + The sidecar stores tag filters by name, and ``Reprint`` is the + only tag model whose name lives on ``series_name`` rather than + ``name`` — a dump that reached for ``name`` would raise. + """ + from codex.models.named import Reprint + from codex.models.settings import ( + SettingsBrowser, + SettingsBrowserFilters, + SettingsBrowserShow, + ) + + user = User.objects.create_user(username="alice", password=_TEST_PASSWORD) + show, _ = SettingsBrowserShow.objects.get_or_create() + browser = SettingsBrowser.objects.create(user=user, show=show) + reprint = Reprint.objects.create( + series_name="Kapitän Wissenschaft", language="de" + ) + SettingsBrowserFilters.objects.create(browser=browser, reprints=[reprint.pk]) + snapshot = self._snapshot_sidecar() + + # Rebuilding the library re-creates the tag row under a new PK, + # which is the whole reason the sidecar keys on names. + SettingsBrowser.objects.all().delete() + reprint.delete() + rebuilt = Reprint.objects.create( + series_name="Kapitän Wissenschaft", language="de" + ) + assert rebuilt.pk != reprint.pk + + restore(sidecar_path=snapshot) + filters = SettingsBrowserFilters.objects.get(browser__user=user) + assert filters.reprints == [rebuilt.pk] + def test_restore_is_idempotent(self) -> None: self._seed_main_db() snapshot = self._snapshot_sidecar() diff --git a/tests/test_user_data_store.py b/tests/test_user_data_store.py index 8b0e2a270..ca9b0aedb 100644 --- a/tests/test_user_data_store.py +++ b/tests/test_user_data_store.py @@ -114,6 +114,41 @@ def test_is_empty_tracks_inserts(self) -> None: self.store.upsert("users", ("username",), {"username": "alice"}) assert not self.store.is_empty() + def test_schema_gains_columns_added_by_a_release(self) -> None: + """ + A sidecar written before a new filter column still accepts it. + + ``schema.sql`` is all ``CREATE TABLE IF NOT EXISTS``, so an + upgraded install re-runs it against a file that predates the + column and nothing happens. Simulate that by dropping the + column from a freshly created sidecar and reconnecting. + """ + conn = self.store.connection() + conn.execute("ALTER TABLE settings_filters DROP COLUMN reprints") + columns = { + column["name"] + for column in conn.execute("PRAGMA table_info(settings_filters)") + } + assert "reprints" not in columns + self.store.close() + + store = SidecarStore(self.sidecar_path) + try: + store.upsert( + "settings_filters", + ("username", "client", "name"), + { + "username": "alice", + "client": "web", + "name": "", + "reprints": '["Kapitän Wissenschaft"]', + }, + ) + rows = store.fetchall("settings_filters") + finally: + store.close() + assert rows[0]["reprints"] == '["Kapitän Wissenschaft"]' + def test_upsert_empty_data_raises(self) -> None: """``upsert`` with no columns is a programming error.""" with pytest.raises(ValueError, match="no columns"): diff --git a/uv.lock b/uv.lock index 27790ebc6..ca8d5f7b2 100644 --- a/uv.lock +++ b/uv.lock @@ -567,7 +567,7 @@ wheels = [ [[package]] name = "codex" -version = "2.2.2" +version = "2.2.3" source = { editable = "." } dependencies = [ { name = "adrf" }, @@ -653,7 +653,7 @@ requires-dist = [ { name = "adrf", specifier = "~=0.1.12" }, { name = "bidict", specifier = "~=0.23" }, { name = "channels", specifier = "~=4.2" }, - { name = "comicbox", extras = ["pdf"], specifier = "~=4.5.1" }, + { name = "comicbox", extras = ["pdf"], specifier = "~=4.6.1" }, { name = "cryptography", specifier = ">=48.0.0" }, { name = "dateparser", specifier = "~=1.2" }, { name = "django", specifier = "~=6.0" }, @@ -713,7 +713,7 @@ lint = [ { name = "mbake", specifier = "~=1.4.5" }, { name = "pathspec", specifier = "~=1.1.0" }, { name = "radon", specifier = "~=6.0" }, - { name = "ruff", specifier = "~=0.15" }, + { name = "ruff", specifier = "~=0.16" }, { name = "ty", specifier = "~=0.0.18" }, { name = "types-python-dateutil", specifier = "~=2.9" }, { name = "vulture", specifier = "~=2.3" }, @@ -739,7 +739,7 @@ wheels = [ [[package]] name = "comicbox" -version = "4.5.1" +version = "4.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "bidict" }, @@ -776,9 +776,9 @@ dependencies = [ { name = "xmltodict" }, { name = "zipremove" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d0/55/0a69c9e807b7c18b39cf4aa3da8280ef9dddb5b64377381999bca6490a1e/comicbox-4.5.1.tar.gz", hash = "sha256:e90c58dfbe9e234fe89c8ad2684ff5414f5045439eca2bcbe771e6b80e9770ff", size = 1009771, upload-time = "2026-07-24T20:15:28.57Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6e/05/1c2369de5b000c963fa119c2515d3fd1c64e55fa518eca2b691a86315c26/comicbox-4.6.1.tar.gz", hash = "sha256:4a4f50df0143c21853356a8defa767335434d870da6cdc6c56fef9a77c1f8c87", size = 1015362, upload-time = "2026-07-26T21:26:50.629Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/d0/e14bf8b03fbc139c75855996fa9e7f28b9291beb4ed79f58bba40079e7bd/comicbox-4.5.1-py3-none-any.whl", hash = "sha256:e7d188e0520330eae76298b3f3cfa1a43fa164dd22ea9fb771e1597dd3f6503c", size = 340303, upload-time = "2026-07-24T20:15:26.84Z" }, + { url = "https://files.pythonhosted.org/packages/cf/24/6acdbbc6c3ef7bb9023bbeb586d442649218f72177e3013b26a0b1dadc3e/comicbox-4.6.1-py3-none-any.whl", hash = "sha256:8d4c288f775063858620679dd0e4b61291ef14180fbc24767adf5b15684208f7", size = 342787, upload-time = "2026-07-26T21:26:49.016Z" }, ] [package.optional-dependencies] @@ -2175,15 +2175,15 @@ wheels = [ [[package]] name = "mokkari" -version = "4.2.0" +version = "4.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/14/4b/487fcb4862b5b1d4aaeb48ce55e09473dfa178257fbe49468fd8da0477ab/mokkari-4.2.0.tar.gz", hash = "sha256:41120f12f9143da0a7b22907f20b2d2291f66878133077235f736e755126c643", size = 78532, upload-time = "2026-07-21T13:38:33.015Z" } +sdist = { url = "https://files.pythonhosted.org/packages/56/f5/b61a9ea4a145cb9b4a52906332ed6817dd46e744024fdd5115c251ff0f68/mokkari-4.3.0.tar.gz", hash = "sha256:339e664f0b250c98eadecaeced698ad812ba702c90a55687e6df2a80ee6c9672", size = 78743, upload-time = "2026-07-25T12:16:27.065Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/65/7f/6e6373abdcc1ea168ecd57b5c80e70b03ff74fb2451a0276592ab9b4b3d2/mokkari-4.2.0-py3-none-any.whl", hash = "sha256:305d9eba8a9becf055e74cbe9ad386b67cbded9a6c5089ac944590602004fea2", size = 59176, upload-time = "2026-07-21T13:38:32.015Z" }, + { url = "https://files.pythonhosted.org/packages/03/db/a20299019eef6a77c77520f83bb3ae6d53fd852c48f5ade97fee2a7cd77e/mokkari-4.3.0-py3-none-any.whl", hash = "sha256:7a3b75844a4668e5524305353ebc11df2c9bc5a63b497013603e1098de89d2f3", size = 59257, upload-time = "2026-07-25T12:16:25.996Z" }, ] [[package]] @@ -2654,14 +2654,14 @@ wheels = [ [[package]] name = "prompt-toolkit" -version = "3.0.52" +version = "3.0.53" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "wcwidth" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/ea/39b988c938f75cb75d7045b5c69f8bfed47ee2152c8837fb403de29d6fb8/prompt_toolkit-3.0.53.tar.gz", hash = "sha256:9ec8a0ad96d5c56148b3f914aa79c1564c3fde5d2e6b876e7bc327e353cf8fa6", size = 435492, upload-time = "2026-07-26T20:56:14.758Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, + { url = "https://files.pythonhosted.org/packages/54/6f/84908cad2d6aa5144abcf7b42709fe4fdb459bc640ec7ac5786e7693dabc/prompt_toolkit-3.0.53-py3-none-any.whl", hash = "sha256:01c0891d7f9237d5e339f7d3e42cdae80b7534abb1c7c0e3352efba6231492f2", size = 392288, upload-time = "2026-07-26T20:56:12.512Z" }, ] [[package]] @@ -3086,11 +3086,11 @@ wheels = [ [[package]] name = "pytz" -version = "2026.2" +version = "2026.3.post1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ff/46/dd499ec9038423421951e4fad73051febaa13d2df82b4064f87af8b8c0c3/pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a", size = 320861, upload-time = "2026-05-04T01:35:29.667Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fb/48/fb042503b6ca6cd271261dc559fd6432f7d8c713153e9ec5c591af4dfc1c/pytz-2026.3.post1.tar.gz", hash = "sha256:2211d3fcf9a797d3405cac96ac7f61d80e6a644f72a3309607282fe8a2010c5d", size = 319745, upload-time = "2026-07-25T15:12:07.385Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126", size = 510141, upload-time = "2026-05-04T01:35:27.408Z" }, + { url = "https://files.pythonhosted.org/packages/0f/7b/39c34ca613b0b198cb866466651b26b045e2009864c5183c979a3b83f383/pytz-2026.3.post1-py2.py3-none-any.whl", hash = "sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815", size = 508283, upload-time = "2026-07-25T15:12:05.782Z" }, ] [[package]]