diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9264620..033d445 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,7 +125,9 @@ jobs: with: python-version: "3.11" cache: pip - cache-dependency-path: tools/import/pyproject.toml + cache-dependency-path: | + tools/import/pyproject.toml + tools/themes/pyproject.toml - name: Dépendances run: pip install -e ".[dev]" @@ -135,3 +137,12 @@ jobs: env: DATABASE_URL: postgres://palacards:change-me@localhost:5433/palacards run: pytest -q + + # Atelier des boosters à thème (tools/themes) : fiche YAML, sélection DuckDB, script de création. + - name: Atelier des thèmes (dépendances) + working-directory: tools/themes + run: pip install -e ".[dev]" + + - name: Atelier des thèmes (tests) + working-directory: tools/themes + run: pytest -q diff --git a/apps/api/src/services/themes.ts b/apps/api/src/services/themes.ts index f96ec68..a4ec390 100644 --- a/apps/api/src/services/themes.ts +++ b/apps/api/src/services/themes.ts @@ -87,13 +87,21 @@ async function buildTheme(ctx: Ctx, adminId: string, input: ThemeInput) { .returning(); // Les titres du dump ont leur première lettre en majuscule : « jeu vidéo » retrouve « Jeu vidéo ». const byTitle = titles.map((t) => t.charAt(0).toUpperCase() + t.slice(1)); + // Rareté et rang par rareté écrits dès l'insertion : un seul passage, au lieu d'insérer puis de tout + // mettre à jour (plus de 11 min pour 150 000 articles le 2026-11-03, lignes fraîches sans statistiques). const inserted = await tx.execute<{ card_id: string }>(sql` - insert into theme_cards (theme_id, card_id) - select ${theme!.id}, id from cards - where season = ${season} - and (id in (select jsonb_array_elements_text(${JSON.stringify([...pageIds])}::jsonb)::bigint) - or title in (select jsonb_array_elements_text(${JSON.stringify(byTitle)}::jsonb))) - on conflict do nothing + with wanted as ( + select x::bigint as id from jsonb_array_elements_text(${JSON.stringify([...pageIds])}::jsonb) x + union + select c.id from cards c + where c.season = ${season} + and c.title in (select jsonb_array_elements_text(${JSON.stringify(byTitle)}::jsonb)) + ) + insert into theme_cards (theme_id, card_id, rarity, pos) + select ${theme!.id}, c.id, c.rarity, + (row_number() over (partition by c.rarity order by c.id) - 1)::int + from cards c join wanted w on w.id = c.id + where c.season = ${season} returning card_id `); if (inserted.length < THEME_MIN_CARDS) @@ -106,7 +114,7 @@ async function buildTheme(ctx: Ctx, adminId: string, input: ThemeInput) { ...theme!, categories, cardCount: inserted.length, - byRarity: (await indexTheme(tx, theme!.id, season)).byRarity, + byRarity: (await summarizeIndex(tx, theme!.id, season)).byRarity, dryRun: !!input.dryRun, }; if (input.dryRun) throw new DryRun(result); @@ -207,6 +215,11 @@ async function indexTheme(tx: Tx, themeId: number, season: number): Promise { const counts = await tx.execute<{ rarity: Rarity; n: number }>(sql` select rarity, count(*)::int as n from theme_cards where theme_id = ${themeId} and rarity is not null group by rarity diff --git a/apps/api/test/events.test.ts b/apps/api/test/events.test.ts index f295c9e..aa26636 100644 --- a/apps/api/test/events.test.ts +++ b/apps/api/test/events.test.ts @@ -582,8 +582,8 @@ describe("boosters à thème", () => { it("réunit plusieurs catégories (et l'ancien champ `category`), avec un essai à blanc", async () => { const admin = await signUpAdmin(app, ctx); - const pool = await ctx.db.execute<{ id: string }>(sql` - select id from cards where season = (select id from seasons where status = 'active') order by id limit 12 + const pool = await ctx.db.execute<{ id: string; title: string }>(sql` + select id, title from cards where season = (select id from seasons where status = 'active') order by id limit 12 `); const ids = pool.map((r) => Number(r.id)); const members: Record = { @@ -600,7 +600,8 @@ describe("boosters à thème", () => { name: "Essai", categories: ["Catégorie:Chanteur_français", "Chanteuse française"], depth: 1, - titles: [], + // Déjà dans une catégorie : compté une seule fois (rareté et rang posés à l'insertion). + titles: [pool[0]!.title], price: 250, startsAt: new Date(window.startsAt), endsAt: new Date(window.endsAt), diff --git a/apps/web/src/app/(game)/collection/page.tsx b/apps/web/src/app/(game)/collection/page.tsx index b84f8a4..f36a468 100644 --- a/apps/web/src/app/(game)/collection/page.tsx +++ b/apps/web/src/app/(game)/collection/page.tsx @@ -2,7 +2,7 @@ import { countByRarity, MAX_LEVEL, RARITIES, RARITY_LABELS, recycleValue, type Rarity } from "@palacards/game"; import type { CardDTO, Page } from "@palacards/shared"; -import { ChevronsUp } from "lucide-react"; +import { ChevronDown, ChevronsUp } from "lucide-react"; import Link from "next/link"; import { useSearchParams } from "next/navigation"; import { Suspense, useCallback, useState } from "react"; @@ -28,6 +28,7 @@ import { RecycleBreakdown } from "@/components/RecycleBreakdown"; import { CardSkeletons, ConfirmDialog, Empty, ErrorBox, LoadMore } from "@/components/ui"; import { api, ApiError } from "@/lib/api"; import { fmt } from "@/lib/format"; +import { useStoredFlag } from "@/lib/use-stored-flag"; interface Summary { season: number; @@ -74,6 +75,49 @@ function Completion({ summary }: { summary: Summary }) { ); } +/** + * Encart de complétion. Grand écran : colonne de droite, toujours ouvert. Téléphone et tablette : au-dessus des + * filtres, replié par défaut (le pourcentage global reste visible) ; sous la grille, le chargement continu des + * cartes le repoussait sans fin. + */ +function CompletionBox({ summary }: { summary: Summary | undefined }) { + const [open, setOpen] = useStoredFlag("pc-collection-completion-open"); + const owned = summary?.byRarity.reduce((s, r) => s + r.owned, 0) ?? 0; + const total = summary?.byRarity.reduce((s, r) => s + r.total, 0) ?? 0; + const ratio = total ? (owned / total) * 100 : 0; + // Une décimale sous 10 % : avec des milliers d'articles, un début de collection afficherait « 0 % ». + const pct = ratio.toLocaleString("fr-FR", { maximumFractionDigits: ratio < 10 ? 1 : 0 }); + const title = `Complétion · saison ${summary?.season ?? "…"}`; + return ( + + ); +} + export default function CollectionPage() { return ( }> @@ -526,12 +570,7 @@ function Collection() { )} - + {selecting && ( diff --git a/apps/web/src/app/(game)/upgrade/page.tsx b/apps/web/src/app/(game)/upgrade/page.tsx index 9d3d961..d31ee9a 100644 --- a/apps/web/src/app/(game)/upgrade/page.tsx +++ b/apps/web/src/app/(game)/upgrade/page.tsx @@ -166,6 +166,16 @@ function Upgrader() { const skipped = useRef(false); const lastTick = useRef(0); const dialRef = useRef(null); + /** Boutons de l'établi : hors de l'écran, une barre collée en bas les remplace pendant le choix des cartes. */ + const actionsRef = useRef(null); + const [actionsVisible, setActionsVisible] = useState(true); + useEffect(() => { + const el = actionsRef.current; + if (!el) return; + const io = new IntersectionObserver(([entry]) => setActionsVisible(!!entry?.isIntersecting)); + io.observe(el); + return () => io.disconnect(); + }, []); const target = nextRarity(rarity)!; const count = picked.length; @@ -561,7 +571,10 @@ function Upgrader() { -
+
{phase === "done" ? ( <> {result?.success && revealed && ( @@ -735,6 +748,26 @@ function Upgrader() { )} + {/* Téléphone et tablette : l'établi est loin au-dessus de la grille ; on tente sans remonter. */} + {phase === "idle" && !seriesAdding && count > 0 && !actionsVisible && ( +
+
+

+ {fmt(count)}/{fmt(useful)} cartes +

+ +
+
+ )}
); diff --git a/apps/web/src/app/globals.css b/apps/web/src/app/globals.css index 1b76542..543a471 100644 --- a/apps/web/src/app/globals.css +++ b/apps/web/src/app/globals.css @@ -1693,3 +1693,15 @@ button:disabled .pc-pack-foil { mask-image: linear-gradient(to right, transparent, #000 75%); pointer-events: none; } + +/* + * iPhone : Safari zoome sur tout champ de moins de 16 px quand il prend le focus (recherche de la Collection, + * filtres, messages…). Hors couche, cette règle passe devant les utilitaires `text-sm` des champs. + */ +@media (pointer: coarse) { + input:not([type="checkbox"], [type="radio"], [type="range"], [type="color"], [type="file"]), + select, + textarea { + font-size: max(16px, 1em); + } +} diff --git a/apps/web/src/components/PackOpener.tsx b/apps/web/src/components/PackOpener.tsx index 4ae9a9c..210d39b 100644 --- a/apps/web/src/components/PackOpener.tsx +++ b/apps/web/src/components/PackOpener.tsx @@ -18,7 +18,7 @@ import { memo, useCallback, useEffect, useRef, useState } from "react"; import { toast } from "sonner"; import { api, ApiError, thumbSrc } from "@/lib/api"; import { fmt } from "@/lib/format"; -import { useMe } from "@/lib/game"; +import { useMe, useQuietToasts } from "@/lib/game"; import { useCardMedia } from "@/lib/media"; import { play } from "@/lib/sfx"; import { PHONE_QUERY, useMediaQuery } from "@/lib/use-media-query"; @@ -777,6 +777,8 @@ export const PackOpener = memo(function PackOpener({ const [cursor, setCursor] = useState(0); const [busy, setBusy] = useState(false); const [announce, setAnnounce] = useState(""); + /** Ce que le serveur a fait du paquet (pity, recyclage auto, wishlist) : affiché sous les cartes, pas en toast. */ + const [notes, setNotes] = useState([]); const timers = useRef[]>([]); const region = useRef(null); const buying = !!theme && theme.owned === 0; @@ -821,6 +823,7 @@ export const PackOpener = memo(function PackOpener({ timers.current = []; setBusy(true); setAnnounce(""); + setNotes([]); setRecycled({}); if (speed !== "instant") { setPhase("tearing"); @@ -853,16 +856,18 @@ export const PackOpener = memo(function PackOpener({ if (res.cards.some((c) => c.shiny)) play("shiny", 60); else if (isFxTier(best.rarity)) play(JINGLE[best.rarity], 60); } - if (res.pityTriggered) toast("Pity déclenchée : UR ou mieux garantie !"); setWished(Object.fromEntries((res.wishlistKept ?? []).map((id) => [id, true]))); + // Sur téléphone, un toast à chaque paquet couvrait la frise de rareté : ces infos restent sous les cartes. const kept = res.wishlistKept?.length ?? 0; - const keptNote = kept ? `${kept} carte${kept > 1 ? "s" : ""} de ta wishlist gardée${kept > 1 ? "s" : ""}` : ""; - if (res.autoRecycled) { - const n = res.autoRecycled.instanceIds.length; - toast.success( - `Recyclage auto : ${n} carte${n > 1 ? "s" : ""}, +${fmt(res.autoRecycled.gain)} PW${kept ? ` · ${keptNote}` : ""}`, - ); - } else if (kept) toast.success(`Recyclage auto : ${keptNote}`); + const recycledN = res.autoRecycled?.instanceIds.length ?? 0; + setNotes( + [ + res.pityTriggered && "Pity déclenchée : UR ou mieux garantie", + res.autoRecycled && + `Recyclage auto : ${recycledN} carte${recycledN > 1 ? "s" : ""}, +${fmt(res.autoRecycled.gain)} PW`, + kept > 0 && `${kept} carte${kept > 1 ? "s" : ""} de ta wishlist gardée${kept > 1 ? "s" : ""}`, + ].filter((x): x is string => !!x), + ); if (buying) play("coin"); if (res.theme) onOpened?.(res.theme); void mutateMe((m) => (m ? { ...m, packs: res.packs } : m), { revalidate: false }); @@ -878,9 +883,9 @@ export const PackOpener = memo(function PackOpener({ if (!card.instanceId) return; setRecycled((d) => ({ ...d, [card.instanceId!]: true })); try { - const res = await api<{ gain: number }>("/collection/recycle", { body: { instanceIds: [card.instanceId] } }); + await api<{ gain: number }>("/collection/recycle", { body: { instanceIds: [card.instanceId] } }); + // Pas de toast : la case de la carte affiche déjà « Recyclée · +N PW ». play("coin"); - toast.success(`+${res.gain} PW : ${card.title} recyclée.`); } catch (err) { setRecycled(({ [card.instanceId!]: _, ...rest }) => rest); toast.error(err instanceof ApiError ? err.message : "Recyclage impossible."); @@ -894,6 +899,9 @@ export const PackOpener = memo(function PackOpener({ const dealt = { cards, revealed, onReveal: reveal, recycled, onRecycle: askRecycle, speed }; /** Téléphone, cartes encore en pile : la pile porte ses propres boutons. */ const decking = phone && phase === "dealt" && cursor < cards.length; + // Pendant la pile, les notifications attendent : en haut de l'écran, elles cachaient la frise du paquet. + // Dès la requête : le serveur envoie les succès du paquet avant même de répondre. + useQuietToasts(phone && (busy || decking)); return (
{!decking && ( -
- {phase === "dealt" && !allRevealed && ( - + // Téléphone : barre collée en bas de l'écran, « Ouvrir le suivant » reste sous le pouce même après le + // récapitulatif (dix lignes, plus haut que l'écran) et sous la barre de Safari. +
+ {phase === "dealt" && notes.length > 0 && ( +

{notes.join(" · ")}

)} - + )} + - {phase === "dealt" && allRevealed && ( - - Voir ma collection - - )} + : "Ouvrir un paquet"} + + {phase === "dealt" && allRevealed && ( + + {phone ? "Collection" : "Voir ma collection"} + + )} +
)} diff --git a/apps/web/src/lib/game.tsx b/apps/web/src/lib/game.tsx index f80c0a7..8dd3eff 100644 --- a/apps/web/src/lib/game.tsx +++ b/apps/web/src/lib/game.tsx @@ -34,6 +34,8 @@ interface GameContextValue { holdNews: () => () => void; /** Lance `fn` tout de suite, ou à la fin de la retenue en cours. */ afterHold: (fn: () => void) => void; + /** Met en attente les toasts d'ambiance (voir `useQuietToasts`) ; renvoie la fonction qui les relâche. */ + quietToasts: () => () => void; } /** @@ -92,6 +94,16 @@ export function useHoldNews() { return hold; } +/** + * Tant que `active` est vrai, les toasts d'ambiance (notifications, gros tirages des potes) attendent : sur + * téléphone, ils couvriraient la pile de cartes en cours d'ouverture. Ils s'affichent dès que `active` + * repasse à faux ou que le composant est démonté. Rien d'autre n'attend (solde, compteurs, duels). + */ +export function useQuietToasts(active: boolean) { + const { quietToasts } = useGame(); + useEffect(() => (active ? quietToasts() : undefined), [active, quietToasts]); +} + /** Écoute un événement temps réel tant que le composant est monté. */ export function useSocketEvent(event: E, handler: ServerToClientEvents[E]) { const { socket, afterHold } = useGame(); @@ -162,6 +174,26 @@ export function GameProvider({ children }: { children: ReactNode }) { }; }, []); + // Toasts d'ambiance en attente (voir `useQuietToasts`). + const quiet = useRef(0); + const quieted = useRef<(() => void)[]>([]); + const afterQuiet = useCallback((fn: () => void) => { + if (quiet.current) quieted.current.push(fn); + else fn(); + }, []); + const quietToasts = useCallback(() => { + quiet.current++; + let released = false; + return () => { + if (released) return; + released = true; + if (--quiet.current > 0) return; + const queue = quieted.current; + quieted.current = []; + for (const fn of queue) fn(); + }; + }, []); + useEffect(() => { if (error instanceof ApiError && error.status === 401) router.replace("/login"); }, [error, router]); @@ -198,9 +230,11 @@ export function GameProvider({ children }: { children: ReactNode }) { news((n) => { void mutateMe((m) => (m ? { ...m, unreadNotifications: n.unread } : m), { revalidate: false }); const { text, href } = describe(n); - const sound = NOTIFICATION_SFX[n.type]; - if (sound) play(sound); - toast(text, { action: { label: "Voir", onClick: () => router.push(href) } }); + afterQuiet(() => { + const sound = NOTIFICATION_SFX[n.type]; + if (sound) play(sound); + toast(text, { action: { label: "Voir", onClick: () => router.push(href) } }); + }); }), ); s.on("message:new", refreshMe); @@ -247,9 +281,11 @@ export function GameProvider({ children }: { children: ReactNode }) { if (item.card.rarity !== "L" && !item.card.shiny) return; const what = `${RARITY_LABELS[item.card.rarity]}${item.card.shiny ? " brillante" : ""}`; const verb = item.source === "shop" ? "a acheté en boutique" : "a tiré"; - toast(`${item.user.name} ${verb} ${item.card.title} (${what}) !`, { - action: { label: "Voir", onClick: () => router.push("/feed") }, - }); + afterQuiet(() => + toast(`${item.user.name} ${verb} ${item.card.title} (${what}) !`, { + action: { label: "Voir", onClick: () => router.push("/feed") }, + }), + ); }), ); // Exposé aux composants une fois connecté ; après une coupure, on relit l'état manqué. @@ -262,7 +298,7 @@ export function GameProvider({ children }: { children: ReactNode }) { if (meTimer) clearTimeout(meTimer); s.disconnect(); }; - }, [mutateMe, router, afterHold]); + }, [mutateMe, router, afterHold, afterQuiet]); // Bonus de connexion du jour, réclamé une fois par chargement (le serveur l'accorde une fois par jour). // Aussi au retour sur l'onglet et après une reconnexion : un onglet ouvert passé minuit le touche. @@ -286,7 +322,9 @@ export function GameProvider({ children }: { children: ReactNode }) { }, [loggedIn, connection]); return ( - + {children} ); diff --git a/tools/import/palacards_import/sqlextract.py b/tools/import/palacards_import/sqlextract.py index 1a2d436..91c3be5 100644 --- a/tools/import/palacards_import/sqlextract.py +++ b/tools/import/palacards_import/sqlextract.py @@ -13,14 +13,19 @@ from collections.abc import Iterable, Iterator from typing import BinaryIO -_FIRST_ID = re.compile(rb"VALUES \((\d+),") +_FIRST_ID = re.compile(rb"^\((\d+),") _STR = rb"'(?:[^'\\]|\\.)*'" def _lines(stream: BinaryIO, table: str, expected_columns: list[str]) -> Iterator[bytes]: - """Lignes INSERT de `table`, après avoir vérifié les colonnes du CREATE TABLE.""" - prefix = f"INSERT INTO `{table}` VALUES ".encode() + """Lignes de valeurs des INSERT de `table`, après avoir vérifié les colonnes du CREATE TABLE. + + Deux formats : l'ancien (`INSERT … VALUES (…),(…);` sur une seule ligne) et celui des dumps de + novembre 2026 (`INSERT … VALUES` seul sur sa ligne, puis un tuple par ligne jusqu'au `;`). + """ + prefix = f"INSERT INTO `{table}` VALUES".encode() columns: list[str] = [] + inside = False with gzip.open(stream, "rb") as f: for line in f: if line.startswith(b" `"): @@ -28,7 +33,16 @@ def _lines(stream: BinaryIO, table: str, expected_columns: list[str]) -> Iterato elif line.startswith(prefix): if columns[: len(expected_columns)] != expected_columns: raise SystemExit(f"Format inattendu pour `{table}` : {columns} (attendu {expected_columns})") - yield line + rest = line[len(prefix) :].strip() + inside = not rest.endswith(b";") + if rest: + yield rest + elif inside: + if line.startswith(b"("): + yield line + inside = not line.rstrip().endswith(b";") + else: + inside = False def _stop(line: bytes, max_page: int | None) -> bool: diff --git a/tools/import/tests/test_sqlextract.py b/tools/import/tests/test_sqlextract.py index a2b09ac..7281b45 100644 --- a/tools/import/tests/test_sqlextract.py +++ b/tools/import/tests/test_sqlextract.py @@ -80,3 +80,42 @@ def test_category_target_ids_stops_once_all_titles_found(): 7: "Article_de_qualité", 9: "Bon_article", } + + +# Dumps de novembre 2026 : `INSERT … VALUES` seul sur sa ligne, puis un tuple par ligne. +PAGE_PROPS_2026 = rb"""CREATE TABLE `page_props` ( + `pp_page` int(10) unsigned NOT NULL, + `pp_propname` varbinary(60) NOT NULL DEFAULT '', + `pp_value` blob NOT NULL, + `pp_sortkey` float DEFAULT NULL, + PRIMARY KEY (`pp_page`,`pp_propname`) +) ENGINE=InnoDB; +INSERT INTO `page_props` VALUES +(3,'defaultsort','Meillet, Antoine',NULL), +(5,'disambiguation','',NULL), +(12,'disambiguation','',NULL); +INSERT INTO `page_props` VALUES +(40,'disambiguation','',NULL); +""" + +CATEGORYLINKS_2026 = rb"""CREATE TABLE `categorylinks` ( + `cl_from` int(8) unsigned NOT NULL DEFAULT 0, + `cl_sortkey` varbinary(230) NOT NULL DEFAULT '', + `cl_timestamp` timestamp NOT NULL, + `cl_sortkey_prefix` varbinary(255) NOT NULL DEFAULT '', + `cl_type` enum('page','subcat','file') NOT NULL DEFAULT 'page', + `cl_collation_id` smallint(5) unsigned NOT NULL DEFAULT 0, + `cl_target_id` bigint(20) unsigned NOT NULL, + PRIMARY KEY (`cl_from`,`cl_target_id`) +); +INSERT INTO `categorylinks` VALUES +(3,'\xff\n\'),(\x1a','2025-11-15 15:34:27','Meillet','page',1,7), +(4,'y','2025-11-15 15:34:27','','subcat',1,9), +(50,'z','2025-11-15 15:34:27','','page',1,9); +""" + + +def test_one_tuple_per_line_format(): + assert disambiguation_ids(gz(PAGE_PROPS_2026)) == {5, 12, 40} + assert disambiguation_ids(gz(PAGE_PROPS_2026), max_page=10) == {5} + assert category_members(gz(CATEGORYLINKS_2026), [7, 9]) == {7: {3}, 9: {50}} diff --git a/tools/themes/README.md b/tools/themes/README.md new file mode 100644 index 0000000..344a965 --- /dev/null +++ b/tools/themes/README.md @@ -0,0 +1,31 @@ +# Atelier des boosters à thème + +Crée un booster à thème en une dizaine de minutes au lieu de plus d'une heure : tout est lu dans un **index local** +(DuckDB) construit une fois par saison, sans parcourir l'API Wikipédia ni interroger Wikidata par lots. + +Installé sur PalaServer : `/srv/nas/palacards-themes` (données sur le NAS, ~5 Go), lancé par `./run …`. + +## L'index (une fois par saison) + +```sh +cd /srv/nas/palacards-themes && nohup ./run index > data/index.log 2>&1 & +``` + +1. Dumps du Wikipédia FR (`page`, `categorylinks`, `linktarget`, `page_props`, ~1,6 Go) : arbre des catégories, + articles de chaque catégorie, élément Wikidata de chaque article. +2. Wikidata en quelques requêtes en bloc sur [QLever](https://qlever.dev) : nature (P31), métier (P106), sport (P641), + nationalité (P27), pays (P17), genre (P136), sous-classes (P279, en entier), descriptions et libellés en français. +3. Cartes de la saison active, exportées de la prod. + +## Un booster + +1. Écrire la fiche `data/themes/.yaml` (modèle : [`exemple.yaml`](exemple.yaml)). Pour trouver les noms : + `./run find-cat "jeu vidéo"`, `./run find-class "console"`, `./run card "Tetris"`. +2. `./run preview data/themes/.yaml` (quelques secondes) : cartes par rareté et par famille, légendaires, UR, + descriptions suspectes, catégories qui apportent le plus, échantillons. Rapport complet dans + `data/themes//preview.md`. Ajuster la fiche, recommencer. +3. `./run create data/themes/.yaml` : essai en prod dans une transaction annulée. +4. `./run create data/themes/.yaml --go` : sauvegarde (`palacards-backup.service`), puis création en une + transaction (thème, cartes avec rareté et rang, index de tirage), en quelques secondes. + +Règles de prix (doc 22 du projet Infra Palawi) : 200 PW avec assez de légendaires, sinon 150 PW ; jamais sous 150. diff --git a/tools/themes/exemple.yaml b/tools/themes/exemple.yaml new file mode 100644 index 0000000..5f287b9 --- /dev/null +++ b/tools/themes/exemple.yaml @@ -0,0 +1,62 @@ +# Fiche d'exemple : le booster « Jeux vidéo » (n° 15, octobre 2026) refait avec l'atelier. +# Copier dans data/themes/.yaml, puis : ./run preview data/themes/.yaml +# +# Trouver les noms : ./run find-cat "jeu vidéo" · ./run find-class "console" · ./run card "Tetris" +# Astuce : les catégories cachées « Portail:/Articles liés » regroupent presque tout un sujet. + +name: Jeux vidéo +description: Jeux, consoles, personnages, studios et créateurs. +display_category: Jeux vidéo # affichage seulement (colonne themes.category) +price: 150 # règle : assez de légendaires -> 200, sinon 150 ; jamais sous 150 +starts: now # ou une date ISO en UTC (2026-11-05T08:00Z) +days: 7 # ou ends: 2026-11-11T22:59Z + +families: + # Une carte compte dans la première famille qui la contient (l'ordre compte pour l'aperçu). + - name: Jeux + wikidata: + P31: [Q7889, Q7058673] # jeu vidéo, série de jeux vidéo (+ toutes leurs sous-classes) + keep_desc: ["jeu", "jeux", "vidéo", "console", "franchise"] # garde si la description en parle + drop_desc: ['\bfilm\b', "série télévisée", "manga", "roman", "bande dessinée", "comics"] + + - name: Personnages et univers + wikidata: + P31: [Q1569167] # personnage de jeu vidéo + categories: + - { name: Personnage de jeu vidéo, depth: 2 } + - { name: Univers de jeu vidéo, depth: 1 } + keep_desc: ["jeu", "jeux", "vidéo", "personnage", "univers", "franchise"] + drop_desc: ["manga", "comics", "bande dessinée"] + + - name: Consoles, studios et éditeurs + categories: + - { name: Console de jeux vidéo, depth: 2 } + - { name: Matériel de jeu vidéo, depth: 1 } + - { name: Développeur de jeux vidéo, depth: 2 } + - { name: Éditeur de jeux vidéo, depth: 2 } + - { name: Entreprise de jeux vidéo, depth: 1 } + - { name: Genre de jeux vidéo, depth: 0 } + - { name: Événement du jeu vidéo, depth: 1 } + + - name: Créateurs et e-sport + wikidata: + P106: [Q3630699, Q57414145, Q4379701] # concepteur de jeux, streamer, joueur pro + P641: [Q300920] # sport électronique + categories: + - { name: Personnalité liée au secteur du jeu vidéo, depth: 2 } + - { name: Esport, depth: 2 } + - { name: Vidéaste web vidéoludique, depth: 1 } + people: any + keep_desc: ["jeu", "jeux", "vidéo", "stream", "twitch", "youtub", "esport", "e-sport", "gam"] + keep_no_desc: true + +exclude: + desc: ["acteur pornographique", "actrice pornographique"] + titles: [] + +include: # titres forcés (légendaires ajoutées, cartes phares) + - Squeezie + - Gotaga + - ZeratoR + - Nintendo + - PlayStation 5 diff --git a/tools/themes/palacards_themes/__init__.py b/tools/themes/palacards_themes/__init__.py new file mode 100644 index 0000000..90897d4 --- /dev/null +++ b/tools/themes/palacards_themes/__init__.py @@ -0,0 +1 @@ +"""Atelier des boosters à thème : index local du Wikipédia FR et de Wikidata, fiches YAML, aperçu, création.""" diff --git a/tools/themes/palacards_themes/cli.py b/tools/themes/palacards_themes/cli.py new file mode 100644 index 0000000..6e50f30 --- /dev/null +++ b/tools/themes/palacards_themes/cli.py @@ -0,0 +1,155 @@ +"""palacards-themes : index | preview | create | find-cat | find-class | card. + +Exemples (sur PalaServer, depuis /srv/nas/palacards-themes) : + ./run index # une fois par saison, ~15 à 25 min, en arrière-plan + ./run preview themes/jeux-video.yaml + ./run create themes/jeux-video.yaml # essai (transaction annulée) + ./run create themes/jeux-video.yaml --go # sauvegarde puis création + ./run find-cat "jeu vidéo" + ./run find-class "console de jeux" + ./run card "Tetris" +""" + +from __future__ import annotations + +import argparse +import sys +import time + +from . import create as create_mod +from . import index, paths, prod, report, spec +from .select import select + + +def _meta(con) -> dict[str, str]: + return dict(con.execute("SELECT key, value FROM meta").fetchall()) + + +def cmd_index(a) -> None: + index.build(a.season, download=not a.no_download, wikidata_fetch=not a.no_wikidata) + + +def _run(a): + s = spec.load(a.spec) + con = index.connect() + meta = _meta(con) + t = time.time() + sel = select(con, s) + return s, con, meta, sel, time.time() - t + + +def cmd_preview(a) -> None: + s, con, meta, sel, took = _run(a) + md, summary = report.preview(con, s, sel, int(meta["season"]), meta["dump"]) + out = paths.THEMES / s.slug + out.mkdir(parents=True, exist_ok=True) + (out / "preview.md").write_text(md, encoding="utf-8") + (out / "ids.txt").write_text("\n".join(str(i) for i in sorted(sel.ids)) + "\n") + print(summary) + print(f"Sélection en {took:.1f} s · aperçu : {out / 'preview.md'}") + + +def cmd_create(a) -> None: + s, _con, meta, sel, _ = _run(a) + season = int(meta["season"]) + price = a.price or s.price + if not price: + raise SystemExit("Prix manquant : `price:` dans la fiche ou --price.") + if price < 150: + raise SystemExit("Pas sous 150 PW : acheter puis recycler rapporterait des PW (doc 22).") + if len(sel.ids) < 5: + raise SystemExit("Moins de 5 cartes : thème trop petit.") + sql = create_mod.script(s, sorted(sel.ids), season, price, a.by, commit=a.go) + out = paths.THEMES / s.slug + out.mkdir(parents=True, exist_ok=True) + (out / ("create.sql" if a.go else "create-dry-run.sql")).write_text(sql, encoding="utf-8") + if a.go: + print("Sauvegarde de la base…") + prod.backup() + t = time.time() + res = prod.psql(sql) + print(res.strip()) + print(f"{'Créé' if a.go else 'Essai annulé (rien d’écrit)'} en {time.time() - t:.1f} s : " + f"{s.name}, {len(sel.ids):,} cartes, {price} PW, du {s.starts:%Y-%m-%d %H:%M} au {s.ends:%Y-%m-%d %H:%M} UTC") + + +def cmd_find_cat(a) -> None: + con = index.connect() + rows = con.execute( + """WITH m AS (SELECT cat AS title, count(*) n FROM cat_member WHERE cat ILIKE ? GROUP BY 1), + e AS (SELECT parent AS title, count(*) n FROM cat_edge WHERE parent ILIKE ? GROUP BY 1) + SELECT coalesce(m.title, e.title) t, coalesce(m.n, 0), coalesce(e.n, 0) + FROM m FULL JOIN e ON e.title = m.title + ORDER BY coalesce(m.n, 0) + 20 * coalesce(e.n, 0) DESC LIMIT ?""", + [f"%{a.text}%", f"%{a.text}%", a.limit], + ).fetchall() + for title, n, sub in rows: + print(f"{n:>7,} articles {sub:>5,} sous-cat. {title}") + + +def cmd_find_class(a) -> None: + con = index.connect() + rows = con.execute( + """WITH l AS (SELECT q, label FROM labels WHERE label ILIKE ?), + a AS (SELECT v, count(*) n FROM p31 WHERE v IN (SELECT q FROM l) GROUP BY v), + b AS (SELECT v, count(*) n FROM p106 WHERE v IN (SELECT q FROM l) GROUP BY v), + c AS (SELECT v, count(*) n FROM p641 WHERE v IN (SELECT q FROM l) GROUP BY v) + SELECT l.q, l.label, coalesce(a.n, 0), coalesce(b.n, 0), coalesce(c.n, 0) FROM l + LEFT JOIN a ON a.v = l.q LEFT JOIN b ON b.v = l.q LEFT JOIN c ON c.v = l.q + ORDER BY coalesce(a.n, 0) + coalesce(b.n, 0) + coalesce(c.n, 0) DESC LIMIT ?""", + [f"%{a.text}%", a.limit], + ).fetchall() + print("Comptes directs (sans les sous-classes) : P31 nature, P106 métier, P641 sport") + for q, label, p31, p106, p641 in rows: + print(f"Q{q:<10} P31 {p31:>7,} P106 {p106:>7,} P641 {p641:>6,} {label}") + + +def cmd_card(a) -> None: + con = index.connect() + row = con.execute("SELECT id, title, rarity, views, q, descr FROM art WHERE title = ?", + [spec.title_name(a.title)]).fetchone() + if not row: + raise SystemExit("Carte introuvable dans la saison.") + cid, title, rarity, views, q, descr = row + print(f"{title} · id {cid} · {rarity} · {views:,} vues · Q{q} · {descr or '—'}") + for prop in ["P31", "P106", "P641", "P136", "P27"]: + vals = con.execute(f"""SELECT 'Q' || t.v, coalesce(l.label, '?') FROM {prop.lower()} t + LEFT JOIN labels l ON l.q = t.v WHERE t.q = ?""", [q]).fetchall() + if vals: + print(f" {prop} : " + ", ".join(f"{label} ({v})" for v, label in vals)) + cats = [r[0] for r in con.execute("SELECT cat FROM cat_member WHERE page_id = ? ORDER BY cat", [cid]).fetchall()] + print(" Catégories : " + " · ".join(cats)) + + +def main(argv: list[str] | None = None) -> None: + p = argparse.ArgumentParser(prog="palacards-themes", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = p.add_subparsers(dest="cmd", required=True) + s = sub.add_parser("index", help="construit l'index local de la saison") + s.add_argument("--season", type=int) + s.add_argument("--no-download", action="store_true", help="réutilise les dumps déjà analysés") + s.add_argument("--no-wikidata", action="store_true", help="réutilise les données Wikidata déjà téléchargées") + s.set_defaults(fn=cmd_index) + s = sub.add_parser("preview", help="aperçu d'une fiche") + s.add_argument("spec") + s.set_defaults(fn=cmd_preview) + s = sub.add_parser("create", help="crée le booster (essai sans --go)") + s.add_argument("spec") + s.add_argument("--go", action="store_true", help="sauvegarde puis création pour de vrai") + s.add_argument("--price", type=int) + s.add_argument("--by", default="palawi", help="pseudo de l'admin créateur") + s.set_defaults(fn=cmd_create) + for name, fn in [("find-cat", cmd_find_cat), ("find-class", cmd_find_class)]: + s = sub.add_parser(name) + s.add_argument("text") + s.add_argument("--limit", type=int, default=30) + s.set_defaults(fn=fn) + s = sub.add_parser("card", help="tout ce que l'index sait d'une carte") + s.add_argument("title") + s.set_defaults(fn=cmd_card) + a = p.parse_args(argv) + a.fn(a) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/themes/palacards_themes/create.py b/tools/themes/palacards_themes/create.py new file mode 100644 index 0000000..e31bed1 --- /dev/null +++ b/tools/themes/palacards_themes/create.py @@ -0,0 +1,55 @@ +"""Création du booster en prod en une transaction : thème, cartes avec rareté et rang, index de tirage. + +Même résultat que `createTheme` + `indexTheme` de l'API (`apps/api/src/services/themes.ts`), mais en SQL +direct : la rareté et le rang par rareté sont écrits dès l'insertion (150 000 cartes en quelques secondes, +contre plus de 11 min pour `theme.js` le 2026-11-03). +""" + +from __future__ import annotations + +from .spec import Spec + +PREVIEW_SIZE = 4 # comme PREVIEW_SIZE dans themes.ts + + +def _lit(s: str) -> str: + return "'" + s.replace("'", "''") + "'" + + +def script(spec: Spec, ids: list[int], season: int, price: int, by: str, commit: bool) -> str: + data = "\n".join(str(i) for i in sorted(ids)) + return f""" +\\set ON_ERROR_STOP on +BEGIN; +DO $$ BEGIN + IF (SELECT id FROM seasons WHERE status = 'active') <> {season} THEN + RAISE EXCEPTION 'La saison active n''est plus la {season} : refais l''index.'; + END IF; +END $$; +CREATE TEMP TABLE _ids (id bigint PRIMARY KEY) ON COMMIT DROP; +COPY _ids FROM STDIN; +{data} +\\. +INSERT INTO themes (name, description, category, price, starts_at, ends_at, created_by) +VALUES ({_lit(spec.name)}, {_lit(spec.description)}, {_lit(spec.display_category)}, {price}, + {_lit(spec.starts.isoformat())}, {_lit(spec.ends.isoformat())}, + (SELECT id FROM "user" WHERE username = {_lit(by)})) +RETURNING id AS theme_id \\gset +INSERT INTO theme_cards (theme_id, card_id, rarity, pos) +SELECT :theme_id, c.id, c.rarity, + (row_number() OVER (PARTITION BY c.rarity ORDER BY c.id) - 1)::int +FROM cards c JOIN _ids i ON i.id = c.id WHERE c.season = {season}; +UPDATE themes SET + card_count = (SELECT count(*) FROM theme_cards WHERE theme_id = themes.id), + by_rarity = (SELECT jsonb_object_agg(r::text, coalesce(n, 0)) FROM unnest(enum_range(NULL::rarity)) r + LEFT JOIN (SELECT rarity, count(*)::int n FROM theme_cards WHERE theme_id = themes.id + GROUP BY rarity) x ON x.rarity = r), + preview_ids = (SELECT jsonb_agg(id) FROM ( + SELECT c.id FROM theme_cards tc JOIN cards c ON c.season = {season} AND c.id = tc.card_id + WHERE tc.theme_id = themes.id ORDER BY c.rarity DESC, c.views_12m DESC LIMIT {PREVIEW_SIZE}) p), + indexed_season = {season} +WHERE id = :theme_id; +SELECT 'theme', id, card_count, by_rarity, created_by IS NOT NULL FROM themes WHERE id = :theme_id; +{"COMMIT;" if commit else "ROLLBACK;"} +{"ANALYZE theme_cards;" if commit else ""} +""" diff --git a/tools/themes/palacards_themes/dumps.py b/tools/themes/palacards_themes/dumps.py new file mode 100644 index 0000000..88a2863 --- /dev/null +++ b/tools/themes/palacards_themes/dumps.py @@ -0,0 +1,127 @@ +"""Dumps SQL du Wikipédia FR -> TSV : pages de catégories, liens de catégories, identifiants Wikidata. + +Depuis 2025, `categorylinks` ne contient plus le nom de la catégorie : `cl_target_id` renvoie à `linktarget`. +Les dumps sont en `CHARSET=binary` : lecture en octets, ligne par ligne, avec des regex compilées +(même méthode que `tools/import/palacards_import/sqlextract.py`). +""" + +from __future__ import annotations + +import gzip +import re +import time +from pathlib import Path + +import requests + +from . import paths + +BASE = "https://dumps.wikimedia.org/frwiki/latest" +TABLES = ["page", "categorylinks", "linktarget", "page_props"] +_STR = rb"'((?:[^'\\]|\\.)*)'" +_SKIP = rb"'(?:[^'\\]|\\.)*'" +_UNESCAPE = re.compile(rb"\\(.)") + + +def _title(raw: bytes) -> str: + return _UNESCAPE.sub(rb"\1", raw).decode("utf-8", "replace").replace("_", " ") + + +def download(force: bool = False) -> None: + paths.DUMPS.mkdir(parents=True, exist_ok=True) + http = requests.Session() + http.headers["User-Agent"] = paths.USER_AGENT + for table in TABLES: + name = f"frwiki-latest-{table}.sql.gz" + dest = paths.DUMPS / name + size = int(http.head(f"{BASE}/{name}", timeout=30, allow_redirects=True).headers["Content-Length"]) + if dest.exists() and dest.stat().st_size == size and not force: + print(f" {name} : déjà là") + continue + start = time.time() + with http.get(f"{BASE}/{name}", stream=True, timeout=60) as r, open(dest, "wb") as f: + r.raise_for_status() + f.writelines(r.iter_content(1 << 20)) + if dest.stat().st_size != size: + raise SystemExit(f"{name} : taille inattendue") + print(f" {name} : {size / 1e6:.0f} Mo en {time.time() - start:.0f} s") + + +def _lines(table: str): + """Lignes de valeurs des INSERT de `table`. + + Deux formats : l'ancien (`INSERT … VALUES (…),(…);` sur une seule ligne) et celui des dumps de 2026 + (`INSERT … VALUES` seul, puis un tuple par ligne jusqu'au `;`). + """ + prefix = f"INSERT INTO `{table}` VALUES".encode() + inside = False + with gzip.open(paths.DUMPS / f"frwiki-latest-{table}.sql.gz", "rb") as f: + for line in f: + if line.startswith(prefix): + inside = True + rest = line[len(prefix):].strip() + if rest: + yield rest + inside = not rest.endswith(b";") + elif inside: + if line.startswith(b"("): + yield line + if line.rstrip().endswith(b";"): + inside = False + else: + inside = False + + +def _write(name: str, rows) -> int: + paths.RAW.mkdir(parents=True, exist_ok=True) + n = 0 + with open(paths.RAW / name, "w", encoding="utf-8") as out: + for row in rows: + out.write("\t".join(row) + "\n") + n += 1 + return n + + +def parse() -> None: + """Écrit raw/cat_pages.tsv, linktarget.tsv, cl_pages.tsv, cl_subcats.tsv, qids.tsv.""" + t = time.time() + # page : (page_id, page_namespace, page_title, …) ; seules les catégories (14) servent. + cat_page = re.compile(rb"\((\d+),14," + _STR + rb",") + n = _write("cat_pages.tsv", ((pid.decode(), _title(title)) for line in _lines("page") + for pid, title in cat_page.findall(line))) + print(f" pages de catégories : {n:,} ({time.time() - t:.0f} s)") + + t = time.time() + lt = re.compile(rb"\((\d+),14," + _STR + rb"\)") + n = _write("linktarget.tsv", ((i.decode(), _title(title)) for line in _lines("linktarget") + for i, title in lt.findall(line))) + print(f" cibles de catégories : {n:,} ({time.time() - t:.0f} s)") + + t = time.time() + # (cl_from, cl_sortkey, cl_timestamp, cl_sortkey_prefix, cl_type, cl_collation_id, cl_target_id) + cl = re.compile(rb"\((\d+)," + _SKIP + b"," + _SKIP + b"," + _SKIP + rb",'(page|subcat)',\d+,(\d+)\)", + re.DOTALL) + paths.RAW.mkdir(parents=True, exist_ok=True) + pages = subcats = 0 + with open(paths.RAW / "cl_pages.tsv", "w") as fp, open(paths.RAW / "cl_subcats.tsv", "w") as fs: + for line in _lines("categorylinks"): + for src, kind, target in cl.findall(line): + if kind == b"page": + fp.write(f"{src.decode()}\t{target.decode()}\n") + pages += 1 + else: + fs.write(f"{src.decode()}\t{target.decode()}\n") + subcats += 1 + print(f" liens : {pages:,} articles, {subcats:,} sous-catégories ({time.time() - t:.0f} s)") + + t = time.time() + pp = re.compile(rb"\((\d+),'wikibase_item','Q(\d+)'") + n = _write("qids.tsv", ((p.decode(), q.decode()) for line in _lines("page_props") + for p, q in pp.findall(line))) + print(f" identifiants Wikidata : {n:,} ({time.time() - t:.0f} s)") + + +def dump_date() -> str: + """Date de téléchargement du dump des pages (AAAA-MM-JJ).""" + mtime = Path(paths.DUMPS / "frwiki-latest-page.sql.gz").stat().st_mtime + return time.strftime("%Y-%m-%d", time.gmtime(mtime)) diff --git a/tools/themes/palacards_themes/index.py b/tools/themes/palacards_themes/index.py new file mode 100644 index 0000000..e1b29d4 --- /dev/null +++ b/tools/themes/palacards_themes/index.py @@ -0,0 +1,103 @@ +"""Construit l'index local d'une saison (DuckDB) : cartes, arbre des catégories, données Wikidata. + +Une fois par saison (choix de Palawi, 2026-11-04) : ~15 à 25 min, sans aucune requête à l'API Wikipédia. +""" + +from __future__ import annotations + +import time + +import duckdb + +from . import dumps, paths, prod, wikidata + +RARITIES = ["C", "PC", "R", "SR", "UR", "L"] + + +def connect(path=None, read_only: bool = True) -> duckdb.DuckDBPyConnection: + con = duckdb.connect(str(path or paths.current_index()), read_only=read_only) + con.execute("SET memory_limit = '4GB'; SET threads = 4;") + con.execute(f"SET temp_directory = '{paths.DATA / 'tmp'}'") + return con + + +def _tsv(name: str, columns: dict[str, str]) -> str: + cols = ", ".join(f"'{k}': '{v}'" for k, v in columns.items()) + return ( + f"read_csv('{paths.RAW / name}', delim='\\t', header=false, quote='', escape='', " + f"columns={{{cols}}}, ignore_errors=true)" + ) + + +def build(season: int | None = None, download: bool = True, wikidata_fetch: bool = True) -> None: + start = time.time() + season = season or prod.active_season() + print(f"Index de la saison {season}") + if download: + print("1/4 Dumps du Wikipédia FR") + dumps.download() + print("2/4 Analyse des dumps") + dumps.parse() + if wikidata_fetch: + print("3/4 Wikidata (QLever)") + wikidata.fetch_all() + print("4/4 Cartes de la saison et base DuckDB") + prod.export_cards(season) + dest = paths.index_db(season) + tmp = dest.with_suffix(".building") + tmp.unlink(missing_ok=True) + (paths.DATA / "tmp").mkdir(parents=True, exist_ok=True) + con = connect(tmp, read_only=False) + rank = " ".join(f"WHEN '{r}' THEN {i}" for i, r in enumerate(RARITIES)) + con.execute(f""" + CREATE TABLE cards AS + SELECT id, title, rarity, CASE rarity {rank} END::TINYINT AS r, views + FROM read_csv('{paths.RAW / 'cards.csv'}', header=false, delim=',', quote='"', escape='"', + columns={{'id': 'BIGINT', 'title': 'VARCHAR', 'rarity': 'VARCHAR', 'views': 'BIGINT'}}); + CREATE TABLE cat AS SELECT * FROM {_tsv('cat_pages.tsv', {'id': 'BIGINT', 'title': 'VARCHAR'})}; + CREATE TABLE lt AS SELECT * FROM {_tsv('linktarget.tsv', {'lt_id': 'BIGINT', 'title': 'VARCHAR'})}; + CREATE TABLE cat_edge AS + SELECT DISTINCT c.title AS child, l.title AS parent + FROM {_tsv('cl_subcats.tsv', {'src': 'BIGINT', 'lt_id': 'BIGINT'})} s + JOIN cat c ON c.id = s.src JOIN lt l ON l.lt_id = s.lt_id; + CREATE TABLE cat_member AS + SELECT DISTINCT s.src AS page_id, l.title AS cat + FROM {_tsv('cl_pages.tsv', {'src': 'BIGINT', 'lt_id': 'BIGINT'})} s + JOIN lt l ON l.lt_id = s.lt_id + WHERE s.src IN (SELECT id FROM cards); + CREATE TABLE qid AS + SELECT DISTINCT q.page_id, q.q FROM {_tsv('qids.tsv', {'page_id': 'BIGINT', 'q': 'BIGINT'})} q + WHERE q.page_id IN (SELECT id FROM cards); + """) + for prop in wikidata.ITEM_PROPS: + con.execute(f""" + CREATE TABLE {prop.lower()} AS + SELECT DISTINCT t.q, t.v FROM {_tsv(f'{prop}.tsv', {'q': 'BIGINT', 'v': 'BIGINT'})} t + WHERE t.q IN (SELECT q FROM qid) + """) + con.execute(f""" + CREATE TABLE p279 AS SELECT DISTINCT * FROM {_tsv('P279.tsv', {'c': 'BIGINT', 'p': 'BIGINT'})}; + CREATE TABLE descr AS + SELECT q, any_value(text) AS text FROM {_tsv('desc.tsv', {'q': 'BIGINT', 'text': 'VARCHAR'})} + WHERE q IN (SELECT q FROM qid) GROUP BY q; + CREATE TABLE labels AS + SELECT q, any_value(text) AS label FROM ( + SELECT * FROM {_tsv('labels_classes.tsv', {'q': 'BIGINT', 'text': 'VARCHAR'})} + UNION ALL SELECT * FROM {_tsv('labels_values.tsv', {'q': 'BIGINT', 'text': 'VARCHAR'})} + ) GROUP BY q; + CREATE TABLE art AS + SELECT c.id, c.title, c.rarity, c.r, c.views, q.q, d.text AS descr, + EXISTS (SELECT 1 FROM p31 WHERE p31.q = q.q AND p31.v = 5) AS person + FROM cards c LEFT JOIN (SELECT page_id, min(q) AS q FROM qid GROUP BY page_id) q ON q.page_id = c.id + LEFT JOIN descr d ON d.q = q.q; + CREATE INDEX cat_edge_parent ON cat_edge(parent); + CREATE INDEX cat_member_cat ON cat_member(cat); + CREATE TABLE meta AS SELECT * FROM (VALUES + ('season', '{season}'), ('dump', '{dumps.dump_date()}'), + ('built', strftime(now(), '%Y-%m-%d %H:%M'))) t(key, value); + """) + for table in ["cards", "cat_edge", "cat_member", "qid", "p31", "p279", "descr", "labels"]: + print(f" {table} : {con.execute(f'SELECT count(*) FROM {table}').fetchone()[0]:,}") + con.close() + tmp.rename(dest) + print(f"Index prêt : {dest} ({time.time() - start:.0f} s)") diff --git a/tools/themes/palacards_themes/paths.py b/tools/themes/palacards_themes/paths.py new file mode 100644 index 0000000..e1a4552 --- /dev/null +++ b/tools/themes/palacards_themes/paths.py @@ -0,0 +1,26 @@ +"""Emplacements des données (hors dépôt : plusieurs Go).""" + +from __future__ import annotations + +import os +from pathlib import Path + +DATA = Path(os.environ.get("PALACARDS_THEMES_DATA", "/srv/nas/palacards-themes/data")) +DUMPS = DATA / "dumps" +RAW = DATA / "raw" # TSV intermédiaires (dumps analysés, Wikidata) +THEMES = DATA / "themes" # un dossier par booster : fiche, aperçu, ids, SQL + + +def index_db(season: int) -> Path: + return DATA / f"index-s{season}.duckdb" + + +def current_index() -> Path: + """Index le plus récent (saison la plus haute).""" + found = sorted(DATA.glob("index-s*.duckdb"), key=lambda p: int(p.stem.split("-s")[1])) + if not found: + raise SystemExit("Aucun index : lance d'abord `palacards-themes index`.") + return found[-1] + + +USER_AGENT = "PalaCards-themes/0.1 (https://github.com/Palawizard/palacards)" diff --git a/tools/themes/palacards_themes/prod.py b/tools/themes/palacards_themes/prod.py new file mode 100644 index 0000000..2431a7b --- /dev/null +++ b/tools/themes/palacards_themes/prod.py @@ -0,0 +1,44 @@ +"""Accès à la base de prod (vm-apps) par SSH : export des cartes, exécution d'un script SQL.""" + +from __future__ import annotations + +import subprocess + +from . import paths + +STACK = "/opt/dockpanel/stacks/palacards" +PSQL = f"cd {STACK} && docker compose exec -T postgres psql -U palacards -d palacards -v ON_ERROR_STOP=1 -X -q" + + +def psql(script: str, timeout: int = 600) -> str: + """Exécute un script SQL (avec d'éventuelles données COPY … FROM STDIN) ; renvoie la sortie.""" + r = subprocess.run( + ["ssh", "vm-apps", PSQL + " -At"], input=script.encode(), capture_output=True, timeout=timeout, check=False + ) + if r.returncode != 0: + raise SystemExit(f"psql : {r.stderr.decode(errors='replace').strip()}") + return r.stdout.decode() + + +def active_season() -> int: + return int(psql("select id from seasons where status = 'active';").strip()) + + +def export_cards(season: int) -> None: + """raw/cards.csv : id, titre, rareté, vues sur 12 mois des cartes de la saison.""" + paths.RAW.mkdir(parents=True, exist_ok=True) + query = ( + f"\\copy (select id, title, rarity, views_12m from cards where season = {season}) " + "to stdout with (format csv)" + ) + with open(paths.RAW / "cards.csv", "wb") as out: + r = subprocess.run(["ssh", "vm-apps", PSQL + " -At"], input=query.encode(), stdout=out, timeout=900, check=False) + if r.returncode != 0: + raise SystemExit("Export des cartes impossible.") + + +def backup() -> None: + """Sauvegarde de la base avant une création (service habituel de vm-apps).""" + r = subprocess.run(["ssh", "vm-apps", "systemctl start palacards-backup.service"], timeout=900, check=False) + if r.returncode != 0: + raise SystemExit("Sauvegarde impossible : création annulée.") diff --git a/tools/themes/palacards_themes/report.py b/tools/themes/palacards_themes/report.py new file mode 100644 index 0000000..f9633d9 --- /dev/null +++ b/tools/themes/palacards_themes/report.py @@ -0,0 +1,133 @@ +"""Aperçu d'une fiche : chiffres par rareté et par famille, légendaires, bruit probable, échantillons.""" + +from __future__ import annotations + +import duckdb + +from .index import RARITIES +from .select import Selection +from .spec import Spec + +# Mots de description qui signalent souvent un hors-sujet : à regarder, pas à exclure d'office. +NOISE = { + "film": r"\bfilm\b", + "série télévisée": r"série télévisée|feuilleton", + "album / chanson": r"\balbum\b|chanson|\bsingle\b", + "épisode": r"épisode", + "liste": r"^liste\b|\bliste de\b", + "commune / ville": r"\bcommune\b|\bvillage\b|\bville\b", + "footballeur": r"footballeu", + "homme / femme politique": r"(homme|femme|personnalité) politique", + "acteur / actrice": r"\bact(eur|rice)\b", + "chanteur / chanteuse": r"chanteu(r|se)", + "personnage de fiction": r"personnage de fiction|personnage de", + "entreprise": r"entreprise|société", + "taxon / espèce": r"espèce|\bgenre de\b|taxon", + "astéroïde / galaxie": r"astéroïde|galaxie", +} + + +def _md_table(headers: list[str], rows: list[list]) -> str: + out = ["| " + " | ".join(headers) + " |", "|" + "---|" * len(headers)] + out += ["| " + " | ".join(str(c).replace("|", "/") for c in r) + " |" for r in rows] + return "\n".join(out) + + +def _counts(con, ids: list[int]) -> dict[str, int]: + con.execute("CREATE OR REPLACE TEMP TABLE _sel AS SELECT unnest(?::BIGINT[]) AS id", [ids]) + rows = con.execute("SELECT a.rarity, count(*) FROM art a JOIN _sel USING (id) GROUP BY 1").fetchall() + found = dict(rows) + return {r: found.get(r, 0) for r in RARITIES} + + +def price_hint(legendaries: int) -> str: + # Règle de Palawi (2026-10-19) : assez de légendaires -> 200 PW, sinon 150 ; jamais sous 150 + # (valeur de recyclage moyenne d'un booster à thème ~126 PW). + return "200 PW" if legendaries >= 10 else "150 PW (ou 200 + légendaires ajoutées)" + + +def preview(con: duckdb.DuckDBPyConnection, spec: Spec, sel: Selection, season: int, dump: str) -> tuple[str, str]: + """(rapport complet en Markdown, résumé court).""" + ids = sorted(sel.ids) + total = _counts(con, ids) + con.execute("CREATE OR REPLACE TEMP TABLE _fam AS SELECT unnest(?::BIGINT[]) AS id, unnest(?::VARCHAR[]) AS fam", + [list(sel.family_of), list(sel.family_of.values())]) + lines = [f"# Aperçu : {spec.name}", "", + f"Saison {season} · index du dump {dump} · fiche `{spec.path}`", "", + (f"**{len(ids):,} cartes** · prix conseillé : {price_hint(total['L'])}" + f"{f' · prix de la fiche : {spec.price} PW' if spec.price else ''}" + f" · du {spec.starts:%Y-%m-%d %H:%M} au {spec.ends:%Y-%m-%d %H:%M} UTC"), "", + _md_table(RARITIES + ["Total"], [[f"{total[r]:,}" for r in RARITIES] + [f"{len(ids):,}"]]), ""] + if sel.missing_include: + lines += [f"**Ajouts introuvables dans les cartes** : {', '.join(sel.missing_include)}", ""] + if sel.excluded: + lines += ["Exclusions globales : " + ", ".join(f"{k} −{v:,}" for k, v in sel.excluded.items()), ""] + + # Familles + rows = [] + for fr in sel.families: + mine = [i for i, f in sel.family_of.items() if f == fr.family.name] + c = _counts(con, mine) + rows.append([fr.family.name, " + ".join(f"{k} {v:,}" for k, v in fr.sources.items()), + ", ".join(f"{k} −{v:,}" for k, v in fr.dropped.items()) or "—", + *[f"{c[r]:,}" for r in RARITIES], f"{len(mine):,}"]) + lines += ["## Familles", "", "Une carte compte dans la première famille qui la contient.", "", + _md_table(["Famille", "Sources", "Retirées", *RARITIES, "Total"], rows), ""] + for fr in sel.families: + warn = [] + if fr.missing_categories: + warn.append(f"catégories inconnues : {', '.join(fr.missing_categories)}") + if fr.missing_titles: + warn.append(f"titres introuvables : {', '.join(fr.missing_titles)}") + if warn: + lines += [f"- **{fr.family.name}** : " + " ; ".join(warn)] + lines.append("") + + # Légendaires et UR + leg = con.execute("""SELECT a.title, coalesce(a.descr, ''), f.fam FROM art a JOIN _fam f USING (id) + WHERE a.rarity = 'L' ORDER BY a.views DESC""").fetchall() + lines += [f"## Légendaires ({len(leg)})", "", _md_table(["Carte", "Description", "Famille"], leg) if leg + else "Aucune.", ""] + ur = con.execute("""SELECT a.title, coalesce(a.descr, ''), f.fam FROM art a JOIN _fam f USING (id) + WHERE a.rarity = 'UR' ORDER BY a.views DESC LIMIT 80""").fetchall() + lines += [f"## UR ({total['UR']:,}, les 80 plus lues)", "", _md_table(["Carte", "Description", "Famille"], ur), ""] + + # Bruit probable + lines += ["## À vérifier (descriptions souvent hors sujet)", ""] + noise_rows = [] + for label, rx in NOISE.items(): + hits = con.execute(f"""SELECT a.title, a.rarity FROM art a JOIN _fam USING (id) + WHERE regexp_matches(a.descr, '(?i){rx}') ORDER BY a.views DESC""").fetchall() + if hits: + noise_rows.append([label, f"{len(hits):,}", ", ".join(f"{t} ({r})" for t, r in hits[:6])]) + lines += [_md_table(["Motif", "Cartes", "Exemples (les plus lues)"], noise_rows) if noise_rows else "Rien.", ""] + + # Par famille : natures, descriptions, catégories qui apportent le plus, échantillons + for fr in sel.families: + name = fr.family.name + lines += [f"## {name}", ""] + top31 = con.execute("""SELECT coalesce(l.label, 'Q' || p.v), count(*) n FROM _fam f JOIN art a USING (id) + JOIN p31 p ON p.q = a.q LEFT JOIN labels l ON l.q = p.v + WHERE f.fam = ? GROUP BY 1 ORDER BY n DESC LIMIT 12""", [name]).fetchall() + if top31: + lines += ["Natures (P31) : " + ", ".join(f"{label} {n:,}" for label, n in top31), ""] + heads = con.execute("""SELECT array_to_string(list_slice(string_split(lower(a.descr), ' '), 1, 3), ' ') h, + count(*) n FROM _fam f JOIN art a USING (id) + WHERE f.fam = ? AND a.descr IS NOT NULL GROUP BY 1 ORDER BY n DESC LIMIT 15""", + [name]).fetchall() + if heads: + lines += ["Débuts de description : " + ", ".join(f"« {h} » {n:,}" for h, n in heads), ""] + if fr.tree: + cats = con.execute("""SELECT m.cat, count(DISTINCT m.page_id) n FROM cat_member m + JOIN _fam f ON f.id = m.page_id AND f.fam = ? + WHERE m.cat IN (SELECT unnest(?::VARCHAR[])) GROUP BY 1 ORDER BY n DESC LIMIT 20""", + [name, list(fr.tree)]).fetchall() + lines += ["Catégories qui apportent le plus : " + + ", ".join(f"{c} ({fr.tree[c]}) {n:,}" for c, n in cats), ""] + sample = con.execute("""SELECT a.title, a.rarity, coalesce(a.descr, '—') FROM _fam f JOIN art a USING (id) + WHERE f.fam = ? ORDER BY hash(a.id) LIMIT 15""", [name]).fetchall() + lines += [_md_table(["Échantillon", "Rareté", "Description"], sample), ""] + + summary = (f"{spec.name} : {len(ids):,} cartes (" + + ", ".join(f"{r} {total[r]:,}" for r in RARITIES) + f") · prix conseillé {price_hint(total['L'])}") + return "\n".join(lines), summary diff --git a/tools/themes/palacards_themes/select.py b/tools/themes/palacards_themes/select.py new file mode 100644 index 0000000..359a44f --- /dev/null +++ b/tools/themes/palacards_themes/select.py @@ -0,0 +1,206 @@ +"""Sélection des cartes d'une fiche dans l'index local (DuckDB), avec le détail de chaque étape.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import duckdb + +from .index import RARITIES +from .spec import CLASS_PROPS, Categories, Family, Spec + + +def _rx(patterns: list[str]) -> str: + """Plusieurs motifs en une regex RE2 insensible à la casse.""" + return "(?i)" + "|".join(f"(?:{p})" for p in patterns) + + +def subclasses(con: duckdb.DuckDBPyConnection, qids: list[int]) -> list[int]: + """Les classes et toutes leurs sous-classes (P279*), cycles compris.""" + if not qids: + return [] + rows = con.execute( + """WITH RECURSIVE sub(q) AS ( + SELECT unnest(?::BIGINT[]) + UNION SELECT p.c FROM p279 p JOIN sub ON p.p = sub.q) + SELECT q FROM sub""", + [qids], + ).fetchall() + return [r[0] for r in rows] + + +def category_tree(con: duckdb.DuckDBPyConnection, cats: list[Categories]) -> dict[str, int]: + """Catégories et sous-catégories jusqu'à la profondeur demandée -> profondeur atteinte.""" + seen: dict[str, int] = {} + for c in cats: + rows = con.execute( + """WITH RECURSIVE t(title, d) AS ( + SELECT unnest(?::VARCHAR[]), 0 + UNION SELECT e.child, t.d + 1 FROM t JOIN cat_edge e ON e.parent = t.title WHERE t.d < ?) + SELECT title, min(d) FROM t GROUP BY title""", + [c.names, c.depth], + ).fetchall() + for title, d in rows: + seen[title] = min(d, seen.get(title, d)) + return seen + + +def _wikidata_ids(con, props: dict[str, list[int]], exact: bool) -> set[int]: + ids: set[int] = set() + for prop, values in props.items(): + targets = values if exact or prop not in CLASS_PROPS else subclasses(con, values) + rows = con.execute( + f"SELECT DISTINCT a.id FROM art a JOIN {prop.lower()} t ON t.q = a.q WHERE t.v IN (SELECT unnest(?::BIGINT[]))", + [targets], + ).fetchall() + ids.update(r[0] for r in rows) + return ids + + +def _category_ids(con, cats: list[Categories]) -> tuple[set[int], dict[str, int]]: + tree = category_tree(con, cats) + if not tree: + return set(), {} + rows = con.execute( + "SELECT DISTINCT page_id FROM cat_member WHERE cat IN (SELECT unnest(?::VARCHAR[]))", [list(tree)] + ).fetchall() + return {r[0] for r in rows}, tree + + +def _ids_where(con, ids: set[int], where: str, params: list) -> set[int]: + """Parmi `ids`, ceux de `art` qui vérifient `where`.""" + if not ids: + return set() + con.execute("CREATE OR REPLACE TEMP TABLE _ids AS SELECT unnest(?::BIGINT[]) AS id", [list(ids)]) + rows = con.execute(f"SELECT a.id FROM art a JOIN _ids USING (id) WHERE {where}", params).fetchall() + return {r[0] for r in rows} + + +@dataclass +class FamilyResult: + family: Family + sources: dict[str, int] = field(default_factory=dict) + dropped: dict[str, int] = field(default_factory=dict) + ids: set[int] = field(default_factory=set) + tree: dict[str, int] = field(default_factory=dict) + missing_titles: list[str] = field(default_factory=list) + missing_categories: list[str] = field(default_factory=list) + + +def _titles(con, titles: list[str]) -> tuple[set[int], list[str]]: + if not titles: + return set(), [] + rows = con.execute( + "SELECT id, title FROM art WHERE title IN (SELECT unnest(?::VARCHAR[]))", [titles] + ).fetchall() + found = {t for _, t in rows} + return {i for i, _ in rows}, [t for t in titles if t not in found] + + +def family(con: duckdb.DuckDBPyConnection, fam: Family) -> FamilyResult: + res = FamilyResult(fam) + ids: set[int] = set() + if fam.wikidata: + w = _wikidata_ids(con, fam.wikidata, fam.exact) + res.sources["wikidata"] = len(w) + ids |= w + if fam.categories: + c, res.tree = _category_ids(con, fam.categories) + res.sources[f"catégories ({len(res.tree)})"] = len(c) + known = {r[0] for r in con.execute( + "SELECT DISTINCT parent FROM cat_edge WHERE parent IN (SELECT unnest(?::VARCHAR[])) " + "UNION SELECT DISTINCT cat FROM cat_member WHERE cat IN (SELECT unnest(?::VARCHAR[]))", + [[n for cs in fam.categories for n in cs.names]] * 2).fetchall()} + res.missing_categories = [n for cs in fam.categories for n in cs.names if n not in known] + ids |= c + if fam.title_regex: + rows = con.execute("SELECT id FROM art WHERE regexp_matches(title, ?)", [_rx(fam.title_regex)]).fetchall() + res.sources["motifs de titre"] = len(rows) + ids |= {r[0] for r in rows} + if fam.titles: + t, res.missing_titles = _titles(con, fam.titles) + res.sources["titres"] = len(t) + forced = t + else: + forced = set() + ids |= forced + + def drop(label: str, where: str, params: list) -> None: + nonlocal ids + gone = _ids_where(con, ids - forced, where, params) + if gone: + res.dropped[label] = len(gone) + ids -= gone + + rank = RARITIES.index(fam.min_rarity) + if rank: + drop(f"rareté < {fam.min_rarity}", "a.r < ?", [rank]) + if fam.people == "only": + drop("pas une personne", "NOT a.person", []) + elif fam.people == "none": + drop("personne", "a.person", []) + if fam.keep_desc: + cond = "NOT coalesce(regexp_matches(a.descr, ?), false)" + if fam.keep_no_desc: + cond += " AND a.descr IS NOT NULL" + drop("description hors sujet", cond, [_rx(fam.keep_desc)]) + if fam.drop_desc: + drop("description exclue", "coalesce(regexp_matches(a.descr, ?), false)", [_rx(fam.drop_desc)]) + if fam.drop_title_regex: + drop("titre exclu", "regexp_matches(a.title, ?)", [_rx(fam.drop_title_regex)]) + if fam.drop_wikidata: + gone = _wikidata_ids(con, fam.drop_wikidata, fam.exact) & (ids - forced) + if gone: + res.dropped["classe exclue"] = len(gone) + ids -= gone + if fam.drop_categories: + gone = _category_ids(con, fam.drop_categories)[0] & (ids - forced) + if gone: + res.dropped["catégorie exclue"] = len(gone) + ids -= gone + res.ids = ids + return res + + +@dataclass +class Selection: + families: list[FamilyResult] + family_of: dict[int, str] + excluded: dict[str, int] + included: set[int] + missing_include: list[str] + ids: set[int] + + +def select(con: duckdb.DuckDBPyConnection, spec: Spec) -> Selection: + results = [family(con, f) for f in spec.families] + family_of: dict[int, str] = {} + for r in results: + for i in r.ids: + family_of.setdefault(i, r.family.name) + ids = set(family_of) + excluded: dict[str, int] = {} + + def exclude(label: str, gone: set[int]) -> None: + nonlocal ids + gone &= ids + if gone: + excluded[label] = len(gone) + ids -= gone + + if spec.exclude_titles: + exclude("titres", _titles(con, spec.exclude_titles)[0]) + if spec.exclude_title_regex: + exclude("motifs de titre", _ids_where(con, ids, "regexp_matches(a.title, ?)", [_rx(spec.exclude_title_regex)])) + if spec.exclude_desc: + exclude("descriptions", _ids_where(con, ids, "coalesce(regexp_matches(a.descr, ?), false)", + [_rx(spec.exclude_desc)])) + if spec.exclude_wikidata: + exclude("classes", _wikidata_ids(con, spec.exclude_wikidata, False)) + if spec.exclude_categories: + exclude("catégories", _category_ids(con, spec.exclude_categories)[0]) + included, missing = _titles(con, spec.include) + for i in included: + family_of.setdefault(i, "Ajouts") + ids |= included + return Selection(results, {i: family_of[i] for i in ids}, excluded, included, missing, ids) diff --git a/tools/themes/palacards_themes/spec.py b/tools/themes/palacards_themes/spec.py new file mode 100644 index 0000000..16f0048 --- /dev/null +++ b/tools/themes/palacards_themes/spec.py @@ -0,0 +1,179 @@ +"""Fiche d'un booster à thème (YAML) : voir `exemple.yaml` à côté du paquet. + +Un booster = des familles (jeux, personnages, consoles…). Chaque famille réunit des sources +(classes Wikidata, catégories Wikipédia, motifs de titre, titres) puis applique ses filtres +(rareté minimale, personnes, descriptions). Les exclusions et ajouts globaux viennent après. +""" + +from __future__ import annotations + +import re +import unicodedata +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import yaml + +from .index import RARITIES + +PROPS = {"P31", "P106", "P641", "P27", "P17", "P136"} +# Propriétés dont les valeurs sont des classes : on prend aussi leurs sous-classes (P279*). +CLASS_PROPS = {"P31", "P106", "P641", "P136"} + + +def _list(v) -> list: + if v is None: + return [] + return v if isinstance(v, list) else [v] + + +def _qids(v) -> dict[str, list[int]]: + out: dict[str, list[int]] = {} + for prop, values in (v or {}).items(): + prop = str(prop).upper() + if prop not in PROPS: + raise SystemExit(f"Propriété Wikidata non indexée : {prop} (connues : {', '.join(sorted(PROPS))})") + out[prop] = [int(str(q).upper().lstrip("Q")) for q in _list(values)] + return out + + +def category_name(name: str) -> str: + name = re.sub(r"^(Catégorie|Category)\s*:", "", str(name).strip(), flags=re.IGNORECASE).replace("_", " ").strip() + return name[:1].upper() + name[1:] + + +def title_name(title: str) -> str: + title = str(title).strip().replace("_", " ") + return title[:1].upper() + title[1:] + + +@dataclass +class Categories: + names: list[str] + depth: int + + +def _categories(v, default_depth: int) -> list[Categories]: + out: list[Categories] = [] + for item in _list(v): + if isinstance(item, dict): + out.append(Categories([category_name(n) for n in _list(item.get("name") or item.get("names"))], + int(item.get("depth", default_depth)))) + else: + out.append(Categories([category_name(item)], default_depth)) + return out + + +@dataclass +class Family: + name: str + wikidata: dict[str, list[int]] = field(default_factory=dict) + exact: bool = False # sans les sous-classes + categories: list[Categories] = field(default_factory=list) + title_regex: list[str] = field(default_factory=list) + titles: list[str] = field(default_factory=list) + min_rarity: str = "C" + people: str = "any" # any | only | none + keep_desc: list[str] = field(default_factory=list) + keep_no_desc: bool = True + drop_desc: list[str] = field(default_factory=list) + drop_title_regex: list[str] = field(default_factory=list) + drop_wikidata: dict[str, list[int]] = field(default_factory=dict) + drop_categories: list[Categories] = field(default_factory=list) + + +@dataclass +class Spec: + path: Path + name: str + description: str + display_category: str + price: int | None + starts: datetime + ends: datetime + families: list[Family] + exclude_titles: list[str] + exclude_title_regex: list[str] + exclude_desc: list[str] + exclude_wikidata: dict[str, list[int]] + exclude_categories: list[Categories] + include: list[str] + + @property + def slug(self) -> str: + s = unicodedata.normalize("NFKD", self.name).encode("ascii", "ignore").decode().lower() + return re.sub(r"[^a-z0-9]+", "-", s).strip("-") + + +def _date(v, default: datetime) -> datetime: + if v in (None, "now", "maintenant"): + return default + if isinstance(v, datetime): + return v if v.tzinfo else v.replace(tzinfo=UTC) + d = datetime.fromisoformat(str(v)) + return d if d.tzinfo else d.replace(tzinfo=UTC) + + +def _check_regex(patterns: list[str], where: str) -> list[str]: + for p in patterns: + try: + re.compile(p) + except re.error as err: + raise SystemExit(f"Regex invalide dans {where} : {p!r} ({err})") from err + return patterns + + +def load(path: str | Path) -> Spec: + path = Path(path) + raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + if not raw.get("name"): + raise SystemExit("La fiche doit avoir un `name`.") + families = [] + for i, f in enumerate(_list(raw.get("families"))): + depth = int(f.get("depth", 1)) + fam = Family( + name=f.get("name") or f"Famille {i + 1}", + wikidata=_qids(f.get("wikidata")), + exact=bool(f.get("exact", False)), + categories=_categories(f.get("categories"), depth), + title_regex=_check_regex(_list(f.get("title_regex")), f"{f.get('name')}.title_regex"), + titles=[title_name(t) for t in _list(f.get("titles"))], + min_rarity=str(f.get("min_rarity", "C")).upper(), + people=str(f.get("people", "any")), + keep_desc=_check_regex(_list(f.get("keep_desc")), f"{f.get('name')}.keep_desc"), + keep_no_desc=bool(f.get("keep_no_desc", True)), + drop_desc=_check_regex(_list(f.get("drop_desc")), f"{f.get('name')}.drop_desc"), + drop_title_regex=_check_regex(_list(f.get("drop_title_regex")), f"{f.get('name')}.drop_title_regex"), + drop_wikidata=_qids(f.get("drop_wikidata")), + drop_categories=_categories(f.get("drop_categories"), 0), + ) + if fam.min_rarity not in RARITIES: + raise SystemExit(f"{fam.name} : min_rarity parmi {RARITIES}") + if fam.people not in {"any", "only", "none"}: + raise SystemExit(f"{fam.name} : people vaut any, only ou none") + families.append(fam) + if not families and not raw.get("include"): + raise SystemExit("La fiche n'a aucune famille.") + now = datetime.now(UTC).replace(second=0, microsecond=0) + starts = _date(raw.get("starts"), now) + ends = _date(raw.get("ends"), starts + timedelta(days=int(raw.get("days", 7)))) + if ends <= starts: + raise SystemExit("`ends` doit être après `starts`.") + ex = raw.get("exclude") or {} + return Spec( + path=path, + name=str(raw["name"]), + description=str(raw.get("description", "")), + display_category=str(raw.get("display_category") or "|".join(f.name for f in families)), + price=int(raw["price"]) if raw.get("price") else None, + starts=starts, + ends=ends, + families=families, + exclude_titles=[title_name(t) for t in _list(ex.get("titles"))], + exclude_title_regex=_check_regex(_list(ex.get("title_regex")), "exclude.title_regex"), + exclude_desc=_check_regex(_list(ex.get("desc")), "exclude.desc"), + exclude_wikidata=_qids(ex.get("wikidata")), + exclude_categories=_categories(ex.get("categories"), 0), + include=[title_name(t) for t in _list(raw.get("include"))], + ) diff --git a/tools/themes/palacards_themes/wikidata.py b/tools/themes/palacards_themes/wikidata.py new file mode 100644 index 0000000..394e49f --- /dev/null +++ b/tools/themes/palacards_themes/wikidata.py @@ -0,0 +1,140 @@ +"""Données Wikidata des articles du Wikipédia FR, en quelques requêtes en bloc sur QLever. + +QLever (Université de Fribourg) renvoie des millions de lignes en quelques secondes là où le +Wikidata Query Service expire au bout de 60 s : 2,9 millions de descriptions en ~20 s, 3,8 millions +de P31 en ~15 s (mesuré le 2026-11-04). Les requêtes transitives (`P31/P279*`) peuvent dépasser la +mémoire du serveur public : on télécharge donc P279 en entier et on fait la fermeture en local. +""" + +from __future__ import annotations + +import re +import time + +import requests + +from . import paths + +ENDPOINT = "https://qlever.dev/api/wikidata" +PREFIXES = """PREFIX schema: +PREFIX wdt: +PREFIX rdfs: +""" +FRWIKI = "?a schema:about ?item ; schema:isPartOf ." +# Propriétés gardées pour chaque article : nature, métier, sport, nationalité, pays, genre. +ITEM_PROPS = ["P31", "P106", "P641", "P27", "P17", "P136"] +_Q = re.compile(r"") + + +def _fetch(name: str, query: str, attempts: int = 6) -> None: + """Résultat TSV de la requête dans raw/.qlever.tsv (brut).""" + paths.RAW.mkdir(parents=True, exist_ok=True) + dest = paths.RAW / f"{name}.qlever.tsv" + for attempt in range(1, attempts + 1): + t = time.time() + try: + with requests.post( + ENDPOINT, + data={"query": PREFIXES + query}, + headers={"Accept": "text/tab-separated-values", "User-Agent": paths.USER_AGENT}, + stream=True, + timeout=(30, 600), + ) as r: + r.raise_for_status() + with open(dest, "wb") as f: + f.writelines(r.iter_content(1 << 20)) + with open(dest, "rb") as f: + head = f.read(200) + f.seek(max(0, dest.stat().st_size - 1000)) + tail = f.read() + if head.lstrip().startswith(b"{"): + raise RuntimeError(head.decode("utf-8", "replace")) + # Erreur en cours d'export (mémoire du serveur public) : QLever l'ajoute à la fin du résultat. + if b"An error has occurred while exporting" in tail: + raise RuntimeError(tail.decode("utf-8", "replace").strip().splitlines()[-1]) + print(f" {name} : {dest.stat().st_size / 1e6:.0f} Mo en {time.time() - t:.0f} s") + return + except (requests.RequestException, RuntimeError) as err: + print(f" {name} : échec ({err}), essai {attempt}/{attempts}") + time.sleep(30 * attempt) + raise SystemExit(f"QLever : impossible de récupérer {name}") + + +def _pairs(name: str, out: str) -> int: + """Convertit deux colonnes d'entités en TSV d'entiers (Q123 -> 123).""" + n = 0 + with open(paths.RAW / f"{name}.qlever.tsv", encoding="utf-8") as f, open(paths.RAW / out, "w") as w: + next(f) + for line in f: + ids = _Q.findall(line) + if len(ids) == 2: + w.write(f"{ids[0]}\t{ids[1]}\n") + n += 1 + return n + + +def _texts(name: str, out: str) -> int: + """Entité + texte en français ("…"@fr) -> TSV (texte sans tabulation).""" + n = 0 + with open(paths.RAW / f"{name}.qlever.tsv", encoding="utf-8") as f, open(paths.RAW / out, "w") as w: + next(f) + for line in f: + col = line.rstrip("\n").split("\t", 1) + m = _Q.match(col[0]) + if not m or len(col) < 2: + continue + text = col[1] + if text.startswith('"'): + text = text[1 : text.rfind('"')] + text = text.replace("\\\"", '"').replace("\\\\", "\\").replace("\t", " ") + w.write(f"{m.group(1)}\t{text}\n") + n += 1 + return n + + +def fetch_all() -> None: + for prop in ITEM_PROPS: + _fetch(prop, f"SELECT ?item ?v WHERE {{ {FRWIKI} ?item wdt:{prop} ?v . }}") + print(f" {_pairs(prop, f'{prop}.tsv'):,} lignes") + _fetch("P279", "SELECT ?c ?p WHERE { ?c wdt:P279 ?p . }") + print(f" {_pairs('P279', 'P279.tsv'):,} lignes") + _fetch("desc", f'SELECT ?item ?d WHERE {{ {FRWIKI} ?item schema:description ?d . FILTER(LANG(?d) = "fr") }}') + print(f" {_texts('desc', 'desc.tsv'):,} descriptions") + # Libellés des classes (sujets et objets de P279) et des valeurs des propriétés des articles. + _fetch( + "labels_classes", + 'SELECT ?v ?l WHERE { { SELECT DISTINCT ?v WHERE { { ?c wdt:P279 ?v } UNION { ?v wdt:P279 ?x } } } ' + '?v rdfs:label ?l . FILTER(LANG(?l) = "fr") }', + ) + n = _texts("labels_classes", "labels_classes.tsv") + values = " UNION ".join(f"{{ {FRWIKI} ?item wdt:{p} ?v . }}" for p in ["P106", "P641", "P27", "P17", "P136"]) + _fetch( + "labels_values", + f'SELECT ?v ?l WHERE {{ {{ SELECT DISTINCT ?v WHERE {{ {values} }} }} ?v rdfs:label ?l . ' + 'FILTER(LANG(?l) = "fr") }', + ) + n += _texts("labels_values", "labels_values.tsv") + print(f" {n:,} libellés") + + +def labels(qids: list[int]) -> dict[int, str]: + """Libellés français de quelques entités (aperçu, recherche) : une petite requête.""" + out: dict[int, str] = {} + for i in range(0, len(qids), 400): + chunk = " ".join(f"wd:Q{q}" for q in qids[i : i + 400]) + r = requests.post( + ENDPOINT, + data={ + "query": PREFIXES + "PREFIX wd: \n" + f'SELECT ?v ?l WHERE {{ VALUES ?v {{ {chunk} }} ?v rdfs:label ?l . FILTER(LANG(?l) = "fr") }}' + }, + headers={"Accept": "text/tab-separated-values", "User-Agent": paths.USER_AGENT}, + timeout=60, + ) + r.raise_for_status() + for line in r.text.splitlines()[1:]: + col = line.split("\t", 1) + m = _Q.match(col[0]) + if m and len(col) == 2: + out[int(m.group(1))] = col[1][1 : col[1].rfind('"')] + return out diff --git a/tools/themes/pyproject.toml b/tools/themes/pyproject.toml new file mode 100644 index 0000000..aca007c --- /dev/null +++ b/tools/themes/pyproject.toml @@ -0,0 +1,26 @@ +[project] +name = "palacards-themes" +version = "0.1.0" +description = "Atelier des boosters à thème de PalaCards : index local (catégories, Wikidata), fiche YAML, aperçu, création" +requires-python = ">=3.11" +dependencies = [ + "duckdb>=1.2", + "pyyaml>=6", + "requests>=2.32", +] + +[project.optional-dependencies] +dev = ["pytest>=8", "ruff>=0.8"] + +[project.scripts] +palacards-themes = "palacards_themes.cli:main" + +[build-system] +requires = ["setuptools>=69"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = ["palacards_themes"] + +[tool.ruff] +line-length = 110 diff --git a/tools/themes/tests/test_select.py b/tools/themes/tests/test_select.py new file mode 100644 index 0000000..02636e1 --- /dev/null +++ b/tools/themes/tests/test_select.py @@ -0,0 +1,123 @@ +"""Sélection d'une fiche sur un petit index fabriqué à la main.""" + +from __future__ import annotations + +import textwrap + +import duckdb +import pytest + +from palacards_themes import create, report, spec +from palacards_themes.select import category_tree, select, subclasses + + +@pytest.fixture() +def con(): + c = duckdb.connect() + c.execute(""" + CREATE TABLE cards AS SELECT * FROM (VALUES + (1, 'Tetris', 'SR', 3, 50000), (2, 'Mario', 'L', 5, 900000), (3, 'Super Mario Bros. (film)', 'UR', 4, 80000), + (4, 'Pac-Man', 'PC', 1, 900), (5, 'Joueur obscur', 'C', 0, 10), (6, 'Squeezie', 'UR', 4, 70000), + (7, 'Paris', 'L', 5, 2000000), (8, 'Jeu de go', 'R', 2, 5000)) t(id, title, rarity, r, views); + CREATE TABLE cat_edge AS SELECT * FROM (VALUES + ('Jeu de réflexion', 'Jeu vidéo'), ('Jeu vidéo', 'Jeu de réflexion'), ('Jeu de plateau', 'Jeu')) t(child, parent); + CREATE TABLE cat_member AS SELECT * FROM (VALUES + (1, 'Jeu de réflexion'), (4, 'Jeu vidéo'), (8, 'Jeu de plateau')) t(page_id, cat); + CREATE TABLE qid AS SELECT * FROM (VALUES (1, 101), (2, 102), (3, 103), (4, 104), (5, 105), (6, 106)) + t(page_id, q); + CREATE TABLE p31 AS SELECT * FROM (VALUES + (101, 7889), (102, 1569167), (103, 11424), (104, 900), (105, 5), (106, 5)) t(q, v); + CREATE TABLE p106 AS SELECT * FROM (VALUES (105, 4379701), (106, 57414145)) t(q, v); + CREATE TABLE p641 (q BIGINT, v BIGINT); + CREATE TABLE p27 (q BIGINT, v BIGINT); + CREATE TABLE p17 (q BIGINT, v BIGINT); + CREATE TABLE p136 (q BIGINT, v BIGINT); + -- 900 (jeu d'arcade) est une sous-classe de jeu vidéo ; cycle 7889 <-> 901 pour la fermeture. + CREATE TABLE p279 AS SELECT * FROM (VALUES (900, 7889), (901, 7889), (7889, 901)) t(c, p); + CREATE TABLE labels AS SELECT * FROM (VALUES (7889, 'jeu vidéo'), (5, 'être humain')) t(q, label); + CREATE TABLE descr AS SELECT * FROM (VALUES + (101, 'jeu vidéo de réflexion'), (102, 'personnage de jeu vidéo'), (103, 'film américain'), + (104, 'jeu vidéo d''arcade'), (105, 'joueur de jeux vidéo'), (106, 'vidéaste web français')) t(q, text); + CREATE TABLE art AS + SELECT c.id, c.title, c.rarity, c.r, c.views, q.q, d.text AS descr, + EXISTS (SELECT 1 FROM p31 WHERE p31.q = q.q AND p31.v = 5) AS person + FROM cards c LEFT JOIN qid q ON q.page_id = c.id LEFT JOIN descr d ON d.q = q.q; + """) + return c + + +def write(tmp_path, body: str): + p = tmp_path / "fiche.yaml" + p.write_text(textwrap.dedent(body), encoding="utf-8") + return spec.load(p) + + +def test_subclasses_and_cycles(con): + assert sorted(subclasses(con, [7889])) == [900, 901, 7889] + assert category_tree(con, [spec.Categories(["Jeu vidéo"], 5)]) == {"Jeu vidéo": 0, "Jeu de réflexion": 1} + + +def test_families_filters_and_includes(tmp_path, con): + s = write(tmp_path, """ + name: Jeux vidéo + price: 150 + families: + - name: Jeux + wikidata: {P31: [Q7889]} + drop_desc: ['\\bfilm\\b'] + - name: Joueurs + wikidata: {P106: [Q4379701, Q57414145]} + min_rarity: PC + - name: Catégories + categories: [{name: "Catégorie:Jeu_vidéo", depth: 1}] + exclude: + titles: [Pac-Man] + include: [Mario, Inconnu] + """) + sel = select(con, s) + # Tetris et Pac-Man (sous-classe) par P31, Squeezie par métier ; le joueur C est sous la rareté minimale. + assert sel.ids == {1, 6, 2} + assert sel.family_of[1] == "Jeux" and sel.family_of[2] == "Ajouts" + assert sel.missing_include == ["Inconnu"] + assert sel.excluded == {"titres": 1} + jeux = sel.families[0] + assert jeux.sources == {"wikidata": 2} and jeux.dropped == {} + joueurs = sel.families[1] + assert joueurs.dropped == {"rareté < PC": 1} + md, summary = report.preview(con, s, sel, 1, "2026-11-02") + assert "Légendaires (1)" in md and "Mario" in md + assert summary.startswith("Jeux vidéo : 3 cartes") + + +def test_people_and_description_filters(tmp_path, con): + s = write(tmp_path, """ + name: Test + families: + - name: Tout + title_regex: ['^(Mario|Squeezie|Super Mario)'] + people: none + keep_desc: ['jeu'] + """) + assert select(con, s).ids == {2} + + +def test_create_script(tmp_path, con): + s = write(tmp_path, """ + name: L'essai + price: 200 + starts: 2026-11-05T08:00Z + days: 7 + families: [{name: Jeux, wikidata: {P31: [Q7889]}}] + """) + sql = create.script(s, [4, 1], 1, 200, "palawi", commit=False) + assert "'L''essai'" in sql and "2026-11-12T08:00:00+00:00" in sql + assert "COPY _ids FROM STDIN;\n1\n4\n\\." in sql + assert sql.strip().endswith("ROLLBACK;") + assert "COMMIT" in create.script(s, [1], 1, 200, "palawi", commit=True) + + +def test_spec_errors(tmp_path): + with pytest.raises(SystemExit): + write(tmp_path, "name: x\nfamilies: [{name: a, wikidata: {P999: [Q1]}}]\n") + with pytest.raises(SystemExit): + write(tmp_path, "name: x\nfamilies: [{name: a, min_rarity: Z}]\n")