diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e6822c8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +.git +.gitignore +__pycache__ +*.py[cod] +*.egg-info +dist/ +build/ +.venv/ +tests/ +.github/ +.claude/ +CLAUDE.md +README.md +README.rst +LICENSE +.readthedocs.yaml +.DS_Store +Docker/ diff --git a/.github/workflows/auto-tag.yml b/.github/workflows/auto-tag.yml new file mode 100644 index 0000000..968db53 --- /dev/null +++ b/.github/workflows/auto-tag.yml @@ -0,0 +1,71 @@ +name: Auto Tag on Version Change + +on: + push: + branches: + - main + paths: + - 'VERSION' + +permissions: + contents: write + +jobs: + check-version: + name: Check for version change and create tag + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get current version + id: current + run: | + VERSION=$(cat VERSION | tr -d '[:space:]') + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Current version: $VERSION" + + - name: Get previous version + id: previous + run: | + git checkout HEAD^1 -- VERSION 2>/dev/null || echo "0.0.0" > VERSION + PREV_VERSION=$(cat VERSION | tr -d '[:space:]') + git checkout HEAD -- VERSION + echo "version=$PREV_VERSION" >> $GITHUB_OUTPUT + echo "Previous version: $PREV_VERSION" + + - name: Check if version changed + id: changed + run: | + if [ "${{ steps.current.outputs.version }}" != "${{ steps.previous.outputs.version }}" ]; then + echo "changed=true" >> $GITHUB_OUTPUT + echo "Version changed: ${{ steps.previous.outputs.version }} -> ${{ steps.current.outputs.version }}" + else + echo "changed=false" >> $GITHUB_OUTPUT + echo "Version unchanged" + fi + + - name: Check if tag exists + id: tag_exists + if: steps.changed.outputs.changed == 'true' + run: | + TAG="v${{ steps.current.outputs.version }}" + if git rev-parse "$TAG" >/dev/null 2>&1; then + echo "exists=true" >> $GITHUB_OUTPUT + echo "Tag $TAG already exists" + else + echo "exists=false" >> $GITHUB_OUTPUT + fi + + - name: Create and push tag + if: steps.changed.outputs.changed == 'true' && steps.tag_exists.outputs.exists == 'false' + run: | + VERSION="${{ steps.current.outputs.version }}" + TAG="v$VERSION" + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git tag -a "$TAG" -m "Release $VERSION" + git push origin "$TAG" + echo "Created tag: $TAG" diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..2bd1250 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,178 @@ +name: Build and Push Docker Image + +on: + push: + branches: + - main + - develop + paths: + - "unit3dup/**" + - "common/**" + - "view/**" + - "requirements.txt" + - "pyproject.toml" + - "Docker/dockerfile" + - "VERSION" + - ".github/workflows/docker-publish.yml" + workflow_dispatch: + inputs: + tag: + description: "Image tag (default: latest)" + required: false + default: "latest" + +env: + GHCR_IMAGE: ghcr.io/${{ vars.GHCR_USERNAME }}/${{ vars.REPO_NAME }} + +jobs: + build: + runs-on: ${{ matrix.runner }} + permissions: + contents: read + packages: write + + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-24.04 + cache-scope: amd64 + # Uncomment to enable ARM64 builds (slower, uses native ARM runner) + # - platform: linux/arm64 + # runner: ubuntu-24.04-arm + # cache-scope: arm64 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.GHCR_IMAGE }} + + - name: Build and push by digest + id: build + uses: docker/build-push-action@v6 + with: + context: . + file: Docker/dockerfile + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.GHCR_IMAGE }},push-by-digest=true,name-canonical=true,push=true + cache-from: type=gha,scope=${{ matrix.cache-scope }} + cache-to: type=gha,scope=${{ matrix.cache-scope }},mode=max + provenance: false + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.cache-scope }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + runs-on: ubuntu-latest + needs: build + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Read VERSION file + id: version + run: | + VERSION=$(cat VERSION | tr -d '[:space:]') + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Extract GHCR metadata + id: meta-ghcr + uses: docker/metadata-action@v5 + with: + images: ${{ env.GHCR_IMAGE }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=${{ steps.version.outputs.version }},enable={{is_default_branch}} + type=ref,event=branch + type=sha,prefix= + type=raw,value=${{ github.event.inputs.tag }},enable=${{ github.event_name == 'workflow_dispatch' }} + + - name: Extract Docker Hub metadata + id: meta-dockerhub + uses: docker/metadata-action@v5 + with: + images: ${{ secrets.DOCKERHUB_USERNAME }}/${{ vars.REPO_NAME }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=${{ steps.version.outputs.version }},enable={{is_default_branch}} + type=ref,event=branch + type=sha,prefix= + type=raw,value=${{ github.event.inputs.tag }},enable=${{ github.event_name == 'workflow_dispatch' }} + + - name: Create GHCR manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create \ + $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '${{ steps.meta-ghcr.outputs.json }}') \ + $(printf '${{ env.GHCR_IMAGE }}@sha256:%s ' *) + + # Copy multi-arch manifest from GHCR to Docker Hub without rebuilding + - name: Copy manifest to Docker Hub + run: | + GHCR_SOURCE="${{ env.GHCR_IMAGE }}:$(echo '${{ github.sha }}' | cut -c1-7)" + docker buildx imagetools create \ + $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '${{ steps.meta-dockerhub.outputs.json }}') \ + "$GHCR_SOURCE" + + - name: Inspect GHCR image + run: | + docker buildx imagetools inspect ${{ env.GHCR_IMAGE }}:${{ steps.meta-ghcr.outputs.version }} + + - name: Inspect Docker Hub image + run: | + docker buildx imagetools inspect ${{ secrets.DOCKERHUB_USERNAME }}/${{ vars.REPO_NAME }}:${{ steps.meta-dockerhub.outputs.version }} diff --git a/Docker/dockerfile b/Docker/dockerfile index 5fe5775..351866d 100644 --- a/Docker/dockerfile +++ b/Docker/dockerfile @@ -1,6 +1,4 @@ -# 02/05/2025 -# Build "docker build -t unit3dup ." inside the Docker folder first -# Remove "docker images", "docker rmi -f unit3dup:latest" +# Build from repo root: docker build -t unit3dup -f Docker/dockerfile . FROM python:3.11-slim @@ -8,6 +6,7 @@ FROM python:3.11-slim ARG USERNAME=pc # environment +ENV HOME=/home/$USERNAME ENV VIRTUAL_ENV=/home/$USERNAME/venv ENV PATH="$VIRTUAL_ENV/bin:$PATH" @@ -33,21 +32,17 @@ RUN groupadd -g 1000 $USERNAME && \ USER $USERNAME WORKDIR /home/$USERNAME -# Upgrade pip -RUN pip install --upgrade pip +# Create virtual env and upgrade pip +RUN python3 -m venv $VIRTUAL_ENV && \ + pip install --upgrade pip -# A new virtual env -RUN python3 -m venv $VIRTUAL_ENV - -# Activate virtual -RUN . $VIRTUAL_ENV/bin/activate - -# Copy .whl in the container -# COPY unit3dup-0.8.6-py3-none-any.whl /app/ - -# Install .whl -# RUN pip install --no-cache-dir /app/unit3dup-0.8.6-py3-none-any.whl -RUN pip install --no-cache-dir unit3dup +# Copy source and install from source +COPY --chown=$USERNAME:$USERNAME . /home/$USERNAME/app +WORKDIR /home/$USERNAME/app +RUN pip install --no-cache-dir ".[web]" && \ + chmod o+rx /home/$USERNAME && \ + chmod -R o+rX $VIRTUAL_ENV +WORKDIR /home/$USERNAME # Set the entry point ( see pyproject.toml) ENTRYPOINT ["unit3dup"] \ No newline at end of file diff --git a/README.md b/README.md index 06a6b58..70b7e99 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,8 @@ cd ~/unit3dup ```bash python3 -m venv .venv source .venv/bin/activate -pip install -e . +pip install -e . # CLI only +pip install -e ".[web]" # CLI + web dashboard ``` L'option `-e` (editable) permet de recevoir les mises à jour du fork simplement avec un `git pull`, sans réinstaller. @@ -146,7 +147,8 @@ Pas besoin de réinstaller grâce au mode `-e`. Si des nouvelles dépendances on ```bash source .venv/bin/activate -pip install -e . +pip install -e . # CLI only +pip install -e ".[web]" # CLI + web dashboard ``` --- diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..04a373e --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.16.0 diff --git a/common/bittorrent.py b/common/bittorrent.py index 79c94b4..de06bb0 100644 --- a/common/bittorrent.py +++ b/common/bittorrent.py @@ -7,8 +7,10 @@ @dataclass class BittorrentData: - tracker_response: str + tracker_response: str | None torrent_response: Mytorrent | None content: Media - tracker_message: dict + tracker_message: dict | str archive_path: str + release_name: str = "" + qbit_category: str | None = None diff --git a/common/command.py b/common/command.py index 58bc078..7749b31 100644 --- a/common/command.py +++ b/common/command.py @@ -44,8 +44,12 @@ def __init__(self): parser.add_argument("-personal", "--personal", action="store_true", help="Set to personal release") parser.add_argument("-confirm", "--confirm", action="store_true", help="Demande une validation manuelle du release_name avant chaque upload") + parser.add_argument("-skipval", "--skip_validation", action="store_true", + help="Skip tracker rule validation checks") parser.add_argument("-ftp", "--ftp", action="store_true", help="Connect to FTP") + parser.add_argument("-web", "--web", action="store_true", + help="Start web dashboard (use with -watcher for full mode)") # optional parser.add_argument("-dump", "--dump", action="store_true", help="Download all torrent titles") @@ -118,7 +122,11 @@ def __init__(self): if self.args.force.lower() not in [ System.category_list[System.MOVIE], System.category_list[System.GAME], System.category_list[System.TV_SHOW], - System.category_list[System.DOCUMENTARY]]: + System.category_list[System.DOCUMENTARY], + System.category_list[System.ANIMATION], + System.category_list[System.TV_ANIMATION], + System.category_list[System.DOCUMENTARY_FILM], + System.category_list[System.TV_DOCUMENTARY]]: self.args.force = None print("Invalid -force category") exit() diff --git a/common/external_services/ftpx/client.py b/common/external_services/ftpx/client.py index d927d91..8515533 100644 --- a/common/external_services/ftpx/client.py +++ b/common/external_services/ftpx/client.py @@ -255,7 +255,11 @@ def change_path(self, selected_folder: str): def user_input_search(self): custom_console.print("Search '->' ", end='', style='violet bold') - keyword = input() + try: + keyword = input() + except EOFError: + custom_console.bot_warning_log("No interactive input available, skipping search") + return for item in self.page.get_items(): if keyword.lower() in item.name.lower(): custom_console.bot_question_log( diff --git a/common/external_services/igdb/client.py b/common/external_services/igdb/client.py index 30e4010..b7f0415 100644 --- a/common/external_services/igdb/client.py +++ b/common/external_services/igdb/client.py @@ -70,17 +70,21 @@ def select_result(self, results: list) -> int | None: @staticmethod def input_manager(input_message: str) -> int | None: - while True: - custom_console.print(input_message, end='', style='violet bold') - user_choice = input() - if user_choice.upper() == "Q": - exit(1) - if user_choice.upper() == "S": - return None - if user_choice.isdigit(): - user_choice = int(user_choice) - if user_choice < 999999: - return user_choice + try: + while True: + custom_console.print(input_message, end='', style='violet bold') + user_choice = input() + if user_choice.upper() == "Q": + exit(1) + if user_choice.upper() == "S": + return None + if user_choice.isdigit(): + user_choice = int(user_choice) + if user_choice < 999999: + return user_choice + except EOFError: + custom_console.bot_warning_log("No interactive input available, skipping") + return None class IGDBClient: diff --git a/common/external_services/igdb/core/api.py b/common/external_services/igdb/core/api.py index 44ba4bb..d395493 100644 --- a/common/external_services/igdb/core/api.py +++ b/common/external_services/igdb/core/api.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import os from urllib.parse import urljoin from common.external_services.sessions.session import MyHttp from common import config_settings @@ -31,10 +32,11 @@ def login(self)-> bool: custom_console.bot_question_log("No IGDB_ID_SECRET provided\n") return False + cache_dir = str(os.path.join(config_settings.user_preferences.CACHE_PATH, "http_cache")) self.http_client = MyHttp({ "User-Agent": "Unit3D-up/0.0 (Linux 5.10.0-23-amd64)", "Accept": "application/json", - }) + }, cache_dir=cache_dir) response = self.http_client.post(self.oauth, params = { diff --git a/common/external_services/mediaresult.py b/common/external_services/mediaresult.py index 4e13aa3..8d2a586 100644 --- a/common/external_services/mediaresult.py +++ b/common/external_services/mediaresult.py @@ -2,6 +2,10 @@ from datetime import datetime +ANIMATION_GENRE_ID = 16 +DOCUMENTARY_GENRE_ID = 99 + + class MediaResult: def __init__(self, result=None, video_id: int = 0, imdb_id = None, trailer_key: str = None, keywords_list: str = None, season_title = None): @@ -19,6 +23,28 @@ def __init__(self, result=None, video_id: int = 0, imdb_id = None, trailer_key: except ValueError: pass + def is_animation(self) -> bool: + """Check if the TMDB result has the Animation genre (ID 16).""" + if not self.result: + return False + # Search results (Movie/TvShow): genre_ids as list[int] + if hasattr(self.result, 'genre_ids') and self.result.genre_ids: + return ANIMATION_GENRE_ID in self.result.genre_ids + # Details results (MovieDetails/TVShowDetails): genres as list[Genre] + if hasattr(self.result, 'genres') and self.result.genres: + return any(g.id == ANIMATION_GENRE_ID for g in self.result.genres) + return False + + def is_documentary(self) -> bool: + """Check if the TMDB result has the Documentary genre (ID 99).""" + if not self.result: + return False + if hasattr(self.result, 'genre_ids') and self.result.genre_ids: + return DOCUMENTARY_GENRE_ID in self.result.genre_ids + if hasattr(self.result, 'genres') and self.result.genres: + return any(g.id == DOCUMENTARY_GENRE_ID for g in self.result.genres) + return False + diff --git a/common/external_services/theMovieDB/core/api.py b/common/external_services/theMovieDB/core/api.py index 009dd5f..ba281a4 100644 --- a/common/external_services/theMovieDB/core/api.py +++ b/common/external_services/theMovieDB/core/api.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import os +import dataclasses import hashlib from datetime import datetime @@ -7,6 +8,7 @@ from typing import TypeVar from common.external_services.theMovieDB.core.models.tvshow.alternative import Alternative +from common.external_services.theMovieDB.core.models.tvshow.external_ids import ExternalIds from common.external_services.theMovieDB.core.models.tvshow.details import TVShowDetails from common.external_services.theMovieDB.core.models.movie.details import MovieDetails from common.external_services.theMovieDB.core.models.movie.nowplaying import NowPlaying @@ -99,13 +101,16 @@ def keywords(serie_id: int)-> dict: return {'url': f'{base_url}/tv/{serie_id}/keywords', 'datatype': Keyword, 'query': '', 'results': 'results'} + @staticmethod + def external_ids(serie_id: int) -> dict: + return {'url': f'{base_url}/tv/{serie_id}/external_ids', 'datatype': ExternalIds, 'query': ''} class TmdbAPI(MyHttp): params = { "api_key": config.TMDB_APIKEY, - "language": "it-IT", + "language": "fr-FR", } # Mappatura automatica degli endpoint @@ -119,7 +124,8 @@ def __init__(self): Initialize the Api instance with an HTTP client """ headers = Agent.headers() - super().__init__(headers) + cache_dir = str(os.path.join(config_settings.user_preferences.CACHE_PATH, "http_cache")) + super().__init__(headers, cache_dir=cache_dir) self.http_client = self.session def _search(self, query: str, category: str) -> list[T] | None: @@ -184,6 +190,12 @@ def details(self, video_id: int, category: str) -> list[T] | None: print(f"Endpoint for category '{category}' not found.") return [] + def _external_ids(self, video_id: int, category: str) -> list[T] | None: + if endpoint_class := self.ENDPOINTS.get(category): + if hasattr(endpoint_class, 'external_ids'): + request = endpoint_class.external_ids(video_id) + return self.request(endpoint=request) + return None def _keywords(self, video_id: int, category: str) -> list[T] | None: if endpoint_class:=self.ENDPOINTS.get(category): @@ -210,7 +222,12 @@ def request(self, endpoint: dict) -> list[T] | None: else: response_data = response.json().get(endpoint['results'], []) - return [endpoint['datatype'](**attribute) for attribute in response_data] + datatype = endpoint['datatype'] + if dataclasses.is_dataclass(datatype): + allowed = {f.name for f in dataclasses.fields(datatype)} + return [datatype(**{k: v for k, v in attribute.items() if k in allowed}) + for attribute in response_data] + return [datatype(**attribute) for attribute in response_data] else: return [] return None @@ -265,6 +282,24 @@ def is_like(self, results: list[T]) -> T | bool: return result return False + def _fetch_imdb_id(self, tmdb_id: int, category: str) -> int: + """Fetch IMDb ID from TMDB details (movie) or external_ids (tv).""" + if category != 'tv': + details = self.details(video_id=tmdb_id, category=category) + if details and hasattr(details[0], 'imdb_id') and details[0].imdb_id: + try: + return int(str(details[0].imdb_id).replace('tt', '')) + except (ValueError, TypeError): + pass + else: + ext = self._external_ids(video_id=tmdb_id, category=category) + if ext and ext[0].imdb_id: + try: + return int(str(ext[0].imdb_id).replace('tt', '')) + except (ValueError, TypeError): + pass + return 0 + def results_in_string(self, tmdb_id:int, imdb_id:int)-> MediaResult: """ Use id from the string filename or name folder @@ -275,6 +310,8 @@ def results_in_string(self, tmdb_id:int, imdb_id:int)-> MediaResult: if tmdb_id: if tmdb_id > 0: + if not imdb_id: + imdb_id = self._fetch_imdb_id(tmdb_id, self.category) # Request trailer and keywords trailer_key = self.trailer(tmdb_id) keywords_list = self.keywords(tmdb_id) if trailer_key else '' @@ -302,10 +339,10 @@ def search(self) -> MediaResult | None: # or start an on-line search results = self._search(self.query, self.category) - # Use imdb_id when tmdb_id is not available - imdb_id = 0 if results: if result:=self.is_like(results): + # Fetch imdb_id from TMDB details or external_ids + imdb_id = self._fetch_imdb_id(result.id, self.category) # Get the trailer trailer_key = self.trailer(result.id) keywords_list = self.keywords(result.id) @@ -330,7 +367,7 @@ def search(self) -> MediaResult | None: custom_console.bot_warning_log(f"Title: '{self.query}'\ncategory:'{self.category}'") if self.category in 'tv': - serie = f"S{str(self.media.guess_season).zfill(2)}" if self.media.guess_season else '' + serie = f"S{str(self.media.guess_season).zfill(2)}" if self.media.guess_season is not None else '' if not self.media.torrent_pack: serie += f"E{str(self.media.guess_episode).zfill(2)}" custom_console.bot_warning_log(f"details: '{serie}' Pack: '{self.media.torrent_pack}'") diff --git a/common/external_services/theMovieDB/core/models/tvshow/external_ids.py b/common/external_services/theMovieDB/core/models/tvshow/external_ids.py new file mode 100644 index 0000000..f5c48e9 --- /dev/null +++ b/common/external_services/theMovieDB/core/models/tvshow/external_ids.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +from dataclasses import dataclass + + +@dataclass +class ExternalIds: + id: int + imdb_id: str | None = None + freebase_mid: str | None = None + freebase_id: str | None = None + tvdb_id: int | None = None + tvrage_id: int | None = None + wikidata_id: str | None = None + facebook_id: str | None = None + instagram_id: str | None = None + twitter_id: str | None = None diff --git a/common/extractor.py b/common/extractor.py index 3aa44a7..ad16076 100644 --- a/common/extractor.py +++ b/common/extractor.py @@ -40,7 +40,11 @@ def delete_old_rar(self, rar_volumes_list: list): # Ask user for each file while manual_mode: - delete_choice = input("Delete the old file ? (Y/N/All) Q=quit ") + try: + delete_choice = input("Delete the old file ? (Y/N/All) Q=quit ") + except EOFError: + custom_console.bot_warning_log("No interactive input available, keeping file") + break # Wait for an answer if delete_choice: # Only letters diff --git a/common/mediainfo.py b/common/mediainfo.py index 165b18f..e0a022e 100644 --- a/common/mediainfo.py +++ b/common/mediainfo.py @@ -30,6 +30,7 @@ def general_track(self)-> dict: if not self._general_track: for track in self.media_info.to_data().get("tracks", []): if track.get("track_type") == "General": + self._general_track = track return self._general_track self._general_track = {} return self._general_track @@ -209,4 +210,94 @@ def is_silent(self) -> bool: if not audio: return False langs = [t.get("language", "") for t in audio] - return bool(langs) and all(l == "zxx" for l in langs) \ No newline at end of file + return bool(langs) and all(l == "zxx" for l in langs) + + # ── Propriétés ajoutées pour les validators ──────────────────────────── + + @property + def encoding_settings(self) -> str | None: + """Chaîne encoding_settings du premier track vidéo (options x264/x265).""" + video = self.video_track + if video: + return video[0].get("encoding_settings", None) + return None + + @property + def writing_library(self) -> str | None: + """Writing library du premier track vidéo.""" + video = self.video_track + if video: + return video[0].get("writing_library", None) + return None + + @property + def video_format(self) -> str | None: + """Format vidéo (AVC, HEVC, AV1, etc.).""" + video = self.video_track + if video: + return video[0].get("format", None) + return None + + @property + def color_primaries(self) -> str | None: + """Color primaries (BT.601, BT.709, BT.2020, etc.).""" + video = self.video_track + if video: + return video[0].get("color_primaries", None) + return None + + @property + def transfer_characteristics(self) -> str | None: + """Transfer characteristics (PQ / SMPTE ST 2084, HLG, etc.).""" + video = self.video_track + if video: + return video[0].get("transfer_characteristics", None) + return None + + @property + def hdr_format(self) -> str | None: + """HDR format string (Dolby Vision, HDR10, HDR10+, etc.).""" + video = self.video_track + if video: + return video[0].get("hdr_format", None) + return None + + @property + def container_format(self) -> str: + """Extension du fichier (container).""" + _, ext = os.path.splitext(self.file_path) + return ext.lower() + + @property + def multiview_count(self) -> int | None: + """Nombre de vues pour contenu 3D.""" + video = self.video_track + if video: + val = video[0].get("multiview_count", None) + return int(val) if val else None + return None + + @property + def audio_formats(self) -> list[dict]: + """Liste des formats et canaux de chaque piste audio.""" + result = [] + for track in self.audio_track: + result.append({ + "format": track.get("format", ""), + "channels": track.get("channel_s", 0), + "service_kind": track.get("service_kind", ""), + "delay": track.get("delay_relative_to_video", None), + "language": track.get("language", ""), + }) + return result + + @property + def subtitle_formats(self) -> list[dict]: + """Liste des formats de chaque piste sous-titre.""" + result = [] + for track in self.subtitle_track: + result.append({ + "format": track.get("format", ""), + "language": track.get("language", ""), + }) + return result \ No newline at end of file diff --git a/common/settings.py b/common/settings.py index dc604df..11621cb 100644 --- a/common/settings.py +++ b/common/settings.py @@ -13,7 +13,11 @@ from common import trackers config_file = "Unit3Dbot.json" -version = "0.8.21" +try: + from importlib.metadata import version as _get_version + version = _get_version("Unit3Dup") +except Exception: + version = (Path(__file__).resolve().parent.parent / "VERSION").read_text(encoding="utf-8").strip() if os.name == "nt": PW_TORRENT_ARCHIVE_PATH: Path = Path(os.getenv("LOCALAPPDATA", ".")) / "Unit3Dup_config" / "pw_torrent_archive" @@ -58,6 +62,7 @@ class TrackerConfig(BaseModel): Gemini_URL: str Gemini_APIKEY: str | None = None Gemini_PID: str | None = None + Gemini_USERNAME: str | None = None MULTI_TRACKER: list[str] | None = None TMDB_APIKEY: str | None = None IMGBB_KEY: str | None = None @@ -90,9 +95,24 @@ class TorrentClientConfig(BaseModel): SHARED_RTORR_PATH: str | None = None TORRENT_CLIENT: str | None = None TAG: str | None = None + QBIT_SKIP_HASH_CHECK: bool = False +class WatcherFolder(BaseModel): + path: str + category: str | None = None + + +def get_watcher_folders(prefs) -> list[WatcherFolder]: + """Return resolved list of WatcherFolder from config (new or legacy format).""" + if prefs.WATCHER_PATHS: + return prefs.WATCHER_PATHS + if prefs.WATCHER_PATH and "no_path" not in str(prefs.WATCHER_PATH).lower(): + return [WatcherFolder(path=prefs.WATCHER_PATH)] + return [] + + class UserPreferences(BaseModel): PTSCREENS_PRIORITY: int = 0 LENSDUMP_PRIORITY: int = 1 @@ -112,6 +132,7 @@ class UserPreferences(BaseModel): SIZE_TH: int = 50 WATCHER_INTERVAL: int = 60 WATCHER_PATH: str | None = None + WATCHER_PATHS: list[WatcherFolder] | None = None WATCHER_DESTINATION_PATH: str | None = None TORRENT_ARCHIVE_PATH: str | None = None CACHE_PATH: str | None = None @@ -125,6 +146,8 @@ class UserPreferences(BaseModel): CACHE_DBONLINE: bool = False PERSONAL_RELEASE: bool = False FAST_LOAD: int = 0 + WEB_HOST: str = "0.0.0.0" + WEB_PORT: int = 8000 class Options(BaseModel): @@ -155,6 +178,7 @@ class ConsoleOptions(BaseModel): class UploaderTag(BaseModel): """Tags d'équipe autorisés à marquer un upload comme personal_release.""" TAGS_TEAM: list[str] = [] + EXCLUDED_TAGS: list[str] = [] class Validate: @@ -366,7 +390,7 @@ class Config(BaseModel): def set_default_uploader_tag(cls, v): """Fournit uploader_tag par défaut pour les JSON qui ne l'ont pas encore.""" if 'uploader_tag' not in v or v['uploader_tag'] is None: - v['uploader_tag'] = {'TAGS_TEAM': []} + v['uploader_tag'] = {'TAGS_TEAM': [], 'EXCLUDED_TAGS': []} return v @model_validator(mode='before') @@ -420,6 +444,10 @@ def set_default_torrent_client_config(cls, v): if field in ['SHARED_TRASM_PATH', 'SHARED_QBIT_PATH', 'SHARED_RTORR_PATH']: section[field] = Validate.shared_path(path=section[field], field_name=field) + if field in ['QBIT_SKIP_HASH_CHECK']: + if isinstance(section[field], str): + section[field] = section[field].lower() in ('true', '1', 'yes') + return v @@ -441,10 +469,19 @@ def set_default_user_preferences(cls, v): if field in ['TORRENT_COMMENT','WATCHER_PATH','DEFAULT_TRACKER']: section[field] = Validate.string(value=section[field], field_name=field) + if field in ['WEB_HOST']: + if isinstance(section[field], str) and section[field].strip() == '': + defaults = UserPreferences.model_fields + section[field] = defaults[field].default if field in defaults else "0.0.0.0" + if field in ['NUMBER_OF_SCREENSHOTS','COMPRESS_SCSHOT','IMGBB_PRIORITY','FREE_IMAGE_PRIORITY', 'LENSDUMP_PRIORITY','PASSIMA_PRIORITY','IMARIDE_PRIORITY', 'WATCHER_INTERVAL','SIZE_TH', - 'FAST_LOAD']: - section[field] = Validate.integer(value=section[field], field_name=field) + 'FAST_LOAD', 'WEB_PORT']: + if isinstance(section[field], str) and section[field].strip() == '': + defaults = UserPreferences.model_fields + section[field] = defaults[field].default if field in defaults else 0 + else: + section[field] = Validate.integer(value=section[field], field_name=field) if field == 'PREFERRED_LANG': section[field] =Validate.iso3166(value=section[field], field_name=field) @@ -534,6 +571,7 @@ def create_default_json_file(path: Path): "Gemini_URL": "https://gemini-tracker.org", "Gemini_APIKEY": "no_key", "Gemini_PID": "no_key", + "Gemini_USERNAME": "no_key", "MULTI_TRACKER" : ["gemini"], "TMDB_APIKEY": "no_key", "IMGBB_KEY": "no_key", @@ -565,6 +603,7 @@ def create_default_json_file(path: Path): "SHARED_RTORR_PATH": "no_path", "TORRENT_CLIENT": "qbittorrent", "TAG": "ADDED TORRENTS", + "QBIT_SKIP_HASH_CHECK": false, }, "user_preferences": { "PTSCREENS_PRIORITY": 0, @@ -584,6 +623,7 @@ def create_default_json_file(path: Path): "SIZE_TH": 10, "WATCHER_INTERVAL": 60, "WATCHER_PATH": "no_path", + "WATCHER_PATHS": [], "WATCHER_DESTINATION_PATH": "no_path", "TORRENT_ARCHIVE_PATH": "no_path", "CACHE_PATH": "no_path", @@ -623,6 +663,7 @@ def create_default_json_file(path: Path): }, "uploader_tag": { "TAGS_TEAM": [], + "EXCLUDED_TAGS": [], }, } @@ -635,10 +676,6 @@ def create_default_json_file(path: Path): @staticmethod def load_config(): - if not WATCHER_DESTINATION_PATH.exists(): - print(f"Create default destination watcher path: {WATCHER_DESTINATION_PATH}") - os.makedirs(WATCHER_DESTINATION_PATH) - if not WATCHER_PATH.exists(): print(f"Create default watcher path: {WATCHER_PATH}") os.makedirs(WATCHER_PATH) @@ -700,7 +737,7 @@ def __init__(self, default_json_path: Path): self.options_config = self.file_config_data["options"] self.console_options_config = self.file_config_data["console_options"] # uploader_tag peut être absent des anciens JSON → fallback sur valeur par défaut - self.uploader_tag_config = self.file_config_data.get("uploader_tag", {"TAGS_TEAM": []}) + self.uploader_tag_config = self.file_config_data.get("uploader_tag", {"TAGS_TEAM": [], "EXCLUDED_TAGS": []}) # New tracker attribute self.tracker_diff_keys = self.tracker_config.keys() ^ TrackerConfig.__annotations__.keys()\ @@ -733,7 +770,7 @@ def update_tracker_config(self): # Add the new attributes in 'tracker config' if self.tracker_diff_keys: self.updated = True - missing_keys_dict = {key: '' for key in self.tracker_diff_keys} + missing_keys_dict = {key: 'no_key' for key in self.tracker_diff_keys} self.tracker_config.update(missing_keys_dict) @@ -741,7 +778,10 @@ def update_torrent_client_config(self): # Add the new attributes in 'torrent client' if self.torrent_diff_keys: self.updated = True - missing_keys_dict = {key: '' for key in self.torrent_diff_keys} + missing_keys_dict = {} + for key in self.torrent_diff_keys: + field = TorrentClientConfig.model_fields.get(key) + missing_keys_dict[key] = field.default if field and field.default is not None else '' self.torrent_config.update(missing_keys_dict) @@ -749,7 +789,8 @@ def update_user_preferences_config(self): # Add the new attributes in 'user preferences' if self.user_preferences_diff_keys: self.updated = True - missing_keys_dict = {key: '' for key in self.user_preferences_diff_keys} + list_fields = {"WATCHER_PATHS"} + missing_keys_dict = {key: ([] if key in list_fields else '') for key in self.user_preferences_diff_keys} self.user_preferences_config.update(missing_keys_dict) @@ -776,7 +817,7 @@ def update_uploader_tag_config(self): elif self.uploader_tag_diff_keys: self.updated = True missing_keys_dict = { - key: ([] if key == 'TAGS_TEAM' else '') + key: ([] if key in ('TAGS_TEAM', 'EXCLUDED_TAGS') else '') for key in self.uploader_tag_diff_keys } self.uploader_tag_config.update(missing_keys_dict) diff --git a/common/torrent_clients.py b/common/torrent_clients.py index 9e36cdc..72d03c0 100644 --- a/common/torrent_clients.py +++ b/common/torrent_clients.py @@ -42,14 +42,14 @@ def remove_tags(self, infohash_list: list[str]): }, ) - # Force savepath properly via qBittorrent WebAPI (torrents/add) - def add_torrent_file(self, file_buffer, savepath: str, tags: str | None = None): - data = { - "savepath": savepath, - "autoTMM": "false", # prevents categories/templates from overriding savepath (as much as possible) - } + def add_torrent_file(self, file_buffer, tags: str | None = None, category: str | None = None, skip_checking: bool = False): + data = {} if tags: data["tags"] = tags + if category: + data["category"] = category + if skip_checking: + data["skip_checking"] = "true" files = {"torrents": file_buffer} return self._post("torrents/add", data=data, files=files) @@ -64,7 +64,7 @@ def connect(self): raise NotImplementedError @abstractmethod - def send_to_client(self, tracker_data_response: str, torrent: Mytorrent | None, content: Media, archive_path: str): + def send_to_client(self, tracker_data_response: str, torrent: Mytorrent | None, content: Media, archive_path: str, category: str | None = None): raise NotImplementedError @staticmethod @@ -99,7 +99,7 @@ def connect(self) -> transmission_rpc.Client | None: custom_console.bot_error_log(f"{self.__class__.__name__} Please verify your configuration") return None - def send_to_client(self, tracker_data_response: str, torrent: Mytorrent | None, content: Media, archive_path: str): + def send_to_client(self, tracker_data_response: str, torrent: Mytorrent | None, content: Media, archive_path: str, category: str | None = None): # Transmission "shared path" if config_settings.torrent_client_config.SHARED_QBIT_PATH: torr_location = config_settings.torrent_client_config.SHARED_QBIT_PATH @@ -161,37 +161,7 @@ def connect(self) -> MyQbittorrent | None: custom_console.bot_error_log(f"{self.__class__.__name__} Please verify your configuration") return None - def send_to_client(self, tracker_data_response: str, torrent: Mytorrent | None, content: Media, archive_path: str): - # qBittorrent "shared path" - if config_settings.torrent_client_config.SHARED_QBIT_PATH: - torr_location = config_settings.torrent_client_config.SHARED_QBIT_PATH - else: - # content.torrent_path is the most reliable: file or folder release - # Convert to absolute path first to ensure proper detection - base = os.path.abspath(content.torrent_path) if content.torrent_path else None - - if not base: - # Fallback: try to use content.file_name - base = os.path.abspath(content.file_name) if content.file_name else None - - if base: - if os.path.isfile(base): - # It's a single file release, use the parent directory (where the file is located) - torr_location = os.path.dirname(base) - elif os.path.isdir(base): - # It's a folder release (dossier avec fichier vidéo dedans) - # Pour -u avec un dossier, pointer vers le dossier parent du dossier de release - torr_location = os.path.dirname(base) - else: - # Path doesn't exist, try to get parent directory anyway - torr_location = os.path.dirname(base) - else: - # No valid path found, use current directory as fallback - torr_location = os.getcwd() - - torr_location = os.path.normpath(torr_location) - custom_console.bot_warning_log(f"[QbittorrentClient] Forced savepath: {torr_location}") - + def send_to_client(self, tracker_data_response: str, torrent: Mytorrent | None, content: Media, archive_path: str, category: str | None = None): # Compute infohash (for tagging) with open(archive_path, "rb") as file_buffer: torrent_data = file_buffer.read() @@ -199,11 +169,12 @@ def send_to_client(self, tracker_data_response: str, torrent: Mytorrent | None, info_hash = hashlib.sha1(bencode2.bencode(info)).hexdigest() file_buffer.seek(0) - # ✅ IMPORTANT: use torrents/add with autoTMM=false and savepath + # Let qBittorrent category handle the save path automatically self.client.add_torrent_file( file_buffer=file_buffer, - savepath=str(torr_location), tags=config_settings.torrent_client_config.TAG, + category=category, + skip_checking=config_settings.torrent_client_config.QBIT_SKIP_HASH_CHECK, ) # Optional: enforce tags via addTags as well @@ -213,13 +184,13 @@ def send_to_client(self, tracker_data_response: str, torrent: Mytorrent | None, # not fatal pass - def send_file_to_client(self, torrent_path: str, media_location: str): - # Keep a simple path-based call for manual usage + def send_file_to_client(self, torrent_path: str, category: str | None = None): with open(torrent_path, "rb") as fb: self.client.add_torrent_file( file_buffer=fb, - savepath=str(os.path.normpath(media_location)), tags=config_settings.torrent_client_config.TAG, + category=category, + skip_checking=config_settings.torrent_client_config.QBIT_SKIP_HASH_CHECK, ) @@ -262,7 +233,7 @@ def connect(self) -> RTorrent | None: custom_console.bot_error_log(f"{self.__class__.__name__} Socket connection error or wrong OS platform") exit() - def send_to_client(self, tracker_data_response: str, torrent: Mytorrent | None, content: Media, archive_path: str): + def send_to_client(self, tracker_data_response: str, torrent: Mytorrent | None, content: Media, archive_path: str, category: str | None = None): if config_settings.torrent_client_config.SHARED_RTORR_PATH: torr_location = config_settings.torrent_client_config.SHARED_RTORR_PATH else: diff --git a/common/trackers/data.py b/common/trackers/data.py index 1b6bb38..08dd789 100644 --- a/common/trackers/data.py +++ b/common/trackers/data.py @@ -9,6 +9,7 @@ "url": config_settings.tracker_config.Gemini_URL, "api_key": config_settings.tracker_config.Gemini_APIKEY, "pass_key": config_settings.tracker_config.Gemini_PID, + "username": config_settings.tracker_config.Gemini_USERNAME, "announce": f"{config_settings.tracker_config.Gemini_URL}/announce/{config_settings.tracker_config.Gemini_PID}", "source": "Gemini", } diff --git a/common/trackers/gemini.py b/common/trackers/gemini.py index 25c16e3..262b769 100644 --- a/common/trackers/gemini.py +++ b/common/trackers/gemini.py @@ -3,8 +3,11 @@ gemini_data = { "CATEGORY":{ "movie": 1, "tv": 2, - "edicola": 6, - "game": 4}, + "animation": 7, + "tv_animation": 6, + "game": 4, + "documentary": 13, + "tv_documentary": 14}, "FREELECH":{ "size20": 100, "size15": 75, @@ -17,42 +20,35 @@ "vh": 2, "untouched": 2, "bd-untouched": 2, + "vu": 2, "encode": 3, + "bdrip": 3, "bluray": 3, "fullhd": 3, "hevc": 3, "hdrip": 3, - "vu": 2, "web-dl": 4, "webdl": 4, "web": 4, "web-dlmux": 4, "webrip": 5, "hdtv": 6, + "iso": 7, + "dvd": 11, + "tvrip": 12, + "dvdrip": 13, + "altro": 3, + # Game/book types (to be fixed in a future update) "mac": 12, "macos": 12, "windows": 13, "pc": 13, - "cinema-md": 14, - "hdts": 14, - "wrs": 14, - "md": 14, - "altro": 15, "pdf": 16, "nintendo": 17, "nsw": 17, "ps4": 18, "psn": 18, "epub": 19, - "mp4": 20, - "pack": 22, - "avi": 23, - "dvdrip": 24, - "bdrip": 25, - "webmux": 26, - "dlmux": 27, - "bdmux": 29, - "3d": 32, "cbr-cbz": 33, "ps5": 35, "psvr": 35, diff --git a/common/trackers/trackers.py b/common/trackers/trackers.py index f54f7c3..a2dfed4 100644 --- a/common/trackers/trackers.py +++ b/common/trackers/trackers.py @@ -27,20 +27,27 @@ def load_from_module(cls, tracker_name: str) -> "TRACKData": codec=tracker_data.get("CODEC"), ) - def filter_type(self, file_name: str) -> int: + def filter_type(self, file_name: str, resolution: str | None = None) -> int: file_name = ManageTitles.clean(file_name) - # >Clean the releaser sign file_name = file_name.replace("-", " ") word_list = file_name.lower().strip().split(" ") - # Caso 1: Cerca un TYPE_ID nel nome del file + # Case 1: Explicit TYPE_ID keyword in filename for word in word_list: if word in self.type_id: return self.type_id[word] - # Caso 2: Se non trova un TYPE_ID, cerca un codec e ritorna 'encode' + # Case 2: Codec found → infer source from resolution for word in word_list: if word in self.codec: + if resolution: + res_digits = ''.join(c for c in resolution if c.isdigit()) + if res_digits: + res_value = int(res_digits) + if res_value >= 720: + return self.type_id.get("bdrip", self.type_id.get("encode", -1)) + else: + return self.type_id.get("dvdrip", self.type_id.get("encode", -1)) return self.type_id.get("encode", -1) return self.type_id.get("altro", -1) diff --git a/common/utility.py b/common/utility.py index 4a29c35..002bf74 100644 --- a/common/utility.py +++ b/common/utility.py @@ -246,8 +246,19 @@ class System: TV_SHOW = 2 MOVIE = 1 GAME = 3 - - category_list = {MOVIE: 'movie', TV_SHOW: 'tv', GAME : 'game', DOCUMENTARY: 'edicola'} + ANIMATION = 5 + TV_ANIMATION = 6 + DOCUMENTARY_FILM = 7 + TV_DOCUMENTARY = 8 + + category_list = { + MOVIE: 'movie', TV_SHOW: 'tv', GAME: 'game', + DOCUMENTARY: 'edicola', + ANIMATION: 'animation', + TV_ANIMATION: 'tv_animation', + DOCUMENTARY_FILM: 'documentary', + TV_DOCUMENTARY: 'tv_documentary', + } RESOLUTIONS= [ "8640", "4320", "2160", "1080", "720", "576", "480"] RESOLUTION_labels = ["8640p", "4320p", "2160p", "1080p", "1080i", "720p", "720i", "576p", "576i", "480p", "480i"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e61ebd2 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,67 @@ +services: + unit3dup: + image: johandevl/unit3dup:latest + container_name: unit3dup + user: "1000:1000" + + # Change to 'restart: unless-stopped' after successful configuration + restart: "no" + + # Watcher mode: monitors input folder, auto-uploads new files + # Web mode: add -web to enable the web dashboard for manual approval + # Dry mode: add -noup to create torrents without uploading + # Scan mode: add -scan instead of -watcher to analyze without uploading + command: ["-watcher", "-web"] + # command: ["-watcher"] # Watcher only: auto-upload without web UI + # command: ["-watcher", "-noup"] # Dry mode: torrent only, no upload + # command: ["-scan", "/home/pc/data"] # Scan mode: analyze files only + + environment: + - TZ=Europe/Paris # Adjust to your timezone + + ports: + - "8000:8000" + + volumes: + # Config (persists Unit3Dbot.json, cache, torrent archive, + # watcher_state.json and watcher_dryrun.json) + - ./config:/home/pc/Unit3Dup_config + + # Media input (files/folders to upload) + - /path/to/media:/home/pc/data + + # Torrent client: if running on the host, use host IP instead of localhost + # network_mode: host # Alternative: use host networking directly + + # ── First-run setup ────────────────────────────────────────────── + # + # 1. Create config directory: + # mkdir -p ./config + # + # 2. Generate default config (will exit with error — this is expected): + # docker compose run --rm unit3dup + # + # 3. Edit ./config/Unit3Dbot.json: + # - Set tracker URL (Gemini_URL) and API key (Gemini_APIKEY) + # - Set TMDB API key (TMDB_APIKEY) + # - Set torrent client config (QBIT_HOST, QBIT_USER, etc.) + # Note: use host IP (not localhost) for services running on the host + # - Set paths (use container paths): + # WATCHER_PATH: /home/pc/data + # + # Watcher state is stored in ./config/: + # - watcher_state.json: tracks uploaded/skipped entries (not modified in dry-run) + # - watcher_dryrun.json: preview of what would happen (dry-run only) + # + # 4. Start watcher: + # docker compose up -d + # + # 5. Once running, update restart policy: + # Change 'restart: "no"' to 'restart: unless-stopped' + # + # ── One-off commands ───────────────────────────────────────────── + # + # docker compose run --rm unit3dup -u /home/pc/data/MyMovie + # docker compose run --rm unit3dup -f /home/pc/data/MyFolder + # docker compose run --rm unit3dup -scan /home/pc/data + # diff --git a/pyproject.toml b/pyproject.toml index c48cfc9..3f8be01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,9 +3,8 @@ requires = ["setuptools>=61.0", "wheel"] build-backend = "setuptools.build_meta" [project] -dynamic = ["dependencies"] +dynamic = ["dependencies", "version"] name = "Unit3Dup" -version = "0.8.21" description = "An uploader for the Unit3D torrent tracker" readme = "README.rst" requires-python = ">=3.10" @@ -18,13 +17,24 @@ authors = [ [project.urls] Homepage = "https://github.com/31December99/Unit3Dup" +[project.optional-dependencies] +web = [ + "fastapi>=0.115.0", + "uvicorn[standard]>=0.32.0", + "jinja2>=3.1.0", + "python-multipart>=0.0.12", +] [tool.setuptools.packages.find] where = ["."] exclude = ["tests", "docs","http_cache","venv*"] +[tool.setuptools.package-data] +"unit3dup.web" = ["templates/**/*.html", "static/**/*"] + [tool.setuptools.dynamic] dependencies = {file = ["requirements.txt"]} +version = {file = "VERSION"} [project.scripts] diff --git a/requirements.txt b/requirements.txt index b3c0a57..5e7a71f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,3 @@ cinemagoer==2023.5.1 pathvalidate==3.2.3 bencode2==0.3.24 rtorrent-rpc==0.9.4 - - - diff --git a/tests/__init__.py b/tests/__init__.py index 37e39c7..4b6d1b4 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,7 +7,7 @@ from common.trackers.trackers import TRACKData from common.mediainfo import MediaFile from common.command import CommandLine -from common.settings import Load +from common.settings import Load, DEFAULT_JSON_PATH from common.utility import System diff --git a/tests/movieshow.py b/tests/movieshow.py index 288dfd7..88c5534 100644 --- a/tests/movieshow.py +++ b/tests/movieshow.py @@ -38,7 +38,8 @@ def test_tmdb(): for content in contents: """ DUPLICATE """ - assert isinstance(tests.UserContent.is_duplicate(content=content, tracker_name=TRACKER_TEST),bool) + result = tests.UserContent.check_duplicate(content=content, tracker_name=TRACKER_TEST) + assert result is None or isinstance(result, dict) """ VIDEO INFO """ # TMDB diff --git a/tests/watcher.py b/tests/watcher.py index 52525b9..db2f8f6 100644 --- a/tests/watcher.py +++ b/tests/watcher.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import tests +from common.settings import get_watcher_folders # /* ----------------------------------------------------------------------------------------------- */ force_media = 0 @@ -27,7 +28,8 @@ def test_cli_watcher(): mode="auto", trackers_name_list= ['Gemini'] ) + watcher_folders = get_watcher_folders(tests.config.user_preferences) assert bot.watcher(duration=tests.config.user_preferences.WATCHER_INTERVAL, - watcher_path=tests.config.user_preferences.WATCHER_PATH, - destination_path=tests.config.user_preferences.WATCHER_DESTINATION_PATH) == True + watcher_folders=watcher_folders, + state_dir=str(tests.DEFAULT_JSON_PATH.parent)) == True diff --git a/unit3dup/__main__.py b/unit3dup/__main__.py index ece1ae9..428d8d8 100644 --- a/unit3dup/__main__.py +++ b/unit3dup/__main__.py @@ -2,7 +2,7 @@ from common.torrent_clients import TransmissionClient, QbittorrentClient, RTorrentClient from common.command import CommandLine -from common.settings import Load,DEFAULT_JSON_PATH +from common.settings import Load, DEFAULT_JSON_PATH, get_watcher_folders from unit3dup.torrent import View from unit3dup import pvtTracker @@ -23,8 +23,13 @@ def main(): custom_console.bot_log(f"[Configuration] '{DEFAULT_JSON_PATH}'") custom_console.bot_log(f"[*.torrent Archive] '{config.user_preferences.TORRENT_ARCHIVE_PATH}'") custom_console.bot_log(f"[Images,Tmdb cache] '{config.user_preferences.CACHE_PATH}'") - custom_console.bot_log(f"[Watcher] '{config.user_preferences.WATCHER_PATH}'") - custom_console.bot_log(f"[Watcher] '{config.user_preferences.WATCHER_DESTINATION_PATH}'") + watcher_folders = get_watcher_folders(config.user_preferences) + if watcher_folders: + for wf in watcher_folders: + cat_info = f" (category: {wf.category})" if wf.category else "" + custom_console.bot_log(f"[Watcher] '{wf.path}'{cat_info}") + else: + custom_console.bot_log(f"[Watcher] (not configured)") print() # /// Initialize command line interface @@ -117,13 +122,71 @@ def main(): torrent_archive_path=tracker_archive) bot.run() - # Watcher + # Web-only mode (no watcher) + if getattr(cli.args, 'web', False) and not cli.args.watcher: + try: + from unit3dup.web.main import start_web + except ImportError: + custom_console.bot_error_log( + "Web dependencies are not installed. Run: pip install Unit3Dup[web]" + ) + return + import os + from unit3dup.state_db import StateDB + + db_path = os.path.join(str(DEFAULT_JSON_PATH.parent), "unit3dup.db") + state_db = StateDB(db_path=db_path) + custom_console.bot_log(f"[Web] Starting web dashboard on {config.user_preferences.WEB_HOST}:{config.user_preferences.WEB_PORT}") + start_web(state_db, host=config.user_preferences.WEB_HOST, port=config.user_preferences.WEB_PORT) + return + + # Watcher (with optional web dashboard) if cli.args.watcher: + if not watcher_folders: + custom_console.bot_error_log("No watcher folders configured. Set WATCHER_PATHS or WATCHER_PATH in config.") + return + bot = Bot(path='', cli=cli.args, mode="auto", trackers_name_list=tracker_name_list, torrent_archive_path=tracker_archive) - bot.watcher(duration=config.user_preferences.WATCHER_INTERVAL, watcher_path=config.user_preferences.WATCHER_PATH, - destination_path = config.user_preferences.WATCHER_DESTINATION_PATH) + # Watcher + Web mode: run watcher in background thread, web server in main thread + if getattr(cli.args, 'web', False): + try: + from unit3dup.web.main import start_web + except ImportError: + custom_console.bot_error_log( + "Web dependencies are not installed. Run: pip install Unit3Dup[web]" + ) + return + import os + import threading + from unit3dup.state_db import StateDB + + db_path = os.path.join(str(DEFAULT_JSON_PATH.parent), "unit3dup.db") + state_db = StateDB(db_path=db_path) + + # Start watcher in a daemon thread + watcher_thread = threading.Thread( + target=bot.watcher, + kwargs={ + "duration": config.user_preferences.WATCHER_INTERVAL, + "watcher_folders": watcher_folders, + "state_dir": str(DEFAULT_JSON_PATH.parent), + }, + daemon=True, + name="watcher", + ) + watcher_thread.start() + custom_console.bot_log(f"[Web] Watcher started in background thread") + custom_console.bot_log(f"[Web] Starting web dashboard on {config.user_preferences.WEB_HOST}:{config.user_preferences.WEB_PORT}") + + start_web(state_db, host=config.user_preferences.WEB_HOST, port=config.user_preferences.WEB_PORT) + return + + # Standard watcher (no web) + bot.watcher(duration=config.user_preferences.WATCHER_INTERVAL, + watcher_folders=watcher_folders, + state_dir=str(DEFAULT_JSON_PATH.parent)) # ftp and upload if cli.args.ftp: diff --git a/unit3dup/bot.py b/unit3dup/bot.py index 4b7e48a..e934167 100644 --- a/unit3dup/bot.py +++ b/unit3dup/bot.py @@ -24,13 +24,13 @@ class Bot: Methods: run(): Starts the media processing and torrent handling tasks - watcher(duration: int, watcher_path: str): Monitors a folder for changes and processes files + watcher(duration: int, watcher_folders: list): Monitors multiple folders for changes and processes files ftp(): Connects to a remote FTP server and processes files """ # Bot Manager def __init__(self, path: str, cli: argparse.Namespace, trackers_name_list: list, mode="man", - torrent_archive_path = None): + torrent_archive_path=None, qbit_category: str | None = None): """ Initializes the Bot instance with path, command-line interface object, and mode @@ -41,10 +41,17 @@ def __init__(self, path: str, cli: argparse.Namespace, trackers_name_list: list, """ self.trackers_name_list = trackers_name_list self.torrent_archive_path = torrent_archive_path + self.qbit_category = qbit_category self.content_manager = None self.path = path.strip() self.cli = cli self.mode = mode + self.upload_count = 0 + self.skip_reasons: list[dict] = [] + self.release_names: list[str] = [] + self.release_sources: list[str] = [] + self.content_categories: list[str] = [] + self.validation_reports: dict[str, list[dict]] = {} def contents(self) -> bool | list[Media]: @@ -93,7 +100,7 @@ def run(self) -> bool: return False # Instance a new run - torrent_manager = TorrentManager(cli=self.cli, tracker_archive=self.torrent_archive_path) + torrent_manager = TorrentManager(cli=self.cli, tracker_archive=self.torrent_archive_path, qbit_category=self.qbit_category) # Process the torrents content (files) torrent_manager.process(contents=contents) @@ -103,39 +110,110 @@ def run(self) -> bool: else: # otherwise run the torrents creations and the upload process torrent_manager.run(trackers_name_list=self.trackers_name_list) + + self.upload_count = torrent_manager.upload_count + self.skip_reasons = torrent_manager.skip_reasons + self.release_names = torrent_manager.release_names + self.release_sources = torrent_manager.release_sources + self.content_categories = torrent_manager.content_categories + self.validation_reports = torrent_manager.validation_reports return True + def prepare(self) -> list: + """Prepare contents without uploading. Returns list of PreparedItem objects.""" + from unit3dup.prepared_item import PreparedItem + + contents = self.contents() + if not contents: + return [] + + torrent_manager = TorrentManager( + cli=self.cli, + tracker_archive=self.torrent_archive_path, + qbit_category=self.qbit_category, + ) + torrent_manager.process(contents=contents) + prepared_items = torrent_manager.prepare_all(trackers_name_list=self.trackers_name_list) + + self.skip_reasons = torrent_manager.skip_reasons + self.validation_reports = torrent_manager.validation_reports + + return prepared_items - def watcher(self, duration: int, watcher_path: str, destination_path: str)-> bool: + def watcher(self, duration: int, watcher_folders: list, state_dir: str) -> bool: """ - Monitors the watcher path for new files/folders, uploads them one-by-one, - then moves successfully processed items to the destination folder. + Monitors multiple watcher folders for new files/folders, uploads them one-by-one. + Each folder can have its own qBittorrent category. + Uses a persistent JSON state file to skip already-processed entries. Args: duration (int): The time duration in seconds for the watchdog to wait before checking again - watcher_path (str): The path to the folder being monitored for new files - destination_path: Where to move items AFTER a successful upload + watcher_folders (list[WatcherFolder]): List of folder configs to monitor + state_dir (str): Directory where the watcher_state.json is stored (config dir) """ + from unit3dup.watcher_state import WatcherState + try: - # Return if the watcher path doesn't exist - if not os.path.exists(watcher_path): - custom_console.bot_error_log("Watcher path does not exist or is not configured\n") + # Validate folders at startup + valid_folders = [] + for wf in watcher_folders: + if os.path.exists(wf.path): + cat_info = f" (category: {wf.category})" if wf.category else "" + custom_console.bot_log(f"[Watcher] Monitoring: {wf.path}{cat_info}") + valid_folders.append(wf) + else: + custom_console.bot_warning_log(f"[Watcher] Path does not exist, skipping: {wf.path}") + + if not valid_folders: + custom_console.bot_error_log("[Watcher] No valid watcher folders found\n") return False + dry_run = self.cli.noup or self.cli.noseed + watcher_state = WatcherState(state_dir=state_dir) + # In dry-run mode, write to a separate preview file + dryrun_state = WatcherState(state_dir=state_dir, filename="watcher_dryrun.json") if dry_run else None + + state_db = None + if getattr(self.cli, 'web', False): + from unit3dup.state_db import StateDB + db_path = os.path.join(state_dir, "unit3dup.db") + state_db = StateDB(db_path=db_path) + # Migrate existing JSON state if DB is fresh + if not state_db.count_by_status(): + migrated = state_db.migrate_from_json(watcher_state.state_file) + if migrated: + custom_console.bot_log(f"[Watcher] Migrated {migrated} entries from JSON to SQLite") + # Clean up items left in 'analyzing' from a prior crash + recovered = state_db.recover_analyzing() + if recovered: + custom_console.bot_log(f"[Watcher] Cleaned up {recovered} item(s) stuck in 'analyzing' status") + + if dry_run: + custom_console.bot_log("[Watcher] DRY-RUN mode: results written to watcher_dryrun.json only") + custom_console.bot_log( + f"[Watcher] State file: {watcher_state.state_file} " + f"({len(watcher_state.uploaded)} uploaded, {len(watcher_state.skipped)} skipped)" + ) + # Watchdog loop while True: - # Traiter immédiatement les fichiers au démarrage et à chaque cycle - watcher_root = Path(watcher_path) - done_root = Path(destination_path) - done_root.mkdir(parents=True, exist_ok=True) - - # Skip if there are no files in the watcher folder - if not os.listdir(watcher_path): - custom_console.bot_log("The are no files in the Watcher folder\n") - else: - # Process top-level entries one-by-one to avoid moving everything at once. + for watcher_folder in valid_folders: + watcher_path = watcher_folder.path + folder_category = watcher_folder.category + + if not os.path.exists(watcher_path): + continue + + watcher_root = Path(watcher_path) + + # Skip if there are no files in this watcher folder + if not os.listdir(watcher_path): + continue + entries = sorted( - [p for p in watcher_root.iterdir() if p.name and not p.name.startswith(".")], + [p for p in watcher_root.iterdir() + if p.name and not p.name.startswith(".") + and (p.is_dir() or ManageTitles.filter_ext(p.name))], key=lambda p: p.name.lower(), ) @@ -143,6 +221,14 @@ def watcher(self, duration: int, watcher_path: str, destination_path: str)-> bo if not src.exists(): continue + # Check state BEFORE any heavy processing + if state_db: + status = state_db.is_known(src.name) + else: + status = watcher_state.is_known(str(src), folder_path=watcher_path) + if status: + continue + # Upload this single item (folder or file) mode = "folder" if src.is_dir() else "man" custom_console.bot_log(f"\n[Watcher] Processing -> {src}") @@ -153,18 +239,278 @@ def watcher(self, duration: int, watcher_path: str, destination_path: str)-> bo trackers_name_list=self.trackers_name_list, mode=mode, torrent_archive_path=self.torrent_archive_path, + qbit_category=folder_category, ) + if state_db: + # Web mode: prepare only, don't upload + import json + from datetime import datetime + + # Insert a lightweight "analyzing" placeholder so + # the web UI shows this item is being processed. + try: + analyzing_id = state_db.add_item( + source_basename=src.name, + source_path=str(src), + folder_path=watcher_path, + source_type="folder" if src.is_dir() else "file", + status="analyzing", + discovered_at=datetime.now().isoformat(), + ) + except Exception: + analyzing_id = 0 + + try: + prepared_items = single_bot.prepare() + except Exception as exc: + # Transition analyzing record to error so it + # doesn't stay stuck forever. + error_msg = f"prepare() failed: {exc}" + if analyzing_id: + state_db.update_item(analyzing_id, status="error", upload_error=error_msg) + else: + state_db.add_item( + source_basename=src.name, + source_path=str(src), + folder_path=watcher_path, + source_type="folder" if src.is_dir() else "file", + status="error", + upload_error=error_msg, + discovered_at=datetime.now().isoformat(), + ) + custom_console.bot_error_log(f"[Watcher/Web] Error → {src.name} ({exc})") + continue + + # Torrent archive exists = previously uploaded. + # Record as "uploaded" to match non-web watcher behavior. + if prepared_items and all( + getattr(p, 'skip_reason', None) == "already_in_archive" + for p in prepared_items + ): + first = prepared_items[0] + fields = dict( + status="uploaded", + content_category=first.content_category, + qbit_category=first.qbit_category, + display_name=first.display_name, + torrent_name=first.content.torrent_name if first.content else "", + release_name=first.release_name or src.name, + source_tag=first.source_tag, + file_size=first.content.size if first.content else 0, + skip_reason="already_in_archive", + uploaded_at=datetime.now().isoformat(), + ) + if analyzing_id: + state_db.update_item(analyzing_id, **fields) + else: + state_db.add_item( + source_basename=src.name, + source_path=str(src), + folder_path=watcher_path, + source_type="folder" if src.is_dir() else "file", + discovered_at=datetime.now().isoformat(), + **fields, + ) + custom_console.bot_log( + f"[Watcher/Web] Already uploaded → {first.release_name or src.name}" + ) + continue + + for item in prepared_items: + if item.skip_reason: + fields = dict( + status="skipped", + content_category=item.content_category, + qbit_category=item.qbit_category, + display_name=item.display_name, + torrent_name=item.content.torrent_name if item.content else "", + release_name=item.release_name, + source_tag=item.source_tag, + file_size=item.content.size if item.content else 0, + resolution=item.resolution, + tmdb_id=item.tmdb_id, + imdb_id=item.imdb_id, + igdb_id=item.igdb_id, + tmdb_title=item.tmdb_title, + tmdb_year=item.tmdb_year, + description=item.description, + mediainfo=item.mediainfo, + nfo_content=item.nfo_content, + audio_tracks=item.audio_tracks, + subtitle_tracks=item.subtitle_tracks, + tracker_payload=item.tracker_data, + tracker_name=item.tracker_name, + trackers_list=item.trackers_list, + torrent_archive_path=item.torrent_filepath, + validation_report=item.validation_report, + has_errors=int(item.has_errors), + has_warnings=int(item.has_warnings), + skip_reason=item.skip_reason, + duplicate_match=item.duplicate_match, + prepared_at=None, + ) + if analyzing_id: + state_db.update_item(analyzing_id, **fields) + analyzing_id = 0 # Only update once + else: + state_db.add_item( + source_basename=src.name, + source_path=str(src), + folder_path=watcher_path, + source_type="folder" if src.is_dir() else "file", + discovered_at=datetime.now().isoformat(), + **fields, + ) + custom_console.bot_warning_log(f"[Watcher/Web] Skipped → {src.name} ({item.skip_reason})") + else: + fields = dict( + status="pending", + content_category=item.content_category, + qbit_category=item.qbit_category, + display_name=item.display_name, + torrent_name=item.content.torrent_name if item.content else "", + release_name=item.release_name, + source_tag=item.source_tag, + file_size=item.content.size if item.content else 0, + resolution=item.resolution, + tmdb_id=item.tmdb_id, + imdb_id=item.imdb_id, + igdb_id=item.igdb_id, + tmdb_title=item.tmdb_title, + tmdb_year=item.tmdb_year, + description=item.description, + mediainfo=item.mediainfo, + nfo_content=item.nfo_content, + audio_tracks=item.audio_tracks, + subtitle_tracks=item.subtitle_tracks, + tracker_payload=item.tracker_data, + tracker_name=item.tracker_name, + trackers_list=item.trackers_list, + torrent_archive_path=item.torrent_filepath, + validation_report=item.validation_report, + has_errors=int(item.has_errors), + has_warnings=int(item.has_warnings), + duplicate_match=None, + prepared_at=datetime.now().isoformat(), + ) + if analyzing_id: + state_db.update_item(analyzing_id, **fields) + analyzing_id = 0 # Only update once + else: + state_db.add_item( + source_basename=src.name, + source_path=str(src), + folder_path=watcher_path, + source_type="folder" if src.is_dir() else "file", + discovered_at=datetime.now().isoformat(), + **fields, + ) + custom_console.bot_log(f"[Watcher/Web] Pending → {item.release_name or src.name}") + + if not prepared_items: + fields = dict( + status="skipped", + skip_reason="no_processable_media", + ) + if analyzing_id: + state_db.update_item(analyzing_id, **fields) + else: + state_db.add_item( + source_basename=src.name, + source_path=str(src), + folder_path=watcher_path, + source_type="folder" if src.is_dir() else "file", + discovered_at=datetime.now().isoformat(), + **fields, + ) + custom_console.bot_warning_log(f"[Watcher/Web] Skipped → {src.name} (no_processable_media)") + + continue # Skip the normal (non-web) processing below + ok = single_bot.run() - if not ok: + + # In dry-run, only write to the preview file (not the main state) + target_state = dryrun_state if dry_run else watcher_state + + content_cat = single_bot.content_categories[0] if single_bot.content_categories else None + + if ok and single_bot.upload_count > 0: + # Use the normalized release name if available + release_name = single_bot.release_names[0] if single_bot.release_names else src.name + upload_report = single_bot.validation_reports.get(release_name) + release_source = single_bot.release_sources[0] if single_bot.release_sources else None + target_state.mark_uploaded( + source_path=str(src), + torrent_name=release_name, + trackers=self.trackers_name_list, + folder_path=watcher_path, + category=folder_category, + content_category=content_cat, + validation_report=upload_report, + source=release_source, + ) + label = "[Watcher] DRY-RUN uploaded" if dry_run else "[Watcher] Uploaded" + custom_console.bot_log(f"{label} -> {release_name}") + elif single_bot.skip_reasons: + unique_reasons = sorted(set(s["reason"] for s in single_bot.skip_reasons)) + + if unique_reasons == ["already_in_archive"]: + # Torrent exists in archive = content was uploaded before + archive_name = single_bot.skip_reasons[0].get("torrent_name", src.name) + archive_source = next( + (s.get("source") for s in single_bot.skip_reasons if s.get("source")), None + ) + target_state.mark_uploaded( + source_path=str(src), + torrent_name=archive_name, + trackers=self.trackers_name_list, + folder_path=watcher_path, + category=folder_category, + content_category=content_cat, + source=archive_source, + ) + label = "[Watcher] DRY-RUN already uploaded" if dry_run else "[Watcher] Already uploaded" + custom_console.bot_log(f"{label} -> {archive_name}") + else: + reasons = ", ".join(unique_reasons) + skip_report = [] + skip_source = None + for s in single_bot.skip_reasons: + if "validation_report" in s: + skip_report.extend(s["validation_report"]) + if not skip_source and s.get("source"): + skip_source = s["source"] + target_state.mark_skipped( + source_path=str(src), + torrent_name=src.name, + reason=reasons, + folder_path=watcher_path, + category=folder_category, + content_category=content_cat, + validation_report=skip_report or None, + source=skip_source, + ) + custom_console.bot_warning_log( + f"[Watcher] Skipped -> {src.name} ({reasons})" + ) + else: + target_state.mark_skipped( + source_path=str(src), + torrent_name=src.name, + reason="no_processable_media", + folder_path=watcher_path, + category=folder_category, + content_category=content_cat, + ) custom_console.bot_warning_log( - f"[Watcher] Upload failed/skipped, leaving in place -> {src}" + f"[Watcher] Skipped -> {src.name} (no_processable_media)" ) - - # Nettoyer les fichiers .nfo isolés dans le dossier watcher après avoir traité tous les fichiers du cycle - self._cleanup_orphaned_nfo_files(watcher_path) - # Attendre avant le prochain cycle + # Clean orphaned NFO files for this folder + self._cleanup_orphaned_nfo_files(watcher_path) + + # Wait before next cycle print() start_time = time.perf_counter() end_time = start_time + duration @@ -174,7 +520,7 @@ def watcher(self, duration: int, watcher_path: str, destination_path: str)-> bo custom_console.bot_counter_log( f"WATCHDOG: {remaining_time:.1f} seconds Ctrl-c to Exit " ) - time.sleep(0.01) + time.sleep(1) print() except KeyboardInterrupt: diff --git a/unit3dup/compliance/__init__.py b/unit3dup/compliance/__init__.py new file mode 100644 index 0000000..e4bf5c3 --- /dev/null +++ b/unit3dup/compliance/__init__.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +"""Compliance checker for past uploads on UnIT3D trackers.""" + +from unit3dup.compliance.scanner import ( + ComplianceScanner, + UnIT3DClient, + RateLimiter, + check_one_torrent, + mediainfo_facts_from_text, + extract_mediainfo_text, +) + +__all__ = [ + "ComplianceScanner", + "UnIT3DClient", + "RateLimiter", + "check_one_torrent", + "mediainfo_facts_from_text", + "extract_mediainfo_text", +] diff --git a/unit3dup/compliance/scanner.py b/unit3dup/compliance/scanner.py new file mode 100644 index 0000000..81e5d96 --- /dev/null +++ b/unit3dup/compliance/scanner.py @@ -0,0 +1,916 @@ +# -*- coding: utf-8 -*- +"""Compliance scanner: fetches past uploads from UnIT3D and validates their +release names against the G3MINI naming rules. + +Read-only: never issues PATCH/PUT/DELETE. Edits happen via deep-links to the +UnIT3D web edit page. +""" + +from __future__ import annotations + +import json +import re +import threading +import time +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Callable, Iterator, Optional +from urllib.parse import urljoin, urlparse + +import requests + +from common.trackers.data import trackers_api_data +from unit3dup.release_normalizer import normalize_release_name +from unit3dup.state_db import StateDB +from unit3dup.validators.naming_validator import NamingValidator +from unit3dup.validators.encoding_validator import EncodingValidator +from unit3dup.prez import _LANG_NAME_TO_ISO + +try: + from view import custom_console +except Exception: # pragma: no cover - logging fallback + class _Noop: + def bot_log(self, *a, **k): pass + def bot_warning_log(self, *a, **k): pass + def bot_error_log(self, *a, **k): pass + custom_console = _Noop() + + +# ── Severity ranking ─────────────────────────────────────────────────────── + +_SEVERITY_ORDER = {"NONE": 0, "INFO": 1, "WARNING": 2, "ERROR": 3} +_REVERSE_SEVERITY = {v: k for k, v in _SEVERITY_ORDER.items()} + + +def _max_severity(severities: list[str]) -> str: + if not severities: + return "NONE" + best = 0 + for s in severities: + best = max(best, _SEVERITY_ORDER.get(s, 0)) + return _REVERSE_SEVERITY[best] + + +# ── Mediainfo text parsing ───────────────────────────────────────────────── + +_RE_MULTIVIEW = re.compile(r"^\s*MultiView[ _]?Count\s*:\s*(\d+)", re.IGNORECASE | re.MULTILINE) +# Top-level MediaInfo section header. We accept common localizations too so +# a French/Italian/Spanish-generated mediainfo doesn't parse as "0 sections". +_RE_SECTION = re.compile( + r"^\s*(General|Général|Generale|Video|Vidéo|V[ií]deo|Audio|Text|Texto|Testo|Menu|Men[uú])" + r"(?:\s*#\d+)?\s*$", + re.IGNORECASE | re.MULTILINE, +) +_RE_KV = re.compile(r"^\s*([A-Za-z][A-Za-z0-9_ /()-]*?)\s*:\s*(.+?)\s*$") + +# BDInfo dumps use completely different delimiters; parsing them as MediaInfo +# yields 0 audio sections, which would make release_normalizer think the disc +# is silent. We detect them up-front and skip the facts extraction. +_BDINFO_MARKERS = ("DISC INFO:", "Disc Title:", "PLAYLIST REPORT:", "VIDEO:\n") + + +def _looks_like_bdinfo(text: str) -> bool: + if not text: + return False + head = text[:4096] + return any(marker in head for marker in _BDINFO_MARKERS) + + +def extract_mediainfo_text(payload: dict) -> Optional[str]: + """Prefer the MediaInfo dump; fall back to BDInfo if unset. + + UnIT3D's JSON:API resource exposes these under the snake_case keys + `media_info` / `bd_info` (see TorrentResource::toArray). Older/custom + deployments sometimes return the raw DB column names `mediainfo` / + `bdinfo`, so we accept both. + """ + for key in ("media_info", "mediainfo"): + value = payload.get(key) + if value and isinstance(value, str) and value.strip(): + return value + for key in ("bd_info", "bdinfo"): + value = payload.get(key) + if value and isinstance(value, str) and value.strip(): + return value + return None + + +def _iter_sections(mi_text: str) -> Iterator[tuple[str, dict[str, str]]]: + """Yield (section_name_lower, dict-of-lowercased-keys → raw-values) for each + top-level MediaInfo section. Works on the human-readable text representation + returned by pymediainfo --Output="".""" + lines = mi_text.splitlines() + i = 0 + n = len(lines) + while i < n: + line = lines[i] + m = _RE_SECTION.match(line) + if not m: + i += 1 + continue + section = m.group(1).strip().lower() + fields: dict[str, str] = {} + i += 1 + while i < n and not _RE_SECTION.match(lines[i]): + kv = _RE_KV.match(lines[i]) + if kv: + key = kv.group(1).strip().lower() + val = kv.group(2).strip() + if key and key not in fields: + fields[key] = val + i += 1 + yield section, fields + + +def mediainfo_facts_from_text(mi_text: str) -> dict: + """Extract only the facts the naming validator actually uses. + + Returns a dict with: + - multiview_count: int | None + - audio_formats: list[{service_kind, delay, language}] + - audio_track_count: int + - is_bdinfo: bool (True when we cannot extract MediaInfo-style facts) + """ + if not mi_text: + return {"multiview_count": None, "audio_formats": [], "audio_track_count": 0, "is_bdinfo": False} + + # BDInfo uses a different grammar; bail out so callers know not to + # infer is_silent from a zero audio count. + if _looks_like_bdinfo(mi_text): + return { + "multiview_count": None, + "audio_formats": [], + "audio_track_count": 0, + "is_bdinfo": True, + } + + multiview = None + m = _RE_MULTIVIEW.search(mi_text) + if m: + try: + multiview = int(m.group(1)) + except (TypeError, ValueError): + multiview = None + + audio_formats: list[dict[str, Any]] = [] + subtitle_formats: list[dict[str, Any]] = [] + writing_library: Optional[str] = None + video_width: Optional[int] = None + video_height: Optional[int] = None + video_aspect_ratio: Optional[str] = None + for section, fields in _iter_sections(mi_text): + if section == "audio": + track: dict[str, Any] = {} + service_kind = fields.get("servicekind") or fields.get("service kind") or "" + if service_kind: + track["service_kind"] = service_kind + delay_raw = ( + fields.get("delay relative to video") + or fields.get("delay") + or "" + ) + if delay_raw: + num = re.search(r"-?\d+(?:\.\d+)?", delay_raw) + if num: + try: + track["delay"] = float(num.group(0)) + except (TypeError, ValueError): + pass + lang = fields.get("language") or "" + if lang: + track["language"] = lang + audio_formats.append(track) + + elif section == "text": + lang = fields.get("language") or "" + fmt = fields.get("format") or "" + subtitle_formats.append({"language": lang, "format": fmt}) + + elif section in ("video", "vidéo", "video"): + if writing_library is None: + lib = ( + fields.get("writing library") + or fields.get("encoded library name") + or fields.get("encoded_library_name") + or "" + ) + if lib: + writing_library = lib + if video_width is None: + w_raw = fields.get("width") or "" + # MediaInfo formats widths as "1 920 pixels" — strip separators + wm = re.search(r"\d[\d\s\u00a0]*", w_raw) + if wm: + try: + video_width = int(re.sub(r"\s", "", wm.group(0))) + except ValueError: + pass + if video_height is None: + h_raw = fields.get("height") or "" + hm = re.search(r"\d[\d\s\u00a0]*", h_raw) + if hm: + try: + video_height = int(re.sub(r"\s", "", hm.group(0))) + except ValueError: + pass + if video_aspect_ratio is None: + ar = fields.get("display aspect ratio") or fields.get("aspect ratio") or "" + if ar: + video_aspect_ratio = ar + + # Multiview fallback: scan Video section too + if multiview is None: + for section, fields in _iter_sections(mi_text): + if section != "video": + continue + mv = fields.get("multiview_count") or fields.get("multiview count") + if mv: + try: + multiview = int(mv) + break + except (TypeError, ValueError): + pass + + return { + "multiview_count": multiview, + "audio_formats": audio_formats, + "audio_track_count": len(audio_formats), + "subtitle_formats": subtitle_formats, + "writing_library": writing_library, + "video_width": video_width, + "video_height": video_height, + "video_aspect_ratio": video_aspect_ratio, + "is_bdinfo": False, + } + + +@dataclass +class _FakeMediaFile: + """Duck-typed stand-in for common.mediainfo.MediaFile. + + Exposes only the attributes validators read (naming + encoding) + so they can run without file access. + """ + multiview_count: Optional[int] = None + audio_formats: Optional[list[dict]] = None + audio_track_count: int = 0 + subtitle_formats: Optional[list[dict]] = None + writing_library: Optional[str] = None + video_width: Optional[int] = None + video_height: Optional[int] = None + video_aspect_ratio: Optional[str] = None + + +@dataclass +class _PrezMediaFile: + """Duck-typed stand-in for common.mediainfo.MediaFile, shaped for + prez.generate_prez. Built from the raw MediaInfo text returned by the + tracker, so we can render a tool-format description without file access. + """ + video_format: Optional[str] = None + video_bit_depth: Optional[str] = None + video_width: Optional[int] = None + video_height: Optional[int] = None + audio_track: list[dict] = None + subtitle_track: list[dict] = None + + def __post_init__(self): + if self.audio_track is None: + self.audio_track = [] + if self.subtitle_track is None: + self.subtitle_track = [] + + +_RE_LEADING_INT = re.compile(r"-?\d+") + + +def _parse_int_prefix(value: str) -> Optional[int]: + """Extract the leading integer from strings like '6 channels' or '1 920 pixels'.""" + if not value: + return None + cleaned = re.sub(r"[\s\u00a0]", "", value) + m = _RE_LEADING_INT.match(cleaned) + if not m: + return None + try: + return int(m.group(0)) + except (TypeError, ValueError): + return None + + +def _normalize_language_to_iso(value: str) -> str: + """Map a MediaInfo `Language` value to an ISO code when possible. + + MediaInfo text dumps usually store the full English/French name + ("French", "English"), but prez expects the 2-letter ISO code so the + flag lookup works. Returns the original string when no mapping applies. + """ + if not value: + return "" + raw = value.strip() + # Already an ISO code like "fr" or "fr-FR" + short = raw.lower().split("-")[0] + if len(short) == 2 and short.isalpha(): + return short + iso = _LANG_NAME_TO_ISO.get(raw.lower()) + return iso or raw + + +def build_prez_media_file(mi_text: Optional[str]) -> Optional[_PrezMediaFile]: + """Convert a raw MediaInfo text dump into a prez-compatible shim. + + Returns None when the text is missing or looks like BDInfo (which prez + isn't wired to consume). + """ + if not mi_text or _looks_like_bdinfo(mi_text): + return None + + pmf = _PrezMediaFile() + for section, fields in _iter_sections(mi_text): + if section in ("video", "vidéo", "vídeo", "video"): + if not pmf.video_format: + pmf.video_format = fields.get("format") or None + if not pmf.video_bit_depth: + bd = fields.get("bit depth") or "" + if bd: + m = _RE_LEADING_INT.search(bd) + pmf.video_bit_depth = m.group(0) if m else bd + if pmf.video_width is None: + pmf.video_width = _parse_int_prefix(fields.get("width") or "") + if pmf.video_height is None: + pmf.video_height = _parse_int_prefix(fields.get("height") or "") + + elif section == "audio": + channel_raw = ( + fields.get("channel(s)") + or fields.get("channels") + or "" + ) + channel_int = _parse_int_prefix(channel_raw) + pmf.audio_track.append({ + "language": _normalize_language_to_iso(fields.get("language", "")), + "title": fields.get("title", ""), + "format": fields.get("format", ""), + "channel_s": str(channel_int) if channel_int is not None else "", + "bit_rate": fields.get("bit rate", ""), + }) + + elif section in ("text", "texto", "testo"): + pmf.subtitle_track.append({ + "language": _normalize_language_to_iso(fields.get("language", "")), + "title": fields.get("title", ""), + "forced": fields.get("forced", ""), + "format": fields.get("format", ""), + }) + + return pmf + + +# ── Rate limiter ─────────────────────────────────────────────────────────── + +class RateLimiter: + """Thread-safe sleep-based rate limiter. + + Defaults to 2.1s between calls (safe margin under 30 req/min). + """ + + def __init__(self, min_interval_s: float = 2.1): + self._min_interval = max(0.0, float(min_interval_s)) + self._lock = threading.Lock() + self._next_allowed = 0.0 + + def wait(self) -> None: + with self._lock: + now = time.monotonic() + sleep_for = self._next_allowed - now + if sleep_for > 0: + time.sleep(sleep_for) + now = time.monotonic() + self._next_allowed = now + self._min_interval + + +# ── UnIT3D HTTP client ───────────────────────────────────────────────────── + +class UnIT3DClient: + """Minimal read-only HTTP wrapper around the UnIT3D API. + + Reuses the base_url / api_token from `trackers_api_data` for the given + tracker name. Honors 429 via a 60s back-off (matches pvtTracker.Tracker._get). + """ + + _PLACEHOLDER_USERNAMES = {"", "no_key", "no_user", "none"} + + def __init__( + self, + tracker_name: str = "GEMINI", + rate_limiter: Optional[RateLimiter] = None, + timeout: float = 15.0, + stop_event: Optional[threading.Event] = None, + ): + api_data = trackers_api_data.get(tracker_name.upper()) + if not api_data: + raise ValueError(f"Tracker '{tracker_name}' not found in trackers_api_data") + + self.tracker_name = tracker_name.upper() + self.base_url = api_data["url"].rstrip("/") + self.api_token = api_data["api_key"] + raw_username = (api_data.get("username") or "").strip() + # Match the ComplianceService placeholder treatment so we don't + # accidentally fire queries with uploader="no_key". + if raw_username.lower() in self._PLACEHOLDER_USERNAMES: + self.username = "" + else: + self.username = raw_username + self.filter_url = urljoin(self.base_url + "/", "api/torrents/filter") + self.fetch_url = urljoin(self.base_url + "/", "api/torrents/") + + self._rate = rate_limiter or RateLimiter() + self._timeout = timeout + self._stop_event = stop_event + self._headers = { + "User-Agent": "Unit3D-up/compliance/1.0", + "Accept": "application/json", + } + + def set_stop_event(self, event: Optional[threading.Event]) -> None: + """Make HTTP sleeps interruptible by `event.set()`.""" + self._stop_event = event + + # Deep-link to the web edit form (read-only from our side). + def edit_url(self, torrent_id: int) -> str: + return f"{self.base_url}/torrents/{int(torrent_id)}/edit" + + def _get_json(self, url: str, params: Optional[dict] = None, max_retries: int = 5) -> Optional[dict]: + """GET a JSON endpoint with 429/5xx backoff. + + Raises requests.HTTPError on 4xx non-429 after a single pass. Raises + after `max_retries` for repeated 429/5xx/network failures. Respects + the rate limiter on every attempt. + """ + attempt = 0 + while True: + self._rate.wait() + try: + response = requests.get( + url=url, + params=params, + headers=self._headers, + timeout=self._timeout, + ) + except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as exc: + custom_console.bot_warning_log(f"[Compliance] Network error ({exc}); retrying in 10s") + attempt += 1 + if attempt >= max_retries: + raise + self._sleep(10) + continue + + if response.status_code == 429: + custom_console.bot_warning_log("[Compliance] 429 rate limit — sleeping 60s") + self._sleep(60) + attempt += 1 + if attempt >= max_retries: + response.raise_for_status() + continue + + if 500 <= response.status_code < 600: + attempt += 1 + if attempt >= max_retries: + response.raise_for_status() + self._sleep(5 * attempt) + continue + + # 4xx non-429: bubble up immediately so the caller can decide + # whether to drop this one item or abort the whole scan. + response.raise_for_status() + try: + return response.json() + except ValueError: + return None + + def _sleep(self, seconds: float) -> None: + """Interruptible sleep: honors stop_event when present.""" + if self._stop_event is not None: + # wait() returns True as soon as the event is set. + self._stop_event.wait(seconds) + else: + time.sleep(seconds) + + def iter_user_torrents( + self, + uploader: str, + per_page: int = 100, + max_pages: int = 2000, + ) -> Iterator[dict]: + """Stream every torrent for the given uploader, page by page. + + Yields the raw torrent dicts (attributes under `data[].attributes`). + Guards against infinite/cyclic pagination by tracking visited URLs + and capping the total page count. + """ + if not uploader: + raise ValueError("uploader is required") + + params = { + "api_token": self.api_token, + "uploader": uploader, + "perPage": per_page, + "page": 1, + } + + next_url: Optional[str] = self.filter_url + next_params: Optional[dict] = dict(params) + + visited: set[str] = set() + pages = 0 + + while next_url: + pages += 1 + if pages > max_pages: + custom_console.bot_warning_log( + f"[Compliance] Pagination capped at {max_pages} pages" + ) + return + + # Cycle guard: include the resolved next_params so ?page=N differs. + cycle_key = next_url if not next_params else f"{next_url}|{sorted(next_params.items())}" + if cycle_key in visited: + custom_console.bot_warning_log( + f"[Compliance] Pagination cycle detected at page {pages}; stopping" + ) + return + visited.add(cycle_key) + + try: + payload = self._get_json(next_url, params=next_params) + except requests.exceptions.HTTPError as exc: + status = exc.response.status_code if exc.response is not None else "?" + custom_console.bot_warning_log( + f"[Compliance] Page {pages} failed with HTTP {status}; stopping" + ) + return + if not payload: + return + + data = payload.get("data") or [] + for entry in data: + if isinstance(entry, dict) and "attributes" in entry and "id" in entry: + yield self._flatten_entry(entry) + elif isinstance(entry, dict): + # Some deployments return flat objects already + yield entry + + links = payload.get("links") or {} + nxt = links.get("next") + if not nxt: + break + + parsed = urlparse(nxt) + if not parsed.scheme: + nxt = urljoin(self.base_url + "/", nxt.lstrip("/")) + next_url = nxt + # The next URL from Laravel already includes page & uploader in + # its query string; don't double-send them via params. Trust the + # URL and only re-inject api_token when it's missing. + if "api_token=" in (urlparse(next_url).query or ""): + next_params = None + else: + next_params = {"api_token": self.api_token} + + @staticmethod + def _flatten_entry(entry: dict) -> dict: + """Flatten the JSON:API-ish envelope into a single dict.""" + out = dict(entry.get("attributes") or {}) + out["id"] = entry.get("id") + return out + + def get_torrent(self, torrent_id: int) -> Optional[dict]: + url = urljoin(self.fetch_url, str(int(torrent_id))) + payload = self._get_json(url, params={"api_token": self.api_token}) + if not payload: + return None + # /api/torrents/{id} calls TorrentResource::withoutWrapping(), so the + # response is `{type, id, attributes}` with NO outer `data` key. Older + # deployments (or wrapped resources) still expose `{data: {...}}`. + data = payload.get("data") + if isinstance(data, dict): + if "attributes" in data: + return self._flatten_entry(data) + return data + if "attributes" in payload and "id" in payload: + return self._flatten_entry(payload) + return payload + + def find_by_name(self, name: str, uploader: Optional[str] = None, per_page: int = 10) -> list[dict]: + """Used by the post-upload hook when we couldn't extract the id from + the tracker response URL.""" + if not name: + return [] + params: dict[str, Any] = { + "api_token": self.api_token, + "name": name, + "perPage": per_page, + } + if uploader: + params["uploader"] = uploader + payload = self._get_json(self.filter_url, params=params) + if not payload: + return [] + out: list[dict] = [] + for entry in payload.get("data") or []: + if isinstance(entry, dict) and "attributes" in entry and "id" in entry: + out.append(self._flatten_entry(entry)) + elif isinstance(entry, dict): + out.append(entry) + return out + + +# ── Single-torrent check ─────────────────────────────────────────────────── + +def _extract_torrent_id(payload: dict) -> Optional[int]: + raw = payload.get("id") + if raw is None: + return None + try: + return int(raw) + except (TypeError, ValueError): + return None + + +def _extract_category(payload: dict) -> Optional[str]: + cat = payload.get("category") + if isinstance(cat, dict): + return cat.get("name") or cat.get("slug") + if isinstance(cat, str): + return cat + return None + + +def _classify_diff(has_any_violation: bool, normalizer_diff: bool) -> str: + if has_any_violation and normalizer_diff: + return "both" + if has_any_violation: + return "rule_violation" + if normalizer_diff: + return "normalizer_diff" + return "clean" + + +def check_one_torrent( + payload: dict, + *, + tracker_name: str, + db: StateDB, + edit_url: str, + uploader: Optional[str] = None, + linked_item_id: Optional[int] = None, +) -> dict: + """Run the naming checks on one torrent payload and upsert the result. + + Returns the stored compliance row. + """ + torrent_id = _extract_torrent_id(payload) + if torrent_id is None: + raise ValueError("torrent payload missing 'id'") + + # UnIT3D stores the release name in `name`, and the file/folder name + # inside the torrent in `folder`. The folder is what actually hits disk, + # so we check that primarily; when unavailable, fall back to `name`. + current_name = payload.get("folder") or payload.get("name") or "" + release_name = current_name or "" + if not release_name: + raise ValueError(f"torrent #{torrent_id} has no name/folder to check") + + mi_text = extract_mediainfo_text(payload) + facts = mediainfo_facts_from_text(mi_text or "") + + fake_media = _FakeMediaFile( + multiview_count=facts.get("multiview_count"), + audio_formats=facts.get("audio_formats") or [], + audio_track_count=facts.get("audio_track_count", 0), + subtitle_formats=facts.get("subtitle_formats") or [], + writing_library=facts.get("writing_library"), + video_width=facts.get("video_width"), + video_height=facts.get("video_height"), + video_aspect_ratio=facts.get("video_aspect_ratio"), + ) + + violations = [] + for validator in (NamingValidator(), EncodingValidator()): + try: + violations.extend(validator.validate( + media=None, + mediafile=fake_media, + release_name=release_name, + mediainfo_text=mi_text, + )) + except Exception as exc: + custom_console.bot_warning_log( + f"[Compliance] {validator.__class__.__name__} failed for #{torrent_id}: {exc}" + ) + + # Only trust audio_track_count==0 to mean "silent" when we actually + # parsed MediaInfo. With BDInfo (or any unparsable text) we default to + # is_silent=False — otherwise every BD release would be flagged as silent. + if facts.get("is_bdinfo"): + is_silent = False + else: + is_silent = facts.get("audio_track_count", 0) == 0 + + # release_year est exposé par UnIT3D dans `attributes.release_year`. + # On le passe au normalizer comme fallback : utile surtout pour les séries + # dont le nom source n'inclut généralement pas l'année. + raw_year = payload.get("release_year") + year_hint: Optional[str] = None + if raw_year is not None: + try: + year_int = int(raw_year) + if 1900 <= year_int <= 2999: + year_hint = str(year_int) + except (TypeError, ValueError): + year_hint = None + + try: + proposed = normalize_release_name(release_name, mi_text, is_silent=is_silent, year=year_hint) + except Exception as exc: + custom_console.bot_warning_log(f"[Compliance] normalizer failed for torrent #{torrent_id}: {exc}") + proposed = release_name + + normalizer_diff = bool(proposed) and proposed != release_name + # Any violation — including INFO — is treated as non-clean so the severity + # filter and the diff_kind stay consistent with each other. + has_any_violation = bool(violations) + diff_kind = _classify_diff(has_any_violation, normalizer_diff) + + if violations: + severity_max = _max_severity([v.severity for v in violations]) + elif normalizer_diff: + severity_max = "INFO" + else: + severity_max = "NONE" + + violations_serialised = [ + { + "rule": v.rule, + "severity": v.severity, + "message": v.message, + "source_doc": v.source_doc, + } + for v in violations + ] + + description_text = payload.get("description") + if description_text is not None and not isinstance(description_text, str): + description_text = str(description_text) + + fields: dict[str, Any] = dict( + torrent_id=torrent_id, + tracker_name=tracker_name, + current_name=release_name, + proposed_name=proposed, + violations=violations_serialised, + diff_kind=diff_kind, + severity_max=severity_max, + category=_extract_category(payload), + edit_url=edit_url, + description=description_text, + mediainfo=mi_text, + ) + if uploader: + fields["uploader"] = uploader + if linked_item_id is not None: + fields["linked_item_id"] = int(linked_item_id) + + db.upsert_compliance(**fields) + row = db.get_compliance_by_torrent(torrent_id) + return row or fields + + +# ── Scanner façade ───────────────────────────────────────────────────────── + +class ComplianceScanner: + def __init__( + self, + db: StateDB, + tracker_name: str = "GEMINI", + client: Optional[UnIT3DClient] = None, + ): + self.db = db + self.tracker_name = tracker_name.upper() + self.client = client or UnIT3DClient(tracker_name=self.tracker_name) + + def scan_all( + self, + uploader: str, + *, + stop_event: Optional[threading.Event] = None, + progress_cb: Optional[Callable[[dict], None]] = None, + ) -> dict: + """Paginate every torrent for `uploader`, check each one, return a summary.""" + if not uploader: + raise ValueError("uploader is required for scan_all") + + total = 0 + clean = 0 + violations = 0 + errors = 0 + started_at = datetime.now().isoformat() + + # Make HTTP sleeps (429 back-off, retries) interruptible too. + previous_event = getattr(self.client, "_stop_event", None) + if stop_event is not None: + self.client.set_stop_event(stop_event) + + try: + iterator = self.client.iter_user_torrents(uploader) + except Exception as exc: + if stop_event is not None: + self.client.set_stop_event(previous_event) + custom_console.bot_error_log(f"[Compliance] scan_all init failed: {exc}") + return { + "started_at": started_at, + "finished_at": datetime.now().isoformat(), + "total": 0, "clean": 0, "violations": 0, "errors": 1, + "last_error": str(exc), + } + + for payload in iterator: + if stop_event is not None and stop_event.is_set(): + custom_console.bot_log("[Compliance] Scan interrupted by stop_event") + break + try: + torrent_id = _extract_torrent_id(payload) or 0 + edit_url = self.client.edit_url(torrent_id) if torrent_id else "" + row = check_one_torrent( + payload, + tracker_name=self.tracker_name, + db=self.db, + edit_url=edit_url, + uploader=uploader, + ) + if row.get("severity_max") in (None, "NONE"): + clean += 1 + else: + violations += 1 + total += 1 + if progress_cb: + progress_cb({"processed": total, "current": row.get("current_name", "")}) + except Exception as exc: + errors += 1 + custom_console.bot_error_log( + f"[Compliance] Error while checking torrent: {exc}" + ) + + if stop_event is not None: + self.client.set_stop_event(previous_event) + + return { + "started_at": started_at, + "finished_at": datetime.now().isoformat(), + "total": total, + "clean": clean, + "violations": violations, + "errors": errors, + } + + def scan_one(self, torrent_id: int, uploader: Optional[str] = None) -> Optional[dict]: + payload = self.client.get_torrent(int(torrent_id)) + if not payload: + return None + return check_one_torrent( + payload, + tracker_name=self.tracker_name, + db=self.db, + edit_url=self.client.edit_url(int(torrent_id)), + uploader=uploader or self.client.username or None, + ) + + def scan_by_name( + self, + name: str, + uploader: Optional[str] = None, + linked_item_id: Optional[int] = None, + ) -> Optional[dict]: + """Fuzzy-lookup a torrent by name and check it. Used by the post-upload + hook when we can't parse the id from the tracker response URL.""" + uploader = uploader or self.client.username or None + matches = self.client.find_by_name(name=name, uploader=uploader) + # Prefer an exact name or folder match if any; otherwise take the first. + chosen: Optional[dict] = None + for m in matches: + if (m.get("name") == name) or (m.get("folder") == name): + chosen = m + break + if chosen is None and matches: + chosen = matches[0] + if not chosen: + return None + torrent_id = _extract_torrent_id(chosen) or 0 + return check_one_torrent( + chosen, + tracker_name=self.tracker_name, + db=self.db, + edit_url=self.client.edit_url(torrent_id) if torrent_id else "", + uploader=uploader, + linked_item_id=linked_item_id, + ) diff --git a/unit3dup/duplicate.py b/unit3dup/duplicate.py index fd36c6d..ea7516a 100644 --- a/unit3dup/duplicate.py +++ b/unit3dup/duplicate.py @@ -34,7 +34,7 @@ def same_season(self) -> bool: # Compare season and episode only if it is a serie # Return true if they have at least the same season and episode otherwise false - if self.content_file.guessit_season and self.tracker_file.guessit_season: + if self.content_file.guessit_season is not None and self.tracker_file.guessit_season is not None: same_season = self.content_file.guessit_season == self.tracker_file.guessit_season same_episode = self.content_file.guessit_episode == self.tracker_file.guessit_episode return same_season and same_episode @@ -126,6 +126,9 @@ def __init__(self, content: Media, tracker_name: str, cli: argparse.Namespace): # Final result self.flag_already = False + # Match data captured when a duplicate is found (None otherwise) + self.match_data: dict | None = None + # For printing output self.TMDB_ID_WIDTH = 6 self.IGDB_ID_WIDTH = 6 @@ -199,6 +202,9 @@ def user_choose() -> bool: # Skip this media if "s" == user_answer.lower(): return True + except EOFError: + custom_console.bot_warning_log("No interactive input available, skipping duplicate") + return True except KeyboardInterrupt: custom_console.bot_error_log("\nOperation cancelled. Bye !") exit(1) @@ -238,7 +244,7 @@ def _print_output(self, value: dict, delta_size: int, size_th: int): formatted_size_th = f"{delta_size:<{self.DELTA_SIZE_WIDTH}}" output = 'not available' - if self.category in {'movie', 'tv'}: + if self.category in {'movie', 'tv', 'animation', 'tv_animation', 'documentary', 'tv_documentary'}: output = ( f"Tracker - size: '{formatted_size}' " f"delta={formatted_size_th}% - " @@ -284,6 +290,17 @@ def _process_tracker_data(self, data_from_the_tracker) -> bool: else: self._print_output(value=data_from_the_tracker['attributes'], delta_size=delta_size, size_th = config_settings.user_preferences.SIZE_TH) + attrs = data_from_the_tracker.get('attributes', {}) + self.match_data = { + "id": data_from_the_tracker.get("id"), + "name": attrs.get("name"), + "size": attrs.get("size"), + "resolution": attrs.get("resolution"), + "info_hash": attrs.get("info_hash"), + "tmdb_id": attrs.get("tmdb_id"), + "igdb_id": attrs.get("igdb_id"), + "delta_size": delta_size, + } return True return False \ No newline at end of file diff --git a/unit3dup/generate_prez.py b/unit3dup/generate_prez.py index c5ba745..90baf2a 100755 --- a/unit3dup/generate_prez.py +++ b/unit3dup/generate_prez.py @@ -389,9 +389,14 @@ def main(): print("\n" + "="*80) # Option pour sauvegarder dans un fichier - save = input("\nVoulez-vous sauvegarder dans un fichier? (o/n): ").lower() + try: + save = input("\nVoulez-vous sauvegarder dans un fichier? (o/n): ").lower() + except EOFError: + save = "n" if save == 'o': - filename = f"prez_{tmdb_id}.txt" + from common import config_settings + save_dir = str(config_settings.user_preferences.CACHE_PATH) + filename = os.path.join(save_dir, f"prez_{tmdb_id}.txt") with open(filename, 'w', encoding='utf-8') as f: f.write(prez) print(f"Présentation sauvegardée dans {filename}") diff --git a/unit3dup/media.py b/unit3dup/media.py index cd05b79..32c5fd1 100644 --- a/unit3dup/media.py +++ b/unit3dup/media.py @@ -55,6 +55,7 @@ def __init__(self, folder: str, subfolder: str): self._tmdb_id: int | None = None self._imdb_id: int | None = None self._igdb_id: int | None = None + self._tmdb_year: int | None = None self._generate_title: str | None = None @@ -171,6 +172,14 @@ def igdb_id(self) -> int: def igdb_id(self, value): self._igdb_id = value + @property + def tmdb_year(self) -> int | None: + return self._tmdb_year + + @tmdb_year.setter + def tmdb_year(self, value): + self._tmdb_year = value + @property def generate_title(self) -> str: @@ -183,7 +192,7 @@ def generate_title(self) -> str: # Search for Season and Episode o torrent_pack if 'tv' in self.category: - serie = f"S{str(self.guess_season).zfill(2)}" if self.guess_season else '' + serie = f"S{str(self.guess_season).zfill(2)}" if self.guess_season is not None else '' if not self.torrent_pack: serie+= f"E{str(self.guess_episode).zfill(2)}" else: @@ -239,14 +248,14 @@ def guess_title(self, value): @property def guess_season(self): - if not self._guess_season: + if self._guess_season is None: if 'tv' in self.category: self._guess_season = self.guess_filename.guessit_season return self._guess_season @property def guess_episode(self): - if not self._episode: + if self._episode is None: if 'tv' in self.category: self._episode = self.guess_filename.guessit_episode return self._episode @@ -315,7 +324,7 @@ def category(self): return self._category # Search for a tv_show - elif self.guess_filename.guessit_season: + elif self.guess_filename.guessit_season is not None: self._category = System.category_list.get(System.TV_SHOW) else: self._category = System.category_list.get(System.MOVIE) @@ -335,6 +344,10 @@ def mediafile(self): if self.category in { System.category_list.get(System.MOVIE), System.category_list.get(System.TV_SHOW), + System.category_list.get(System.ANIMATION), + System.category_list.get(System.TV_ANIMATION), + System.category_list.get(System.DOCUMENTARY_FILM), + System.category_list.get(System.TV_DOCUMENTARY), }: # Read from the current video file the height field file_path = os.path.join(self.folder, self.file_name) diff --git a/unit3dup/media_manager/ContentManager.py b/unit3dup/media_manager/ContentManager.py index 320f000..190401f 100644 --- a/unit3dup/media_manager/ContentManager.py +++ b/unit3dup/media_manager/ContentManager.py @@ -81,7 +81,7 @@ def get_data(self, media: Media) -> Media | bool: # add category filter to the regex result caused by a possible substring (e.g., S06) in the whole path # and not part of the title - if media.category=='tv': + if media.category in {'tv', 'tv_animation', 'tv_documentary'}: # Search for the first result (Sx) in self.path torrent_pack = bool(re.search(r"(S\d+(?!.*E\d+))|(S\d+E\d+-E?\d+)", self.path)) else: diff --git a/unit3dup/media_manager/DocuManager.py b/unit3dup/media_manager/DocuManager.py index 2a5b75c..0141da5 100644 --- a/unit3dup/media_manager/DocuManager.py +++ b/unit3dup/media_manager/DocuManager.py @@ -9,72 +9,179 @@ from unit3dup.upload import UploadBot from unit3dup import config_settings from unit3dup.media import Media +from unit3dup.prepared_item import PreparedItem from view import custom_console class DocuManager: - def __init__(self, contents: list[Media], cli: argparse.Namespace): + def __init__(self, contents: list[Media], cli: argparse.Namespace, qbit_category: str | None = None): self._my_tmdb = None self.contents: list['Media'] = contents self.cli: argparse = cli + self.qbit_category = qbit_category - def process(self, selected_tracker: str, tracker_name_list: list, tracker_archive: str) -> list[BittorrentData]: + def prepare(self, selected_tracker: str, tracker_name_list: list, tracker_archive: str) -> tuple[list[PreparedItem], list[dict]]: + """Prepare all items for upload without sending them to trackers.""" # -multi : no announce_list . One announce for multi tracker if self.cli.mt: tracker_name_list = [selected_tracker.upper()] - # Init the torrent list - bittorrent_list = [] + prepared_items = [] + skip_reasons = [] + for content in self.contents: # get the archive path archive = os.path.join(tracker_archive, selected_tracker) os.makedirs(archive, exist_ok=True) - torrent_filepath = os.path.join(tracker_archive,selected_tracker, f"{content.torrent_name}.torrent") + torrent_filepath = os.path.join(tracker_archive, selected_tracker, f"{content.torrent_name}.torrent") if self.cli.watcher: - if os.path.exists(content.torrent_path): + if os.path.exists(torrent_filepath): custom_console.bot_log(f"Watcher Active.. skip the old upload '{content.file_name}'") - continue + skip_reasons.append({"torrent_name": content.torrent_name, "reason": "already_in_archive", + "source": content.source or ""}) + prepared_items.append( + PreparedItem( + content=content, + source_path=content.torrent_path or content.file_name, + source_type="folder" if os.path.isdir(content.torrent_path) else "file", + torrent_filepath=torrent_filepath, + skip_reason="already_in_archive", + ) + ) + continue torrent_response = UserContent.torrent(content=content, tracker_name_list=tracker_name_list, selected_tracker=selected_tracker, this_path=torrent_filepath) # Skip if it is a duplicate + duplicate_match = None if ((self.cli.duplicate or config_settings.user_preferences.DUPLICATE_ON) - and UserContent.is_duplicate(content=content, tracker_name=selected_tracker, cli=self.cli)): + and not getattr(self.cli, 'skip_duplicate_check', False)): + duplicate_match = UserContent.check_duplicate( + content=content, tracker_name=selected_tracker, cli=self.cli + ) + if duplicate_match is not None: + skip_reasons.append({"torrent_name": content.torrent_name, "reason": "duplicate_on_tracker", + "source": content.source or ""}) + prepared_items.append( + PreparedItem( + content=content, + source_path=content.torrent_path or content.file_name, + source_type="folder" if os.path.isdir(content.torrent_path) else "file", + content_category=content.category, + tracker_name=selected_tracker, + trackers_list=tracker_name_list, + skip_reason="duplicate_on_tracker", + duplicate_match=duplicate_match or None, + ) + ) continue # print the title will be shown on the torrent page custom_console.bot_log(f"'DISPLAYNAME'...{{{content.display_name}}}\n") - # Don't upload if -noup is set to True - if self.cli.noup: - custom_console.bot_warning_log(f"No Upload active. Done.") - continue - - # Get the cover image + # Get the cover image and description docu_info = PdfImages(content.file_name) docu_info.build_info() - # Tracker payload - unit3d_up = UploadBot(content=content, tracker_name=selected_tracker, cli = self.cli) + unit3d_up = UploadBot(content=content, tracker_name=selected_tracker, cli=self.cli) - # Upload + # Build tracker data unit3d_up.data_docu(document_info=docu_info) - # Get the data - tracker_response, tracker_message = unit3d_up.send(torrent_archive=torrent_filepath) + # Exclusion par tag d'équipe + release_name_check = unit3d_up.tracker.data.get("name", "") + if UploadBot.is_excluded_tag(release_name_check): + tag = release_name_check.rsplit('-', 1)[-1] if '-' in release_name_check else "?" + custom_console.bot_warning_log(f"Tag '{tag}' exclu (EXCLUDED_TAGS). Skip: {release_name_check}") + skip_reasons.append({"torrent_name": content.torrent_name, "reason": "excluded_tag", + "source": content.source or ""}) + prepared_items.append( + PreparedItem( + content=content, + source_path=content.torrent_path or content.file_name, + source_type="folder" if os.path.isdir(content.torrent_path) else "file", + skip_reason="excluded_tag", + ) + ) + continue + + # Create PreparedItem + prepared = PreparedItem( + content=content, + source_path=content.torrent_path or content.file_name, + source_type="folder" if os.path.isdir(content.torrent_path) else "file", + torrent_response=torrent_response, + torrent_filepath=torrent_filepath, + tracker_data=dict(unit3d_up.tracker.data), + tracker_name=selected_tracker, + trackers_list=tracker_name_list, + release_name=unit3d_up.tracker.data.get("name", content.display_name), + display_name=content.display_name, + source_tag=content.source or "", + content_category=content.category, + qbit_category=self.qbit_category, + description=unit3d_up.tracker.data.get("description", ""), + ) + prepared_items.append(prepared) + + return prepared_items, skip_reasons + + @staticmethod + def upload_item(prepared: PreparedItem, cli: argparse.Namespace) -> BittorrentData | None: + """Upload a single prepared item to the tracker.""" + # Create UploadBot + unit3d_up = UploadBot(content=prepared.content, tracker_name=prepared.tracker_name, cli=cli) + + # Set tracker data from prepared + unit3d_up.tracker.data = prepared.tracker_data + + # Send to tracker + tracker_response, tracker_message = unit3d_up.send(torrent_archive=prepared.torrent_filepath) + + return BittorrentData( + tracker_response=tracker_response, + torrent_response=prepared.torrent_response, + content=prepared.content, + tracker_message=tracker_message, + archive_path=prepared.torrent_filepath, + release_name=prepared.release_name, + qbit_category=prepared.qbit_category, + ) + + def process(self, selected_tracker: str, tracker_name_list: list, tracker_archive: str) -> tuple[list[BittorrentData], list[dict]]: + """Process and upload all items to tracker. Backward-compatible wrapper around prepare() + upload_item().""" + prepared_items, skip_reasons = self.prepare(selected_tracker, tracker_name_list, tracker_archive) + + bittorrent_list = [] + + for prepared in prepared_items: + # Skip items that have skip_reason set + if prepared.skip_reason: + continue + + # Handle -noup (dry-run) + if self.cli.noup: + custom_console.bot_warning_log(f"[DRY-RUN] No upload → {prepared.release_name}") + bittorrent_list.append( + BittorrentData( + tracker_response=None, + torrent_response=prepared.torrent_response, + content=prepared.content, + tracker_message="dry-run", + archive_path=prepared.torrent_filepath, + release_name=prepared.release_name, + qbit_category=prepared.qbit_category, + )) + continue - bittorrent_list.append( - BittorrentData( - tracker_response=tracker_response, - torrent_response=torrent_response, - content=content, - tracker_message=tracker_message, - archive_path=torrent_filepath, - )) + # Upload the item + bittorrent_data = self.upload_item(prepared, self.cli) + if bittorrent_data: + bittorrent_list.append(bittorrent_data) - return bittorrent_list \ No newline at end of file + return bittorrent_list, skip_reasons \ No newline at end of file diff --git a/unit3dup/media_manager/GameManager.py b/unit3dup/media_manager/GameManager.py index 5cf8c94..141b814 100644 --- a/unit3dup/media_manager/GameManager.py +++ b/unit3dup/media_manager/GameManager.py @@ -9,29 +9,32 @@ from unit3dup.upload import UploadBot from unit3dup import config_settings from unit3dup.media import Media +from unit3dup.prepared_item import PreparedItem from view import custom_console class GameManager: - def __init__(self, contents: list["Media"], cli: argparse.Namespace): + def __init__(self, contents: list["Media"], cli: argparse.Namespace, qbit_category: str | None = None): """ Initialize the GameManager with the given contents Args: contents (list): List of content media objects cli (argparse.Namespace): user flag Command line + qbit_category (str | None): qBittorrent category to assign to uploaded torrents """ self.contents: list[Media] = contents self.cli: argparse = cli + self.qbit_category = qbit_category self.igdb = IGDBClient() - def process(self, selected_tracker: str, tracker_name_list: list, tracker_archive: str) -> list[BittorrentData]: + def prepare(self, selected_tracker: str, tracker_name_list: list, tracker_archive: str) -> tuple[list[PreparedItem], list[dict]]: """ - Process the game contents to filter duplicates and create torrents + Prepare game contents for upload without sending them. Returns: - list: List of Bittorrent objects created for each content + tuple: (list of PreparedItem objects, list of skip reason dicts) """ login = self.igdb.connect() @@ -44,30 +47,62 @@ def process(self, selected_tracker: str, tracker_name_list: list, tracker_archi if self.cli.upload: custom_console.bot_error_log("Game upload works only with the '-f' flag.You need to specify a folder name.") - return [] + return [], [] + prepared_items = [] + skip_reasons = [] - # Init the torrent list - bittorrent_list = [] for content in self.contents: # get the archive path archive = os.path.join(tracker_archive, selected_tracker) os.makedirs(archive, exist_ok=True) - torrent_filepath = os.path.join(tracker_archive,selected_tracker, f"{content.torrent_name}.torrent") + torrent_filepath = os.path.join(tracker_archive, selected_tracker, f"{content.torrent_name}.torrent") # Filter contents based on existing torrents or duplicates if self.cli.watcher: - if os.path.exists(content.torrent_path): + if os.path.exists(torrent_filepath): custom_console.bot_log(f"Watcher Active.. skip the old upload '{content.file_name}'") - continue + skip_reasons.append({"torrent_name": content.torrent_name, "reason": "already_in_archive", + "source": content.source or ""}) + prepared_items.append(PreparedItem( + content=content, + source_path=content.torrent_path or content.file_name or "", + source_type="folder" if os.path.isdir(content.torrent_path or "") else "file", + display_name=content.display_name, + content_category=content.category, + qbit_category=self.qbit_category, + source_tag=content.source or "", + torrent_filepath=torrent_filepath, + skip_reason="already_in_archive", + )) + continue torrent_response = UserContent.torrent(content=content, tracker_name_list=tracker_name_list, selected_tracker=selected_tracker, this_path=torrent_filepath) - # Skip if it is a duplicate + duplicate_match = None if ((self.cli.duplicate or config_settings.user_preferences.DUPLICATE_ON) - and UserContent.is_duplicate(content=content, tracker_name=selected_tracker, cli=self.cli)): + and not getattr(self.cli, 'skip_duplicate_check', False)): + duplicate_match = UserContent.check_duplicate( + content=content, tracker_name=selected_tracker, cli=self.cli + ) + if duplicate_match is not None: + skip_reasons.append({"torrent_name": content.torrent_name, "reason": "duplicate_on_tracker", + "source": content.source or ""}) + prepared_items.append(PreparedItem( + content=content, + source_path=content.torrent_path or content.file_name or "", + source_type="folder" if os.path.isdir(content.torrent_path or "") else "file", + display_name=content.display_name, + content_category=content.category, + qbit_category=self.qbit_category, + source_tag=content.source or "", + tracker_name=selected_tracker, + trackers_list=tracker_name_list, + skip_reason="duplicate_on_tracker", + duplicate_match=duplicate_match or None, + )) continue # Search for the game on IGDB using the content's title and platform tags @@ -77,29 +112,141 @@ def process(self, selected_tracker: str, tracker_name_list: list, tracker_archi # Skip the upload if there is no valid IGDB if not game_data_results: + skip_reasons.append({"torrent_name": content.torrent_name, "reason": "no_igdb_result", + "source": content.source or ""}) + prepared_items.append(PreparedItem( + content=content, + source_path=content.torrent_path or content.file_name or "", + source_type="folder" if os.path.isdir(content.torrent_path or "") else "file", + display_name=content.display_name, + content_category=content.category, + qbit_category=self.qbit_category, + source_tag=content.source or "", + skip_reason="no_igdb_result", + )) continue # Tracker instance - unit3d_up = UploadBot(content=content, tracker_name=selected_tracker, cli = self.cli) + unit3d_up = UploadBot(content=content, tracker_name=selected_tracker, cli=self.cli) # Get the data unit3d_up.data_game(igdb=game_data_results) - # Don't upload if -noup is set to True - if self.cli.noup: - custom_console.bot_warning_log(f"No Upload active. Done.") + # Exclusion par tag d'équipe + release_name_check = unit3d_up.tracker.data.get("name", "") + if UploadBot.is_excluded_tag(release_name_check): + tag = release_name_check.rsplit('-', 1)[-1] if '-' in release_name_check else "?" + custom_console.bot_warning_log(f"Tag '{tag}' exclu (EXCLUDED_TAGS). Skip: {release_name_check}") + skip_reasons.append({"torrent_name": content.torrent_name, "reason": "excluded_tag", + "source": content.source or ""}) + prepared_items.append(PreparedItem( + content=content, + source_path=content.torrent_path or content.file_name or "", + source_type="folder" if os.path.isdir(content.torrent_path or "") else "file", + display_name=content.display_name, + content_category=content.category, + qbit_category=self.qbit_category, + source_tag=content.source or "", + skip_reason="excluded_tag", + )) continue - # Send to the tracker - tracker_response, tracker_message = unit3d_up.send(torrent_archive=torrent_filepath, nfo_path=content.game_nfo) + # Read NFO content if it exists + nfo_content = None + if os.path.exists(content.game_nfo): + try: + with open(content.game_nfo, 'r', encoding='utf-8') as f: + nfo_content = f.read() + except Exception: + nfo_content = None + + # Determine source type + source_path = content.torrent_path or content.file_name + source_type = "folder" if os.path.isdir(source_path) else "file" + + # Create PreparedItem + prepared = PreparedItem( + content=content, + source_path=source_path, + source_type=source_type, + torrent_response=torrent_response, + torrent_filepath=torrent_filepath, + tracker_data=dict(unit3d_up.tracker.data), + tracker_name=selected_tracker, + trackers_list=tracker_name_list, + release_name=unit3d_up.tracker.data.get("name", content.display_name), + display_name=content.display_name, + source_tag=content.source or "", + content_category=content.category, + qbit_category=self.qbit_category, + description=unit3d_up.tracker.data.get("description", ""), + igdb_id=game_data_results.id if game_data_results else 0, + nfo_content=nfo_content, + ) + prepared_items.append(prepared) + + return prepared_items, skip_reasons + + @staticmethod + def upload_item(prepared: PreparedItem, cli: argparse.Namespace) -> BittorrentData | None: + """ + Upload a prepared game item to the tracker. - bittorrent_list.append( - BittorrentData( - tracker_response=tracker_response, - torrent_response=torrent_response, - content=content, - tracker_message=tracker_message, - archive_path = torrent_filepath, - )) - return bittorrent_list + Args: + prepared: PreparedItem with all required data + cli: Command line arguments + + Returns: + BittorrentData with upload results, or None on failure + """ + # Create UploadBot and restore tracker data + unit3d_up = UploadBot(content=prepared.content, tracker_name=prepared.tracker_name, cli=cli) + unit3d_up.tracker.data = prepared.tracker_data + + # Send to the tracker + tracker_response, tracker_message = unit3d_up.send(torrent_archive=prepared.torrent_filepath, nfo_path=prepared.content.game_nfo) + + return BittorrentData( + tracker_response=tracker_response, + torrent_response=prepared.torrent_response, + content=prepared.content, + tracker_message=tracker_message, + archive_path=prepared.torrent_filepath, + release_name=prepared.release_name, + qbit_category=prepared.qbit_category, + ) + + def process(self, selected_tracker: str, tracker_name_list: list, tracker_archive: str) -> tuple[list[BittorrentData], list[dict]]: + """ + Process the game contents to filter duplicates and create torrents. + Backward-compatible wrapper around prepare() and upload_item(). + + Returns: + tuple: (list of Bittorrent objects, list of skip reasons dicts) + """ + # Prepare all items + prepared_items, skip_reasons = self.prepare(selected_tracker, tracker_name_list, tracker_archive) + + # Upload prepared items + bittorrent_list = [] + for prepared in prepared_items: + # Don't upload if -noup is set to True + if self.cli.noup: + custom_console.bot_warning_log(f"[DRY-RUN] No upload → {prepared.release_name}") + bittorrent_list.append( + BittorrentData( + tracker_response=None, + torrent_response=prepared.torrent_response, + content=prepared.content, + tracker_message="dry-run", + archive_path=prepared.torrent_filepath, + release_name=prepared.release_name, + qbit_category=prepared.qbit_category, + )) + else: + result = self.upload_item(prepared, self.cli) + if result: + bittorrent_list.append(result) + + return bittorrent_list, skip_reasons diff --git a/unit3dup/media_manager/TorrentManager.py b/unit3dup/media_manager/TorrentManager.py index 87faa5e..e3f5845 100644 --- a/unit3dup/media_manager/TorrentManager.py +++ b/unit3dup/media_manager/TorrentManager.py @@ -11,6 +11,7 @@ from unit3dup import config_settings from unit3dup.media import Media +from unit3dup.prepared_item import PreparedItem from common.bittorrent import BittorrentData from common.constants import my_language @@ -21,7 +22,7 @@ class TorrentManager: - def __init__(self, cli: argparse.Namespace, tracker_archive: str): + def __init__(self, cli: argparse.Namespace, tracker_archive: str, qbit_category: str | None = None): self.preferred_lang = my_language(config_settings.user_preferences.PREFERRED_LANG) self.tracker_archive = tracker_archive @@ -29,6 +30,13 @@ def __init__(self, cli: argparse.Namespace, tracker_archive: str): self.games: list[Media] = [] self.doc: list[Media] = [] self.cli = cli + self.qbit_category = qbit_category + self.upload_count = 0 + self.skip_reasons: list[dict] = [] + self.release_names: list[str] = [] + self.release_sources: list[str] = [] + self.content_categories: list[str] = [] + self.validation_reports: dict[str, list[dict]] = {} self.fast_load = config_settings.user_preferences.FAST_LOAD if self.fast_load < 1 or self.fast_load > 150: # full list @@ -59,7 +67,14 @@ def process(self, contents: list) -> None: self.videos = [ content for content in contents - if content.category in {System.category_list.get(System.MOVIE), System.category_list.get(System.TV_SHOW)} + if content.category in { + System.category_list.get(System.MOVIE), + System.category_list.get(System.TV_SHOW), + System.category_list.get(System.ANIMATION), + System.category_list.get(System.TV_ANIMATION), + System.category_list.get(System.DOCUMENTARY_FILM), + System.category_list.get(System.TV_DOCUMENTARY), + } ] @@ -68,6 +83,53 @@ def process(self, contents: list) -> None: content for content in contents if content.category == System.category_list.get(System.DOCUMENTARY) ] + def prepare_all(self, trackers_name_list: list) -> list[PreparedItem]: + """ + Prepare content for each selected tracker using the prepare() method of each manager. + Collects all PreparedItem objects from video, game, and doc managers. + + Args: + trackers_name_list: list of tracker names to prepare content for + + Returns: + list[PreparedItem]: all prepared items combined from all managers and trackers + """ + all_prepared = [] + + for selected_tracker in trackers_name_list: + # Prepare each GAME + if self.games: + game_manager = GameManager(contents=self.games[:self.fast_load], + cli=self.cli, qbit_category=self.qbit_category) + game_prepared, game_skips = game_manager.prepare(selected_tracker=selected_tracker, + tracker_name_list=trackers_name_list, + tracker_archive=self.tracker_archive) + all_prepared.extend(game_prepared) + self.skip_reasons.extend(game_skips) + + # Prepare each VIDEO + if self.videos: + video_manager = VideoManager(contents=self.videos[:self.fast_load], + cli=self.cli, qbit_category=self.qbit_category) + video_prepared, video_skips = video_manager.prepare(selected_tracker=selected_tracker, + tracker_name_list=trackers_name_list, + tracker_archive=self.tracker_archive) + all_prepared.extend(video_prepared) + self.skip_reasons.extend(video_skips) + self.validation_reports.update(video_manager.validation_reports) + + # Prepare each DOC + if self.doc and not self.cli.reseed: + docu_manager = DocuManager(contents=self.doc[:self.fast_load], + cli=self.cli, qbit_category=self.qbit_category) + docu_prepared, docu_skips = docu_manager.prepare(selected_tracker=selected_tracker, + tracker_name_list=trackers_name_list, + tracker_archive=self.tracker_archive) + all_prepared.extend(docu_prepared) + self.skip_reasons.extend(docu_skips) + + return all_prepared + def run(self, trackers_name_list: list): """ @@ -85,26 +147,43 @@ def run(self, trackers_name_list: list): # Build the torrent file and upload each GAME to the tracker if self.games: game_manager = GameManager(contents=self.games[:self.fast_load], - cli=self.cli) - game_process_results = game_manager.process(selected_tracker=selected_tracker, + cli=self.cli, qbit_category=self.qbit_category) + game_process_results, game_skips = game_manager.process(selected_tracker=selected_tracker, tracker_name_list=trackers_name_list, tracker_archive=self.tracker_archive) + self.upload_count += len(game_process_results) + self.release_names.extend(r.release_name for r in game_process_results if r.release_name) + self.release_sources.extend(r.content.source or "" for r in game_process_results if r.content) + self.content_categories.extend(r.content.category for r in game_process_results if r.content) + self.skip_reasons.extend(game_skips) # Build the torrent file and upload each VIDEO to the trackers if self.videos: video_manager = VideoManager(contents=self.videos[:self.fast_load], - cli=self.cli) - video_process_results = video_manager.process(selected_tracker=selected_tracker, + cli=self.cli, qbit_category=self.qbit_category) + video_process_results, video_skips = video_manager.process(selected_tracker=selected_tracker, tracker_name_list=trackers_name_list, tracker_archive=self.tracker_archive) + self.upload_count += len(video_process_results) + self.release_names.extend(r.release_name for r in video_process_results if r.release_name) + self.release_sources.extend(r.content.source or "" for r in video_process_results if r.content) + self.content_categories.extend(r.content.category for r in video_process_results if r.content) + self.skip_reasons.extend(video_skips) + # Validation reports are video-only (Game/Doc don't run ValidationRunner) + self.validation_reports.update(video_manager.validation_reports) # Build the torrent file and upload each DOC to the tracker if self.doc and not self.cli.reseed: docu_manager = DocuManager(contents=self.doc[:self.fast_load], - cli=self.cli) - docu_process_results = docu_manager.process(selected_tracker=selected_tracker, + cli=self.cli, qbit_category=self.qbit_category) + docu_process_results, docu_skips = docu_manager.process(selected_tracker=selected_tracker, tracker_name_list=trackers_name_list, tracker_archive=self.tracker_archive) + self.upload_count += len(docu_process_results) + self.release_names.extend(r.release_name for r in docu_process_results if r.release_name) + self.release_sources.extend(r.content.source or "" for r in docu_process_results if r.content) + self.content_categories.extend(r.content.category for r in docu_process_results if r.content) + self.skip_reasons.extend(docu_skips) # No seeding if self.cli.noseed or self.cli.noup: diff --git a/unit3dup/media_manager/VideoManager.py b/unit3dup/media_manager/VideoManager.py index cb09bdd..6132605 100644 --- a/unit3dup/media_manager/VideoManager.py +++ b/unit3dup/media_manager/VideoManager.py @@ -4,89 +4,161 @@ from common.external_services.theMovieDB.core.api import DbOnline from common.bittorrent import BittorrentData -from common.utility import ManageTitles +from common.utility import ManageTitles, System from unit3dup.media_manager.common import UserContent from unit3dup.upload import UploadBot from unit3dup import config_settings from unit3dup.pvtVideo import Video from unit3dup.media import Media +from unit3dup.prepared_item import PreparedItem from view import custom_console class VideoManager: - def __init__(self, contents: list[Media], cli: argparse.Namespace): + def __init__(self, contents: list[Media], cli: argparse.Namespace, qbit_category: str | None = None): """ Initialize the VideoManager with the given contents Args: contents (list): List of content media objects cli (argparse.Namespace): user flag Command line + qbit_category (str | None): qBittorrent category to assign to uploaded torrents """ self.torrent_found:bool = False self.contents: list[Media] = contents self.cli: argparse = cli + self.qbit_category = qbit_category + self.validation_reports: dict[str, list[dict]] = {} - def process(self, selected_tracker: str, tracker_name_list: list, tracker_archive: str) -> list[BittorrentData]: + def prepare(self, selected_tracker: str, tracker_name_list: list, tracker_archive: str) -> tuple[list[PreparedItem], list[dict]]: """ - Process the video contents to filter duplicates and create torrents + Prepare video contents without uploading. Returns PreparedItem objects and skip reasons. - Returns: - list: List of Bittorrent objects created for each content + Returns: + tuple: (list of PreparedItem objects, list of skip reasons dicts) """ # -multi : no announce_list . One announce for multi tracker if self.cli.mt: tracker_name_list = [selected_tracker.upper()] - # Init the torrent list - bittorrent_list = [] - for content in self.contents : + # Init the lists + prepared_items = [] + skip_reasons = [] + for content in self.contents: # get the archive path archive = os.path.join(tracker_archive, selected_tracker) os.makedirs(archive, exist_ok=True) - torrent_filepath = os.path.join(tracker_archive,selected_tracker, f"{content.torrent_name}.torrent") + torrent_filepath = os.path.join(tracker_archive, selected_tracker, f"{content.torrent_name}.torrent") # Filter contents based on existing torrents or duplicates if UserContent.is_preferred_language(content=content): - if self.cli.watcher: + if self.cli.watcher and not self.cli.noup: if os.path.exists(torrent_filepath): custom_console.bot_log(f"Watcher Active.. skip the old upload '{content.file_name}'") + skip_reasons.append({"torrent_name": content.torrent_name, "reason": "already_in_archive", + "source": content.source or ""}) + prepared_items.append(PreparedItem( + content=content, + source_path=content.torrent_path or content.file_name or "", + source_type="folder" if os.path.isdir(content.torrent_path or "") else "file", + display_name=content.display_name, + content_category=content.category, + qbit_category=self.qbit_category, + source_tag=content.source or "", + torrent_filepath=torrent_filepath, + skip_reason="already_in_archive", + )) continue torrent_response = UserContent.torrent(content=content, tracker_name_list=tracker_name_list, selected_tracker=selected_tracker, this_path=torrent_filepath) # Skip(S) if it is a duplicate or let the user choose to continue (C) - if (self.cli.duplicate or config_settings.user_preferences.DUPLICATE_ON - and UserContent.is_duplicate(content=content, tracker_name=selected_tracker, - cli=self.cli)): + duplicate_match = None + if ((self.cli.duplicate or config_settings.user_preferences.DUPLICATE_ON) + and not getattr(self.cli, 'skip_duplicate_check', False)): + duplicate_match = UserContent.check_duplicate( + content=content, tracker_name=selected_tracker, cli=self.cli + ) + if duplicate_match is not None: + skip_reasons.append({"torrent_name": content.torrent_name, "reason": "duplicate_on_tracker", + "source": content.source or ""}) + prepared_items.append(PreparedItem( + content=content, + source_path=content.torrent_path or content.file_name or "", + source_type="folder" if os.path.isdir(content.torrent_path or "") else "file", + display_name=content.display_name, + content_category=content.category, + qbit_category=self.qbit_category, + source_tag=content.source or "", + tracker_name=selected_tracker, + trackers_list=tracker_name_list, + skip_reason="duplicate_on_tracker", + duplicate_match=duplicate_match or None, + )) continue # Search for VIDEO ID - db_online = DbOnline(media=content,category=content.category, no_title=self.cli.notitle) + db_online = DbOnline(media=content, category=content.category, no_title=self.cli.notitle) db = db_online.media_result # If it is 'None' we skipped the imdb search (-notitle) if not db: + skip_reasons.append({"torrent_name": content.torrent_name, "reason": "no_tmdb_result", + "source": content.source or ""}) + prepared_items.append(PreparedItem( + content=content, + source_path=content.torrent_path or content.file_name or "", + source_type="folder" if os.path.isdir(content.torrent_path or "") else "file", + display_name=content.display_name, + content_category=content.category, + qbit_category=self.qbit_category, + source_tag=content.source or "", + skip_reason="no_tmdb_result", + )) continue + # Propage l'année TMDB au content pour qu'elle soit utilisée + # par le normalizer (notamment pour les séries dont le nom + # source ne contient généralement pas l'année). + content.tmdb_year = db.year + + # Override category for animated content based on TMDB genre + if db.is_animation(): + if content.category == System.category_list[System.MOVIE]: + content.category = System.category_list[System.ANIMATION] + custom_console.bot_log("Category → 'animation' (TMDB genre)") + elif content.category == System.category_list[System.TV_SHOW]: + content.category = System.category_list[System.TV_ANIMATION] + custom_console.bot_log("Category → 'tv_animation' (TMDB genre)") + + # Override category for documentary content based on TMDB genre + if db.is_documentary(): + if content.category == System.category_list[System.MOVIE]: + content.category = System.category_list[System.DOCUMENTARY_FILM] + custom_console.bot_log("Category → 'documentary' (TMDB genre)") + elif content.category == System.category_list[System.TV_SHOW]: + content.category = System.category_list[System.TV_DOCUMENTARY] + custom_console.bot_log("Category → 'tv_documentary' (TMDB genre)") + # Update display name with Serie Title when requested by the user (-notitle) if self.cli.notitle: # Add generated metadata to the display_title if self.cli.gentitle: content.display_name = (f"{db_online.media_result.result.get_title()} " f"{db_online.media_result.year} ") - content.display_name+= " " + content.generate_title + content.display_name += " " + content.generate_title else: # otherwise keep the old meta_data and add the new display_title to it - print() - content.display_name = (f"{db_online.media_result.result.get_title()}" - f" {db_online.media_result.year} {content.guess_title}") + print() + content.display_name = (f"{db_online.media_result.result.get_title()}" + f" {db_online.media_result.year} {content.guess_title}") # Get meta from the media video video_info = Video(media=content, tmdb_id=db.video_id, trailer_key=db.trailer_key) @@ -95,148 +167,278 @@ def process(self, selected_tracker: str, tracker_name_list: list, tracker_archiv custom_console.bot_log(f"'DISPLAYNAME'...{{{content.display_name}}}\n") # Tracker instance - unit3d_up = UploadBot(content=content, tracker_name=selected_tracker, cli = self.cli) + unit3d_up = UploadBot(content=content, tracker_name=selected_tracker, cli=self.cli) # Get the data unit3d_up.data(show_id=db.video_id, imdb_id=db.imdb_id, show_keywords_list=db.keywords_list, video_info=video_info) - # ── Confirmation interactive (-confirm) ─────────────────────── - if getattr(self.cli, 'confirm', False): - release_name = unit3d_up.tracker.data.get("name", content.display_name) - custom_console.rule("[bold cyan]Validation release[/bold cyan]") - custom_console.bot_log(f" Fichier : {content.display_name}") - custom_console.bot_question_log( - f"\n Release name : {release_name}\n\n" - f" Confirmer l'upload ? [o/N] : " + # ── Exclusion par tag d'équipe ──────────────────────────────── + release_name_check = unit3d_up.tracker.data.get("name", "") + if UploadBot.is_excluded_tag(release_name_check): + tag = release_name_check.rsplit('-', 1)[-1] if '-' in release_name_check else "?" + custom_console.bot_warning_log(f"Tag '{tag}' exclu (EXCLUDED_TAGS). Skip: {release_name_check}") + skip_reasons.append({"torrent_name": content.torrent_name, "reason": "excluded_tag", + "source": content.source or ""}) + prepared_items.append(PreparedItem( + content=content, + source_path=content.torrent_path or content.file_name or "", + source_type="folder" if os.path.isdir(content.torrent_path or "") else "file", + display_name=content.display_name, + content_category=content.category, + qbit_category=self.qbit_category, + source_tag=content.source or "", + skip_reason="excluded_tag", + )) + continue + + # ── Validation des règles tracker ───────────────────────────── + val_results = None + runner = None + if not getattr(self.cli, 'skip_validation', False): + from unit3dup.validators import ValidationRunner, create_default_validators + runner = ValidationRunner(create_default_validators()) + val_results = runner.validate( + media=content, + mediafile=getattr(content, 'mediafile', None), + release_name=unit3d_up.tracker.data.get("name", ""), + mediainfo_text=content.mediafile.info if getattr(content, 'mediafile', None) else None, ) - try: - answer = input().strip().lower() - except KeyboardInterrupt: - custom_console.bot_error_log("\nOpération annulée par l'utilisateur.") - break - if answer not in ("o", "oui", "y", "yes"): - custom_console.bot_warning_log(f" ✗ Upload annulé → {release_name}\n") - custom_console.rule() - continue - custom_console.bot_log(f" ✓ Upload confirmé → {release_name}\n") - custom_console.rule() + if val_results: + runner.print_report(custom_console) + # Store warnings/infos even if there are errors (web UI will show them) + self.validation_reports[release_name_check] = runner.to_dicts() - # Don't upload if -noup is set to True - if self.cli.noup: - custom_console.bot_warning_log(f"No Upload active. Done.") - continue + # ── Create PreparedItem ────────────────────────────────────────── + source_type = "folder" if os.path.isdir(content.torrent_path) else "file" - # ── Recherche / génération du NFO ──────────────────────────── - nfo_path = None - nfo_generated = False # True si c'est nous qui l'avons créé → à supprimer après + prepared_item = PreparedItem( + content=content, + source_path=content.torrent_path or content.file_name, + source_type=source_type, + torrent_response=torrent_response, + torrent_filepath=torrent_filepath, + tracker_data=dict(unit3d_up.tracker.data), # Make a copy! + tracker_name=selected_tracker, + trackers_list=tracker_name_list, + release_name=unit3d_up.tracker.data.get("name", content.display_name), + display_name=content.display_name, + source_tag=content.source or "", + resolution=content.screen_size or content.resolution or "", + content_category=content.category, + qbit_category=self.qbit_category, + description=video_info.description, + mediainfo=video_info.mediainfo, + nfo_content=None, + audio_tracks=video_info.audio_tracks, + subtitle_tracks=video_info.subtitle_tracks, + tmdb_id=db.video_id if db else 0, + imdb_id=db.imdb_id if db else 0, + tmdb_title=db.result.get_title() if db and db.result else None, + tmdb_year=db.year if db else None, + validation_report=runner.to_dicts() if val_results else [], + has_errors=runner.has_errors() if val_results else False, + has_warnings=bool(val_results) and not runner.has_errors() if val_results else False, + ) + + prepared_items.append(prepared_item) + + # // end content + return prepared_items, skip_reasons + + @staticmethod + def upload_item(prepared: PreparedItem, cli: argparse.Namespace) -> BittorrentData | None: + """ + Execute the upload for a PreparedItem. + + Args: + prepared: PreparedItem containing all prepared data + cli: Command line arguments + + Returns: + BittorrentData object or None if upload failed + """ + + # Reconstruct UploadBot with the tracker data from prepared + unit3d_up = UploadBot(content=prepared.content, tracker_name=prepared.tracker_name, cli=cli) + # Restore the tracker payload + unit3d_up.tracker.data = prepared.tracker_data + + # ── Search / generate NFO ──────────────────────────── + nfo_path = None + nfo_generated = False + + media_file_path = prepared.content.file_name if prepared.content.file_name else prepared.content.torrent_path + + if media_file_path: + media_file_path = os.path.abspath(media_file_path) - # Déterminer le chemin du fichier média et son répertoire - media_file_path = content.file_name if content.file_name else content.torrent_path - - if not media_file_path: - custom_console.bot_warning_log(f"[NFO] Aucun chemin média disponible pour générer le NFO") + if os.path.isdir(media_file_path): + # Cas d'un dossier (release pack) + media_dir = media_file_path + + # Chercher un fichier .nfo dans le dossier (n'importe quel .nfo) + nfo_files = [f for f in os.listdir(media_dir) if f.lower().endswith('.nfo')] + + if nfo_files: + # Utiliser le premier .nfo trouvé dans le dossier + nfo_path = os.path.join(media_dir, nfo_files[0]) + custom_console.bot_log(f"[NFO] Fichier NFO existant trouvé et utilisé: {nfo_path}") + # nfo_generated reste False → on ne supprimera pas ce fichier else: - # Convertir en chemin absolu - media_file_path = os.path.abspath(media_file_path) - - # Déterminer si c'est un fichier ou un dossier - if os.path.isdir(media_file_path): - # Cas d'un dossier (release pack) - media_dir = media_file_path - - # Chercher un fichier .nfo dans le dossier (n'importe quel .nfo) - nfo_files = [f for f in os.listdir(media_dir) if f.lower().endswith('.nfo')] - - if nfo_files: - # Utiliser le premier .nfo trouvé dans le dossier - nfo_path = os.path.join(media_dir, nfo_files[0]) - custom_console.bot_log(f"[NFO] Fichier NFO existant trouvé et utilisé: {nfo_path}") - # nfo_generated reste False → on ne supprimera pas ce fichier - else: - # Aucun NFO trouvé → en générer un temporaire - nfo_filename = f"{content.torrent_name}.nfo" - nfo_path = os.path.join(media_dir, nfo_filename) - custom_console.bot_log(f"[NFO] Aucun NFO trouvé, génération du fichier NFO temporaire: {nfo_path}") - from common.mediainfo import MediaFile - try: - # Chercher le premier fichier vidéo dans le dossier - video_files = [f for f in os.listdir(media_dir) - if ManageTitles.filter_ext(f)] - if video_files: - video_file_path = os.path.join(media_dir, video_files[0]) - media_file = MediaFile(video_file_path) - if Video.generate_nfo_file(media_file, nfo_path): - custom_console.bot_log(f"[NFO] Fichier NFO temporaire généré avec succès") - nfo_generated = True # Marquer comme temporaire → à supprimer après - else: - custom_console.bot_warning_log(f"[NFO] Échec de la génération du NFO") - nfo_path = None - else: - custom_console.bot_warning_log(f"[NFO] Aucun fichier vidéo trouvé dans le dossier pour générer le NFO") - nfo_path = None - except Exception as e: - custom_console.bot_warning_log(f"[NFO] Erreur lors de la génération du NFO: {e}") + # Aucun NFO trouvé → en générer un temporaire + nfo_filename = f"{prepared.content.torrent_name}.nfo" + nfo_path = os.path.join(media_dir, nfo_filename) + custom_console.bot_log(f"[NFO] Aucun NFO trouvé, génération du fichier NFO temporaire: {nfo_path}") + from common.mediainfo import MediaFile + try: + # Chercher le premier fichier vidéo dans le dossier + video_files = [f for f in os.listdir(media_dir) + if ManageTitles.filter_ext(f)] + if video_files: + video_file_path = os.path.join(media_dir, video_files[0]) + media_file = MediaFile(video_file_path) + if Video.generate_nfo_file(media_file, nfo_path): + custom_console.bot_log(f"[NFO] Fichier NFO temporaire généré avec succès") + nfo_generated = True # Marquer comme temporaire → à supprimer après + else: + custom_console.bot_warning_log(f"[NFO] Échec de la génération du NFO") nfo_path = None - elif os.path.isfile(media_file_path): - # Cas d'un fichier unique - file_dir = os.path.dirname(media_file_path) - file_base = os.path.splitext(os.path.basename(media_file_path))[0] - nfo_candidate = os.path.join(file_dir, f"{file_base}.nfo") - - if os.path.isfile(nfo_candidate): - nfo_path = nfo_candidate - custom_console.bot_log(f"[NFO] Fichier NFO existant trouvé et utilisé: {nfo_path}") else: - # Générer un NFO temporaire - nfo_path = nfo_candidate - custom_console.bot_log(f"[NFO] Aucun NFO trouvé, génération du fichier NFO temporaire: {nfo_path}") - from common.mediainfo import MediaFile - try: - media_file = MediaFile(media_file_path) - if Video.generate_nfo_file(media_file, nfo_path): - custom_console.bot_log(f"[NFO] Fichier NFO temporaire généré avec succès") - nfo_generated = True - else: - custom_console.bot_warning_log(f"[NFO] Échec de la génération du NFO") - nfo_path = None - except Exception as e: - custom_console.bot_warning_log(f"[NFO] Erreur lors de la génération du NFO: {e}") - nfo_path = None - else: - custom_console.bot_warning_log(f"[NFO] Chemin média invalide (ni fichier ni dossier): {media_file_path}") - - # Send to the tracker - # Vérifier que le NFO existe avant l'envoi - if nfo_path: - if os.path.isfile(nfo_path): - custom_console.bot_log(f"[NFO] Envoi du fichier NFO au tracker: {nfo_path}") - else: - custom_console.bot_warning_log(f"[NFO] Fichier NFO introuvable avant l'envoi: {nfo_path}") + custom_console.bot_warning_log(f"[NFO] Aucun fichier vidéo trouvé dans le dossier pour générer le NFO") + nfo_path = None + except Exception as e: + custom_console.bot_warning_log(f"[NFO] Erreur lors de la génération du NFO: {e}") nfo_path = None - else: - custom_console.bot_warning_log(f"[NFO] Aucun fichier NFO à envoyer") - - tracker_response, tracker_message = unit3d_up.send(torrent_archive=torrent_filepath, nfo_path=nfo_path) + elif os.path.isfile(media_file_path): + # Cas d'un fichier unique + file_dir = os.path.dirname(media_file_path) + file_base = os.path.splitext(os.path.basename(media_file_path))[0] + nfo_candidate = os.path.join(file_dir, f"{file_base}.nfo") - # Supprimer UNIQUEMENT le NFO temporaire généré par le script (ne pas supprimer les .nfo existants) - if nfo_generated and nfo_path and os.path.isfile(nfo_path): + if os.path.isfile(nfo_candidate): + nfo_path = nfo_candidate + custom_console.bot_log(f"[NFO] Fichier NFO existant trouvé et utilisé: {nfo_path}") + else: + # Générer un NFO temporaire + nfo_path = nfo_candidate + custom_console.bot_log(f"[NFO] Aucun NFO trouvé, génération du fichier NFO temporaire: {nfo_path}") + from common.mediainfo import MediaFile try: - os.remove(nfo_path) - custom_console.bot_log(f"[NFO] Fichier NFO temporaire supprimé: {nfo_path}") + media_file = MediaFile(media_file_path) + if Video.generate_nfo_file(media_file, nfo_path): + custom_console.bot_log(f"[NFO] Fichier NFO temporaire généré avec succès") + nfo_generated = True + else: + custom_console.bot_warning_log(f"[NFO] Échec de la génération du NFO") + nfo_path = None except Exception as e: - custom_console.bot_warning_log(f"[NFO] Impossible de supprimer le NFO temporaire: {e}") + custom_console.bot_warning_log(f"[NFO] Erreur lors de la génération du NFO: {e}") + nfo_path = None + else: + custom_console.bot_warning_log(f"[NFO] Chemin média invalide (ni fichier ni dossier): {media_file_path}") + + # Send to the tracker + # Vérifier que le NFO existe avant l'envoi + if nfo_path: + if os.path.isfile(nfo_path): + custom_console.bot_log(f"[NFO] Envoi du fichier NFO au tracker: {nfo_path}") + else: + custom_console.bot_warning_log(f"[NFO] Fichier NFO introuvable avant l'envoi: {nfo_path}") + nfo_path = None + else: + custom_console.bot_warning_log(f"[NFO] Aucun fichier NFO à envoyer") + + tracker_response, tracker_message = unit3d_up.send(torrent_archive=prepared.torrent_filepath, nfo_path=nfo_path) + + # Supprimer UNIQUEMENT le NFO temporaire généré par le script (ne pas supprimer les .nfo existants) + if nfo_generated and nfo_path and os.path.isfile(nfo_path): + try: + os.remove(nfo_path) + custom_console.bot_log(f"[NFO] Fichier NFO temporaire supprimé: {nfo_path}") + except Exception as e: + custom_console.bot_warning_log(f"[NFO] Impossible de supprimer le NFO temporaire: {e}") + # Store response for the torrent clients + return BittorrentData( + tracker_response=tracker_response, + torrent_response=prepared.torrent_response, + content=prepared.content, + tracker_message=tracker_message, + archive_path=prepared.torrent_filepath, + release_name=prepared.release_name, + qbit_category=prepared.qbit_category, + ) - # Store response for the torrent clients + def process(self, selected_tracker: str, tracker_name_list: list, tracker_archive: str) -> tuple[list[BittorrentData], list[dict]]: + """ + Process the video contents to filter duplicates and create torrents + + Returns: + tuple: (list of Bittorrent objects, list of skip reasons dicts) + """ + + # Call prepare() to get all prepared items and skip reasons + prepared_items, skip_reasons = self.prepare(selected_tracker, tracker_name_list, tracker_archive) + + # Init the torrent list + bittorrent_list = [] + + for prepared in prepared_items: + # ── Handle validation errors ───────────────────────────── + if prepared.has_errors: + custom_console.bot_error_log("Validation errors found. Skipping upload. Use -skipval to bypass.") + skip_reasons.append({"torrent_name": prepared.content.torrent_name, "reason": "validation_error", + "validation_report": prepared.validation_report, + "source": prepared.source_tag}) + continue + + # ── Confirmation interactive (-confirm) ─────────────────────── + if getattr(self.cli, 'confirm', False): + custom_console.rule("[bold cyan]Validation release[/bold cyan]") + custom_console.bot_log(f" Fichier : {prepared.display_name}") + if prepared.tmdb_id: + custom_console.bot_log(f" TMDB : {prepared.tmdb_id}") + if prepared.imdb_id: + custom_console.bot_log(f" IMDb : tt{prepared.imdb_id:07d}") + custom_console.bot_question_log( + f"\n Release name : {prepared.release_name}\n\n" + f" Confirmer l'upload ? [o/N] : " + ) + try: + answer = input().strip().lower() + except EOFError: + custom_console.bot_warning_log("No interactive input available, skipping confirmation") + continue + except KeyboardInterrupt: + custom_console.bot_error_log("\nOpération annulée par l'utilisateur.") + break + if answer not in ("o", "oui", "y", "yes"): + custom_console.bot_warning_log(f" ✗ Upload annulé → {prepared.release_name}\n") + custom_console.rule() + continue + custom_console.bot_log(f" ✓ Upload confirmé → {prepared.release_name}\n") + custom_console.rule() + + # Don't upload if -noup is set to True + if self.cli.noup: + custom_console.bot_warning_log(f"[DRY-RUN] No upload → {prepared.release_name}") bittorrent_list.append( BittorrentData( - tracker_response=tracker_response, - torrent_response=torrent_response, - content=content, - tracker_message = tracker_message, - archive_path=torrent_filepath, + tracker_response=None, + torrent_response=prepared.torrent_response, + content=prepared.content, + tracker_message="dry-run", + archive_path=prepared.torrent_filepath, + release_name=prepared.release_name, + qbit_category=prepared.qbit_category, )) + continue + + # Upload the item + bittorrent_data = self.upload_item(prepared, self.cli) + if bittorrent_data: + bittorrent_list.append(bittorrent_data) # // end content - return bittorrent_list + return bittorrent_list, skip_reasons diff --git a/unit3dup/media_manager/common.py b/unit3dup/media_manager/common.py index cfc969e..0dd6d47 100644 --- a/unit3dup/media_manager/common.py +++ b/unit3dup/media_manager/common.py @@ -145,7 +145,7 @@ def torrent(content: Media, tracker_name_list: list, selected_tracker: str, this return None @staticmethod - def is_duplicate(content: Media, tracker_name: str, cli: argparse.Namespace) -> bool: + def check_duplicate(content: Media, tracker_name: str, cli: argparse.Namespace) -> dict | None: """ Search for a duplicate. Delta = config.SIZE_TH @@ -154,7 +154,8 @@ def is_duplicate(content: Media, tracker_name: str, cli: argparse.Namespace) -> content (Contents): The content object media tracker_name: The name of the tracker Returns: - my_torrent object + match data dict (id, name, size, resolution, info_hash, ...) when a + duplicate is detected and the user chose to skip; None otherwise. """ duplicate = Duplicate(content=content, tracker_name=tracker_name, cli=cli) if duplicate.process(): @@ -162,9 +163,8 @@ def is_duplicate(content: Media, tracker_name: str, cli: argparse.Namespace) -> f"\n*** User chose to skip '{content.display_name}' ***\n" ) custom_console.rule() - return True - else: - return False + return duplicate.match_data or {} + return None @staticmethod def can_ressed(content: Media, tracker_name: str, cli: argparse.Namespace, tmdb_id :int) -> list[requests.Response]: @@ -229,6 +229,7 @@ def send_to_bittorrent_worker(bittorrent_file: BittorrentData, client: Qbittorre torrent=bittorrent_file.torrent_response, content=bittorrent_file.content, archive_path=archive_path_to_use, + category=bittorrent_file.qbit_category, ) # Nettoyer le fichier temporaire après utilisation diff --git a/unit3dup/prepared_item.py b/unit3dup/prepared_item.py new file mode 100644 index 0000000..e4bc262 --- /dev/null +++ b/unit3dup/prepared_item.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +"""Data class holding all information gathered during the 'prepare' phase. + +A PreparedItem is the output of VideoManager.prepare() / GameManager.prepare() / +DocuManager.prepare(). It contains everything needed to either display a preview +in the web dashboard or execute the actual upload to the tracker. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from unit3dup.media import Media + + +@dataclass +class PreparedItem: + """Snapshot of a fully-analysed media item, ready for review or upload.""" + + # ── Source identity ────────────────────────────────────────────── + content: Media + source_path: str # full path to the source file/folder + source_type: str = "" # 'file' | 'folder' + + # ── Torrent ────────────────────────────────────────────────────── + torrent_response: Any = None # Mytorrent | None + torrent_filepath: str = "" # path to the .torrent file in archive + + # ── Tracker payload ────────────────────────────────────────────── + tracker_data: dict = field(default_factory=dict) # complete tracker upload dict + tracker_name: str = "" + trackers_list: list[str] = field(default_factory=list) + + # ── Release info ───────────────────────────────────────────────── + release_name: str = "" + display_name: str = "" + source_tag: str = "" # WEB-DL, BluRay, etc. + resolution: str = "" + content_category: str = "" # movie, tv, game, documentary... + qbit_category: str | None = None + + # ── Rich content ───────────────────────────────────────────────── + description: str = "" # BBCode prez + mediainfo: str = "" # raw mediainfo text + nfo_content: str | None = None # NFO file contents (read at prepare time) + audio_tracks: list[dict] = field(default_factory=list) + subtitle_tracks: list[dict] = field(default_factory=list) + + # ── External IDs ───────────────────────────────────────────────── + tmdb_id: int = 0 + imdb_id: int = 0 + igdb_id: int = 0 + tmdb_title: str | None = None + tmdb_year: int | None = None + + # ── Validation ─────────────────────────────────────────────────── + validation_report: list[dict] = field(default_factory=list) + has_errors: bool = False + has_warnings: bool = False + + # ── Skip / error ───────────────────────────────────────────────── + skip_reason: str | None = None # if set, item was not uploadable + duplicate_match: dict | None = None # tracker torrent matched as duplicate diff --git a/unit3dup/prez.py b/unit3dup/prez.py new file mode 100644 index 0000000..b0f1075 --- /dev/null +++ b/unit3dup/prez.py @@ -0,0 +1,391 @@ +# -*- coding: utf-8 -*- +""" +Generate BBCode presentation (prez style 3) for torrent descriptions. +Matches the format used in mediatorr's prezStyle3. +""" + +import re + +from common.mediainfo import MediaFile + + +# ── Language mappings ────────────────────────────────────────────────── + +ISO_TO_LANG_NAME = { + "fr": "Français", "fr-CA": "Français (Canada)", "fr-BE": "Français (Belgique)", + "en": "Anglais", "de": "Allemand", "es": "Espagnol", + "it": "Italien", "ja": "Japonais", "ko": "Coréen", "pt": "Portugais", + "ru": "Russe", "zh": "Chinois", "ar": "Arabe", "nl": "Néerlandais", + "pl": "Polonais", "tr": "Turc", "hi": "Hindi", "sv": "Suédois", + "no": "Norvégien", "da": "Danois", "fi": "Finnois", "el": "Grec", + "hu": "Hongrois", "ro": "Roumain", "cs": "Tchèque", "he": "Hébreu", + "th": "Thaï", "vi": "Vietnamien", "id": "Indonésien", "ms": "Malais", + "bg": "Bulgare", "hr": "Croate", "sr": "Serbe", "sk": "Slovaque", + "sl": "Slovène", "uk": "Ukrainien", "ca": "Catalan", "eu": "Basque", + "et": "Estonien", "lv": "Letton", "lt": "Lituanien", "is": "Islandais", + "ka": "Géorgien", "hy": "Arménien", "fa": "Persan", "bn": "Bengali", + "ta": "Tamoul", "te": "Télougou", "ur": "Ourdou", "tl": "Tagalog", +} + +LANG_TO_COUNTRY = { + "Français": "FR", "Français (Canada)": "CA", "Français (Belgique)": "BE", + "Anglais": "GB", "Allemand": "DE", "Espagnol": "ES", + "Italien": "IT", "Japonais": "JP", "Coréen": "KR", "Portugais": "PT", + "Russe": "RU", "Chinois": "CN", "Arabe": "SA", "Néerlandais": "NL", + "Polonais": "PL", "Turc": "TR", "Hindi": "IN", "Suédois": "SE", + "Norvégien": "NO", "Danois": "DK", "Finnois": "FI", "Grec": "GR", + "Hongrois": "HU", "Roumain": "RO", "Tchèque": "CZ", "Hébreu": "IL", + "Thaï": "TH", "Vietnamien": "VN", "Indonésien": "ID", "Malais": "MY", + "Bulgare": "BG", "Croate": "HR", "Serbe": "RS", "Slovaque": "SK", + "Slovène": "SI", "Ukrainien": "UA", "Catalan": "ES", "Basque": "ES", + "Estonien": "EE", "Letton": "LV", "Lituanien": "LT", "Islandais": "IS", + "Géorgien": "GE", "Arménien": "AM", "Persan": "IR", "Bengali": "BD", + "Tamoul": "LK", "Télougou": "IN", "Ourdou": "PK", "Tagalog": "PH", +} + +VARIANT_COUNTRY = {"VFQ": "CA", "VFB": "BE"} + +_LANG_NAME_TO_ISO: dict[str, str] = {v.lower(): k for k, v in ISO_TO_LANG_NAME.items()} +_LANG_NAME_TO_ISO.update({ + "english": "en", "french": "fr", "german": "de", "spanish": "es", + "italian": "it", "japanese": "ja", "korean": "ko", "portuguese": "pt", + "russian": "ru", "chinese": "zh", "arabic": "ar", "dutch": "nl", + "polish": "pl", "turkish": "tr", "hindi": "hi", "swedish": "sv", + "norwegian": "no", "danish": "da", "finnish": "fi", "greek": "el", + "hungarian": "hu", "romanian": "ro", "czech": "cs", "hebrew": "he", + "thai": "th", "vietnamese": "vi", "indonesian": "id", "malay": "ms", + "bulgarian": "bg", "croatian": "hr", "serbian": "sr", "slovak": "sk", + "slovenian": "sl", "ukrainian": "uk", "catalan": "ca", "basque": "eu", + "estonian": "et", "latvian": "lv", "lithuanian": "lt", "icelandic": "is", + "georgian": "ka", "armenian": "hy", "persian": "fa", "bengali": "bn", + "tamil": "ta", "telugu": "te", "urdu": "ur", "tagalog": "tl", +}) + +# ── Codec normalization ─────────────────────────────────────────────── + +CODEC_SHORT = { + "E-AC-3": "EAC3", "E-AC-3 JOC": "EAC3 Atmos", "AC-3": "AC3", + "TrueHD": "TrueHD", "MLP FBA": "TrueHD", "MLP FBA 16-ch": "TrueHD Atmos", + "DTS-HD MA": "DTS-HD MA", "DTS-HD": "DTS-HD", "DTS": "DTS", + "DTS:X": "DTS:X", "AAC": "AAC", "HE-AAC": "HE-AAC", + "FLAC": "FLAC", "Opus": "Opus", "Vorbis": "Vorbis", + "PCM": "PCM", "LPCM": "LPCM", +} + +VIDEO_CODEC_LABEL = { + "AVC": "x264", "HEVC": "x265", "AV1": "AV1", + "VP9": "VP9", "MPEG-4 Visual": "XviD", +} + +SUB_FORMAT_NORMALIZE = { + "UTF-8": "SRT", "SUBRIP": "SRT", "ASS": "ASS", "SSA": "SSA", + "PGS": "PGS", "VOBSUB": "VobSub", "HDMV_PGS": "PGS", + "S_TEXT/UTF8": "SRT", "S_TEXT/ASS": "ASS", "S_TEXT/SSA": "SSA", + "S_HDMV/PGS": "PGS", "S_VOBSUB": "VobSub", +} + + +# ── Language normalization ──────────────────────────────────────────── + +# ISO 639-2 (3-letter) → ISO 639-1 (2-letter). Only common audio/sub +# languages pymediainfo emits in the wild. +_ISO3_TO_ISO1 = { + "fra": "fr", "fre": "fr", "eng": "en", "deu": "de", "ger": "de", + "spa": "es", "ita": "it", "jpn": "ja", "kor": "ko", "por": "pt", + "rus": "ru", "zho": "zh", "chi": "zh", "ara": "ar", "nld": "nl", + "dut": "nl", "pol": "pl", "tur": "tr", "hin": "hi", "swe": "sv", + "nor": "no", "dan": "da", "fin": "fi", "ell": "el", "gre": "el", + "hun": "hu", "ron": "ro", "rum": "ro", "ces": "cs", "cze": "cs", + "heb": "he", "tha": "th", "vie": "vi", "ind": "id", "msa": "ms", + "may": "ms", "bul": "bg", "hrv": "hr", "srp": "sr", "slk": "sk", + "slo": "sk", "slv": "sl", "ukr": "uk", "cat": "ca", "eus": "eu", + "baq": "eu", "est": "et", "lav": "lv", "lit": "lt", "isl": "is", + "ice": "is", "kat": "ka", "geo": "ka", "hye": "hy", "arm": "hy", + "fas": "fa", "per": "fa", "ben": "bn", "tam": "ta", "tel": "te", + "urd": "ur", "tgl": "tl", +} + +# Region word/code → ISO country code, used to resolve regional variants +# of the same base language (e.g. fr-CA, fr-BE). +_REGION_TO_COUNTRY = { + "ca": "CA", "can": "CA", "canada": "CA", "fq": "CA", "quebec": "CA", + "québec": "CA", "qc": "CA", + "be": "BE", "bel": "BE", "belgium": "BE", "belgique": "BE", +} + + +def normalize_lang_code(raw: str) -> str: + """Convert a raw mediainfo language value to a canonical ISO_TO_LANG_NAME key. + + Handles 2-letter ISO 639-1, 3-letter ISO 639-2, English names, hyphen/ + underscore separated regional variants ("fr-CA", "fr_CA"), and + parenthesised region names ("French (Canada)"). Returns "" when no + confident mapping is found. + """ + if not raw: + return "" + s = str(raw).strip() + if not s: + return "" + + # Pull out a region token from "French (Canada)" → region_word="canada". + region_word = "" + m = re.match(r'^\s*([^()]+?)\s*\(\s*([^()]+)\s*\)\s*$', s) + if m: + s = m.group(1).strip() + region_word = m.group(2).strip().lower() + + s_norm = s.replace("_", "-").lower() + parts = s_norm.split("-", 1) + base = parts[0].strip() + suffix = parts[1].strip() if len(parts) == 2 else "" + + if base in ISO_TO_LANG_NAME and "-" not in base: + base_iso = base + elif base in _LANG_NAME_TO_ISO: + base_iso = _LANG_NAME_TO_ISO[base] + elif base in _ISO3_TO_ISO1: + base_iso = _ISO3_TO_ISO1[base] + else: + return "" + + region_code = _REGION_TO_COUNTRY.get(suffix) or _REGION_TO_COUNTRY.get(region_word) + if region_code: + regional_key = f"{base_iso}-{region_code}" + if regional_key in ISO_TO_LANG_NAME: + return regional_key + return base_iso + + +# ── Helpers ─────────────────────────────────────────────────────────── + +def _country_to_flag(country_code: str) -> str: + return "".join(chr(0x1F1E6 + ord(c) - ord("A")) for c in country_code.upper()) + + +def _lang_name(iso_code: str) -> str: + if not iso_code: + return "Inconnu" + canonical = normalize_lang_code(iso_code) + if canonical and canonical in ISO_TO_LANG_NAME: + return ISO_TO_LANG_NAME[canonical] + cleaned = re.sub(r'\s*\([^)]*\)', '', iso_code).strip() + return cleaned.capitalize() if cleaned else "Inconnu" + + +def _infer_lang_from_title(title: str) -> str: + """Try to extract an ISO language code from a track title.""" + if not title: + return "" + # Try exact match first + key = title.strip().lower() + if key in _LANG_NAME_TO_ISO: + return _LANG_NAME_TO_ISO[key] + # Split on common delimiters and check each token + import re + for token in re.split(r'[\s(),/\-]+', key): + iso = _LANG_NAME_TO_ISO.get(token) + if iso: + return iso + return "" + + +def _lang_flag(lang_name: str, variant: str | None = None) -> str: + if variant: + vc = VARIANT_COUNTRY.get(variant) + if vc: + return _country_to_flag(vc) + cc = LANG_TO_COUNTRY.get(lang_name) + return _country_to_flag(cc) if cc else "\U0001f3f3\ufe0f" + + +def _format_channels(channel_count) -> str: + try: + ch = int(channel_count) + except (ValueError, TypeError): + return "" + if ch > 2: + return f"{ch - 1}.1" + return "2.0" if ch == 2 else "1.0" + + +def _format_bitrate(bps) -> str: + try: + bps_int = int(bps) + except (ValueError, TypeError): + return str(bps) if bps else "" + if bps_int >= 1_000_000: + return f"{bps_int / 1_000_000:.1f} Mb/s" + return f"{bps_int // 1_000} kb/s" + + +def _detect_audio_type(title: str | None) -> str | None: + if not title: + return None + upper = title.upper() + for variant in ("VFF", "VFQ", "VFI", "VF2", "VOF", "VOST", "VFB", "VF", "VO"): + if variant in upper.split() or variant in upper.replace(",", " ").split(): + return variant + return None + + +def _detect_sub_qualifier(title: str | None, forced: str | None) -> str: + if title: + t = title.lower() + if "forced" in t or "forcé" in t: + return " forcés" + if "full" in t or "complet" in t: + return " complets" + if "sdh" in t or "cc" in t: + return " SDH" + if forced and forced.lower() == "yes": + return " forcés" + return "" + + +def _normalize_sub_format(fmt: str | None) -> str: + if not fmt: + return "" + return SUB_FORMAT_NORMALIZE.get(fmt.upper().strip(), fmt.upper().strip()) + + +def _quality_label(width, height) -> str: + """Determine quality label from video dimensions, preferring width.""" + try: + w = int(width) if width else 0 + except (ValueError, TypeError): + w = 0 + try: + h = int(height) if height else 0 + except (ValueError, TypeError): + h = 0 + + if not w and not h: + return "" + + # Width-based (primary) — matches Mediatorr logic + if w >= 3800: + return "UHD 2160p" + if w >= 1900: + return "HD 1080p" + if w >= 1200: + return "HD 720p" + + # Height-based fallback (if width unavailable) + if h >= 2160: + return "UHD 2160p" + if h >= 1080: + return "HD 1080p" + if h >= 720: + return "HD 720p" + if h >= 480: + return "SD 480p" + return f"SD {h}p" + + +def _codec_label(video_format: str | None) -> str: + if not video_format: + return "" + return VIDEO_CODEC_LABEL.get(video_format, video_format) + + +# ── Main generator ──────────────────────────────────────────────────── + +def generate_prez(media_file: MediaFile, *, audio_tracks=None, sub_tracks=None) -> str: + """Generate prez style 3 BBCode from a MediaFile instance.""" + bb = "" + + # ── Technique card ── + codec = _codec_label(media_file.video_format) + bit_depth_raw = media_file.video_bit_depth + bit_depth = f"{bit_depth_raw} bits" if bit_depth_raw and bit_depth_raw != "Unknown" else "" + quality = _quality_label(media_file.video_width, media_file.video_height) + + badges = [] + if codec: + badges.append(f"[badge=red][size=13]{codec}[/size][/badge]") + if bit_depth: + badges.append(f"[badge=gray][size=13]{bit_depth}[/size][/badge]") + if quality: + badges.append(f"[badge=blue][size=13]{quality}[/size][/badge]") + + if badges: + bb += "[card]\n" + bb += "[card-title][color=#e74c3c][size=18][b]Technique[/b][/size][/color][/card-title]\n" + bb += "[card-body]\n" + bb += " ".join(badges) + "\n" + bb += "[/card-body]\n[/card]\n\n" + + # ── Audio + Subtitles grid ── + audio_tracks = audio_tracks if audio_tracks is not None else media_file.audio_track + sub_tracks = sub_tracks if sub_tracks is not None else media_file.subtitle_track + + if not audio_tracks and not sub_tracks: + return bb + + bb += "[grid]\n" + + # ── Audio column ── + if audio_tracks: + bb += "[col]\n[card]\n" + bb += "[card-title][color=#2ecc71][size=18][b]\U0001f50a Audio[/b][/size][/color][/card-title]\n" + bb += "[card-body]\n" + + audio_entries = [] + for track in audio_tracks: + lang = track.get("language", "") + title = track.get("title", "") + fmt = track.get("format", "") + channels = track.get("channel_s", "") + bitrate = track.get("bit_rate", "") + + if not lang and title: + lang = _infer_lang_from_title(title) + name = _lang_name(lang) + audio_type = _detect_audio_type(title) + flag = _lang_flag(name, audio_type) + type_str = f" ({audio_type})" if audio_type else "" + + codec_name = CODEC_SHORT.get(fmt, fmt) if fmt else "" + ch_str = _format_channels(channels) + br_str = _format_bitrate(bitrate) + + details = " \u2022 ".join(filter(None, [codec_name, ch_str, br_str])) + entry = f"{flag} {name}{type_str}" + if details: + entry += f"\n[size=11][color=#7f8c8d]{details}[/color][/size]" + audio_entries.append(entry) + + bb += "\n\n".join(audio_entries) + "\n" + bb += "[/card-body]\n[/card]\n[/col]\n" + + # ── Subtitles column ── + if sub_tracks: + bb += "\n[col]\n[card]\n" + bb += "[card-title][color=#f39c12][size=18][b]\U0001f4dd Sous-titres[/b][/size][/color][/card-title]\n" + bb += "[card-body]\n" + + sub_entries = [] + for track in sub_tracks: + lang = track.get("language", "") + title = track.get("title", "") + forced = track.get("forced", "") + fmt = track.get("format", "") + + if not lang and title: + lang = _infer_lang_from_title(title) + name = _lang_name(lang) + flag = _lang_flag(name) + qualifier = _detect_sub_qualifier(title, forced) + sub_format = _normalize_sub_format(fmt) + + entry = f"{flag} {name}{qualifier}" + if sub_format: + entry += f"\n[size=11][color=#7f8c8d]{sub_format}[/color][/size]" + sub_entries.append(entry) + + bb += "\n\n".join(sub_entries) + "\n" + bb += "[/card-body]\n[/card]\n[/col]\n" + + bb += "[/grid]\n" + return bb diff --git a/unit3dup/pvtTorrent.py b/unit3dup/pvtTorrent.py index 0f90481..6b6757f 100644 --- a/unit3dup/pvtTorrent.py +++ b/unit3dup/pvtTorrent.py @@ -37,10 +37,23 @@ def __init__(self, contents: Media, meta: str, trackers_list = None): self.mytorr.created_by = "https://github.com/31December99/Unit3Dup" self.mytorr.private = True self.mytorr.source= trackers_api_data[trackers_list[0]]['source'] - self.mytorr.segments = 16 * 1024 * 1024 + # Piece size set dynamically in hash() based on content size + @staticmethod + def _compute_piece_size(size_bytes: int) -> int: + """Compute optimal piece size based on total content size (upload.md rules).""" + size_mb = size_bytes / (1024 * 1024) + size_gb = size_bytes / (1024 ** 3) + if size_gb > 20: return 16 * 1024 * 1024 # 16 MB + if size_gb > 8: return 8 * 1024 * 1024 # 8 MB + if size_gb > 4: return 4 * 1024 * 1024 # 4 MB + if size_gb > 2: return 2 * 1024 * 1024 # 2 MB + if size_gb > 1: return 1 * 1024 * 1024 # 1 MB + if size_mb > 500: return 512 * 1024 # 512 KB + return 256 * 1024 # 256 KB def hash(self): + self.mytorr.segments = self._compute_piece_size(self.mytorr.size) # Calculate the torrent size size = round(self.mytorr.size / (1024 ** 3), 2) # Print a message for the user diff --git a/unit3dup/pvtVideo.py b/unit3dup/pvtVideo.py index bc9a866..d435e76 100644 --- a/unit3dup/pvtVideo.py +++ b/unit3dup/pvtVideo.py @@ -1,21 +1,16 @@ # -*- coding: utf-8 -*- import hashlib -import os.path import re -import diskcache - -from common.external_services.imageHost import Build from common.mediainfo import MediaFile -from common.frames import VideoFrame +from unit3dup.prez import generate_prez from view import custom_console -from unit3dup import config_settings from unit3dup.media import Media class Video: - """ Build a description for the torrent page: screenshots, mediainfo, trailers, metadata """ + """ Build a description for the torrent page: prez BBCode, mediainfo, metadata """ def __init__(self, media: Media, tmdb_id: int, trailer_key=None): self.file_name: str = media.file_name @@ -23,31 +18,25 @@ def __init__(self, media: Media, tmdb_id: int, trailer_key=None): self.tmdb_id: int = tmdb_id self.trailer_key: int = trailer_key - self.cache = diskcache.Cache(str(config_settings.user_preferences.CACHE_PATH)) - - # Create a cache key for tmdb_id - self.key = f"{self.tmdb_id}.{self.display_name}" - self.cache_key = self.hash_key(self.key) - - # Load the video frames - # if web_enabled is off set the number of screenshots to an even number - if not config_settings.user_preferences.WEBP_ENABLED: - if config_settings.user_preferences.NUMBER_OF_SCREENSHOTS % 2 != 0: - config_settings.user_preferences.NUMBER_OF_SCREENSHOTS += 1 - - samples_n = max(2, min(config_settings.user_preferences.NUMBER_OF_SCREENSHOTS, 10)) - self.video_frames: VideoFrame = VideoFrame(self.file_name, num_screenshots=samples_n) # Init self.is_hd: int = 0 self.description: str = '' self.mediainfo: str = '' + self.audio_tracks: list[dict] = [] + self.subtitle_tracks: list[dict] = [] @staticmethod def hash_key(key: str) -> str: """ Generate a hashkey for the cache index """ return hashlib.md5(key.encode('utf-8')).hexdigest() + @staticmethod + def _strip_mediainfo_path(mediainfo_text: str) -> str: + """Replace the full file path with just the filename in the 'Complete name' field.""" + pattern = r'(Complete name\s+:\s+)(.+[/\\])([^/\\]+\.\w+)' + return re.sub(pattern, lambda m: m.group(1) + m.group(3), mediainfo_text) + @staticmethod def generate_nfo_file(media_info: MediaFile, output_path: str) -> bool: """Génère un fichier NFO avec la sortie brute de Mediainfo. @@ -56,22 +45,10 @@ def generate_nfo_file(media_info: MediaFile, output_path: str) -> bool: - Remplace le chemin complet par juste le nom du fichier dans "Complete name" """ try: - mediainfo_output = media_info.info - - # Pattern pour trouver "Complete name" suivi du chemin complet - pattern = r'(Complete name\s+:\s+)(.+[/\\])([^/\\]+\.\w+)' + mediainfo_output = Video._strip_mediainfo_path(media_info.info) - def replace_path(match: re.Match) -> str: - # Garder seulement le nom de fichier - return match.group(1) + match.group(3) - - # Appliquer le remplacement - mediainfo_output = re.sub(pattern, replace_path, mediainfo_output) - - # Écrire le fichier NFO avec la sortie modifiée with open(output_path, 'w', encoding='utf-8') as f: f.write(mediainfo_output) - return True except Exception as e: custom_console.bot_warning_log(f"[NFO] Erreur lors de la génération du NFO: {e}") @@ -82,42 +59,16 @@ def build_info(self): # media_info media_info = MediaFile(self.file_name) - self.mediainfo = media_info.info - - if config_settings.user_preferences.CACHE_SCR: - description = self.cache.get(self.cache_key) - if description: - custom_console.bot_warning_log(f"\n<> Using cached images for '{self.key}'") - self.description = description.get('description', '') - self.is_hd = description.get('is_hd', 0) - - if not self.description: - # If no description found generate it - custom_console.bot_log(f"\n[GENERATING IMAGES..] [HD {'ON' if self.is_hd == 0 else 'OFF'}]") - # Extract the frames - extracted_frames, is_hd = self.video_frames.create() - # Create a webp file if it's enabled in the config json - extracted_frames_webp = [] - if config_settings.user_preferences.WEBP_ENABLED: - extracted_frames_webp = self.video_frames.create_webp_from_video( - video_path=self.file_name, - start_time=90, - duration=10, - output_path=os.path.join(config_settings.user_preferences.CACHE_PATH, "file.webp"), - ) - custom_console.bot_log("Done.") - - # Build the description - build_description = Build(extracted_frames=extracted_frames_webp + extracted_frames, filename=self.display_name) - self.description = build_description.description() - - if self.trailer_key: - self.description += ( - f"[b][spoiler=Spoiler: PLAY TRAILER][center][youtube]{self.trailer_key}[/youtube]" - f"[/center][/spoiler][/b]" - ) - self.is_hd = is_hd - - # Caching - if config_settings.user_preferences.CACHE_SCR: - self.cache[self.cache_key] = {'tmdb_id': self.tmdb_id, 'description': self.description, 'is_hd': self.is_hd} \ No newline at end of file + self.mediainfo = self._strip_mediainfo_path(media_info.info) + self.audio_tracks = media_info.audio_track + self.subtitle_tracks = media_info.subtitle_track + + # Generate prez BBCode description + self.description = generate_prez(media_info) + + # Determine SD flag: 0 = HD (>=720p), 1 = SD + try: + height = int(media_info.video_height) if media_info.video_height else 0 + except (ValueError, TypeError): + height = 0 + self.is_hd = 0 if height >= 720 else 1 \ No newline at end of file diff --git a/unit3dup/release_normalizer.py b/unit3dup/release_normalizer.py index 8421152..489369c 100644 --- a/unit3dup/release_normalizer.py +++ b/unit3dup/release_normalizer.py @@ -7,6 +7,7 @@ """ import re +import unicodedata from typing import Optional @@ -25,16 +26,23 @@ def _normalize_lang(raw: str) -> str: if r in ("MULTI.VFQ", "MULTI-VFQ"): return "MULTi.VFQ" if r in ("MULTI.VF2", "MULTI-VF2"): return "MULTi.VF2" if r in ("MULTI.VFB", "MULTI-VFB"): return "MULTi.VFB" + if r in ("MULTI.VOF", "MULTI-VOF"): return "MULTi.VOF" + if r in ("MULTI.VOQ", "MULTI-VOQ"): return "MULTi.VOQ" + if r in ("MULTI.VOB", "MULTI-VOB"): return "MULTi.VOB" if r in ("MULTI", "MULTIC"): return "MULTi" - if r in ("FRENCH", "VFF", "VFI"): return "VFF" + if r in ("FRENCH", "VFF"): return "VFF" if r == "VFQ": return "VFQ" if r == "VF2": return "VF2" if r == "VFB": return "VFB" if r == "VOF": return "VOF" if r == "VOQ": return "VOQ" if r == "VOB": return "VOB" + if r in ("FRENCH.VOF", "FRENCH-VOF"): return "VOF" + if r in ("FRENCH.VOQ", "FRENCH-VOQ"): return "VOQ" + if r in ("FRENCH.VOB", "FRENCH-VOB"): return "VOB" if r == "VOSTFR": return "VOSTFR" if r == "SUBFRENCH": return "SUBFRENCH" + if r == "SUBFORCED": return "SUBFORCED" return raw @@ -42,14 +50,15 @@ def _normalize_source(raw: str) -> str: r = raw.upper() if r in ("BLURAY", "BLU-RAY"): return "BluRay" if r == "BDRIP": return "BDRip" + if r == "BRRIP": return "BRRip" if r == "4KLIGHT": return "4KLight" if r in ("HDLIGHT", "MHD"): return "HDLight" if r == "WEBRIP": return "WEBRip" if r in ("WEB-DL", "WEBDL", "WEB"): return "WEB" if r == "HDRIP": return "HDRip" if r == "HDTV": return "HDTV" - if r in ("TVRIP", "TVHDRIP"): return "TVRip" - if r in ("DVDRIP", "DVD"): return "DVDRip" + if r in ("TVRIP", "TVHDRIP", "HDTVRIP"): return "TVRip" + if r in ("DVDRIP", "DVD", "DVD9", "DVD5"): return "DVDRip" if r == "REMUX": return "REMUX" return raw @@ -57,7 +66,11 @@ def _normalize_source(raw: str) -> str: def _clean_title(t: str) -> str: t = t.strip() t = t.replace(" ", ".") + t = unicodedata.normalize('NFD', t) + t = ''.join(c for c in t if unicodedata.category(c) != 'Mn') t = re.sub(r'[^a-zA-Z0-9._-]', '', t) + # Remove isolated dashes: ".-." or leading/trailing dashes + t = re.sub(r'\.?-\.?', '.', t) t = re.sub(r'\.{2,}', '.', t) t = t.strip('.') return t @@ -144,24 +157,34 @@ def _get_lang_from_mediainfo(mi: str) -> str: VFF, VFQ, VF2, VFB, VOF, VOQ, VOB ou '' """ if not mi: return "" - vff = vfq = vfb = vof = voq = vob = False + # Titres explicites (taggés par l'uploader) — priorité haute + vff_title = vfq_title = vfb_title = vof = voq = vob = False + # Codes de langue génériques (Language: French (FR)/(CA)) — fallback + vff_lang = vfq_lang = False for line in mi.splitlines(): - if re.search(r'Language\s*:\s*French\s*\(FR\)', line, re.IGNORECASE): vff = True - elif re.search(r'Language\s*:\s*French\s*\(CA\)', line, re.IGNORECASE): vfq = True - elif re.search(r'Title\s*:.*\b(VFF|VFI|TrueFrench|French\s*\(France\))\b', line, re.IGNORECASE): vff = True - elif re.search(r'Title\s*:.*\b(VFB|French\s*\(Belgique\))\b', line, re.IGNORECASE): vfb = True - elif re.search(r'Title\s*:.*\b(VOF)\b', line, re.IGNORECASE): vof = True - elif re.search(r'Title\s*:.*\b(VFQ|French\s*\(Canadien\))\b', line, re.IGNORECASE): vfq = True + if re.search(r'Title\s*:.*\b(VOF)\b', line, re.IGNORECASE): vof = True elif re.search(r'Title\s*:.*\b(VOQ|French\s*\(Québec\))\b', line, re.IGNORECASE): voq = True elif re.search(r'Title\s*:.*\b(VOB|French\s*\(Belgique\s*VO\))\b', line, re.IGNORECASE): vob = True + elif re.search(r'Title\s*:.*\b(VFF|VFI|TrueFrench|French\s*\(France\))\b', line, re.IGNORECASE): vff_title = True + elif re.search(r'Title\s*:.*\b(VFQ|French\s*\(Canadien\))\b', line, re.IGNORECASE): vfq_title = True + elif re.search(r'Title\s*:.*\b(VFB|French\s*\(Belgique\))\b', line, re.IGNORECASE): vfb_title = True + elif re.search(r'Language\s*:\s*French\s*\(FR\)', line, re.IGNORECASE): vff_lang = True + elif re.search(r'Language\s*:\s*French\s*\(CA\)', line, re.IGNORECASE): vfq_lang = True + + # Les titres VO explicites dominent : "Title: VOF" + "Language: French (FR)" + # signifie audio FR original (pas un doublage) → VOF, pas VFF. + if vof: return "VOF" + if voq: return "VOQ" + if vob: return "VOB" + + vff = vff_title or vff_lang + vfq = vfq_title or vfq_lang + vfb = vfb_title if vff and vfq: return "VF2" if vff: return "VFF" if vfq: return "VFQ" if vfb: return "VFB" - if vof: return "VOF" - if voq: return "VOQ" - if vob: return "VOB" return "" @@ -195,7 +218,18 @@ def _is_silent_from_mediainfo(mi: str) -> bool: m = re.search(r'Language\s*:\s*(\S+)', line, re.IGNORECASE) if m: audio_langs.append(m.group(1).strip().lower()) - return bool(audio_langs) and all(l == 'zxx' for l in audio_langs) + return bool(audio_langs) and all(l == 'zxx' for l in audio_langs) + + +def _count_audio_tracks_from_mediainfo(mi: str) -> int: + """Compte le nombre de pistes audio dans le texte brut MediaInfo.""" + if not mi: + return 0 + count = 0 + for line in mi.splitlines(): + if re.match(r'^Audio\s*$|^Audio #', line): + count += 1 + return count # ══════════════════════════════════════════════════════════════════════════════ @@ -205,14 +239,15 @@ def _is_silent_from_mediainfo(mi: str) -> bool: # Séparateurs utilisés en step 5b pour décoller les tokens collés. # Ordre important : plus long avant plus court dans chaque famille. _TAGS = ( - r'BluRay|BDRip|WEBRip|WEB|4KLight|HDLight|HDRip|TVRip|DVDRip|HDTV|REMUX|CAM' + r'BluRay|BDRip|BRRip|WEBRip|WEB|4KLight|HDLight|HDRip|HDTVRip|TVRip|DVDRip|DVD9|DVD5|HDTV|REMUX|BDMV|CAM' r'|2160p|1080p|1080i|720p|576p|480p|4K|UHD' r'|HDR10P|HDR10|SDR|DV|HLG|PQ10|HDR' r'|x265|x264|H265|H264|HEVC|AVC|AV1|VP9|VC1' - r'|DTS-HDMA|DTS-HD|DTS|AC3|DDP|TrueHD|Atmos|AAC' - r'|MULTi|VFF|VFQ|VF2|VFB|VOSTFR|SUBFRENCH|VOF|VOQ|VOB|FRENCH' - r'|EXTENDED|PROPER|REPACK|UNRATED|UNCUT|REMASTERED|INTERNAL|NoTAG|iNTEGRALE' + r'|DTS-HDMA|DTS-HDHRA|DTS-HD|DTS|AC3|DDP|TrueHD|Atmos|AAC|OPUS' + r'|MULTi|VFF|VFQ|VF2|VFB|VOSTFR|SUBFRENCH|SUBFORCED|VOF|VOQ|VOB|FRENCH' + r'|EXTENDED|PROPER|REPACK|UNRATED|UNCUT|REMASTERED|INTERNAL|NoTAG|iNTEGRALE|COMPLETE' r'|8bit|10bit|12bit' + r'|3D|SBS|HSBS|TAB|HTAB|MVC|CUSTOM|NoGRP' ) # Du plus spécifique au moins spécifique @@ -222,13 +257,16 @@ def _is_silent_from_mediainfo(mi: str) -> bool: r'VFB-[A-Za-z]+(?:-[A-Za-z]+)*', r'VF2-[A-Za-z]+(?:-[A-Za-z]+)*', r'MULTi\.VFF', r'MULTi\.VFQ', r'MULTi\.VF2', r'MULTi\.VFB', + r'MULTi\.VOF', r'MULTi\.VOQ', r'MULTi\.VOB', + r'FRENCH\.VOF', r'FRENCH\.VOQ', r'FRENCH\.VOB', r'MULTi', - r'FRENCH', r'VFF', r'VFQ', r'VF2', r'VFB', r'VOF', r'VOQ', r'VOB', - r'VOSTFR', r'SUBFRENCH', + r'FRENCH', r'VFF', r'VFQ', r'VF2', r'VFB', + r'VOSTFR', r'SUBFRENCH', r'SUBFORCED', ] _EXTRAS_MAP = { + '4K REMASTER': '4K.REMASTER', 'EXTENDED': 'EXTENDED', 'THEATRICAL': 'THEATRICAL', 'PROPER': 'PROPER', @@ -237,11 +275,13 @@ def _is_silent_from_mediainfo(mi: str) -> bool: 'UNCUT': 'UNCUT', 'REMASTERED': 'REMASTERED', 'INTERNAL': 'INTERNAL', - 'NOTAG': 'NoTAG', 'INTEGRALE': 'iNTEGRALE', + 'COMPLETE': 'COMPLETE', 'LIMITED': 'LIMITED', 'IMAX EDITION': 'IMAX.EDITION', 'IMAX': 'IMAX', + 'CUSTOM': 'CUSTOM', + 'UPSCALED': 'UpScaled', } _CODEC_LIST = [ @@ -265,12 +305,19 @@ def _is_silent_from_mediainfo(mi: str) -> bool: _SOURCE_LIST = [ "UHD BluRay", "BluRay", "Blu-Ray", - "BDRip", + "BDRip", "BRRip", "WEB-DL", "WEBRip", - "HDTV", "HDRip", "TVRip", - "WEB", "DVDRip", "DVD", + "HDTVRip", "HDTV", "HDRip", "TVRip", + "WEB", + "DVDRip", "DVD9", "DVD5", "DVD", ] +# Codecs connus — utilisés pour exclure les faux positifs team tag +_CODEC_NAMES = frozenset({ + "X264", "X265", "H264", "H265", "HEVC", "AVC", "AV1", "VP9", "VC1", + "MPEG2", "MPEG-2", +}) + # Qualificatifs de source : peuvent coexister avec une source principale. # Ex: "4KLight BluRay" → source = "4KLight.BluRay" _SOURCE_QUAL_LIST = ["4KLight", "HDLight", "mHD"] @@ -280,8 +327,16 @@ def _is_silent_from_mediainfo(mi: str) -> bool: # PARSER PRINCIPAL # ══════════════════════════════════════════════════════════════════════════════ -def _parse_release(original: str, mi: Optional[str] = None, is_silent: bool = False) -> str: +def _parse_release( + original: str, + mi: Optional[str] = None, + is_silent: bool = False, + year: Optional[str] = None, +) -> str: name = original + year_hint = str(year).strip() if year else "" + if year_hint and not re.fullmatch(r'[12][0-9]{3}', year_hint): + year_hint = "" # ── 1. Extension ───────────────────────────────────────────────────────── ext = "" @@ -290,28 +345,47 @@ def _parse_release(original: str, mi: Optional[str] = None, is_silent: bool = Fa ext = "." + m.group(1) name = name[:-len(ext)] + # ── 1b. Pré-nettoyage : séparateurs et bruit ──────────────────────────── + # " - " ou ".-." → espace (évite les artefacts .-.-. dans le titre) + name = re.sub(r'[.\s]+-[.\s]+', ' ', name) + # @bitrate (ex: AC3@640Kbps) → supprimer le suffixe @bitrate + name = re.sub(r'@\d+[kKmM]bps', '', name, flags=re.IGNORECASE) + # Bitrates standalone (384kbps, 2Mbps…) : bruit sans valeur pour le nommage + name = re.sub(r'-?\d+[kKmM]bps', '', name, flags=re.IGNORECASE) + # Codecs avec point interne (H.264, H.265) : fusionner AVANT l'extraction + # team et la conversion points→espaces (sinon "H" et "264" fuient dans le titre). + name = re.sub(r'(?= 0: name = name[:cut_pos] + name[cut_pos + len(suffix) + 1:] + if not team: + team = "NoTag" + # ── 3. Normalisation séparateurs : points & underscores → espaces ───────── name = name.replace('.', ' ').replace('_', ' ') name = _ws(name) @@ -321,31 +395,48 @@ def _parse_release(original: str, mi: Optional[str] = None, is_silent: bool = Fa name = re.sub(r'(? str: - return _parse_release(release_name, mediainfo_text, is_silent) \ No newline at end of file + return _parse_release(release_name, mediainfo_text, is_silent, year=year) \ No newline at end of file diff --git a/unit3dup/state_db.py b/unit3dup/state_db.py new file mode 100644 index 0000000..b77d4c9 --- /dev/null +++ b/unit3dup/state_db.py @@ -0,0 +1,699 @@ +# -*- coding: utf-8 -*- +"""SQLite-backed state database for tracking media items through the +prepare → review → upload lifecycle. + +Replaces the JSON-based WatcherState for web-mode operation. +Uses WAL journal mode for concurrent read/write access (watcher thread + web server). +""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import threading +from datetime import datetime +from typing import Any + + +_ALLOWED_COLUMNS = { + "source_basename", "source_path", "folder_path", "source_type", + "status", "content_category", "qbit_category", "display_name", + "torrent_name", "release_name", "source_tag", "file_size", "resolution", + "tmdb_id", "imdb_id", "igdb_id", "tmdb_title", "tmdb_year", + "description", "mediainfo", "nfo_content", + "audio_tracks", "subtitle_tracks", + "tracker_payload", "tracker_name", "trackers_list", "torrent_archive_path", + "validation_report", "has_errors", "has_warnings", + "rejection_reason", "user_edited_name", "user_edited_desc", + "discovered_at", "prepared_at", "decided_at", "uploaded_at", + "tracker_response", "upload_error", "skip_reason", "duplicate_match", +} + +_JSON_FIELDS = ("tracker_payload", "trackers_list", "validation_report", "audio_tracks", "subtitle_tracks", "duplicate_match") + +_COMPLIANCE_JSON_FIELDS = ("violations",) + +_COMPLIANCE_ALLOWED_COLUMNS = { + "torrent_id", "tracker_name", "uploader", "category", + "current_name", "proposed_name", "violations", "diff_kind", + "severity_max", "checked_at", "first_seen_at", "ack_status", + "edit_url", "linked_item_id", + "description", "mediainfo", +} + + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_basename TEXT NOT NULL UNIQUE, + source_path TEXT NOT NULL, + folder_path TEXT, + source_type TEXT, + + status TEXT NOT NULL DEFAULT 'pending', + + content_category TEXT, + qbit_category TEXT, + display_name TEXT, + torrent_name TEXT, + release_name TEXT, + source_tag TEXT, + file_size INTEGER, + resolution TEXT, + + tmdb_id INTEGER, + imdb_id INTEGER, + igdb_id INTEGER, + tmdb_title TEXT, + tmdb_year INTEGER, + + description TEXT, + mediainfo TEXT, + nfo_content TEXT, + audio_tracks TEXT, + subtitle_tracks TEXT, + + tracker_payload TEXT, + tracker_name TEXT, + trackers_list TEXT, + torrent_archive_path TEXT, + + validation_report TEXT, + has_errors INTEGER DEFAULT 0, + has_warnings INTEGER DEFAULT 0, + + rejection_reason TEXT, + user_edited_name TEXT, + user_edited_desc TEXT, + + discovered_at TEXT, + prepared_at TEXT, + decided_at TEXT, + uploaded_at TEXT, + + tracker_response TEXT, + upload_error TEXT, + skip_reason TEXT, + duplicate_match TEXT +); + +CREATE INDEX IF NOT EXISTS idx_items_status ON items(status); +CREATE INDEX IF NOT EXISTS idx_items_discovered ON items(discovered_at); +CREATE INDEX IF NOT EXISTS idx_items_source_basename ON items(source_basename); + +CREATE TABLE IF NOT EXISTS compliance ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + torrent_id INTEGER NOT NULL UNIQUE, + tracker_name TEXT NOT NULL, + uploader TEXT, + category TEXT, + current_name TEXT NOT NULL, + proposed_name TEXT, + violations TEXT, + diff_kind TEXT, + severity_max TEXT, + checked_at TEXT NOT NULL, + first_seen_at TEXT NOT NULL, + ack_status TEXT NOT NULL DEFAULT 'unchecked', + edit_url TEXT, + linked_item_id INTEGER, + description TEXT, + mediainfo TEXT +); + +CREATE INDEX IF NOT EXISTS idx_compliance_severity ON compliance(severity_max); +CREATE INDEX IF NOT EXISTS idx_compliance_ack ON compliance(ack_status); +CREATE INDEX IF NOT EXISTS idx_compliance_torrent ON compliance(torrent_id); +""" + + +class StateDB: + """Thread-safe SQLite state database.""" + + def __init__(self, db_path: str): + self._db_path = db_path + self._lock = threading.Lock() + self._ensure_schema() + + # ── Connection helpers ──────────────────────────────────────────── + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self._db_path, timeout=30) + conn.execute("PRAGMA foreign_keys=ON") + conn.row_factory = sqlite3.Row + return conn + + def _ensure_schema(self): + with self._lock: + conn = self._connect() + try: + conn.execute("PRAGMA journal_mode=WAL") + conn.executescript(_SCHEMA) + conn.commit() + finally: + conn.close() + self._ensure_columns() + + def _ensure_columns(self): + """Add columns that may be missing from older databases.""" + items_new_cols = [("audio_tracks", "TEXT"), ("subtitle_tracks", "TEXT"), ("duplicate_match", "TEXT")] + compliance_new_cols = [("description", "TEXT"), ("mediainfo", "TEXT")] + with self._lock: + conn = self._connect() + try: + items_existing = {row[1] for row in conn.execute("PRAGMA table_info(items)").fetchall()} + for col_name, col_type in items_new_cols: + if col_name not in items_existing: + conn.execute(f"ALTER TABLE items ADD COLUMN {col_name} {col_type}") + + compliance_existing = {row[1] for row in conn.execute("PRAGMA table_info(compliance)").fetchall()} + for col_name, col_type in compliance_new_cols: + if col_name not in compliance_existing: + conn.execute(f"ALTER TABLE compliance ADD COLUMN {col_name} {col_type}") + conn.commit() + finally: + conn.close() + + # ── Query helpers ───────────────────────────────────────────────── + + @staticmethod + def _row_to_dict(row: sqlite3.Row | None) -> dict | None: + if row is None: + return None + d = dict(row) + # Deserialize JSON fields + for field in _JSON_FIELDS: + if d.get(field): + try: + d[field] = json.loads(d[field]) + except (json.JSONDecodeError, TypeError): + pass + return d + + # ── Public API: queries ─────────────────────────────────────────── + + def is_known(self, source_basename: str) -> str | None: + """Check if a source entry is already tracked. + + Returns the status string or None if unknown. + """ + conn = self._connect() + try: + row = conn.execute( + "SELECT status FROM items WHERE source_basename = ?", + (source_basename,), + ).fetchone() + return row["status"] if row else None + finally: + conn.close() + + def get_item(self, item_id: int) -> dict | None: + conn = self._connect() + try: + row = conn.execute("SELECT * FROM items WHERE id = ?", (item_id,)).fetchone() + return self._row_to_dict(row) + finally: + conn.close() + + def get_item_by_basename(self, source_basename: str) -> dict | None: + conn = self._connect() + try: + row = conn.execute( + "SELECT * FROM items WHERE source_basename = ?", (source_basename,) + ).fetchone() + return self._row_to_dict(row) + finally: + conn.close() + + def list_items( + self, + status: str | None = None, + category: str | None = None, + page: int = 1, + per_page: int = 50, + ) -> list[dict]: + """List items with optional filtering and pagination. + + Returns dicts WITHOUT the large text fields (description, mediainfo, nfo_content) + for performance. Use get_item() for full detail. + """ + conditions = [] + params: list[Any] = [] + if status: + conditions.append("status = ?") + params.append(status) + if category: + conditions.append("content_category = ?") + params.append(category) + + where = f"WHERE {' AND '.join(conditions)}" if conditions else "" + offset = (page - 1) * per_page + params.extend([per_page, offset]) + + conn = self._connect() + try: + rows = conn.execute( + f"""SELECT id, source_basename, source_path, folder_path, source_type, + status, content_category, qbit_category, display_name, + torrent_name, release_name, source_tag, file_size, resolution, + tmdb_id, imdb_id, igdb_id, tmdb_title, tmdb_year, + tracker_name, trackers_list, torrent_archive_path, + has_errors, has_warnings, validation_report, + rejection_reason, user_edited_name, user_edited_desc, + discovered_at, prepared_at, decided_at, uploaded_at, + upload_error, skip_reason, + json_extract(tracker_payload, '$.season_number') as season_number, + json_extract(tracker_payload, '$.episode_number') as episode_number + FROM items {where} + ORDER BY discovered_at DESC + LIMIT ? OFFSET ?""", + params, + ).fetchall() + return [self._row_to_dict(r) for r in rows] + finally: + conn.close() + + def count_by_status(self) -> dict[str, int]: + conn = self._connect() + try: + rows = conn.execute( + "SELECT status, COUNT(*) as cnt FROM items GROUP BY status" + ).fetchall() + return {r["status"]: r["cnt"] for r in rows} + finally: + conn.close() + + # ── Public API: writes ──────────────────────────────────────────── + + def add_item(self, **kwargs) -> int: + """Insert a new item. Serializes JSON fields automatically. + + Returns the new item ID. + """ + invalid = set(kwargs.keys()) - _ALLOWED_COLUMNS + if invalid: + raise ValueError(f"Invalid column names: {invalid}") + + for field in _JSON_FIELDS: + if field in kwargs and kwargs[field] is not None and not isinstance(kwargs[field], str): + kwargs[field] = json.dumps(kwargs[field], ensure_ascii=False) + + columns = ", ".join(kwargs.keys()) + placeholders = ", ".join(["?"] * len(kwargs)) + + with self._lock: + conn = self._connect() + try: + cursor = conn.execute( + f"INSERT OR IGNORE INTO items ({columns}) VALUES ({placeholders})", + list(kwargs.values()), + ) + conn.commit() + return cursor.lastrowid + finally: + conn.close() + + def update_item(self, item_id: int, **kwargs) -> bool: + """Update specific fields on an item.""" + invalid = set(kwargs.keys()) - _ALLOWED_COLUMNS + if invalid: + raise ValueError(f"Invalid column names: {invalid}") + + for field in _JSON_FIELDS: + if field in kwargs and kwargs[field] is not None and not isinstance(kwargs[field], str): + kwargs[field] = json.dumps(kwargs[field], ensure_ascii=False) + + if not kwargs: + return False + + set_clause = ", ".join(f"{k} = ?" for k in kwargs) + values = list(kwargs.values()) + [item_id] + + with self._lock: + conn = self._connect() + try: + conn.execute(f"UPDATE items SET {set_clause} WHERE id = ?", values) + conn.commit() + return True + finally: + conn.close() + + def mark_uploaded(self, item_id: int, tracker_response: str = "") -> bool: + return self.update_item( + item_id, + status="uploaded", + uploaded_at=datetime.now().isoformat(), + decided_at=datetime.now().isoformat(), + tracker_response=tracker_response, + ) + + def mark_rejected(self, item_id: int, reason: str) -> bool: + return self.update_item( + item_id, + status="rejected", + decided_at=datetime.now().isoformat(), + rejection_reason=reason, + ) + + def mark_error(self, item_id: int, error: str) -> bool: + return self.update_item( + item_id, + status="error", + upload_error=error, + ) + + def mark_queued(self, item_id: int) -> bool: + return self.update_item( + item_id, + status="queued", + decided_at=datetime.now().isoformat(), + ) + + def retry_item(self, item_id: int) -> bool: + """Move a rejected/error/skipped item back to pending for reprocessing.""" + return self.update_item( + item_id, + status="pending", + decided_at=None, + uploaded_at=None, + rejection_reason=None, + upload_error=None, + skip_reason=None, + tracker_response=None, + ) + + def recover_analyzing(self) -> int: + """Delete items stuck in 'analyzing' (orphaned by a prior crash). + + These items have no prepared data, so the watcher will + re-discover and re-process them on the next cycle. + """ + with self._lock: + conn = self._connect() + try: + cursor = conn.execute("DELETE FROM items WHERE status = 'analyzing'") + conn.commit() + return cursor.rowcount + finally: + conn.close() + + def delete_item(self, item_id: int) -> bool: + with self._lock: + conn = self._connect() + try: + conn.execute("DELETE FROM items WHERE id = ?", (item_id,)) + conn.commit() + return True + finally: + conn.close() + + def atomic_transition(self, item_id: int, from_statuses: tuple[str, ...], to_status: str, **extra_fields) -> bool: + """Atomically transition an item's status. Returns True if the transition was applied.""" + invalid = set(extra_fields.keys()) - _ALLOWED_COLUMNS + if invalid: + raise ValueError(f"Invalid column names: {invalid}") + + for field in _JSON_FIELDS: + if field in extra_fields and extra_fields[field] is not None and not isinstance(extra_fields[field], str): + extra_fields[field] = json.dumps(extra_fields[field], ensure_ascii=False) + + placeholders_str = ", ".join(f"{k} = ?" for k in extra_fields) + set_clause = f"status = ?" + if placeholders_str: + set_clause += f", {placeholders_str}" + + where_in = ", ".join("?" for _ in from_statuses) + + values = [to_status] + list(extra_fields.values()) + [item_id] + list(from_statuses) + + with self._lock: + conn = self._connect() + try: + cursor = conn.execute( + f"UPDATE items SET {set_clause} WHERE id = ? AND status IN ({where_in})", + values, + ) + conn.commit() + return cursor.rowcount > 0 + finally: + conn.close() + + # ── Compliance API ──────────────────────────────────────────────── + + @staticmethod + def _compliance_row_to_dict(row: sqlite3.Row | None) -> dict | None: + if row is None: + return None + d = dict(row) + for field in _COMPLIANCE_JSON_FIELDS: + if d.get(field): + try: + d[field] = json.loads(d[field]) + except (json.JSONDecodeError, TypeError): + pass + return d + + def upsert_compliance(self, **kwargs) -> int: + """Insert or update a compliance row keyed on torrent_id. + + Preserves first_seen_at, ack_status, and checked_at (unless + checked_at was explicitly passed) on updates. This matters because + metadata-only updates (e.g. attaching linked_item_id) must NOT + bump the timestamp. + """ + invalid = set(kwargs.keys()) - _COMPLIANCE_ALLOWED_COLUMNS + if invalid: + raise ValueError(f"Invalid compliance column names: {invalid}") + if "torrent_id" not in kwargs: + raise ValueError("torrent_id is required") + + for field in _COMPLIANCE_JSON_FIELDS: + if field in kwargs and not isinstance(kwargs[field], str): + kwargs[field] = json.dumps(kwargs[field], ensure_ascii=False) + + now = datetime.now().isoformat() + checked_at_explicit = "checked_at" in kwargs + kwargs.setdefault("checked_at", now) + kwargs.setdefault("first_seen_at", now) + kwargs.setdefault("ack_status", "unchecked") + + columns = list(kwargs.keys()) + placeholders = ", ".join(["?"] * len(columns)) + col_list = ", ".join(columns) + + # UPDATE clause must NOT touch first_seen_at or ack_status. And skip + # checked_at unless the caller provided it explicitly. + protected = {"torrent_id", "first_seen_at", "ack_status"} + if not checked_at_explicit: + protected.add("checked_at") + update_cols = [c for c in columns if c not in protected] + if update_cols: + update_clause = ", ".join(f"{c} = excluded.{c}" for c in update_cols) + conflict_clause = f"ON CONFLICT(torrent_id) DO UPDATE SET {update_clause}" + else: + conflict_clause = "ON CONFLICT(torrent_id) DO NOTHING" + + sql = ( + f"INSERT INTO compliance ({col_list}) VALUES ({placeholders}) " + f"{conflict_clause}" + ) + + with self._lock: + conn = self._connect() + try: + cursor = conn.execute(sql, list(kwargs.values())) + conn.commit() + if cursor.lastrowid: + return cursor.lastrowid + row = conn.execute( + "SELECT id FROM compliance WHERE torrent_id = ?", + (kwargs["torrent_id"],), + ).fetchone() + return int(row["id"]) if row else 0 + finally: + conn.close() + + def list_compliance( + self, + severity: str | None = None, + ack_status: str | None = None, + diff_kind: str | None = None, + page: int = 1, + per_page: int = 50, + ) -> list[dict]: + conditions: list[str] = [] + params: list[Any] = [] + if severity: + conditions.append("severity_max = ?") + params.append(severity) + if ack_status: + conditions.append("ack_status = ?") + params.append(ack_status) + if diff_kind: + conditions.append("diff_kind = ?") + params.append(diff_kind) + + where = f"WHERE {' AND '.join(conditions)}" if conditions else "" + offset = (page - 1) * per_page + params.extend([per_page, offset]) + + conn = self._connect() + try: + rows = conn.execute( + f"""SELECT * FROM compliance {where} + ORDER BY + CASE severity_max + WHEN 'ERROR' THEN 0 + WHEN 'WARNING' THEN 1 + WHEN 'INFO' THEN 2 + ELSE 3 + END, + checked_at DESC + LIMIT ? OFFSET ?""", + params, + ).fetchall() + return [self._compliance_row_to_dict(r) for r in rows] + finally: + conn.close() + + def get_compliance(self, row_id: int) -> dict | None: + conn = self._connect() + try: + row = conn.execute("SELECT * FROM compliance WHERE id = ?", (row_id,)).fetchone() + return self._compliance_row_to_dict(row) + finally: + conn.close() + + def get_compliance_by_torrent(self, torrent_id: int) -> dict | None: + conn = self._connect() + try: + row = conn.execute( + "SELECT * FROM compliance WHERE torrent_id = ?", (torrent_id,) + ).fetchone() + return self._compliance_row_to_dict(row) + finally: + conn.close() + + def attach_compliance_linked_item(self, torrent_id: int, linked_item_id: int) -> bool: + """Set linked_item_id without touching any other column. + + Used by the post-upload hook to link back to the source item in + `items`, without bumping `checked_at` or any scan metadata. + """ + with self._lock: + conn = self._connect() + try: + cursor = conn.execute( + "UPDATE compliance SET linked_item_id = ? WHERE torrent_id = ?", + (int(linked_item_id), int(torrent_id)), + ) + conn.commit() + return cursor.rowcount > 0 + finally: + conn.close() + + def delete_compliance(self, row_id: int) -> bool: + with self._lock: + conn = self._connect() + try: + cursor = conn.execute("DELETE FROM compliance WHERE id = ?", (row_id,)) + conn.commit() + return cursor.rowcount > 0 + finally: + conn.close() + + def set_compliance_ack(self, row_id: int, ack_status: str) -> bool: + if ack_status not in ("unchecked", "acknowledged", "ignored", "fixed"): + raise ValueError(f"Invalid ack_status: {ack_status}") + with self._lock: + conn = self._connect() + try: + cursor = conn.execute( + "UPDATE compliance SET ack_status = ? WHERE id = ?", + (ack_status, row_id), + ) + conn.commit() + return cursor.rowcount > 0 + finally: + conn.close() + + def count_compliance_by_severity(self, only_unchecked: bool = False) -> dict[str, int]: + where = "WHERE ack_status = 'unchecked'" if only_unchecked else "" + conn = self._connect() + try: + rows = conn.execute( + f"SELECT severity_max, COUNT(*) as cnt FROM compliance {where} GROUP BY severity_max" + ).fetchall() + return {r["severity_max"] or "NONE": r["cnt"] for r in rows} + finally: + conn.close() + + def count_compliance_total(self) -> int: + conn = self._connect() + try: + row = conn.execute("SELECT COUNT(*) as cnt FROM compliance").fetchone() + return int(row["cnt"]) if row else 0 + finally: + conn.close() + + # ── Migration from WatcherState JSON ────────────────────────────── + + def migrate_from_json(self, json_path: str) -> int: + """Import entries from a watcher_state.json file. + + Returns the number of entries migrated. + """ + if not os.path.exists(json_path): + return 0 + + try: + with open(json_path, "r", encoding="utf-8") as f: + data = json.load(f) + except (json.JSONDecodeError, IOError): + return 0 + + count = 0 + now = datetime.now().isoformat() + + for basename, entry in data.get("uploaded", {}).items(): + if self.is_known(basename): + continue + self.add_item( + source_basename=basename, + source_path=entry.get("folder_path", "") or basename, + folder_path=entry.get("folder_path"), + source_type=entry.get("type", ""), + status="uploaded", + content_category=entry.get("content_category", ""), + torrent_name=entry.get("torrent_name", ""), + release_name=entry.get("torrent_name", ""), + source_tag=entry.get("source", ""), + tracker_name="", + trackers_list=entry.get("trackers", []), + validation_report=entry.get("validation_report", []), + discovered_at=entry.get("timestamp", now), + uploaded_at=entry.get("timestamp", now), + ) + count += 1 + + for basename, entry in data.get("skipped", {}).items(): + if self.is_known(basename): + continue + self.add_item( + source_basename=basename, + source_path=entry.get("folder_path", "") or basename, + folder_path=entry.get("folder_path"), + source_type=entry.get("type", ""), + status="skipped", + content_category=entry.get("content_category", ""), + torrent_name=entry.get("torrent_name", ""), + release_name=entry.get("torrent_name", ""), + source_tag=entry.get("source", ""), + skip_reason=entry.get("reason", ""), + validation_report=entry.get("validation_report", []), + discovered_at=entry.get("timestamp", now), + ) + count += 1 + + return count diff --git a/unit3dup/torrent.py b/unit3dup/torrent.py index 29d6891..34ef767 100644 --- a/unit3dup/torrent.py +++ b/unit3dup/torrent.py @@ -211,7 +211,10 @@ def page_view(self, tracker_data: dict, tracker: pvtTracker, info=False, inkey=T custom_console.bot_question_log( f"\n Prossima Pagina '{page}' - Premi un tasto per continuare, Q(quit) - " ) - if input().lower() == "q": + try: + if input().lower() == "q": + break + except EOFError: break else: # otherwise wait for 2 seconds ( 30 request/ 60sec max) dirty diff --git a/unit3dup/upload.py b/unit3dup/upload.py index dde340a..6795e51 100644 --- a/unit3dup/upload.py +++ b/unit3dup/upload.py @@ -23,7 +23,7 @@ def __init__(self, content: Media, tracker_name: str, cli: argparse): self.tracker_data = TRACKData.load_from_module(tracker_name=tracker_name) self.tracker = Unit3d(tracker_name=tracker_name) - def normalize_release_name(self, release_name: str) -> str: + def normalize_release_name(self, release_name: str, year: str | int | None = None) -> str: mediainfo_text: str | None = None is_silent: bool = False @@ -33,7 +33,12 @@ def normalize_release_name(self, release_name: str) -> str: if self.content.mediafile and hasattr(self.content.mediafile, 'is_silent'): is_silent = self.content.mediafile.is_silent - return _normalize_release_name(release_name, mediainfo_text, is_silent) + # Fallback année : tmdb_year sur le content si rien n'est passé explicitement + if year is None: + year = getattr(self.content, 'tmdb_year', None) + year_str = str(year) if year else None + + return _normalize_release_name(release_name, mediainfo_text, is_silent, year=year_str) def _check_personal_release_by_tag(self, release_name: str) -> int: """ @@ -70,6 +75,33 @@ def _check_personal_release_by_tag(self, release_name: str) -> int: return personal_release + @staticmethod + def is_excluded_tag(release_name: str) -> bool: + """ + Vérifie si le tag d'équipe de la release est dans la liste d'exclusion. + + Même logique d'extraction que _check_personal_release_by_tag : + - Extrait le tag après le dernier '-' + - Compare (insensible à la casse) avec EXCLUDED_TAGS + - Retourne True si le tag est exclu + """ + excluded_tags: list[str] = [ + t.upper() + for t in getattr(config_settings.uploader_tag, 'EXCLUDED_TAGS', []) + ] + + if not excluded_tags: + return False + + parts = release_name.rsplit('-', 1) + if len(parts) == 2: + tag = parts[1].strip().upper() + # Retirer l'extension éventuelle (.mkv, .mp4, etc.) + tag = re.sub(r'\.\w{2,4}$', '', tag).upper() + return tag in excluded_tags + + return False + def message(self,tracker_response: requests.Response, torrent_archive: str) -> (requests, dict): name_error = '' @@ -138,7 +170,10 @@ def data(self,show_id: int , imdb_id: int, show_keywords_list: str, video_info: self.tracker.data["mediainfo"] = video_info.mediainfo self.tracker.data["description"] = video_info.description self.tracker.data["sd"] = video_info.is_hd - self.tracker.data["type_id"] = self.tracker_data.filter_type(self.content.file_name) + effective_resolution = self.content.screen_size or self.content.resolution + self.tracker.data["type_id"] = self.tracker_data.filter_type( + self.content.title, resolution=effective_resolution + ) self.tracker.data["season_number"] = self.content.guess_season self.tracker.data["episode_number"] = (self.content.guess_episode if not self.content.torrent_pack else 0) self.tracker.data["personal_release"] = self._check_personal_release_by_tag(release_name) @@ -168,7 +203,7 @@ def data_docu(self, document_info: PdfImages) -> Unit3d | None: self.tracker.data["category_id"] = self.tracker_data.category.get(self.content.category) self.tracker.data["anonymous"] = int(config_settings.user_preferences.ANON) self.tracker.data["description"] = document_info.description - self.tracker.data["type_id"] = self.tracker_data.filter_type(self.content.file_name) + self.tracker.data["type_id"] = self.tracker_data.filter_type(self.content.title) self.tracker.data["resolution_id"] = "" self.tracker.data["personal_release"] = self._check_personal_release_by_tag(release_name) return self.tracker diff --git a/unit3dup/validators/__init__.py b/unit3dup/validators/__init__.py new file mode 100644 index 0000000..16fea20 --- /dev/null +++ b/unit3dup/validators/__init__.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- +""" +validators — Validation des releases selon les règles du tracker G3MINI + +Modules : + - naming_validator : Règles de nommage (nommage.md) + - encoding_validator : Règles d'encodage (encodage.md) + - upload_validator : Règles d'upload (upload.md) +""" + +import logging +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Optional + +from rich.table import Table + +logger = logging.getLogger(__name__) + + +@dataclass +class ValidationResult: + """Résultat d'une vérification de règle.""" + rule: str # ex: "encoding.crf_range" + severity: str # "ERROR", "WARNING", "INFO" + message: str # Explication lisible + source_doc: str # "nommage", "encodage", "upload" + + +class BaseValidator(ABC): + """Classe de base pour tous les validators.""" + + @abstractmethod + def validate( + self, + media, # unit3dup.media.Media + mediafile, # common.mediainfo.MediaFile | None + release_name: str, + mediainfo_text: Optional[str] = None, + ) -> list[ValidationResult]: + ... + + +class ValidationRunner: + """Agrégateur : exécute une liste de validators et produit un rapport.""" + + _SEVERITY_STYLE = { + "ERROR": "[bold red]ERROR[/bold red]", + "WARNING": "[bold yellow]WARN[/bold yellow]", + "INFO": "[cyan]INFO[/cyan]", + } + + def __init__(self, validators: list[BaseValidator]): + self._validators = validators + self._results: list[ValidationResult] = [] + + def validate( + self, + media, + mediafile, + release_name: str, + mediainfo_text: Optional[str] = None, + ) -> list[ValidationResult]: + self._results = [] + for v in self._validators: + try: + self._results.extend( + v.validate(media, mediafile, release_name, mediainfo_text) + ) + except Exception as e: + logger.warning("Validator %s failed: %s", v.__class__.__name__, e) + return self._results + + def has_errors(self) -> bool: + return any(r.severity == "ERROR" for r in self._results) + + def has_warnings(self) -> bool: + return any(r.severity == "WARNING" for r in self._results) + + def print_report(self, console) -> None: + """Affiche un tableau Rich des résultats.""" + if not self._results: + return + + table = Table(title="Validation Report", show_lines=False) + table.add_column("Sev", style="bold", width=7) + table.add_column("Rule", min_width=25) + table.add_column("Message") + table.add_column("Doc", width=10) + + for r in self._results: + sev_display = self._SEVERITY_STYLE.get(r.severity, r.severity) + table.add_row(sev_display, r.rule, r.message, r.source_doc) + + console.print(table) + + errors = sum(1 for r in self._results if r.severity == "ERROR") + warnings = sum(1 for r in self._results if r.severity == "WARNING") + infos = sum(1 for r in self._results if r.severity == "INFO") + console.print( + f" [bold red]{errors} error(s)[/bold red] " + f"[bold yellow]{warnings} warning(s)[/bold yellow] " + f"[cyan]{infos} info(s)[/cyan]" + ) + + def to_dicts(self) -> list[dict]: + """Serialize results to plain dicts for JSON storage.""" + return [ + {"severity": r.severity, "rule": r.rule, "message": r.message, "doc": r.source_doc} + for r in self._results + ] + + +def create_default_validators() -> list[BaseValidator]: + """Factory : retourne la liste de tous les validators actifs.""" + from unit3dup.validators.naming_validator import NamingValidator + from unit3dup.validators.encoding_validator import EncodingValidator + from unit3dup.validators.upload_validator import UploadValidator + + return [ + NamingValidator(), + EncodingValidator(), + UploadValidator(), + ] diff --git a/unit3dup/validators/encoding_validator.py b/unit3dup/validators/encoding_validator.py new file mode 100644 index 0000000..1ea8599 --- /dev/null +++ b/unit3dup/validators/encoding_validator.py @@ -0,0 +1,479 @@ +# -*- coding: utf-8 -*- +""" +encoding_validator.py — Validation des règles d'encodage (encodage.md) +""" + +import re +from typing import Optional + +from unit3dup.validators import BaseValidator, ValidationResult + +_X264_PRESET_ORDER = [ + "ultrafast", "superfast", "veryfast", "faster", "fast", + "medium", "slow", "slower", "veryslow", "placebo", +] +_X265_PRESET_ORDER = [ + "ultrafast", "superfast", "veryfast", "faster", "fast", + "medium", "slow", "slower", "veryslow", "placebo", +] + +_CRF_RANGES = { + "x264": {"720p": (16, 20), "1080p": (17, 21)}, + "x265": {"720p": (18, 22), "1080p": (20, 24), "2160p": (22, 26)}, + "AV1": {"720p": (24, 28), "1080p": (26, 30), "2160p": (28, 32)}, +} + + +def _parse_encoding_settings(text: str) -> dict[str, str]: + """Parse x264/x265 encoding_settings strings (key=value / key=value).""" + if not text: + return {} + result = {} + parts = text.split(" / ") + for part in parts: + if "=" in part: + key, value = part.split("=", 1) + result[key.strip()] = value.strip() + return result + + +def _detect_source_type(release_name: str) -> str: + name_upper = release_name.upper() + if any(p in name_upper for p in ("BDRIP", "WEBRIP", "TVRIP", "HDLIGHT", "4KLIGHT", "HDRIP")): + return "encode" + # Encoder-library tags (lowercase x264/x265, AV1) are scene convention + # for a re-encode. REMUX releases use HEVC/H.265/H.264 — never x264/x265. + # Match on word boundaries so "X265" in "X265-GROUP" or ".x265." matches + # but e.g. "HEVC" or "H265" does not trigger. + if re.search(r'(?:^|[\W_])(?:[xX]26[45]|AV1)(?:$|[\W_])', release_name): + return "encode" + if "REMUX" in name_upper: + return "remux" + if "WEB" in name_upper and "WEBRIP" not in name_upper: + return "web" + if "BLURAY" in name_upper: + return "bluray" + if "HDTV" in name_upper: + return "hdtv" + return "unknown" + + +def _detect_codec(release_name: str) -> str: + name_upper = release_name.upper() + if "X264" in name_upper: + return "x264" + if "X265" in name_upper: + return "x265" + if "AV1" in name_upper: + return "AV1" + return "" + + +def _detect_resolution(release_name: str) -> str: + for r in ("2160p", "1080p", "1080i", "720p"): + if r in release_name: + return r + return "" + + +class EncodingValidator(BaseValidator): + """Validator pour les règles d'encodage G3MINI.""" + + def validate( + self, + media, + mediafile, + release_name: str, + mediainfo_text: Optional[str] = None, + ) -> list[ValidationResult]: + results: list[ValidationResult] = [] + + # Upscale tag check — runs for ALL source types + results.extend(self._check_upscale_tag(release_name)) + + try: + source_type = _detect_source_type(release_name) + codec = _detect_codec(release_name) + resolution = _detect_resolution(release_name) + + if source_type == "encode": + results.extend(self._check_crf_range(mediafile, codec, resolution)) + results.extend(self._check_preset_minimum(mediafile, codec)) + results.extend(self._check_x264_2160p(codec, resolution)) + results.extend(self._check_pgs_vobsub(mediafile)) + results.extend(self._check_hdr_metadata(release_name, mediafile)) + results.extend(self._check_abr_cbr(mediafile)) + results.extend(self._check_upscale(mediafile, resolution)) + results.extend(self._check_crop(mediafile)) + + if source_type in ("encode", "remux"): + results.extend(self._check_container(mediafile)) + + results.extend(self._check_forbidden_audio(mediafile)) + + # Source-vs-writing-library consistency (encodage.md §1: + # "Interdiction d'encoder depuis une source deja reencodee") + results.extend(self._check_source_library_mismatch(source_type, mediafile)) + + except Exception: + pass + + return results + + # ── Check 1: CRF range ───────────────────────────────────────────────── + + def _check_crf_range(self, mediafile, codec: str, resolution: str) -> list[ValidationResult]: + if not mediafile or not getattr(mediafile, 'encoding_settings', None): + return [] + if not codec or not resolution: + return [] + + settings = _parse_encoding_settings(mediafile.encoding_settings) + if "crf" not in settings: + return [] + + # 1080i uses same CRF ranges as 1080p + lookup_res = "1080p" if resolution == "1080i" else resolution + crf_range = _CRF_RANGES.get(codec, {}).get(lookup_res) + if not crf_range: + return [] + + try: + crf_value = float(settings["crf"]) + lo, hi = crf_range + if crf_value < lo or crf_value > hi: + return [ValidationResult( + rule="encoding.crf_range", + severity="WARNING", + message=f"CRF {crf_value} hors plage recommandee ({lo}-{hi}) pour {codec} en {resolution}", + source_doc="encodage", + )] + except (ValueError, KeyError): + pass + return [] + + # ── Check 2: Preset minimum ──────────────────────────────────────────── + + def _check_preset_minimum(self, mediafile, codec: str) -> list[ValidationResult]: + if not mediafile or not getattr(mediafile, 'encoding_settings', None): + return [] + + settings = _parse_encoding_settings(mediafile.encoding_settings) + preset = settings.get("preset", "").lower() + if not preset: + return [] + + if codec == "x264": + order = _X264_PRESET_ORDER + min_idx = 6 # "slow" + min_name = "slow" + elif codec == "x265": + order = _X265_PRESET_ORDER + min_idx = 5 # "medium" + min_name = "medium" + elif codec == "AV1": + # SVT-AV1 uses numeric presets: lower = slower = better + # Minimum preset 4 (Slower) — higher number = faster = worse + try: + preset_num = int(preset) + if preset_num > 4: + return [ValidationResult( + rule="encoding.preset_minimum", + severity="WARNING", + message=f"AV1 preset {preset_num} trop rapide (minimum: 4/Slower)", + source_doc="encodage", + )] + except ValueError: + pass + return [] + else: + return [] + + try: + idx = order.index(preset) + if idx < min_idx: + return [ValidationResult( + rule="encoding.preset_minimum", + severity="WARNING", + message=f"Preset '{preset}' trop rapide pour {codec} (minimum: {min_name})", + source_doc="encodage", + )] + except ValueError: + pass + return [] + + # ── Check 3: x264 at 2160p ───────────────────────────────────────────── + + @staticmethod + def _check_x264_2160p(codec: str, resolution: str) -> list[ValidationResult]: + if codec == "x264" and resolution == "2160p": + return [ValidationResult( + rule="encoding.x264_2160p", + severity="WARNING", + message="x264 deconseille en 2160p", + source_doc="encodage", + )] + return [] + + # ── Check 4: Container .mkv ──────────────────────────────────────────── + + @staticmethod + def _check_container(mediafile) -> list[ValidationResult]: + if not mediafile or not getattr(mediafile, 'container_format', None): + return [] + if mediafile.container_format != ".mkv": + return [ValidationResult( + rule="encoding.container_mkv", + severity="ERROR", + message=f"Container doit etre .mkv, detecte: {mediafile.container_format}", + source_doc="encodage", + )] + return [] + + # ── Check 5: Forbidden audio codecs ──────────────────────────────────── + + @staticmethod + def _check_forbidden_audio(mediafile) -> list[ValidationResult]: + if not mediafile or not getattr(mediafile, 'audio_formats', None): + return [] + + results = [] + for track in mediafile.audio_formats: + fmt = track.get("format", "").upper() + channels = track.get("channels", 0) + try: + channels = int(channels) + except (ValueError, TypeError): + channels = 0 + + if fmt in ("MPEG AUDIO", "MP3"): + results.append(ValidationResult( + rule="encoding.forbidden_audio", + severity="ERROR", + message="MP3 interdit", + source_doc="encodage", + )) + break + + if fmt == "FLAC" and channels > 2: + results.append(ValidationResult( + rule="encoding.forbidden_audio", + severity="ERROR", + message=f"FLAC interdit pour surround ({channels} canaux)", + source_doc="encodage", + )) + break + + return results + + # ── Check 6: PGS/VOBSUB subtitles ───────────────────────────────────── + + @staticmethod + def _check_pgs_vobsub(mediafile) -> list[ValidationResult]: + if not mediafile or not getattr(mediafile, 'subtitle_formats', None): + return [] + + has_pgs_vobsub = False + has_text = False + for track in mediafile.subtitle_formats: + fmt = track.get("format", "").upper() + if fmt in ("PGS", "VOBSUB"): + has_pgs_vobsub = True + if fmt in ("UTF-8", "ASS", "SSA", "SUBRIP"): + has_text = True + + if has_pgs_vobsub and not has_text: + return [ValidationResult( + rule="encoding.pgs_vobsub_encode", + severity="WARNING", + message="PGS/VobSub sans sous-titres texte (SRT/ASS) — interdit pour les encodes", + source_doc="encodage", + )] + return [] + + # ── Check 7: HDR metadata ────────────────────────────────────────────── + + @staticmethod + def _check_hdr_metadata(release_name: str, mediafile) -> list[ValidationResult]: + hdr_tokens = ("HDR", "HDR10", "HDR10P", "DV", "HLG", "HYBRID") + if not any(t in release_name.upper() for t in hdr_tokens): + return [] + if not mediafile: + return [] + + issues = [] + + bit_depth = getattr(mediafile, 'video_bit_depth', None) + if bit_depth and bit_depth != "Unknown": + try: + if int(str(bit_depth).split()[0]) < 10: + issues.append("bit depth < 10") + except (ValueError, AttributeError): + pass + + color = getattr(mediafile, 'color_primaries', None) + if color and "BT.2020" not in str(color): + issues.append("color primaries != BT.2020") + + transfer = getattr(mediafile, 'transfer_characteristics', None) + if transfer: + t_upper = str(transfer).upper() + if "PQ" not in t_upper and "2084" not in t_upper and "HLG" not in t_upper: + issues.append("transfer characteristics manquantes (PQ/2084/HLG)") + + if issues: + return [ValidationResult( + rule="encoding.hdr_metadata", + severity="WARNING", + message=f"HDR metadata: {'; '.join(issues)}", + source_doc="encodage", + )] + return [] + + # ── Check 8: ABR 1-pass / CBR ────────────────────────────────────────── + + @staticmethod + def _check_abr_cbr(mediafile) -> list[ValidationResult]: + if not mediafile or not getattr(mediafile, 'encoding_settings', None): + return [] + + settings = _parse_encoding_settings(mediafile.encoding_settings) + + # CRF mode is fine + if "crf" in settings: + return [] + + # 2-pass ABR is fine + if "pass" in settings: + return [] + + # If we have bitrate/rc settings but no CRF and no 2-pass → likely ABR 1-pass or CBR + rc = settings.get("rc", "").lower() + if rc == "cbr": + return [ValidationResult( + rule="encoding.abr_cbr", + severity="ERROR", + message="Encodage CBR interdit — utiliser CRF", + source_doc="encodage", + )] + + # Only flag if we see explicit bitrate evidence without CRF + if "bitrate" in settings or "vbv-maxrate" in settings: + return [ValidationResult( + rule="encoding.abr_cbr", + severity="ERROR", + message="ABR 1-pass detecte — utiliser CRF ou ABR 2-pass", + source_doc="encodage", + )] + + return [] + + # ── Check 9: Upscale tag in release name ──────────────────────────────── + + @staticmethod + def _check_upscale_tag(release_name: str) -> list[ValidationResult]: + if re.search(r'UpScal', release_name, re.IGNORECASE): + return [ValidationResult( + rule="encoding.upscale_forbidden", + severity="ERROR", + message="Contenu upscale interdit — aucun upscale autorise", + source_doc="encodage", + )] + return [] + + # ── Check 12: Crop (no black borders) ────────────────────────────────── + # encodage.md §1: "Les videos doivent etre crop (aucune bordure noire)". + # Reliable detection from MediaInfo alone is limited. Heuristic: + # encode stored as full 16:9 (1920×1080 / 3840×2160 / 1280×720) for a + # movie with typical cinema aspect (>= 1.85:1) strongly suggests bars. + # We only flag the most obvious cases as INFO (manual review required). + @staticmethod + def _check_crop(mediafile) -> list[ValidationResult]: + if not mediafile: + return [] + try: + w = int(str(getattr(mediafile, 'video_width', '') or '').split()[0]) + h = int(str(getattr(mediafile, 'video_height', '') or '').split()[0]) + except (ValueError, AttributeError, IndexError): + return [] + if not w or not h: + return [] + # Only flag strict 16:9 standard resolutions — bars can't be detected + # reliably without content analysis, so we just alert on suspicious + # exact 16:9 when aspect ratio metadata hints at wider content. + ratio = w / h + if abs(ratio - (16 / 9)) > 0.02: + return [] # Not 16:9, already likely cropped + dar = str(getattr(mediafile, 'video_aspect_ratio', '') or '').lower() + # If DAR explicitly reports >= 1.85 cinematic ratios, bars are likely. + if any(hint in dar for hint in ("2.35", "2.39", "2.40", "21:9", "1.85", "1.90")): + return [ValidationResult( + rule="encoding.crop_suspected", + severity="INFO", + message=f"Video stored 16:9 ({w}x{h}) but DAR={dar} — possible black borders, verify crop", + source_doc="encodage", + )] + return [] + + # ── Check 11: Source tag vs writing library mismatch ─────────────────── + # encodage.md §1: untouched sources (REMUX, FULL, BluRay, WEB-DL, HDTV) + # must NOT come from an already re-encoded file. If the file carries an + # x264/x265 writing library but the release is tagged as untouched, the + # source is effectively a re-encode — which is forbidden. + @staticmethod + def _check_source_library_mismatch(source_type: str, mediafile) -> list[ValidationResult]: + if source_type not in ("remux", "web", "bluray", "hdtv"): + return [] + if not mediafile: + return [] + lib = getattr(mediafile, 'writing_library', None) + if not lib: + return [] + lib_upper = str(lib).upper() + if lib_upper.startswith("X264") or lib_upper.startswith("X265"): + return [ValidationResult( + rule="encoding.source_reencoded", + severity="ERROR", + message=f"Source taguee untouched ({source_type}) mais writing library = {lib} — encodage depuis une source deja reencodee est interdit", + source_doc="encodage", + )] + return [] + + # ── Check 10: Upscale detection (height-based) ────────────────────────── + + @staticmethod + def _check_upscale(mediafile, resolution: str) -> list[ValidationResult]: + if not mediafile or not resolution: + return [] + + # Map tagged resolution to expected max height + res_map = {"2160p": 2160, "1080p": 1080, "1080i": 1080, "720p": 720} + expected_height = res_map.get(resolution) + if not expected_height: + return [] + + actual_height = getattr(mediafile, 'video_height', None) + if actual_height is None: + return [] + + try: + actual_height = int(actual_height) + except (ValueError, TypeError): + return [] + + # If actual height significantly exceeds what the source type would normally provide, + # and actual height matches the tagged resolution, we can't detect upscale from height alone. + # But if actual height is LESS than tagged → possible wrong tag (not upscale). + # If actual height matches but source is suspicious (e.g., WEB at 2160p for x264), that's + # already covered by x264_2160p check. + # Real upscale detection: if stored_height < height (rare in MediaInfo) + # For now: warn if actual height doesn't match tagged resolution + if actual_height < expected_height * 0.9: + return [ValidationResult( + rule="encoding.upscale", + severity="WARNING", + message=f"Resolution reelle ({actual_height}p) inferieure a la resolution taguee ({resolution}) — possible upscale", + source_doc="encodage", + )] + + return [] diff --git a/unit3dup/validators/naming_validator.py b/unit3dup/validators/naming_validator.py new file mode 100644 index 0000000..b47a4cd --- /dev/null +++ b/unit3dup/validators/naming_validator.py @@ -0,0 +1,299 @@ +# -*- coding: utf-8 -*- +""" +naming_validator.py — Validation des règles de nommage (nommage.md) +""" + +from typing import Optional +import re + +from unit3dup.validators import BaseValidator, ValidationResult + + +class NamingValidator(BaseValidator): + """Validator pour les règles de nommage G3MINI.""" + + def validate( + self, + media, + mediafile, + release_name: str, + mediainfo_text: Optional[str] = None, + ) -> list[ValidationResult]: + """ + Valide le nommage selon les règles : + - Check 1: 3D content detection + - Check 2: Audio Description detection + - Check 3: CUSTOM tag suggestion + - Check 4: NoGRP/NoTAG for unknown teams + """ + results: list[ValidationResult] = [] + + try: + # Check 1: 3D content detection + results.extend(self._check_3d_tag(mediafile, release_name)) + + # Check 2: Audio Description detection + results.extend(self._check_ad_tag(mediafile, release_name)) + + # Check 3: CUSTOM tag suggestion + results.extend(self._check_custom_tag(mediafile, release_name)) + + # Check 4: NoGRP/NoTAG for unknown teams + results.extend(self._check_nogrp_tag(release_name)) + + # Check 5: Reject releases with invalid team (NoTag, NoGRP, or year) + results.extend(self._check_invalid_team(release_name)) + + # Check 6: HARDSUB content requires SUBFRENCH tag (upload.md §3) + results.extend(self._check_hardsub_tag(release_name)) + + # Check 7: MULTi invariant — VO + VF + ST FR (encodage.md §1) + results.extend(self._check_multi_invariant(mediafile, release_name)) + + except Exception: + # Wrap all checks in try/except to be safe + pass + + return results + + def _check_3d_tag( + self, mediafile, release_name: str + ) -> list[ValidationResult]: + """Check 1: 3D content detection.""" + results: list[ValidationResult] = [] + + if mediafile is None: + return results + + # Check if 3D content (multiview_count > 1) + try: + is_3d = ( + hasattr(mediafile, 'multiview_count') and + mediafile.multiview_count is not None and + mediafile.multiview_count > 1 + ) + except Exception: + is_3d = False + + if not is_3d: + return results + + # Check if any 3D tag exists in release_name (dots as separators) + tags_3d = ("3D", "SBS", "HSBS", "TAB", "HTAB", "MVC") + has_3d_tag = any( + re.search(rf'(?:^|[\s.])' + re.escape(tag) + r'(?:[\s.]|$)', release_name, re.IGNORECASE) + for tag in tags_3d + ) + + if not has_3d_tag: + results.append( + ValidationResult( + rule="naming.3d_tag", + severity="WARNING", + message="3D content detected but no 3D tag found in release name", + source_doc="nommage", + ) + ) + + return results + + def _check_ad_tag(self, mediafile, release_name: str) -> list[ValidationResult]: + """Check 2: Audio Description detection.""" + results: list[ValidationResult] = [] + + if mediafile is None: + return results + + # Check for visually impaired audio + has_ad_audio = False + try: + if hasattr(mediafile, 'audio_formats') and mediafile.audio_formats: + for audio in mediafile.audio_formats: + if isinstance(audio, dict): + service_kind = audio.get('service_kind', '') + if service_kind and 'visually impaired' in service_kind.lower(): + has_ad_audio = True + break + except Exception: + pass + + if not has_ad_audio: + return results + + # Check if AD tag exists in release_name (word boundary via dots/spaces) + has_ad_tag = bool(re.search(r'(?:^|[\s.])AD(?:[\s.]|$)', release_name, re.IGNORECASE)) + + if not has_ad_tag: + results.append( + ValidationResult( + rule="naming.ad_tag", + severity="WARNING", + message="Audio Description track detected but no AD tag in release name", + source_doc="nommage", + ) + ) + + return results + + def _check_custom_tag( + self, mediafile, release_name: str + ) -> list[ValidationResult]: + """Check 3: CUSTOM tag suggestion.""" + results: list[ValidationResult] = [] + + if mediafile is None: + return results + + # Check for audio delay + has_delay = False + try: + if hasattr(mediafile, 'audio_formats') and mediafile.audio_formats: + for audio in mediafile.audio_formats: + if isinstance(audio, dict): + delay = audio.get('delay') + if delay is not None and delay != 0: + has_delay = True + break + except Exception: + pass + + if not has_delay: + return results + + # Check if CUSTOM tag exists in release_name + has_custom_tag = "CUSTOM" in release_name.upper() + + if not has_custom_tag: + results.append( + ValidationResult( + rule="naming.custom_tag", + severity="INFO", + message="Audio delay detected, consider adding CUSTOM tag", + source_doc="nommage", + ) + ) + + return results + + # Suffixes audio hyphenés qui ne sont PAS des tags de team + _AUDIO_HYPHEN_SUFFIXES = {"HDMA", "HDHRA", "HD", "AC3"} + + # Placeholders that indicate no real release group (acceptable per upload.md §1). + _PLACEHOLDER_TEAMS = {"NOTAG", "NOGRP"} + + def _check_invalid_team(self, release_name: str) -> list[ValidationResult]: + """Check 5: Reject releases whose team is a year. + + NoTag/NoGRP placeholders are ALLOWED by upload.md §1 for unknown releases, + so they are not flagged here (they are merely the fallback, not an error). + """ + name_no_ext = re.sub(r'\.(mkv|mp4|avi|ts|m2ts|iso)$', '', release_name, flags=re.IGNORECASE) + m = re.search(r'-([A-Za-z0-9]{2,})$', name_no_ext) + if not m: + return [] + + team = m.group(1) + if team.upper() in self._AUDIO_HYPHEN_SUFFIXES: + return [] + + if team.upper() in self._PLACEHOLDER_TEAMS: + # Accepted placeholder — compliant with docs. + return [] + + if re.fullmatch(r'(?:19|20)\d{2}', team): + return [ValidationResult( + rule="naming.invalid_team", + severity="ERROR", + message=f"Release team appears to be a year ({team}) — missing release group", + source_doc="upload", + )] + + return [] + + # ── Check 6: HARDSUB → SUBFRENCH ────────────────────────────────────── + # upload.md §3: "Les sous-titres incrustes HARDSUBS sont interdits. + # Sauf si presents dans la source officielle. Dans ce cas, utiliser le tag SUBFRENCH." + def _check_hardsub_tag(self, release_name: str) -> list[ValidationResult]: + has_hardsub = bool(re.search( + r'(?:^|[\s.])(?:HARDSUBS?|HC)(?:[\s.]|$)', + release_name, re.IGNORECASE, + )) + if not has_hardsub: + return [] + has_subfrench = bool(re.search( + r'(?:^|[\s.])SUBFRENCH(?:[\s.]|$)', + release_name, re.IGNORECASE, + )) + if has_subfrench: + return [] + return [ValidationResult( + rule="naming.hardsub_subfrench", + severity="ERROR", + message="HARDSUB detected without SUBFRENCH tag — hardsubs are forbidden unless sourced officially (then use SUBFRENCH)", + source_doc="upload", + )] + + # ── Check 7: MULTi invariant (VO + VF + ST FR) ──────────────────────── + # encodage.md §1: "Une release MULTi contient au minimum la VO, la VF + # et les sous-titres FR complets" + def _check_multi_invariant(self, mediafile, release_name: str) -> list[ValidationResult]: + if not re.search(r'(?:^|[\s.])MULTi(?:[\s.]|$)', release_name, re.IGNORECASE): + return [] + if mediafile is None: + return [] + + audio_langs = [] + try: + for a in (getattr(mediafile, 'audio_formats', None) or []): + if isinstance(a, dict): + audio_langs.append(str(a.get('language', '')).strip().lower()) + except Exception: + return [] + + sub_langs = [] + try: + for s in (getattr(mediafile, 'subtitle_formats', None) or []): + if isinstance(s, dict): + sub_langs.append(str(s.get('language', '')).strip().lower()) + except Exception: + sub_langs = [] + + # Need at least 2 audio tracks for MULTi + if len(audio_langs) < 2: + return [] # Not enough info; skip silently + + def _is_fr(lang: str) -> bool: + return lang.startswith('fr') or lang.startswith('french') + + has_fr_audio = any(_is_fr(l) for l in audio_langs) + has_non_fr_audio = any(l and not _is_fr(l) and l != 'zxx' for l in audio_langs) + has_fr_sub = any(_is_fr(l) for l in sub_langs) + + missing = [] + if not has_non_fr_audio: missing.append("VO audio") + if not has_fr_audio: missing.append("VF audio") + if not has_fr_sub: missing.append("FR subtitles") + if missing: + return [ValidationResult( + rule="naming.multi_invariant", + severity="WARNING", + message=f"MULTi tag requires VO + VF + FR subtitles; missing: {', '.join(missing)}", + source_doc="encodage", + )] + return [] + + def _check_nogrp_tag(self, release_name: str) -> list[ValidationResult]: + """Check 4: Releases without team should have NoGRP or NoTAG.""" + name_no_ext = re.sub(r'\.(mkv|mp4|avi|ts|m2ts|iso)$', '', release_name, flags=re.IGNORECASE) + m = re.search(r'-([A-Za-z0-9]{2,})$', name_no_ext) + has_team = bool(m) and m.group(1).upper() not in self._AUDIO_HYPHEN_SUFFIXES + has_nogrp = bool(re.search(r'(?:^|[\s.])(?:NoGRP|NoTAG)(?:[\s.]|$)', release_name, re.IGNORECASE)) + + if not has_team and not has_nogrp: + return [ValidationResult( + rule="naming.nogrp_tag", + severity="WARNING", + message="No team tag found — unknown releases must be tagged NoGRP or NoTAG", + source_doc="upload", + )] + return [] diff --git a/unit3dup/validators/upload_validator.py b/unit3dup/validators/upload_validator.py new file mode 100644 index 0000000..e8c8546 --- /dev/null +++ b/unit3dup/validators/upload_validator.py @@ -0,0 +1,258 @@ +# -*- coding: utf-8 -*- +""" +upload_validator.py — Validation des règles d'upload (upload.md) +""" + +import os +import re +from typing import Optional + +from unit3dup.validators import BaseValidator, ValidationResult + + +class UploadValidator(BaseValidator): + """Validator pour les règles d'upload G3MINI.""" + + def validate( + self, + media, + mediafile, + release_name: str, + mediainfo_text: Optional[str] = None, + ) -> list[ValidationResult]: + """ + Valide les règles d'upload : + - Check 1: External subtitle files detection + - Check 2: Archive files detection + - Check 3: Season completeness (for packs) + - Check 4: Multi-format/multi-tag detection + """ + results: list[ValidationResult] = [] + + try: + # Check 1: External subtitle files + results.extend(self._check_external_subtitles(media)) + + # Check 2: Archive files + results.extend(self._check_archive_files(media)) + + # Check 3: Season completeness + results.extend(self._check_season_completeness(media)) + + # Check 4: Multi-format/multi-tag detection + results.extend(self._check_multi_format(media)) + + # Check 5: Pack consistency — same team and language across files + results.extend(self._check_pack_consistency(media)) + + except Exception: + pass + + return results + + def _check_external_subtitles(self, media) -> list[ValidationResult]: + """Check 1: External subtitle files detection.""" + results: list[ValidationResult] = [] + + if media is None or not hasattr(media, 'torrent_path'): + return results + + try: + torrent_path = media.torrent_path + if os.path.isfile(torrent_path): + directory = os.path.dirname(torrent_path) + else: + directory = torrent_path + + files = os.listdir(directory) + subtitle_extensions = ('.srt', '.ass', '.ssa', '.sub') + video_extensions = ('.mkv', '.mp4', '.avi', '.mov', '.flv', '.wmv') + + has_subtitles = any( + f.lower().endswith(subtitle_extensions) for f in files + ) + has_videos = any( + f.lower().endswith(video_extensions) for f in files + ) + + if has_subtitles and has_videos: + results.append( + ValidationResult( + rule="upload.external_subtitles", + severity="ERROR", + message="Fichiers sous-titres externes detectes - ils doivent etre muxes dans le MKV", + source_doc="upload", + ) + ) + + except Exception: + pass + + return results + + def _check_archive_files(self, media) -> list[ValidationResult]: + """Check 2: Archive files detection.""" + results: list[ValidationResult] = [] + + if media is None or not hasattr(media, 'torrent_path'): + return results + + try: + torrent_path = media.torrent_path + if os.path.isfile(torrent_path): + directory = os.path.dirname(torrent_path) + else: + directory = torrent_path + + files = os.listdir(directory) + archive_extensions = ('.rar', '.zip', '.7z', '.tar', '.gz') + + has_archives = any( + f.lower().endswith(archive_extensions) for f in files + ) + + if has_archives: + results.append( + ValidationResult( + rule="upload.archive_files", + severity="ERROR", + message="Archives interdites dans le torrent", + source_doc="upload", + ) + ) + + except Exception: + pass + + return results + + def _check_season_completeness(self, media) -> list[ValidationResult]: + """Check 3: Season completeness (for packs/folders).""" + results: list[ValidationResult] = [] + + if media is None or not hasattr(media, 'torrent_path'): + return results + + try: + torrent_path = media.torrent_path + if not os.path.isdir(torrent_path): + return results + + files = os.listdir(torrent_path) + season_episodes: dict[int, list[int]] = {} + + for file in files: + match = re.search(r'[Ss](\d{1,2})[Ee](\d{1,3})', file) + if match: + season = int(match.group(1)) + episode = int(match.group(2)) + if season not in season_episodes: + season_episodes[season] = [] + season_episodes[season].append(episode) + + for season, episodes in season_episodes.items(): + episodes = sorted(set(episodes)) + if len(episodes) > 1: + expected = list(range(episodes[0], episodes[-1] + 1)) + if episodes != expected: + results.append( + ValidationResult( + rule="upload.season_incomplete", + severity="WARNING", + message=f"Season {season} has gaps in episode numbers", + source_doc="upload", + ) + ) + break + + except Exception: + pass + + return results + + # upload.md §4: + # "Pas de multi-tag (plusieurs teams dans un meme pack)" + # "Pas de mix de langues (FRENCH et MULTi dans un meme pack)" + def _check_pack_consistency(self, media) -> list[ValidationResult]: + results: list[ValidationResult] = [] + if media is None or not hasattr(media, 'torrent_path'): + return results + try: + torrent_path = media.torrent_path + if not os.path.isdir(torrent_path): + return results + + video_ext = ('.mkv', '.mp4', '.avi', '.mov', '.flv', '.wmv', '.ts', '.m2ts') + teams: set[str] = set() + lang_categories: set[str] = set() # "MULTi" or "MONO" (FRENCH/VFF/VFQ/...) + + for file in os.listdir(torrent_path): + if not any(file.lower().endswith(e) for e in video_ext): + continue + stem = re.sub(r'\.[^.]+$', '', file) + + team_m = re.search(r'-([A-Za-z0-9]{2,})$', stem) + if team_m: + t = team_m.group(1).upper() + if t not in ("HDMA", "HDHRA", "HD", "AC3", "NOTAG", "NOGRP"): + teams.add(t) + + if re.search(r'(?:^|[\s.])MULTi(?:[\s.]|$)', stem, re.IGNORECASE): + lang_categories.add("MULTi") + elif re.search(r'(?:^|[\s.])(?:FRENCH|VFF|VFQ|VFB|VF2|VOF|VOQ|VOB|VOSTFR)(?:[\s.]|$)', stem, re.IGNORECASE): + lang_categories.add("MONO") + + if len(teams) > 1: + results.append(ValidationResult( + rule="upload.multi_team_pack", + severity="WARNING", + message=f"Multi-team detecte dans le pack: {', '.join(sorted(teams))}", + source_doc="upload", + )) + if len(lang_categories) > 1: + results.append(ValidationResult( + rule="upload.mixed_languages_pack", + severity="WARNING", + message="Mix de langues detecte (MULTi + tag mono) dans le meme pack", + source_doc="upload", + )) + except Exception: + pass + return results + + def _check_multi_format(self, media) -> list[ValidationResult]: + """Check 4: Multi-format/multi-tag detection (for packs).""" + results: list[ValidationResult] = [] + + if media is None or not hasattr(media, 'torrent_path'): + return results + + try: + torrent_path = media.torrent_path + if not os.path.isdir(torrent_path): + return results + + files = os.listdir(torrent_path) + video_extensions = ('.mkv', '.mp4', '.avi', '.mov', '.flv', '.wmv') + resolutions = set() + + for file in files: + if any(file.lower().endswith(ext) for ext in video_extensions): + resolution_match = re.search(r'(\d{3,4})p', file, re.IGNORECASE) + if resolution_match: + resolutions.add(resolution_match.group(1)) + + if len(resolutions) > 1: + results.append( + ValidationResult( + rule="upload.multi_format", + severity="WARNING", + message="Multi-resolution detecte dans le pack", + source_doc="upload", + ) + ) + + except Exception: + pass + + return results diff --git a/unit3dup/watcher_state.py b/unit3dup/watcher_state.py new file mode 100644 index 0000000..668413a --- /dev/null +++ b/unit3dup/watcher_state.py @@ -0,0 +1,150 @@ +# -*- coding: utf-8 -*- + +import json +import os +import sys +from datetime import datetime + +# Legacy delimiter used in old composite keys (folder_path||basename). +_KEY_SEP = "||" + + +class WatcherState: + """Persistent state for the watcher to track processed entries and avoid reprocessing. + + Stores two categories in a JSON file: + - uploaded: entries successfully uploaded to at least one tracker + - skipped: entries that could not be uploaded (validation, encoding, etc.) + + The JSON file is stored in the config directory so it persists across + Docker restarts (mounted volume). + + Keys are the source basename. The folder_path is stored inside each entry. + """ + + def __init__(self, state_dir: str, filename: str = "watcher_state.json"): + self.state_file = os.path.join(state_dir, filename) + self._state = self._load() + self._migrate_legacy_keys() + + def _load(self) -> dict: + if os.path.exists(self.state_file): + try: + with open(self.state_file, 'r', encoding='utf-8') as f: + data = json.load(f) + if "uploaded" in data and "skipped" in data: + return data + except (json.JSONDecodeError, IOError) as e: + print(f"[WatcherState] WARNING: corrupt state file {self.state_file}: {e}", file=sys.stderr) + return {"uploaded": {}, "skipped": {}} + + def _save(self): + tmp = self.state_file + ".tmp" + with open(tmp, 'w', encoding='utf-8') as f: + json.dump(self._state, f, indent=4, ensure_ascii=False) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, self.state_file) + + def _migrate_legacy_keys(self): + """Migrate old composite keys (folder_path||basename) to basename-only keys.""" + changed = False + for section in ("uploaded", "skipped"): + migrated = {} + for key, value in self._state[section].items(): + if _KEY_SEP in key: + new_key = key.split(_KEY_SEP, 1)[1] + migrated[new_key] = value + changed = True + else: + migrated[key] = value + self._state[section] = migrated + if changed: + self._save() + + @staticmethod + def _make_key(source_path: str, folder_path: str | None = None) -> str: + return os.path.basename(source_path) + + def is_known(self, source_path: str, folder_path: str | None = None) -> str | None: + """Check if a source entry is already tracked. + + Returns: + "uploaded", "skipped", or None if unknown. + """ + key = self._make_key(source_path) + + if key in self._state["uploaded"]: + return "uploaded" + if key in self._state["skipped"]: + return "skipped" + return None + + + def mark_uploaded(self, source_path: str, torrent_name: str, trackers: list[str], + folder_path: str | None = None, category: str | None = None, + content_category: str | None = None, + validation_report: list[dict] | None = None, + source: str | None = None): + """Record a successfully uploaded entry.""" + key = self._make_key(source_path) + # Promote from skipped to uploaded if previously skipped + self._state["skipped"].pop(key, None) + entry = { + "torrent_name": torrent_name, + "source_name": key, + "folder_path": folder_path, + "category": category, + "content_category": content_category, + "type": "folder" if os.path.isdir(source_path) else "file", + "trackers": trackers, + "timestamp": datetime.now().isoformat(), + } + if validation_report: + entry["validation_report"] = validation_report + if source: + entry["source"] = source + self._state["uploaded"][key] = entry + self._save() + + def mark_skipped(self, source_path: str, torrent_name: str, reason: str, + folder_path: str | None = None, category: str | None = None, + content_category: str | None = None, + validation_report: list[dict] | None = None, + source: str | None = None): + """Record a skipped entry with the reason it was not uploaded.""" + key = self._make_key(source_path) + # Never downgrade an uploaded entry to skipped + if key in self._state["uploaded"]: + return + entry = { + "torrent_name": torrent_name, + "source_name": key, + "folder_path": folder_path, + "category": category, + "content_category": content_category, + "type": "folder" if os.path.isdir(source_path) else "file", + "reason": reason, + "timestamp": datetime.now().isoformat(), + } + if validation_report: + entry["validation_report"] = validation_report + if source: + entry["source"] = source + self._state["skipped"][key] = entry + self._save() + + def remove(self, source_name: str, folder_path: str | None = None): + """Remove an entry from the state to allow reprocessing.""" + key = self._make_key(source_name) + self._state["uploaded"].pop(key, None) + self._state["skipped"].pop(key, None) + self._save() + + @property + def uploaded(self) -> dict: + return self._state["uploaded"] + + @property + def skipped(self) -> dict: + return self._state["skipped"] diff --git a/unit3dup/web/api.py b/unit3dup/web/api.py new file mode 100644 index 0000000..6334d2a --- /dev/null +++ b/unit3dup/web/api.py @@ -0,0 +1,452 @@ +# -*- coding: utf-8 -*- +"""JSON API endpoints for the web dashboard.""" + +from __future__ import annotations + +from fastapi import APIRouter, Body, HTTPException + +from unit3dup.state_db import StateDB +from unit3dup.web.models import ( + ApproveRequest, RejectRequest, BulkApproveRequest, BulkRejectRequest, BulkRescanRequest, + RescanTmdbRequest, UpdateCategoryRequest, UpdateSourceTypeRequest, UpdateResolutionRequest, + UpdateSeasonEpisodeRequest, UpdateTracksRequest, + StatsResponse, ItemDetail, ItemListResponse, ItemSummary, QueueStatusResponse, + ComplianceListResponse, ComplianceScanStatus, ComplianceAckRequest, + BulkComplianceDeleteRequest, +) +from unit3dup.web.upload_service import UploadService +from unit3dup.web.compliance_service import ComplianceService +from unit3dup.web.bbcode_renderer import bbcode_to_html +from unit3dup.prez import generate_prez +from unit3dup.compliance.scanner import build_prez_media_file + +router = APIRouter(prefix="/api/v1", tags=["api"]) + +# These will be set by the app factory +_state_db: StateDB | None = None +_upload_service: UploadService | None = None +_compliance_service: ComplianceService | None = None + + +def init_api( + state_db: StateDB, + upload_service: UploadService, + compliance_service: ComplianceService | None = None, +): + global _state_db, _upload_service, _compliance_service + _state_db = state_db + _upload_service = upload_service + _compliance_service = compliance_service + + +def _db() -> StateDB: + if _state_db is None: + raise HTTPException(500, "Database not initialized") + return _state_db + + +def _svc() -> UploadService: + if _upload_service is None: + raise HTTPException(500, "Upload service not initialized") + return _upload_service + + +@router.get("/stats", response_model=StatsResponse) +def get_stats(): + counts = _db().count_by_status() + return StatsResponse( + pending=counts.get("pending", 0), + queued=counts.get("queued", 0), + rescanning=counts.get("rescanning", 0), + uploaded=counts.get("uploaded", 0), + rejected=counts.get("rejected", 0), + skipped=counts.get("skipped", 0), + error=counts.get("error", 0), + total=sum(counts.values()), + ) + + +@router.get("/items") +def list_items(status: str | None = None, category: str | None = None, + page: int = 1, per_page: int = 50): + items = _db().list_items(status=status, category=category, page=page, per_page=per_page) + return { + "items": items, + "total": len(items), + "page": page, + "per_page": per_page, + } + + +@router.get("/items/{item_id}") +def get_item(item_id: int): + item = _db().get_item(item_id) + if not item: + raise HTTPException(404, "Item not found") + return item + + +@router.post("/items/{item_id}/approve") +def approve_item(item_id: int, req: ApproveRequest = Body(default=ApproveRequest())): + result = _svc().approve_and_upload(item_id, req.release_name, req.description) + if not result["success"]: + raise HTTPException(400, result["message"]) + return result + + +@router.post("/items/{item_id}/reject") +def reject_item(item_id: int, req: RejectRequest): + result = _svc().reject_item(item_id, req.reason) + if not result["success"]: + raise HTTPException(400, result["message"]) + return result + + +@router.post("/items/{item_id}/retry") +def retry_item(item_id: int): + result = _svc().retry_item(item_id) + if not result["success"]: + raise HTTPException(400, result["message"]) + return result + + +@router.post("/items/{item_id}/cancel") +def cancel_item(item_id: int): + result = _svc().cancel_item(item_id) + if not result["success"]: + raise HTTPException(400, result["message"]) + return result + + +@router.post("/items/{item_id}/reset") +def reset_uploaded_item(item_id: int): + result = _svc().reset_uploaded_item(item_id) + if not result["success"]: + raise HTTPException(400, result["message"]) + return result + + +@router.post("/items/{item_id}/save") +def save_item(item_id: int, req: ApproveRequest): + """Save release name / description edits without changing status.""" + item = _db().get_item(item_id) + if not item: + raise HTTPException(404, "Item not found") + updates = {} + if req.release_name is not None: + updates["user_edited_name"] = req.release_name + if req.description is not None: + updates["user_edited_desc"] = req.description + if updates: + _db().update_item(item_id, **updates) + return {"success": True, "message": "Changes saved"} + + +@router.post("/items/{item_id}/rescan") +def rescan_item(item_id: int): + """Re-run the full prepare pipeline on the source file.""" + result = _svc().rescan_item(item_id) + if not result["success"]: + raise HTTPException(400, result["message"]) + return result + + +@router.post("/items/{item_id}/force-rescan") +def force_rescan_item(item_id: int): + """Re-run the prepare pipeline, bypassing the duplicate-on-tracker check.""" + result = _svc().force_rescan_item(item_id) + if not result["success"]: + raise HTTPException(400, result["message"]) + return result + + +@router.post("/items/{item_id}/update-category") +def update_category(item_id: int, req: UpdateCategoryRequest): + """Update the category (category_id) for an item.""" + import json + item = _db().get_item(item_id) + if not item: + raise HTTPException(404, "Item not found") + tracker_payload = item.get("tracker_payload") + if isinstance(tracker_payload, str): + tracker_payload = json.loads(tracker_payload) + if tracker_payload: + tracker_payload["category_id"] = req.category_id + _db().update_item(item_id, content_category=req.category_label, tracker_payload=tracker_payload) + return {"success": True, "message": f"Category updated to {req.category_label}"} + + +@router.post("/items/{item_id}/update-resolution") +def update_resolution(item_id: int, req: UpdateResolutionRequest): + """Update the resolution (resolution_id) for an item.""" + import json + item = _db().get_item(item_id) + if not item: + raise HTTPException(404, "Item not found") + tracker_payload = item.get("tracker_payload") + if isinstance(tracker_payload, str): + tracker_payload = json.loads(tracker_payload) + if tracker_payload: + tracker_payload["resolution_id"] = req.resolution_id + _db().update_item(item_id, resolution=req.resolution_label, tracker_payload=tracker_payload) + return {"success": True, "message": f"Resolution updated to {req.resolution_label}"} + + +@router.post("/items/{item_id}/update-source-type") +def update_source_type(item_id: int, req: UpdateSourceTypeRequest): + """Update the source type (type_id) for an item.""" + import json + item = _db().get_item(item_id) + if not item: + raise HTTPException(404, "Item not found") + tracker_payload = item.get("tracker_payload") + if isinstance(tracker_payload, str): + tracker_payload = json.loads(tracker_payload) + if tracker_payload: + tracker_payload["type_id"] = req.type_id + _db().update_item(item_id, source_tag=req.source_label, tracker_payload=tracker_payload) + return {"success": True, "message": f"Source type updated to {req.source_label}"} + + +@router.post("/items/{item_id}/update-season-episode") +def update_season_episode(item_id: int, req: UpdateSeasonEpisodeRequest): + """Update the season_number and episode_number for an item.""" + import json + item = _db().get_item(item_id) + if not item: + raise HTTPException(404, "Item not found") + tracker_payload = item.get("tracker_payload") + if isinstance(tracker_payload, str): + tracker_payload = json.loads(tracker_payload) + if not isinstance(tracker_payload, dict): + tracker_payload = {} + tracker_payload["season_number"] = req.season_number + tracker_payload["episode_number"] = req.episode_number + _db().update_item(item_id, tracker_payload=tracker_payload) + return {"success": True, "message": f"Season/episode updated to S{req.season_number:02d}E{req.episode_number:02d}"} + + +@router.post("/items/{item_id}/update-tracks") +def update_tracks(item_id: int, req: UpdateTracksRequest): + """Update audio/subtitle tracks and regenerate the prez description.""" + result = _svc().regenerate_prez( + item_id, + [t.model_dump() for t in req.audio_tracks], + [t.model_dump() for t in req.subtitle_tracks], + ) + if not result["success"]: + raise HTTPException(400, result["message"]) + return result + + +@router.post("/items/{item_id}/rescan-tmdb") +def rescan_tmdb(item_id: int, req: RescanTmdbRequest): + result = _svc().rescan_tmdb(item_id, req.tmdb_id) + if not result["success"]: + raise HTTPException(400, result["message"]) + return result + + +@router.delete("/items/{item_id}") +def delete_item(item_id: int): + success = _db().delete_item(item_id) + if not success: + raise HTTPException(404, "Item not found") + return {"success": True, "message": "Item deleted"} + + +@router.get("/queue/status", response_model=QueueStatusResponse) +def queue_status(): + return _svc().queue_status() + + +@router.post("/items/bulk-approve") +def bulk_approve(req: BulkApproveRequest): + return _svc().bulk_approve(req.ids) + + +@router.post("/items/bulk-reject") +def bulk_reject(req: BulkRejectRequest): + return _svc().bulk_reject(req.ids, req.reason) + + +@router.post("/items/bulk-rescan") +def bulk_rescan(req: BulkRescanRequest): + return _svc().bulk_rescan(req.ids) + + +# ── Compliance ────────────────────────────────────────────────────── + +def _compliance() -> ComplianceService: + if _compliance_service is None: + raise HTTPException(503, "Compliance service not initialized") + return _compliance_service + + +@router.get("/compliance/items", response_model=ComplianceListResponse) +def compliance_list( + severity: str | None = None, + ack_status: str | None = None, + diff_kind: str | None = None, + page: int = 1, + per_page: int = 100, +): + items = _db().list_compliance( + severity=severity, + ack_status=ack_status, + diff_kind=diff_kind, + page=page, + per_page=per_page, + ) + counts = _db().count_compliance_by_severity(only_unchecked=True) + return { + "items": items, + "total": _db().count_compliance_total(), + "page": page, + "per_page": per_page, + "counts": counts, + } + + +@router.get("/compliance/items/{row_id}") +def compliance_get(row_id: int): + row = _db().get_compliance(row_id) + if not row: + raise HTTPException(404, "Compliance row not found") + row["description_html"] = bbcode_to_html(row.get("description")) + + # Regenerate a fresh description from the stored MediaInfo text, using the + # same generator the pending/upload flow uses. Done on the fly so it stays + # in sync with the tool's prez template without a DB migration. + generated_description = "" + audio_tracks: list[dict] = [] + sub_tracks: list[dict] = [] + try: + shim = build_prez_media_file(row.get("mediainfo")) + if shim is not None: + generated_description = generate_prez(shim) or "" + audio_tracks = list(shim.audio_track or []) + sub_tracks = list(shim.subtitle_track or []) + except Exception: + generated_description = "" + row["generated_description"] = generated_description + row["generated_description_html"] = bbcode_to_html(generated_description) + row["audio_tracks"] = audio_tracks + row["sub_tracks"] = sub_tracks + return row + + +@router.post("/compliance/items/{row_id}/generate-description") +def compliance_generate_description(row_id: int, body: dict | None = Body(default=None)): + """Regenerate the prez-format description, optionally with per-track overrides. + + Body (optional): {"audio_tracks": [...], "sub_tracks": [...]}. + Each track is a dict with keys expected by generate_prez (language, title, + format, channel_s, bit_rate, forced). + """ + row = _db().get_compliance(row_id) + if not row: + raise HTTPException(404, "Compliance row not found") + + audio_override = (body or {}).get("audio_tracks") + sub_override = (body or {}).get("sub_tracks") + + try: + shim = build_prez_media_file(row.get("mediainfo")) + except Exception: + shim = None + if shim is None: + raise HTTPException(422, "MediaInfo unavailable or BDInfo format — cannot regenerate") + + audio_tracks = audio_override if isinstance(audio_override, list) else shim.audio_track + sub_tracks = sub_override if isinstance(sub_override, list) else shim.subtitle_track + + try: + generated = generate_prez(shim, audio_tracks=audio_tracks, sub_tracks=sub_tracks) or "" + except Exception as exc: + raise HTTPException(500, f"Generation failed: {exc}") + + return { + "generated_description": generated, + "generated_description_html": bbcode_to_html(generated), + } + + +@router.post("/compliance/scan") +def compliance_scan(): + return _compliance().enqueue_full_scan() + + +@router.post("/compliance/items/{row_id}/ack") +def compliance_ack(row_id: int, req: ComplianceAckRequest | None = Body(default=None)): + status = req.status if req else "acknowledged" + ok = _db().set_compliance_ack(row_id, status) + if not ok: + raise HTTPException(404, "Compliance row not found") + return {"success": True, "message": f"Status set to {status}"} + + +@router.post("/compliance/items/{row_id}/ignore") +def compliance_ignore(row_id: int): + ok = _db().set_compliance_ack(row_id, "ignored") + if not ok: + raise HTTPException(404, "Compliance row not found") + return {"success": True, "message": "Status set to ignored"} + + +@router.post("/compliance/items/{row_id}/recheck") +def compliance_recheck(row_id: int): + row = _db().get_compliance(row_id) + if not row: + raise HTTPException(404, "Compliance row not found") + return _compliance().enqueue_check_one(int(row["torrent_id"])) + + +@router.delete("/compliance/items/{row_id}") +def compliance_delete_and_rescan(row_id: int): + row = _db().get_compliance(row_id) + if not row: + raise HTTPException(404, "Compliance row not found") + torrent_id = int(row["torrent_id"]) + _db().delete_compliance(row_id) + result = _compliance().enqueue_check_one(torrent_id) + return { + "success": result.get("success", True), + "message": result.get("message", f"Row deleted, fresh check queued for torrent #{torrent_id}"), + } + + +@router.post("/compliance/items/bulk-delete") +def compliance_bulk_delete_and_rescan(req: BulkComplianceDeleteRequest): + deleted = 0 + queued = 0 + failed_queue = 0 + missing = 0 + for row_id in req.ids: + row = _db().get_compliance(row_id) + if not row: + missing += 1 + continue + torrent_id = int(row["torrent_id"]) + _db().delete_compliance(row_id) + deleted += 1 + result = _compliance().enqueue_check_one(torrent_id) + if result.get("success"): + queued += 1 + else: + failed_queue += 1 + return { + "success": deleted > 0, + "deleted": deleted, + "queued": queued, + "failed_queue": failed_queue, + "missing": missing, + "message": f"Deleted {deleted} row(s), queued {queued} fresh check(s)" + + (f", {failed_queue} queue failure(s)" if failed_queue else "") + + (f", {missing} not found" if missing else ""), + } + + +@router.get("/compliance/scan-status", response_model=ComplianceScanStatus) +def compliance_status(): + return _compliance().scan_status() diff --git a/unit3dup/web/bbcode_renderer.py b/unit3dup/web/bbcode_renderer.py new file mode 100644 index 0000000..6a8a461 --- /dev/null +++ b/unit3dup/web/bbcode_renderer.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +"""Convert BBCode (including Unit3Dup custom tags) to HTML for web preview.""" + +from __future__ import annotations + +import html +import re + + +# Order matters: process inner tags before outer tags +_RULES: list[tuple[str, str]] = [ + # Custom Unit3Dup tags + (r'\[badge=(\w+)\](.*?)\[/badge\]', r'\2'), + (r'\[card-title\](.*?)\[/card-title\]', r'
\1'),
+ (r'\[quote\](.*?)\[/quote\]', r'\1'), +] + +# Compile patterns with DOTALL so . matches newlines +_COMPILED_RULES = [(re.compile(p, re.DOTALL | re.IGNORECASE), r) for p, r in _RULES] + +_SAFE_URL_RE = re.compile(r'^https?://', re.IGNORECASE) + +def _sanitize_urls(html_text: str) -> str: + """Remove dangerous href/src attributes that don't start with http(s).""" + def _check_attr(match): + attr = match.group(1) # href or src + url = match.group(2) + if _SAFE_URL_RE.match(url): + return match.group(0) # keep safe URLs + return f'{attr}="#"' # replace dangerous URLs + + return re.sub(r'((?:href|src))="([^"]*)"', _check_attr, html_text) + + +def bbcode_to_html(text: str | None) -> str: + """Convert BBCode text to HTML.""" + if not text: + return "" + + result = html.escape(text) + + # Apply rules (multiple passes for nested tags) + for _ in range(3): + prev = result + for pattern, replacement in _COMPILED_RULES: + result = pattern.sub(replacement, result) + if result == prev: + break + + # Sanitize URLs to prevent javascript: and data: XSS + result = _sanitize_urls(result) + + # Convert newlines to
+ Audits past uploads against the G3MINI naming rules. Read-only — edits happen on the tracker. +
+Gemini_USERNAME in your Unit3Dbot.json so we know which uploader to audit.
+
+| + + | +Severity | +Current name | +Proposed | +Category | +Checked | +Status | +Actions | +
|---|
| Name ↕ | +Status ↕ | +Category ↕ | +Source ↕ | +Discovered ↕ | +
|---|---|---|---|---|
| + + {{ item.release_name or item.display_name or item.source_basename }} + + | +{{ item.status }} | +{{ item.content_category | category_label }} | +{{ item.source_tag or "—" }} | +{{ item.discovered_at | datefmt }} | +
| No items yet. Start the watcher to discover media. | ||||
{{ items | length }} item(s)
+| Name ↕ | +Status ↕ | +Category ↕ | +Source ↕ | +Resolution ↕ | +Reason ↕ | +Date ↕ | +
|---|---|---|---|---|---|---|
| + + {{ item.release_name or item.display_name or item.source_basename }} + + | +{{ item.status }} | +{{ item.content_category | category_label }} | +{{ item.source_tag or "—" }} | +{{ item.resolution or "—" }} | +{{ (item.rejection_reason or item.skip_reason) | reason_label }} | +{{ (item.uploaded_at or item.discovered_at) | datefmt }} | +
| No items in history. | ||||||
| Source Name | {{ item.source_basename }} |
| Release Name | {{ item.release_name or "—" }} |
| Display Name | {{ item.display_name or "—" }} |
| Category | +
+ {{ item.content_category | category_label }}
+ {% if item.status == 'pending' %}
+ {% set current_cat_id = item.tracker_payload.category_id if item.tracker_payload and item.tracker_payload.category_id else '' %}
+
+
+
+
+ {% endif %}
+ |
+
| Resolution | +
+ {{ item.resolution or "—" }}
+ {% if item.status == 'pending' %}
+ {% set current_res_id = item.tracker_payload.resolution_id if item.tracker_payload and item.tracker_payload.resolution_id else '' %}
+
+
+
+
+ {% endif %}
+ |
+
| Source | +
+ {{ item.source_tag or "—" }}
+ {% if item.status == 'pending' %}
+ {% set current_type_id = item.tracker_payload.type_id if item.tracker_payload and item.tracker_payload.type_id else '' %}
+
+
+
+
+ {% endif %}
+ |
+
| Season / Episode | +
+
+ {% if sn is not none and en is not none %}
+ {% if sn == 0 and en == 0 %}
+ Complete Series
+ {% elif en == 0 %}
+ Complete Season {{ sn }}
+ {% else %}
+ S{{ '%02d' % sn }}E{{ '%02d' % en }}
+ {% endif %}
+ {% else %}
+ Not set
+ {% endif %}
+
+ {% if item.status == 'pending' %}
+
+
+
+
+
+
+
+ {% endif %}
+ |
+
| Size | {{ item.file_size | filesize }} |
| Tracker | {{ item.tracker_name or "—" }} |
| Similar | View similar torrents on tracker |
| TMDB | +
+ {% if item.tmdb_id %}
+
+ {{ item.tmdb_title or "" }} ({{ item.tmdb_year or "" }})
+
+ ID: {{ item.tmdb_id }}
+ {% else %}
+ Not found
+ {% endif %}
+ {% if item.status == 'pending' %}
+
+
+
+
+ {% endif %}
+ |
+
| IMDB | tt{{ '%07d' % item.imdb_id if item.imdb_id else item.imdb_id }} |
| IGDB | {{ item.igdb_id }} |
| Discovered | {{ item.discovered_at | datefmt }} |
| Uploaded | {{ item.uploaded_at | datefmt }} |
| Skip Reason | {{ item.skip_reason | reason_label }} |
| Duplicate match | ++ {% if dup_url %} + + + {{ item.duplicate_match.name or 'View on tracker' }} + + {% else %} + {{ item.duplicate_match.name or '—' }} + {% endif %} + {% if item.duplicate_match.size %} + — {{ item.duplicate_match.size | filesize }} + {% endif %} + {% set _ds = item.duplicate_match.get('delta_size') %} + {% if _ds is not none %} + (Δ {{ _ds }}%) + {% endif %} + | +
| Rejected | {{ item.rejection_reason }} |
| Error | {{ item.upload_error }} |
| Severity | Rule | Message |
|---|---|---|
| {{ v.severity if v.severity else "INFO" }} | +{{ v.rule if v.rule else "—" }} | +{{ v.message if v.message else v }} | +
| Langue | Format | Canaux | Débit | |
|---|---|---|---|---|
| + + | +{{ track.get('format', '') }} | +{{ track.get('channel_s', '') }} | +{{ track.get('bit_rate', '') }} | ++ |
| Langue | Format | Forcé | |
|---|---|---|---|
| + + | +{{ track.get('format', '') }} | ++ | + |
{{ item.mediainfo }}
+ Queued for upload
+This item is waiting in the upload queue. It will be processed automatically.
+ +Uploading...
+This item is currently being uploaded to the tracker.
+Move this item back to pending for re-review and re-upload.
+| + + | +Source ↕ | +Release Name ↕ | +Size ↕ | +Category ↕ | +Source Tag ↕ | +Resolution ↕ | +Validation | +Reason ↕ | +Discovered ↕ | ++ |
|---|---|---|---|---|---|---|---|---|---|---|
| + {% if item.status == 'analyzing' or item.status == 'rescanning' %} + + {% else %} + + {% endif %} + | +{{ item.source_basename }} | ++ {% if item.status == 'analyzing' %} + {{ item.source_basename }} + + Analyzing + + {% elif item.status == 'rescanning' %} + {{ item.source_basename }} + + Rescanning + + {% else %} + + {{ item.release_name or item.display_name or "—" }} + + {% if item.status == 'queued' %} + Queued + {% elif item.status == 'approved' %} + Uploading + {% endif %} + {% endif %} + | +{{ item.file_size | filesize }} | +{{ item.content_category | category_label }} | +{{ item.source_tag or "—" }} | +{{ item.resolution or "—" }} | ++ {% if item.status == 'analyzing' %} + ... + {% elif item.status == 'rescanning' %} + ... + {% elif item.has_errors %}Error + {% elif item.has_warnings %}Warn + {% else %}OK{% endif %} + | +{{ item.skip_reason | reason_label }} | +{{ item.discovered_at | datefmt }} | ++ {% if item.status == 'analyzing' %} + Analyzing... + {% elif item.status == 'rescanning' %} + Rescanning... + {% else %} + Review + {% endif %} + | +
| No pending items. Waiting for watcher to discover new media... | ||||||||||
Queue is empty
+Approve items from the pending list to start uploading.
+Analyzing
+Rescanning
+Pending
+Queued
+Uploaded
+Rejected / Skipped
+Errors
+{{ items | length }} item(s) waiting for review
+{{ items | length }} item(s) in the upload pipeline
+