From 0404d2d4e9af03eeee53c2e8df54bda3c33ee351 Mon Sep 17 00:00:00 2001 From: D-VR <26770468+D-VR@users.noreply.github.com> Date: Thu, 25 Jun 2026 22:15:05 +0200 Subject: [PATCH] Add behavioral sync fixtures --- .github/workflows/test.yaml | 7 +- pyproject.toml | 4 + syncmymoodle/__main__.py | 16 +- tests/__init__.py | 1 + tests/fixtures/html/external_overview.html | 6 + tests/fixtures/html/h5p_iframe.html | 6 + tests/fixtures/html/h5p_view.html | 6 + tests/fixtures/html/page_module.html | 12 + .../assignment_opencast_assignments.json | 10 + .../moodle/assignment_opencast_course.json | 13 + tests/fixtures/moodle/courses.json | 17 + tests/fixtures/moodle/mixed_assignments.json | 17 + tests/fixtures/moodle/mixed_course.json | 100 ++++++ tests/fixtures/moodle/mixed_courses.json | 7 + tests/fixtures/moodle/mixed_folders.json | 6 + .../moodle/mixed_submission_files.json | 8 + .../fixtures/moodle/nested_folder_course.json | 31 ++ .../fixtures/moodle/opencast_lti_course.json | 18 ++ tests/fixtures/moodle/skip_rules_course.json | 89 ++++++ tests/fixtures/opencast/episode_series_a.json | 17 + tests/fixtures/opencast/episode_series_b.json | 26 ++ tests/fixtures/opencast/episode_single.json | 34 ++ tests/fixtures/opencast/lti_series.html | 11 + tests/fixtures/opencast/lti_single.html | 11 + tests/fixtures/opencast/series.json | 16 + tests/fixtures/sciebo/propfind_root.xml | 30 ++ tests/fixtures/sciebo/propfind_slides.xml | 22 ++ tests/fixtures/sciebo/public_share.html | 7 + tests/helpers.py | 228 +++++++++++++ tests/snapshots/assignment_opencast_tree.txt | 5 + tests/snapshots/mixed_course_tree.txt | 20 ++ tests/snapshots/nested_folder_tree.txt | 8 + tests/snapshots/opencast_lti_tree.txt | 7 + tests/snapshots/skip_rules_tree.txt | 4 + tests/test_course_prefix_handling.py | 210 ++++++------ tests/test_download_behavior.py | 252 +++++++++++++++ tests/test_storage_safety.py | 75 +++++ tests/test_sync_filtering.py | 51 +++ tests/test_sync_fixtures.py | 299 ++++++++++++++++++ 39 files changed, 1602 insertions(+), 105 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/fixtures/html/external_overview.html create mode 100644 tests/fixtures/html/h5p_iframe.html create mode 100644 tests/fixtures/html/h5p_view.html create mode 100644 tests/fixtures/html/page_module.html create mode 100644 tests/fixtures/moodle/assignment_opencast_assignments.json create mode 100644 tests/fixtures/moodle/assignment_opencast_course.json create mode 100644 tests/fixtures/moodle/courses.json create mode 100644 tests/fixtures/moodle/mixed_assignments.json create mode 100644 tests/fixtures/moodle/mixed_course.json create mode 100644 tests/fixtures/moodle/mixed_courses.json create mode 100644 tests/fixtures/moodle/mixed_folders.json create mode 100644 tests/fixtures/moodle/mixed_submission_files.json create mode 100644 tests/fixtures/moodle/nested_folder_course.json create mode 100644 tests/fixtures/moodle/opencast_lti_course.json create mode 100644 tests/fixtures/moodle/skip_rules_course.json create mode 100644 tests/fixtures/opencast/episode_series_a.json create mode 100644 tests/fixtures/opencast/episode_series_b.json create mode 100644 tests/fixtures/opencast/episode_single.json create mode 100644 tests/fixtures/opencast/lti_series.html create mode 100644 tests/fixtures/opencast/lti_single.html create mode 100644 tests/fixtures/opencast/series.json create mode 100644 tests/fixtures/sciebo/propfind_root.xml create mode 100644 tests/fixtures/sciebo/propfind_slides.xml create mode 100644 tests/fixtures/sciebo/public_share.html create mode 100644 tests/helpers.py create mode 100644 tests/snapshots/assignment_opencast_tree.txt create mode 100644 tests/snapshots/mixed_course_tree.txt create mode 100644 tests/snapshots/nested_folder_tree.txt create mode 100644 tests/snapshots/opencast_lti_tree.txt create mode 100644 tests/snapshots/skip_rules_tree.txt create mode 100644 tests/test_download_behavior.py create mode 100644 tests/test_storage_safety.py create mode 100644 tests/test_sync_filtering.py create mode 100644 tests/test_sync_fixtures.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b9be0b5..4146162 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -32,8 +32,11 @@ jobs: - name: Check formatting with black and isort run: | - black --check syncmymoodle - isort --check-only syncmymoodle + black --check syncmymoodle tests + isort --check-only syncmymoodle tests + + - name: Run tests + run: python -m pytest # Disabled for now, until we refactor the main project # - name: Lint with flake8 diff --git a/pyproject.toml b/pyproject.toml index 28b6afb..0b125dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,12 +39,16 @@ test = [ "flake8", "flake8-bugbear", "mypy", + "pytest", "types-requests" ] [tool.isort] profile = "black" +[tool.black] +target-version = ["py311"] + [tool.mypy] warn_unused_configs = true warn_redundant_casts = true diff --git a/syncmymoodle/__main__.py b/syncmymoodle/__main__.py index 40405e5..fe47223 100755 --- a/syncmymoodle/__main__.py +++ b/syncmymoodle/__main__.py @@ -132,7 +132,7 @@ def add_child( ): if url: url = url.replace("?forcedownload=1", "").replace( - "mod_page/content/3", "mod_page/content" + "mod_page/content/3/", "mod_page/content/" ) url = url.replace("webservice/pluginfile.php", "pluginfile.php") @@ -1825,7 +1825,7 @@ def sync(self): info_res = bs( self.session.get(info_url).text, features="lxml" ) - attempts = info_res.findAll( + attempts = info_res.find_all( "a", { "title": "Überprüfung der eigenen Antworten dieses Versuchs" @@ -2055,6 +2055,7 @@ def download_file(self, node): local_conflict = False old_etag = getattr(old_node, "etag", None) if old_node is not None else None + etag_check_failed = False if old_etag: # Prefer using the old ETag (hash) to detect whether the local file # still matches the previously downloaded version. @@ -2062,11 +2063,12 @@ def download_file(self, node): if not self._local_file_matches_etag(downloadpath, old_etag): local_conflict = True except Exception: - # If we cannot safely compare using the ETag, fall back to the - # timestamp-based heuristic below. - local_conflict = False + # A faulty/unusable ETag cache is treated as if we had no + # cached ETag at all: fall back to the timestamp/HEAD + # heuristic below to decide whether this is a conflict. + etag_check_failed = True - if not old_etag: + if not old_etag or etag_check_failed: if cached_timemodified is not None: # Fallback: compare local mtime with the previous Moodle timestamp. try: @@ -2267,7 +2269,7 @@ def _extract_opencast_episode_id(self, url): def _extract_lti_form_data(self, soup): return { input_tag["name"]: input_tag.get("value", "") - for input_tag in soup.findAll("input") + for input_tag in soup.find_all("input") if input_tag.get("name") } diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/fixtures/html/external_overview.html b/tests/fixtures/html/external_overview.html new file mode 100644 index 0000000..e354849 --- /dev/null +++ b/tests/fixtures/html/external_overview.html @@ -0,0 +1,6 @@ + + + + overview video + + diff --git a/tests/fixtures/html/h5p_iframe.html b/tests/fixtures/html/h5p_iframe.html new file mode 100644 index 0000000..6e9f00f --- /dev/null +++ b/tests/fixtures/html/h5p_iframe.html @@ -0,0 +1,6 @@ + + + + h5p video + + diff --git a/tests/fixtures/html/h5p_view.html b/tests/fixtures/html/h5p_view.html new file mode 100644 index 0000000..5dc5bdd --- /dev/null +++ b/tests/fixtures/html/h5p_view.html @@ -0,0 +1,6 @@ + + + + + + diff --git a/tests/fixtures/html/page_module.html b/tests/fixtures/html/page_module.html new file mode 100644 index 0000000..38d07d4 --- /dev/null +++ b/tests/fixtures/html/page_module.html @@ -0,0 +1,12 @@ + + + +
+ +
+ page video + + + diff --git a/tests/fixtures/moodle/assignment_opencast_assignments.json b/tests/fixtures/moodle/assignment_opencast_assignments.json new file mode 100644 index 0000000..b01e85e --- /dev/null +++ b/tests/fixtures/moodle/assignment_opencast_assignments.json @@ -0,0 +1,10 @@ +{ + "assignments": [ + { + "id": 402, + "cmid": 302, + "intro": "

Watch the embedded recording.

", + "introattachments": [] + } + ] +} diff --git a/tests/fixtures/moodle/assignment_opencast_course.json b/tests/fixtures/moodle/assignment_opencast_course.json new file mode 100644 index 0000000..4ea79e8 --- /dev/null +++ b/tests/fixtures/moodle/assignment_opencast_course.json @@ -0,0 +1,13 @@ +[ + { + "id": 202, + "name": "Assignments", + "modules": [ + { + "id": 302, + "name": "Video reflection", + "modname": "assign" + } + ] + } +] diff --git a/tests/fixtures/moodle/courses.json b/tests/fixtures/moodle/courses.json new file mode 100644 index 0000000..9bd0ec7 --- /dev/null +++ b/tests/fixtures/moodle/courses.json @@ -0,0 +1,17 @@ +[ + { + "id": 101, + "shortname": "(VO) Data Science", + "idnumber": "26ss-data-science" + }, + { + "id": 102, + "shortname": "(UE) Software Quality", + "idnumber": "26ss-software-quality" + }, + { + "id": 103, + "shortname": "Robust Moodle Fixtures", + "idnumber": "26ss-fixtures" + } +] diff --git a/tests/fixtures/moodle/mixed_assignments.json b/tests/fixtures/moodle/mixed_assignments.json new file mode 100644 index 0000000..eb062af --- /dev/null +++ b/tests/fixtures/moodle/mixed_assignments.json @@ -0,0 +1,17 @@ +{ + "assignments": [ + { + "id": 412, + "cmid": 312, + "intro": "

Use the template and upload your answer.

", + "introattachments": [ + { + "filename": "essay-template.docx", + "filepath": "/", + "fileurl": "https://moodle.rwth-aachen.de/pluginfile.php/104/mod_assign/introattachment/essay-template.docx", + "timemodified": 1710000105 + } + ] + } + ] +} diff --git a/tests/fixtures/moodle/mixed_course.json b/tests/fixtures/moodle/mixed_course.json new file mode 100644 index 0000000..8c289b5 --- /dev/null +++ b/tests/fixtures/moodle/mixed_course.json @@ -0,0 +1,100 @@ +[ + { + "id": 210, + "name": "Materials", + "modules": [ + { + "id": 310, + "name": "Lecture slides", + "modname": "resource", + "contents": [ + { + "type": "file", + "filename": "lecture-slides.pdf", + "filepath": "/", + "fileurl": "https://moodle.rwth-aachen.de/pluginfile.php/104/mod_resource/content/1/lecture-slides.pdf", + "mimetype": "application/pdf", + "timemodified": 1710000100 + } + ] + }, + { + "id": 311, + "name": "Data folder", + "modname": "folder", + "contents": [ + { + "type": "file", + "filename": "measurements.csv", + "filepath": "/Data/Raw/", + "fileurl": "https://moodle.rwth-aachen.de/pluginfile.php/104/mod_folder/content/0/Data/Raw/measurements.csv", + "mimetype": "text/csv", + "timemodified": 1710000101 + } + ] + }, + { + "id": 312, + "name": "Essay upload", + "modname": "assign" + }, + { + "id": 313, + "name": "Direct external PDF", + "modname": "url", + "contents": [ + { + "type": "file", + "filename": "direct.pdf", + "filepath": "/", + "fileurl": "https://files.example.test/direct.pdf", + "mimetype": "application/pdf", + "timemodified": 1710000102 + } + ] + }, + { + "id": 314, + "name": "External HTML page", + "modname": "url", + "contents": [ + { + "type": "file", + "filename": "overview.html", + "filepath": "/", + "fileurl": "https://files.example.test/overview.html", + "mimetype": "text/html", + "timemodified": 1710000103 + } + ] + }, + { + "id": 315, + "name": "Page module", + "modname": "page", + "url": "https://moodle.rwth-aachen.de/mod/page/view.php?id=315", + "contents": [ + { + "type": "file", + "filename": "index.html", + "filepath": "/", + "fileurl": "https://moodle.rwth-aachen.de/pluginfile.php/104/mod_page/content/315/index.html", + "mimetype": "text/html", + "timemodified": 1710000104 + } + ] + }, + { + "id": 316, + "name": "Label module", + "modname": "label", + "description": "

Label video https://youtu.be/labelvid001

" + }, + { + "id": 317, + "name": "H5P module", + "modname": "h5pactivity" + } + ] + } +] diff --git a/tests/fixtures/moodle/mixed_courses.json b/tests/fixtures/moodle/mixed_courses.json new file mode 100644 index 0000000..b6afef7 --- /dev/null +++ b/tests/fixtures/moodle/mixed_courses.json @@ -0,0 +1,7 @@ +[ + { + "id": 104, + "shortname": "Comprehensive Sync", + "idnumber": "26ss-comprehensive" + } +] diff --git a/tests/fixtures/moodle/mixed_folders.json b/tests/fixtures/moodle/mixed_folders.json new file mode 100644 index 0000000..72f6ef1 --- /dev/null +++ b/tests/fixtures/moodle/mixed_folders.json @@ -0,0 +1,6 @@ +[ + { + "coursemodule": 311, + "intro": "

Folder intro video https://youtu.be/foldervid01

" + } +] diff --git a/tests/fixtures/moodle/mixed_submission_files.json b/tests/fixtures/moodle/mixed_submission_files.json new file mode 100644 index 0000000..44bd8a5 --- /dev/null +++ b/tests/fixtures/moodle/mixed_submission_files.json @@ -0,0 +1,8 @@ +[ + { + "filename": "feedback.txt", + "filepath": "/Feedback/", + "fileurl": "https://moodle.rwth-aachen.de/pluginfile.php/104/mod_assign/feedback/feedback.txt", + "timemodified": 1710000106 + } +] diff --git a/tests/fixtures/moodle/nested_folder_course.json b/tests/fixtures/moodle/nested_folder_course.json new file mode 100644 index 0000000..fbc2e6d --- /dev/null +++ b/tests/fixtures/moodle/nested_folder_course.json @@ -0,0 +1,31 @@ +[ + { + "id": 201, + "name": "General", + "modules": [ + { + "id": 301, + "name": "Dataset folder", + "modname": "folder", + "contents": [ + { + "type": "file", + "filename": "anscombe.csv", + "filepath": "/Data/CSV/", + "fileurl": "https://moodle.rwth-aachen.de/pluginfile.php/101/mod_folder/content/0/Data/CSV/anscombe.csv?forcedownload=1", + "mimetype": "text/csv", + "timemodified": 1710000000 + }, + { + "type": "file", + "filename": "iris.csv", + "filepath": "/Data/CSV/", + "fileurl": "https://moodle.rwth-aachen.de/pluginfile.php/101/mod_folder/content/0/Data/CSV/iris.csv?forcedownload=1", + "mimetype": "text/csv", + "timemodified": 1710000001 + } + ] + } + ] + } +] diff --git a/tests/fixtures/moodle/opencast_lti_course.json b/tests/fixtures/moodle/opencast_lti_course.json new file mode 100644 index 0000000..0f34ee9 --- /dev/null +++ b/tests/fixtures/moodle/opencast_lti_course.json @@ -0,0 +1,18 @@ +[ + { + "id": 220, + "name": "Opencast", + "modules": [ + { + "id": 501, + "name": "Single LTI", + "modname": "lti" + }, + { + "id": 502, + "name": "Series LTI", + "modname": "lti" + } + ] + } +] diff --git a/tests/fixtures/moodle/skip_rules_course.json b/tests/fixtures/moodle/skip_rules_course.json new file mode 100644 index 0000000..2c7d035 --- /dev/null +++ b/tests/fixtures/moodle/skip_rules_course.json @@ -0,0 +1,89 @@ +[ + { + "id": 203, + "name": "General", + "modules": [ + { + "id": 303, + "name": "Visible PDF", + "modname": "resource", + "contents": [ + { + "type": "file", + "filename": "visible.pdf", + "filepath": "/", + "fileurl": "https://moodle.rwth-aachen.de/pluginfile.php/103/mod_resource/content/1/visible.pdf", + "mimetype": "application/pdf", + "timemodified": 1710000002 + } + ] + }, + { + "id": 304, + "name": "Skip Module", + "modname": "resource", + "contents": [ + { + "type": "file", + "filename": "skipped-by-module.pdf", + "filepath": "/", + "fileurl": "https://moodle.rwth-aachen.de/pluginfile.php/103/mod_resource/content/1/skipped-by-module.pdf", + "mimetype": "application/pdf", + "timemodified": 1710000003 + } + ] + }, + { + "id": 305, + "name": "Excluded Link", + "modname": "resource", + "contents": [ + { + "type": "file", + "filename": "excluded.pdf", + "filepath": "/", + "fileurl": "https://moodle.rwth-aachen.de/pluginfile.php/103/mod_resource/content/1/excluded.pdf", + "mimetype": "application/pdf", + "timemodified": 1710000004 + } + ] + }, + { + "id": 306, + "name": "External Direct File", + "modname": "resource", + "contents": [ + { + "type": "file", + "filename": "external.pdf", + "filepath": "/", + "fileurl": "https://external.example.test/download/external.pdf", + "mimetype": "application/pdf", + "timemodified": 1710000005 + } + ] + } + ] + }, + { + "id": 204, + "name": "Hidden Week", + "modules": [ + { + "id": 307, + "name": "Hidden PDF", + "modname": "resource", + "contents": [ + { + "type": "file", + "filename": "hidden.pdf", + "filepath": "/", + "fileurl": "https://moodle.rwth-aachen.de/pluginfile.php/103/mod_resource/content/1/hidden.pdf", + "mimetype": "application/pdf", + "timemodified": 1710000006 + } + ] + } + ] + } +] diff --git a/tests/fixtures/opencast/episode_series_a.json b/tests/fixtures/opencast/episode_series_a.json new file mode 100644 index 0000000..b261876 --- /dev/null +++ b/tests/fixtures/opencast/episode_series_a.json @@ -0,0 +1,17 @@ +{ + "result": [ + { + "mediapackage": { + "media": { + "track": { + "mimetype": "video/mp4", + "url": "https://video.example.test/opencast/series-a.mp4", + "video": { + "resolution": "1280x720" + } + } + } + } + } + ] +} diff --git a/tests/fixtures/opencast/episode_series_b.json b/tests/fixtures/opencast/episode_series_b.json new file mode 100644 index 0000000..53f1531 --- /dev/null +++ b/tests/fixtures/opencast/episode_series_b.json @@ -0,0 +1,26 @@ +{ + "result": [ + { + "mediapackage": { + "media": { + "track": [ + { + "mimetype": "video/webm", + "url": "https://video.example.test/opencast/series-b.webm", + "video": { + "resolution": "1920x1080" + } + }, + { + "mimetype": "video/mp4", + "url": "https://video.example.test/opencast/series-b.mp4", + "video": { + "resolution": "1920x1080" + } + } + ] + } + } + } + ] +} diff --git a/tests/fixtures/opencast/episode_single.json b/tests/fixtures/opencast/episode_single.json new file mode 100644 index 0000000..7820f3a --- /dev/null +++ b/tests/fixtures/opencast/episode_single.json @@ -0,0 +1,34 @@ +{ + "result": [ + { + "mediapackage": { + "media": { + "track": [ + { + "mimetype": "video/mp4", + "url": "https://video.example.test/opencast/single-low.mp4", + "video": { + "resolution": "640x360" + } + }, + { + "mimetype": "video/mp4", + "url": "https://video.example.test/opencast/single-high.mp4", + "video": { + "resolution": "1920x1080" + } + }, + { + "mimetype": "video/mp4", + "url": "https://video.example.test/opencast/single-stream.mp4", + "transport": "hls", + "video": { + "resolution": "3840x2160" + } + } + ] + } + } + } + ] +} diff --git a/tests/fixtures/opencast/lti_series.html b/tests/fixtures/opencast/lti_series.html new file mode 100644 index 0000000..f9eeab2 --- /dev/null +++ b/tests/fixtures/opencast/lti_series.html @@ -0,0 +1,11 @@ + + + +
+ + + + +
+ + diff --git a/tests/fixtures/opencast/lti_single.html b/tests/fixtures/opencast/lti_single.html new file mode 100644 index 0000000..056a78c --- /dev/null +++ b/tests/fixtures/opencast/lti_single.html @@ -0,0 +1,11 @@ + + + +
+ + + + +
+ + diff --git a/tests/fixtures/opencast/series.json b/tests/fixtures/opencast/series.json new file mode 100644 index 0000000..ddd384d --- /dev/null +++ b/tests/fixtures/opencast/series.json @@ -0,0 +1,16 @@ +{ + "result": [ + { + "mediapackage": { + "id": "bbbbbbbb-cccc-4ddd-8eee-ffffffffffff", + "title": "Series episode A" + } + }, + { + "mediapackage": { + "id": "cccccccc-dddd-4eee-8fff-aaaaaaaaaaaa", + "title": "Series episode B" + } + } + ] +} diff --git a/tests/fixtures/sciebo/propfind_root.xml b/tests/fixtures/sciebo/propfind_root.xml new file mode 100644 index 0000000..0c88d59 --- /dev/null +++ b/tests/fixtures/sciebo/propfind_root.xml @@ -0,0 +1,30 @@ + + + + /public.php/webdav/ + + + "folder-root" + + + + + /public.php/webdav/readme.pdf + + + "etag-readme" + + SHA1:1111111111111111111111111111111111111111 + + + + + + /public.php/webdav/slides/ + + + "folder-slides" + + + + diff --git a/tests/fixtures/sciebo/propfind_slides.xml b/tests/fixtures/sciebo/propfind_slides.xml new file mode 100644 index 0000000..72a63ac --- /dev/null +++ b/tests/fixtures/sciebo/propfind_slides.xml @@ -0,0 +1,22 @@ + + + + /public.php/webdav/slides/ + + + "folder-slides" + + + + + /public.php/webdav/slides/deck.pdf + + + "etag-deck" + + SHA1:2222222222222222222222222222222222222222 + + + + + diff --git a/tests/fixtures/sciebo/public_share.html b/tests/fixtures/sciebo/public_share.html new file mode 100644 index 0000000..3872221 --- /dev/null +++ b/tests/fixtures/sciebo/public_share.html @@ -0,0 +1,7 @@ + + + + + + + diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 0000000..a802980 --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +import json +import os +from collections import Counter +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from syncmymoodle.__main__ import Node, SyncMyMoodle + +FIXTURES = Path(__file__).parent / "fixtures" +SNAPSHOTS = Path(__file__).parent / "snapshots" + +# Set SMM_UPDATE_SNAPSHOTS=1 to rewrite snapshot files from the actual rows +# instead of asserting against them. Use this after an intentional change to +# the sync behavior, then review the resulting snapshot diff. +UPDATE_SNAPSHOTS = os.environ.get("SMM_UPDATE_SNAPSHOTS") not in (None, "", "0") + + +DEFAULT_CONFIG = { + "basedir": "./", + "selected_courses": [], + "skip_courses": [], + "only_sync_semester": [], + "course_prefix_handling": "keep", + "nolinks": False, + "used_modules": { + "assign": True, + "resource": True, + "url": {"youtube": True, "opencast": True, "sciebo": True, "quiz": False}, + "folder": True, + }, + "exclude_filetypes": [], + "exclude_files": [], + "exclude_links": [], + "allowed_domains": [], + "exclude_sections": [], + "exclude_modules": [], + "updatefiles": False, + "update_files_conflict": "rename", +} + + +@dataclass +class FakeResponse: + text: str = "" + status_code: int = 200 + headers: dict[str, str] = field(default_factory=dict) + url: str | None = None + json_payload: Any = None + chunks: list[bytes] | None = None + + def json(self) -> Any: + if self.json_payload is not None: + return self.json_payload + return json.loads(self.text) + + def iter_content(self, block_size: int): + del block_size + yield from self.chunks or [] + + def close(self) -> None: + pass + + +RouteResult = FakeResponse | Callable[[str, dict[str, Any]], FakeResponse] + + +class FakeSession: + def __init__(self) -> None: + self.routes: dict[tuple[str, str], RouteResult] = {} + self.calls: list[tuple[str, str]] = [] + + def add(self, method: str, url: str, response: RouteResult) -> None: + self.routes[(method.upper(), url)] = response + + def count(self, method: str, url: str | None = None) -> int: + method = method.upper() + if url is None: + return sum(1 for call_method, _ in self.calls if call_method == method) + return Counter(self.calls)[(method, url)] + + def _dispatch(self, method: str, url: str, **kwargs: Any) -> FakeResponse: + method = method.upper() + self.calls.append((method, url)) + route = self.routes.get((method, url)) + if route is None: + raise AssertionError(f"Unexpected fake HTTP request: {method} {url}") + response = route(url, kwargs) if callable(route) else route + if response.url is None: + response.url = url + return response + + def get(self, url: str, **kwargs: Any) -> FakeResponse: + return self._dispatch("GET", url, **kwargs) + + def head(self, url: str, **kwargs: Any) -> FakeResponse: + return self._dispatch("HEAD", url, **kwargs) + + def post(self, url: str, **kwargs: Any) -> FakeResponse: + return self._dispatch("POST", url, **kwargs) + + def request(self, method: str, url: str, **kwargs: Any) -> FakeResponse: + return self._dispatch(method, url, **kwargs) + + +def load_fixture(*parts: str) -> str: + return (FIXTURES.joinpath(*parts)).read_text(encoding="utf-8") + + +def load_json_fixture(*parts: str) -> Any: + return json.loads(load_fixture(*parts)) + + +def load_snapshot(name: str) -> list[str]: + return [ + line + for line in (SNAPSHOTS / name).read_text(encoding="utf-8").splitlines() + if line + ] + + +def assert_snapshot(name: str, actual: list[str]) -> None: + """Compare ``actual`` against a stored snapshot. + + When ``SMM_UPDATE_SNAPSHOTS`` is set the snapshot is (re)written from + ``actual`` instead of asserted, so regenerating after an intentional change + is a one-liner rather than a hand-edit of the pipe-delimited files. + """ + if UPDATE_SNAPSHOTS: + path = SNAPSHOTS / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(actual) + "\n", encoding="utf-8") + return + assert actual == load_snapshot(name) + + +def make_syncer(config: dict[str, Any] | None = None) -> SyncMyMoodle: + merged_config = DEFAULT_CONFIG.copy() + if config: + merged_config.update(config) + syncer = SyncMyMoodle(merged_config) + syncer.wstoken = "fake-webservice-token" + syncer.user_id = 10001 + syncer.session_key = "fake-sesskey" + return syncer + + +def install_moodle_fixtures( + syncer: SyncMyMoodle, + courses: list[dict[str, Any]], + course_contents: dict[int, list[dict[str, Any]]], + assignments: dict[int, dict[str, Any] | None] | None = None, + submission_files: dict[int, list[dict[str, Any]]] | None = None, + folders: dict[int, list[dict[str, Any]]] | None = None, +) -> None: + syncer.get_all_courses = lambda: courses # type: ignore[method-assign] + syncer.get_course = lambda course_id: course_contents[int(course_id)] # type: ignore[method-assign] + syncer.get_assignment = lambda course_id: (assignments or {}).get( # type: ignore[method-assign] + int(course_id) + ) + syncer.get_assignment_submission_files = lambda assignment_id: ( # type: ignore[method-assign] + submission_files or {} + ).get( + int(assignment_id), [] + ) + syncer.get_folders_by_courses = lambda course_id: (folders or {}).get( # type: ignore[method-assign] + int(course_id), [] + ) + + +def node_rows(root: Node) -> list[str]: + rows = [] + + def walk(node: Node) -> None: + for child in node.children: + path = "/".join(part for part in child.get_path() if part) + rows.append( + " | ".join( + [ + child.type, + path, + child.url or "", + str(child.timemodified or ""), + str(child.etag or ""), + ] + ) + ) + walk(child) + + walk(root) + return rows + + +def build_single_file_tree( + filename: str, + url: str, + *, + timemodified: int | None = None, + etag: str | None = None, + semester: str = "26ss", + course: str = "Download Course", + course_id: int = 301, + section: str = "General", + section_id: int = 401, + file_type: str = "Linked file [application/pdf]", +) -> tuple[Node, Node]: + """Build a Root/Semester/Course/Section/ tree. + + Returns the root node and the leaf file node so tests can drive + ``download_file`` against a realistic, course-scoped path (which the cache + lookups in ``download_file`` rely on). + """ + root = Node("", -1, "Root", None) + semester_node = root.add_child(semester, None, "Semester") + course_node = semester_node.add_child(course, course_id, "Course") + section_node = course_node.add_child(section, section_id, "Section") + file_node = section_node.add_child( + filename, + url, + file_type, + url=url, + timemodified=timemodified, + etag=etag, + ) + return root, file_node diff --git a/tests/snapshots/assignment_opencast_tree.txt b/tests/snapshots/assignment_opencast_tree.txt new file mode 100644 index 0000000..59ce3ce --- /dev/null +++ b/tests/snapshots/assignment_opencast_tree.txt @@ -0,0 +1,5 @@ +Semester | 26ss | | | +Course | 26ss/(UE) Software Quality | | | +Section | 26ss/(UE) Software Quality/Assignments | | | +Assignment | 26ss/(UE) Software Quality/Assignments/Video reflection | | | +Opencast | 26ss/(UE) Software Quality/Assignments/Video reflection/Video reflection | https://video.example.test/11111111-2222-4333-8444-555555555555/presentation.mp4 | | diff --git a/tests/snapshots/mixed_course_tree.txt b/tests/snapshots/mixed_course_tree.txt new file mode 100644 index 0000000..3fcdb68 --- /dev/null +++ b/tests/snapshots/mixed_course_tree.txt @@ -0,0 +1,20 @@ +Semester | 26ss | | | +Course | 26ss/Comprehensive Sync | | | +Section | 26ss/Comprehensive Sync/Materials | | | +Linked file [application/pdf] | 26ss/Comprehensive Sync/Materials/lecture-slides.pdf | https://moodle.rwth-aachen.de/pluginfile.php/104/mod_resource/content/1/lecture-slides.pdf | 1710000100 | +Folder | 26ss/Comprehensive Sync/Materials/Data folder | | | +Youtube | 26ss/Comprehensive Sync/Materials/Data folder/Youtube: https://youtu.be/foldervid01 | https://youtu.be/foldervid01 | | +Folder | 26ss/Comprehensive Sync/Materials/Data folder/Data | | | +Folder | 26ss/Comprehensive Sync/Materials/Data folder/Data/Raw | | | +Folder File | 26ss/Comprehensive Sync/Materials/Data folder/Data/Raw/measurements.csv | https://moodle.rwth-aachen.de/pluginfile.php/104/mod_folder/content/0/Data/Raw/measurements.csv | 1710000101 | +Assignment | 26ss/Comprehensive Sync/Materials/Essay upload | | | +Assignment File | 26ss/Comprehensive Sync/Materials/Essay upload/essay-template.docx | https://moodle.rwth-aachen.de/pluginfile.php/104/mod_assign/introattachment/essay-template.docx | 1710000105 | +Folder | 26ss/Comprehensive Sync/Materials/Essay upload/Feedback | | | +Assignment File | 26ss/Comprehensive Sync/Materials/Essay upload/Feedback/feedback.txt | https://moodle.rwth-aachen.de/pluginfile.php/104/mod_assign/feedback/feedback.txt | 1710000106 | +Linked file [application/pdf] | 26ss/Comprehensive Sync/Materials/direct.pdf | https://files.example.test/direct.pdf | | +Youtube | 26ss/Comprehensive Sync/Materials/Youtube: External HTML page | https://www.youtube.com/watch?v=directvid01 | | +Opencast | 26ss/Comprehensive Sync/Materials/Page module | https://video.example.test/33333333-4444-4555-8666-777777777777/presentation.mp4 | | +Embedded videojs | 26ss/Comprehensive Sync/Materials/page-video.mp4 | https://moodle.rwth-aachen.de/pluginfile.php/104/mod_page/content/315/page-video.mp4 | | +Youtube | 26ss/Comprehensive Sync/Materials/Youtube: Page module | https://www.youtube.com/watch?v=pagevideo01 | | +Youtube | 26ss/Comprehensive Sync/Materials/Youtube: Label module | https://youtu.be/labelvid001 | | +Youtube | 26ss/Comprehensive Sync/Materials/Youtube: h5pactivity | https://www.youtube.com/watch?v=h5pvideo001 | | diff --git a/tests/snapshots/nested_folder_tree.txt b/tests/snapshots/nested_folder_tree.txt new file mode 100644 index 0000000..e7c71e6 --- /dev/null +++ b/tests/snapshots/nested_folder_tree.txt @@ -0,0 +1,8 @@ +Semester | 26ss | | | +Course | 26ss/(VO) Data Science | | | +Section | 26ss/(VO) Data Science/General | | | +Folder | 26ss/(VO) Data Science/General/Dataset folder | | | +Folder | 26ss/(VO) Data Science/General/Dataset folder/Data | | | +Folder | 26ss/(VO) Data Science/General/Dataset folder/Data/CSV | | | +Folder File | 26ss/(VO) Data Science/General/Dataset folder/Data/CSV/anscombe.csv | https://moodle.rwth-aachen.de/pluginfile.php/101/mod_folder/content/0/Data/CSV/anscombe.csv | 1710000000 | +Folder File | 26ss/(VO) Data Science/General/Dataset folder/Data/CSV/iris.csv | https://moodle.rwth-aachen.de/pluginfile.php/101/mod_folder/content/0/Data/CSV/iris.csv | 1710000001 | diff --git a/tests/snapshots/opencast_lti_tree.txt b/tests/snapshots/opencast_lti_tree.txt new file mode 100644 index 0000000..fa7fff0 --- /dev/null +++ b/tests/snapshots/opencast_lti_tree.txt @@ -0,0 +1,7 @@ +Semester | 26ss | | | +Course | 26ss/(VO) Data Science | | | +Section | 26ss/(VO) Data Science/Opencast | | | +Opencast | 26ss/(VO) Data Science/Opencast/Single recording | https://video.example.test/opencast/single-high.mp4 | | +Section | 26ss/(VO) Data Science/Series recordings | | | +Opencast | 26ss/(VO) Data Science/Series recordings/Series episode A | https://video.example.test/opencast/series-a.mp4 | | +Opencast | 26ss/(VO) Data Science/Series recordings/Series episode B | https://video.example.test/opencast/series-b.mp4 | | diff --git a/tests/snapshots/skip_rules_tree.txt b/tests/snapshots/skip_rules_tree.txt new file mode 100644 index 0000000..5e68043 --- /dev/null +++ b/tests/snapshots/skip_rules_tree.txt @@ -0,0 +1,4 @@ +Semester | 26ss | | | +Course | 26ss/Robust Moodle Fixtures | | | +Section | 26ss/Robust Moodle Fixtures/General | | | +Linked file [application/pdf] | 26ss/Robust Moodle Fixtures/General/visible.pdf | https://moodle.rwth-aachen.de/pluginfile.php/103/mod_resource/content/1/visible.pdf | 1710000002 | diff --git a/tests/test_course_prefix_handling.py b/tests/test_course_prefix_handling.py index c57d11f..2cd9b98 100644 --- a/tests/test_course_prefix_handling.py +++ b/tests/test_course_prefix_handling.py @@ -1,100 +1,118 @@ -import unittest +import logging from syncmymoodle.__main__ import Node, SyncMyMoodle -class CoursePrefixHandlingTest(unittest.TestCase): - def format_course_name(self, handling, name): - smm = SyncMyMoodle({"course_prefix_handling": handling}) - return smm._format_course_name(name) - - def test_keep_preserves_course_name(self): - self.assertEqual( - self.format_course_name("keep", "(VO) Analysis"), - "(VO) Analysis", - ) - - def test_remove_strips_two_character_prefix(self): - self.assertEqual( - self.format_course_name("remove", "(VO) Analysis"), - "Analysis", - ) - - def test_suffix_moves_two_character_prefix_to_end(self): - self.assertEqual( - self.format_course_name("suffix", "(VU) Software Quality Assurance"), - "Software Quality Assurance (VU)", - ) - - def test_other_two_character_prefixes_are_supported(self): - self.assertEqual( - self.format_course_name("suffix", "(RE) Exercise Session"), - "Exercise Session (RE)", - ) - - def test_non_matching_names_are_preserved(self): - self.assertEqual(self.format_course_name("remove", "Analysis"), "Analysis") - self.assertEqual( - self.format_course_name("remove", "(VO)Analysis"), "(VO)Analysis" - ) - self.assertEqual( - self.format_course_name("remove", "(V) Analysis"), "(V) Analysis" - ) - self.assertEqual( - self.format_course_name("remove", "(ABC) Analysis"), - "(ABC) Analysis", - ) - - def test_invalid_mode_preserves_course_name(self): - with self.assertLogs("syncmymoodle.__main__", level="WARNING"): - self.assertEqual( - self.format_course_name("invalid", "(VO) Analysis"), - "(VO) Analysis", - ) - - -class CourseNameClashTest(unittest.TestCase): - def test_same_course_folder_name_without_url_gets_stable_suffixes(self): - root = Node("", -1, "Root", None) - semester = root.add_child("26ss", None, "Semester") - semester.add_child("Software Quality Assurance", 101, "Course") - semester.add_child("Software Quality Assurance", 102, "Course") - - root.remove_children_nameclashes() - - names = [course.name for course in semester.children] - self.assertEqual(len(names), 2) - self.assertEqual(len(set(names)), 2) - self.assertNotIn("Software Quality Assurance", names) - for name in names: - self.assertTrue(name.startswith("Software Quality Assurance_")) - - def test_same_section_name_without_url_keeps_legacy_merged_path(self): - root = Node("", -1, "Root", None) - course = root.add_child("Course", 100, "Course") - course.add_child("Case Study", 201, "Section") - course.add_child("Case Study", 202, "Section") - - root.remove_children_nameclashes() - - names = [section.name for section in course.children] - self.assertEqual(names, ["Case Study", "Case Study"]) - - def test_same_name_with_different_urls_still_gets_stable_suffixes(self): - root = Node("", -1, "Root", None) - section = root.add_child("General", None, "Section") - section.add_child("Slides", 201, "URL", url="https://example.com/slides-a") - section.add_child("Slides", 202, "URL", url="https://example.com/slides-b") - - root.remove_children_nameclashes() - - names = [link.name for link in section.children] - self.assertEqual(len(names), 2) - self.assertEqual(len(set(names)), 2) - self.assertNotIn("Slides", names) - for name in names: - self.assertTrue(name.startswith("Slides_")) - - -if __name__ == "__main__": - unittest.main() +def format_course_name(handling, name): + smm = SyncMyMoodle({"course_prefix_handling": handling}) + return smm._format_course_name(name) + + +def test_keep_preserves_course_name(): + assert format_course_name("keep", "(VO) Analysis") == "(VO) Analysis" + + +def test_remove_strips_two_character_prefix(): + assert format_course_name("remove", "(VO) Analysis") == "Analysis" + + +def test_suffix_moves_two_character_prefix_to_end(): + assert ( + format_course_name("suffix", "(VU) Software Quality Assurance") + == "Software Quality Assurance (VU)" + ) + + +def test_other_two_character_prefixes_are_supported(): + assert ( + format_course_name("suffix", "(RE) Exercise Session") == "Exercise Session (RE)" + ) + + +def test_non_matching_names_are_preserved(): + assert format_course_name("remove", "Analysis") == "Analysis" + assert format_course_name("remove", "(VO)Analysis") == "(VO)Analysis" + assert format_course_name("remove", "(V) Analysis") == "(V) Analysis" + assert format_course_name("remove", "(ABC) Analysis") == "(ABC) Analysis" + + +def test_invalid_mode_preserves_course_name(caplog): + with caplog.at_level(logging.WARNING, logger="syncmymoodle.__main__"): + assert format_course_name("invalid", "(VO) Analysis") == "(VO) Analysis" + assert any(record.levelno == logging.WARNING for record in caplog.records) + + +def test_page_content_url_normalization_preserves_larger_content_ids(): + root = Node("", -1, "Root", None) + + child = root.add_child( + "Video", + 101, + "Embedded videojs", + url=( + "https://moodle.rwth-aachen.de/pluginfile.php/104/" + "mod_page/content/315/page-video.mp4" + ), + ) + normalized_child = root.add_child( + "Legacy page file", + 102, + "Linked file [application/pdf]", + url=( + "https://moodle.rwth-aachen.de/pluginfile.php/104/" + "mod_page/content/3/legacy.pdf?forcedownload=1" + ), + ) + + assert child.url == ( + "https://moodle.rwth-aachen.de/pluginfile.php/104/" + "mod_page/content/315/page-video.mp4" + ) + assert normalized_child.url == ( + "https://moodle.rwth-aachen.de/pluginfile.php/104/" + "mod_page/content/legacy.pdf" + ) + + +def test_same_course_folder_name_without_url_gets_stable_suffixes(): + root = Node("", -1, "Root", None) + semester = root.add_child("26ss", None, "Semester") + semester.add_child("Software Quality Assurance", 101, "Course") + semester.add_child("Software Quality Assurance", 102, "Course") + + root.remove_children_nameclashes() + + names = [course.name for course in semester.children] + assert len(names) == 2 + assert len(set(names)) == 2 + assert "Software Quality Assurance" not in names + for name in names: + assert name.startswith("Software Quality Assurance_") + + +def test_same_section_name_without_url_keeps_legacy_merged_path(): + root = Node("", -1, "Root", None) + course = root.add_child("Course", 100, "Course") + course.add_child("Case Study", 201, "Section") + course.add_child("Case Study", 202, "Section") + + root.remove_children_nameclashes() + + names = [section.name for section in course.children] + assert names == ["Case Study", "Case Study"] + + +def test_same_name_with_different_urls_still_gets_stable_suffixes(): + root = Node("", -1, "Root", None) + section = root.add_child("General", None, "Section") + section.add_child("Slides", 201, "URL", url="https://example.com/slides-a") + section.add_child("Slides", 202, "URL", url="https://example.com/slides-b") + + root.remove_children_nameclashes() + + names = [link.name for link in section.children] + assert len(names) == 2 + assert len(set(names)) == 2 + assert "Slides" not in names + for name in names: + assert name.startswith("Slides_") diff --git a/tests/test_download_behavior.py b/tests/test_download_behavior.py new file mode 100644 index 0000000..fb87852 --- /dev/null +++ b/tests/test_download_behavior.py @@ -0,0 +1,252 @@ +import hashlib +import os + +from syncmymoodle.__main__ import Node + +from .helpers import FakeResponse, FakeSession, build_single_file_tree, make_syncer + +URL = ( + "https://moodle.rwth-aachen.de/pluginfile.php/301/mod_resource/content/1/slides.pdf" +) + + +def sha1(data: bytes) -> str: + return hashlib.sha1(data).hexdigest() + + +def seed_course_cache(config, *, timemodified, etag): + """Write a per-course cache to disk describing a previously synced file.""" + cache_syncer = make_syncer(config) + cached_root, _ = build_single_file_tree( + "slides.pdf", URL, timemodified=timemodified, etag=etag + ) + cache_syncer.root_node = cached_root + cache_syncer.cache_root_node() + + +def make_run_syncer(config, *, timemodified): + """Return a syncer plus the leaf node for the current (changed) sync run.""" + syncer = make_syncer(config) + syncer.session = FakeSession() + _, file_node = build_single_file_tree("slides.pdf", URL, timemodified=timemodified) + return syncer, file_node + + +# -------------------------------------------------------------------------- +# Actual download happy path (gap 2) +# -------------------------------------------------------------------------- + + +def test_download_streams_chunks_to_disk_and_records_metadata(tmp_path): + config = {"basedir": str(tmp_path)} + syncer, file_node = make_run_syncer(config, timemodified=1710000500) + download_path = syncer.get_sanitized_node_path(file_node) + chunks = [b"%PDF-1.4 first-chunk ", b"second-chunk ", b"third-chunk"] + etag = '"deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"' + syncer.session.add( + "GET", + URL, + FakeResponse( + headers={"Content-Type": "application/pdf", "ETag": etag}, + chunks=chunks, + ), + ) + + assert syncer.download_file(file_node) is True + + assert download_path.read_bytes() == b"".join(chunks) + # The temporary part-file is renamed away once the download completes. + assert not download_path.with_suffix(download_path.suffix + ".temp").exists() + # mtime is aligned with Moodle's timemodified so later runs detect changes. + assert int(download_path.stat().st_mtime) == 1710000500 + # The ETag is persisted on the node for the next run's change detection. + assert file_node.etag == etag + assert syncer.session.count("GET", URL) == 1 + + +def test_download_is_skipped_for_excluded_filetypes(tmp_path): + config = {"basedir": str(tmp_path), "exclude_filetypes": ["pdf"]} + syncer, file_node = make_run_syncer(config, timemodified=1710000500) + download_path = syncer.get_sanitized_node_path(file_node) + + # No GET route registered: a request would raise in the fake session. + assert syncer.download_file(file_node) is True + assert not download_path.exists() + assert syncer.session.calls == [] + + +def test_download_path_is_deduplicated_within_a_run(tmp_path): + # Two distinct nodes that resolve to the same on-disk path must download + # only once, exercising the lazy-initialised ``_downloaded_paths`` guard. + config = {"basedir": str(tmp_path)} + syncer = make_syncer(config) + syncer.session = FakeSession() + syncer.session.add( + "GET", + URL, + FakeResponse(headers={"Content-Type": "application/pdf"}, chunks=[b"data"]), + ) + _, first_node = build_single_file_tree("dup.pdf", URL) + section = first_node.parent + # Bypass add_child's duplicate-url guard to get a second node at the same path. + second_node = Node( + "dup.pdf", URL + "?v=2", "Linked file [application/pdf]", section, url=URL + ) + section.children.append(second_node) + + assert syncer.download_file(first_node) is True + assert syncer.download_file(second_node) is True + assert syncer.session.count("GET", URL) == 1 + + +# -------------------------------------------------------------------------- +# update_files_conflict handling (gap 1) +# -------------------------------------------------------------------------- + + +def _setup_conflict(tmp_path, conflict_mode): + """Cache a file, then locally modify it so the next run sees a conflict.""" + original = b"original remote content" + local_modified = b"locally edited content" + config = { + "basedir": str(tmp_path), + "updatefiles": True, + "update_files_conflict": conflict_mode, + } + seed_course_cache(config, timemodified=1710000300, etag=sha1(original)) + + syncer, file_node = make_run_syncer(config, timemodified=1710000400) + download_path = syncer.get_sanitized_node_path(file_node) + download_path.parent.mkdir(parents=True, exist_ok=True) + download_path.write_bytes(local_modified) + return syncer, file_node, download_path, local_modified + + +def _add_new_remote(syncer, body=b"updated remote content"): + syncer.session.add( + "GET", + URL, + FakeResponse(headers={"Content-Type": "application/pdf"}, chunks=[body]), + ) + return body + + +def test_conflict_keep_preserves_local_file_and_skips_download(tmp_path): + syncer, file_node, download_path, local_modified = _setup_conflict(tmp_path, "keep") + + # No GET registered: keep mode must not contact the server at all. + assert syncer.download_file(file_node) is True + assert download_path.read_bytes() == local_modified + assert syncer.session.calls == [] + + +def test_conflict_none_behaves_like_keep(tmp_path): + syncer, file_node, download_path, local_modified = _setup_conflict(tmp_path, "none") + + assert syncer.download_file(file_node) is True + assert download_path.read_bytes() == local_modified + assert syncer.session.calls == [] + + +def test_conflict_overwrite_replaces_local_file(tmp_path): + syncer, file_node, download_path, _ = _setup_conflict(tmp_path, "overwrite") + new_body = _add_new_remote(syncer) + + assert syncer.download_file(file_node) is True + assert download_path.read_bytes() == new_body + # Overwrite mode leaves no side-car conflict copy behind. + assert list(download_path.parent.glob("*.syncconflict.*")) == [] + assert syncer.session.count("GET", URL) == 1 + + +def test_conflict_rename_moves_local_file_aside_before_download(tmp_path): + syncer, file_node, download_path, local_modified = _setup_conflict( + tmp_path, "rename" + ) + new_body = _add_new_remote(syncer) + + assert syncer.download_file(file_node) is True + + # The fresh remote content lands at the canonical path. + assert download_path.read_bytes() == new_body + # The user's local edits are preserved in a side-car conflict file. + conflicts = list(download_path.parent.glob("*.syncconflict.*")) + assert len(conflicts) == 1 + assert conflicts[0].read_bytes() == local_modified + assert syncer.session.count("GET", URL) == 1 + + +def test_unknown_conflict_mode_defaults_to_rename(tmp_path): + syncer, file_node, download_path, local_modified = _setup_conflict( + tmp_path, "bogus-mode" + ) + new_body = _add_new_remote(syncer) + + assert syncer.download_file(file_node) is True + assert download_path.read_bytes() == new_body + conflicts = list(download_path.parent.glob("*.syncconflict.*")) + assert len(conflicts) == 1 + assert conflicts[0].read_bytes() == local_modified + + +def test_unchanged_timemodified_skips_download_despite_local_edit(tmp_path): + # When Moodle reports the same timemodified as the cache, the file is + # considered unchanged remotely and the local copy is left untouched. + original = b"original remote content" + config = { + "basedir": str(tmp_path), + "updatefiles": True, + "update_files_conflict": "rename", + } + seed_course_cache(config, timemodified=1710000300, etag=sha1(original)) + syncer, file_node = make_run_syncer(config, timemodified=1710000300) + download_path = syncer.get_sanitized_node_path(file_node) + download_path.parent.mkdir(parents=True, exist_ok=True) + download_path.write_bytes(b"locally edited content") + + assert syncer.download_file(file_node) is True + assert syncer.session.calls == [] + assert list(download_path.parent.glob("*.syncconflict.*")) == [] + + +def test_etag_failure_falls_back_to_timestamp_heuristic_conflict(tmp_path, monkeypatch): + # A faulty ETag cache is treated as if there were no cached ETag, so the + # timestamp heuristic decides. Here the local mtime differs from the cached + # Moodle timestamp, so it is a conflict and the local edits are kept aside. + syncer, file_node, download_path, local_modified = _setup_conflict( + tmp_path, "rename" + ) + new_body = _add_new_remote(syncer) + + def boom(path, etag): + raise OSError("cannot read file for hashing") + + monkeypatch.setattr(syncer, "_local_file_matches_etag", boom) + + assert syncer.download_file(file_node) is True + assert download_path.read_bytes() == new_body + conflicts = list(download_path.parent.glob("*.syncconflict.*")) + assert len(conflicts) == 1 + assert conflicts[0].read_bytes() == local_modified + + +def test_etag_failure_falls_back_to_timestamp_heuristic_no_conflict( + tmp_path, monkeypatch +): + # Same fallback, but the local mtime matches the cached Moodle timestamp, so + # the timestamp heuristic reports no local change and the file is updated + # cleanly without leaving a side-car conflict copy behind. + syncer, file_node, download_path, _ = _setup_conflict(tmp_path, "rename") + # Align the local mtime with the cached timemodified the heuristic compares + # against, mimicking a file that was downloaded but never edited locally. + os.utime(download_path, (1710000300, 1710000300)) + new_body = _add_new_remote(syncer) + + def boom(path, etag): + raise OSError("cannot read file for hashing") + + monkeypatch.setattr(syncer, "_local_file_matches_etag", boom) + + assert syncer.download_file(file_node) is True + assert download_path.read_bytes() == new_body + assert list(download_path.parent.glob("*.syncconflict.*")) == [] diff --git a/tests/test_storage_safety.py b/tests/test_storage_safety.py new file mode 100644 index 0000000..47b3be7 --- /dev/null +++ b/tests/test_storage_safety.py @@ -0,0 +1,75 @@ +import gzip +import json +import stat + +from syncmymoodle.__main__ import Node + +from .helpers import FakeSession, make_syncer + + +def test_sanitized_node_path_stays_inside_basedir(tmp_path): + syncer = make_syncer({"basedir": str(tmp_path)}) + root = Node("", -1, "Root", None) + bad_node = root.add_child("%2e%2e", 1, "Section") + + target_path = syncer.get_sanitized_node_path(bad_node) + + assert target_path == tmp_path / "_" + assert target_path.resolve(strict=False).is_relative_to(tmp_path) + + +def test_private_gzip_json_roundtrip_uses_private_permissions(tmp_path): + syncer = make_syncer() + target = tmp_path / "session" + + syncer._write_private_gzip_json(target, {"format": "test", "value": 1}) + + assert stat.S_IMODE(target.stat().st_mode) == 0o600 + with target.open("rb") as handle: + assert json.loads(gzip.decompress(handle.read()).decode("utf-8")) == { + "format": "test", + "value": 1, + } + assert syncer._read_private_gzip_json(target, "test data") == { + "format": "test", + "value": 1, + } + + +def test_download_uses_course_cache_to_skip_unchanged_file(tmp_path): + config = {"basedir": str(tmp_path), "updatefiles": True} + cached_syncer = make_syncer(config) + cached_root = Node("", -1, "Root", None) + semester = cached_root.add_child("26ss", None, "Semester") + course = semester.add_child("Cache Behavior", 301, "Course") + section = course.add_child("General", 401, "Section") + cached_file = section.add_child( + "slides.pdf", + "https://moodle.rwth-aachen.de/pluginfile.php/301/slides.pdf", + "Linked file [application/pdf]", + url="https://moodle.rwth-aachen.de/pluginfile.php/301/slides.pdf", + timemodified=1710000300, + ) + cached_syncer.root_node = cached_root + cached_syncer.cache_root_node() + + download_path = cached_syncer.get_sanitized_node_path(cached_file) + download_path.parent.mkdir(parents=True, exist_ok=True) + download_path.write_bytes(b"already downloaded") + + syncer = make_syncer(config) + syncer.session = FakeSession() + current_root = Node("", -1, "Root", None) + current_semester = current_root.add_child("26ss", None, "Semester") + current_course = current_semester.add_child("Cache Behavior", 301, "Course") + current_section = current_course.add_child("General", 401, "Section") + current_file = current_section.add_child( + "slides.pdf", + "https://moodle.rwth-aachen.de/pluginfile.php/301/slides.pdf", + "Linked file [application/pdf]", + url="https://moodle.rwth-aachen.de/pluginfile.php/301/slides.pdf", + timemodified=1710000300, + ) + + assert syncer.download_file(current_file) is True + assert syncer.session.calls == [] diff --git a/tests/test_sync_filtering.py b/tests/test_sync_filtering.py new file mode 100644 index 0000000..e9d7ee1 --- /dev/null +++ b/tests/test_sync_filtering.py @@ -0,0 +1,51 @@ +from .helpers import FakeSession, make_syncer, node_rows + +FILTER_COURSES = [ + {"id": 201, "shortname": "Current Semester", "idnumber": "26ss-current"}, + {"id": 202, "shortname": "Selected Old Semester", "idnumber": "25ws-selected"}, + {"id": 203, "shortname": "Skipped Current Semester", "idnumber": "26ss-skipped"}, +] + + +def test_selected_courses_override_semester_filter(): + synced_course_ids = [] + syncer = make_syncer( + { + "selected_courses": [ + "https://moodle.rwth-aachen.de/course/view.php?id=202" + ], + "only_sync_semester": ["26ss"], + } + ) + syncer.get_all_courses = lambda: FILTER_COURSES # type: ignore[method-assign] + syncer.get_course = lambda course_id: synced_course_ids.append(course_id) or [] # type: ignore[method-assign] + syncer.session = FakeSession() + + syncer.sync() + + assert synced_course_ids == [202] + assert node_rows(syncer.root_node) == [ + "Semester | 25ws | | | ", + "Course | 25ws/Selected Old Semester | | | ", + ] + + +def test_skip_courses_and_semester_filter_limit_synced_courses(): + synced_course_ids = [] + syncer = make_syncer( + { + "skip_courses": ["https://moodle.rwth-aachen.de/course/view.php?id=203"], + "only_sync_semester": ["26ss"], + } + ) + syncer.get_all_courses = lambda: FILTER_COURSES # type: ignore[method-assign] + syncer.get_course = lambda course_id: synced_course_ids.append(course_id) or [] # type: ignore[method-assign] + syncer.session = FakeSession() + + syncer.sync() + + assert synced_course_ids == [201] + assert node_rows(syncer.root_node) == [ + "Semester | 26ss | | | ", + "Course | 26ss/Current Semester | | | ", + ] diff --git a/tests/test_sync_fixtures.py b/tests/test_sync_fixtures.py new file mode 100644 index 0000000..d6b6815 --- /dev/null +++ b/tests/test_sync_fixtures.py @@ -0,0 +1,299 @@ +from syncmymoodle.__main__ import Node + +from .helpers import ( + FakeResponse, + FakeSession, + assert_snapshot, + install_moodle_fixtures, + load_fixture, + load_json_fixture, + make_syncer, + node_rows, +) + + +def test_nested_moodle_folder_paths_are_preserved(): + courses = [load_json_fixture("moodle", "courses.json")[0]] + syncer = make_syncer() + install_moodle_fixtures( + syncer, + courses, + {101: load_json_fixture("moodle", "nested_folder_course.json")}, + ) + syncer.session = FakeSession() + + syncer.sync() + + assert_snapshot("nested_folder_tree.txt", node_rows(syncer.root_node)) + + +def test_assignment_intro_opencast_embed_is_added_to_assignment_node(): + courses = [load_json_fixture("moodle", "courses.json")[1]] + syncer = make_syncer( + { + "used_modules": { + "assign": True, + "resource": False, + "url": {"youtube": False, "opencast": True, "sciebo": False}, + "folder": False, + } + } + ) + install_moodle_fixtures( + syncer, + courses, + {102: load_json_fixture("moodle", "assignment_opencast_course.json")}, + {102: load_json_fixture("moodle", "assignment_opencast_assignments.json")}, + ) + syncer.session = FakeSession() + + authenticated = [] + syncer._authenticate_opencast_episode = ( # type: ignore[method-assign] + lambda course_id, episode_id: authenticated.append((course_id, episode_id)) + or True + ) + syncer.extractTrackFromEpisode = ( # type: ignore[method-assign] + lambda episode_id: f"https://video.example.test/{episode_id}/presentation.mp4" + ) + + syncer.sync() + + assert authenticated == [(102, "11111111-2222-4333-8444-555555555555")] + assert_snapshot("assignment_opencast_tree.txt", node_rows(syncer.root_node)) + + +def test_skip_rules_apply_to_sections_modules_links_and_domains(): + courses = [load_json_fixture("moodle", "courses.json")[2]] + syncer = make_syncer( + { + "exclude_sections": {"*": ["Hidden*"]}, + "exclude_modules": {"103": ["Skip Module"]}, + "exclude_links": ["*excluded.pdf"], + "allowed_domains": ["moodle.rwth-aachen.de"], + "used_modules": { + "assign": False, + "resource": True, + "url": {"youtube": False, "opencast": False, "sciebo": False}, + "folder": False, + }, + } + ) + install_moodle_fixtures( + syncer, + courses, + {103: load_json_fixture("moodle", "skip_rules_course.json")}, + ) + syncer.session = FakeSession() + + syncer.sync() + + assert_snapshot("skip_rules_tree.txt", node_rows(syncer.root_node)) + + +def test_sciebo_public_share_is_cached_per_sync_run(): + link = "https://rwth-aachen.sciebo.de/s/share-token-123" + public_root = "https://rwth-aachen.sciebo.de/public.php/webdav/" + public_slides = "https://rwth-aachen.sciebo.de/public.php/webdav/slides/" + syncer = make_syncer( + { + "used_modules": { + "assign": False, + "resource": False, + "url": {"youtube": False, "opencast": False, "sciebo": True}, + "folder": False, + } + } + ) + session = FakeSession() + session.add( + "GET", link, FakeResponse(text=load_fixture("sciebo", "public_share.html")) + ) + session.add( + "PROPFIND", + public_root, + FakeResponse(text=load_fixture("sciebo", "propfind_root.xml")), + ) + session.add( + "PROPFIND", + public_slides, + FakeResponse(text=load_fixture("sciebo", "propfind_slides.xml")), + ) + syncer.session = session + + root = Node("", -1, "Root", None) + first_parent = root.add_child("First occurrence", 1, "Section") + second_parent = root.add_child("Second occurrence", 2, "Section") + + syncer.scanForLinks(link, first_parent, 101) + syncer.scanForLinks(link, second_parent, 101) + + assert session.count("GET", link) == 1 + assert session.count("PROPFIND", public_root) == 1 + assert session.count("PROPFIND", public_slides) == 1 + assert [ + row.replace("First occurrence/", "") for row in node_rows(first_parent) + ] == [row.replace("Second occurrence/", "") for row in node_rows(second_parent)] + assert node_rows(first_parent) == [ + "Sciebo Folder | First occurrence/sciebo-share-token-123 | | | ", + "Sciebo File | First occurrence/sciebo-share-token-123/readme.pdf | " + "https://rwth-aachen.sciebo.de/public.php/webdav/readme.pdf | | " + "1111111111111111111111111111111111111111", + "Sciebo Folder | First occurrence/sciebo-share-token-123/slides | | | " + '"folder-slides"', + "Sciebo File | First occurrence/sciebo-share-token-123/slides/deck.pdf | " + "https://rwth-aachen.sciebo.de/public.php/webdav/slides/deck.pdf | | " + "2222222222222222222222222222222222222222", + ] + + +def test_mixed_course_sync_tree_covers_common_module_surfaces(): + courses = load_json_fixture("moodle", "mixed_courses.json") + direct_pdf = "https://files.example.test/direct.pdf" + html_overview = "https://files.example.test/overview.html" + page_url = "https://moodle.rwth-aachen.de/mod/page/view.php?id=315" + h5p_url = "https://moodle.rwth-aachen.de/mod/h5pactivity/view.php?id=317" + h5p_iframe_url = "https://moodle.rwth-aachen.de/h5p/embed/317" + syncer = make_syncer( + { + "used_modules": { + "assign": True, + "resource": True, + "url": {"youtube": True, "opencast": True, "sciebo": False}, + "folder": True, + } + } + ) + install_moodle_fixtures( + syncer, + courses, + {104: load_json_fixture("moodle", "mixed_course.json")}, + {104: load_json_fixture("moodle", "mixed_assignments.json")}, + {412: load_json_fixture("moodle", "mixed_submission_files.json")}, + {104: load_json_fixture("moodle", "mixed_folders.json")}, + ) + session = FakeSession() + session.add( + "HEAD", + direct_pdf, + FakeResponse(headers={"Content-Type": "application/pdf"}), + ) + session.add( + "HEAD", + html_overview, + FakeResponse(headers={"Content-Type": "text/html"}), + ) + session.add( + "GET", + html_overview, + FakeResponse(text=load_fixture("html", "external_overview.html")), + ) + session.add( + "GET", + page_url, + FakeResponse(text=load_fixture("html", "page_module.html")), + ) + session.add( + "GET", h5p_url, FakeResponse(text=load_fixture("html", "h5p_view.html")) + ) + session.add( + "GET", + h5p_iframe_url, + FakeResponse(text=load_fixture("html", "h5p_iframe.html")), + ) + syncer.session = session + syncer._authenticate_opencast_episode = lambda course_id, episode_id: True # type: ignore[method-assign] + syncer.extractTrackFromEpisode = lambda episode_id: ( # type: ignore[method-assign] + f"https://video.example.test/{episode_id}/presentation.mp4" + ) + + syncer.sync() + + assert session.count("HEAD", direct_pdf) == 1 + assert session.count("GET", direct_pdf) == 0 + assert session.count("HEAD", html_overview) == 1 + assert session.count("GET", html_overview) == 1 + assert session.count("GET", page_url) == 1 + assert session.count("GET", h5p_url) == 1 + assert session.count("GET", h5p_iframe_url) == 1 + assert_snapshot("mixed_course_tree.txt", node_rows(syncer.root_node)) + + +def test_opencast_lti_single_and_series_use_lti_and_api_routes(): + courses = [load_json_fixture("moodle", "courses.json")[0]] + single_lti_url = ( + "https://moodle.rwth-aachen.de/mod/lti/launch.php?id=501&triggerview=0" + ) + series_lti_url = ( + "https://moodle.rwth-aachen.de/mod/lti/launch.php?id=502&triggerview=0" + ) + lti_submit_url = "https://engage.streaming.rwth-aachen.de/lti" + single_episode = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" + series_id = "series-1111-2222" + series_episode_a = "bbbbbbbb-cccc-4ddd-8eee-ffffffffffff" + series_episode_b = "cccccccc-dddd-4eee-8fff-aaaaaaaaaaaa" + series_url = ( + "https://engage.streaming.rwth-aachen.de/search/episode.json" + f"?limit=100&offset=0&sid={series_id}" + ) + syncer = make_syncer( + { + "used_modules": { + "assign": False, + "resource": False, + "url": {"youtube": False, "opencast": True, "sciebo": False}, + "folder": False, + } + } + ) + install_moodle_fixtures( + syncer, + courses, + {101: load_json_fixture("moodle", "opencast_lti_course.json")}, + ) + session = FakeSession() + session.add( + "GET", + single_lti_url, + FakeResponse(text=load_fixture("opencast", "lti_single.html")), + ) + session.add( + "GET", + series_lti_url, + FakeResponse(text=load_fixture("opencast", "lti_series.html")), + ) + session.add("POST", lti_submit_url, FakeResponse(text="ok")) + session.add( + "GET", + "https://engage.streaming.rwth-aachen.de/search/episode.json" + f"?id={single_episode}", + FakeResponse(json_payload=load_json_fixture("opencast", "episode_single.json")), + ) + session.add( + "GET", + series_url, + FakeResponse(json_payload=load_json_fixture("opencast", "series.json")), + ) + session.add( + "GET", + "https://engage.streaming.rwth-aachen.de/search/episode.json" + f"?id={series_episode_a}", + FakeResponse( + json_payload=load_json_fixture("opencast", "episode_series_a.json") + ), + ) + session.add( + "GET", + "https://engage.streaming.rwth-aachen.de/search/episode.json" + f"?id={series_episode_b}", + FakeResponse( + json_payload=load_json_fixture("opencast", "episode_series_b.json") + ), + ) + syncer.session = session + + syncer.sync() + + assert session.count("GET", single_lti_url) == 1 + assert session.count("GET", series_lti_url) == 1 + assert session.count("POST", lti_submit_url) == 2 + assert_snapshot("opencast_lti_tree.txt", node_rows(syncer.root_node))