From 7223e4636113d2f51fe5b2ceb66595dd1d288b72 Mon Sep 17 00:00:00 2001 From: pfn-djf Date: Wed, 16 Apr 2025 14:18:08 +0200 Subject: [PATCH 1/6] ignore pattern in multilevel nested histories --- .github/workflows/build+test.yml | 2 +- ascmhl/commands.py | 64 +- ascmhl/generator.py | 196 ++++- ascmhl/history.py | 21 +- ascmhl/utils.py | 7 + tests/conftest.py | 92 +++ tests/test_ignore.py | 19 + tests/test_ignore_extended.py | 1200 ++++++++++++++++++++++++++++++ 8 files changed, 1592 insertions(+), 9 deletions(-) create mode 100644 tests/test_ignore_extended.py diff --git a/.github/workflows/build+test.yml b/.github/workflows/build+test.yml index 137a9c9..86fbb94 100644 --- a/.github/workflows/build+test.yml +++ b/.github/workflows/build+test.yml @@ -5,7 +5,7 @@ name: ascmhl-build-test on: push: - branches: [ master, dev/windowsPathHandling ] + branches: [ master, ignore_pattern_in_nested_histories ] pull_request: branches: [ master ] diff --git a/ascmhl/commands.py b/ascmhl/commands.py index 824ec07..808c46e 100644 --- a/ascmhl/commands.py +++ b/ascmhl/commands.py @@ -12,6 +12,7 @@ import platform import click +import pathspec from lxml import etree from . import logger @@ -35,6 +36,8 @@ from typing import Dict from collections import namedtuple +from .utils import check_path_is_absolute_to_history + @click.command() @click.argument("root_path", type=click.Path(exists=True)) @@ -231,6 +234,9 @@ def create_for_folder_subcommand( # start a verification session on the existing history session = MHLGenerationCreationSession(existing_history, ignore_spec) + # update the ignore spec and include ignores from nested histories + ignore_spec = get_ignore_spec_including_nested_ignores(existing_history, ignore_list, ignore_spec_file) + num_failed_verifications = 0 # store the directory hashes of sub folders so we can use it when calculating the hash of the parent folder # the mapping lookups will follow the dictionary format of [string: [hash_format: hash_value]] where string @@ -239,7 +245,7 @@ def create_for_folder_subcommand( dir_structure_hash_mapping_lookup = {} hash_format_list = sorted(hash_formats) - for folder_path, children in post_order_lexicographic(root_path, session.ignore_spec.get_path_spec()): + for folder_path, children in post_order_lexicographic(root_path, ignore_spec.get_path_spec()): # generate directory hashes dir_hash_context_lookup = {} @@ -695,7 +701,7 @@ def verify_directory_hash_subcommand( existing_history = MHLHistory.load_from_path(root_path) - ignore_spec = ignore.MHLIgnoreSpec(existing_history.latest_ignore_patterns(), ignore_list, ignore_spec_file) + ignore_spec = get_ignore_spec_including_nested_ignores(existing_history, ignore_list, ignore_spec_file) # FIXME: Update once argument signature has been modified to supply a list of formats hash_formats = [] @@ -1031,7 +1037,7 @@ def diff_entire_folder_against_full_history_subcommand(root_path, verbose, ignor num_failed_verifications = 0 num_new_files = 0 - ignore_spec = ignore.MHLIgnoreSpec(existing_history.latest_ignore_patterns(), ignore_list, ignore_spec_file) + ignore_spec = get_ignore_spec_including_nested_ignores(existing_history, ignore_list, ignore_spec_file) for folder_path, children in post_order_lexicographic(root_path, ignore_spec.get_path_spec()): for item_name, is_dir in children: @@ -1572,3 +1578,55 @@ def seal_file_path(existing_history, file_path, hash_formats: [str], session) -> hash_result_lookup[hash_format] = SealPathResult(current_hash_lookup[hash_format], success) return hash_result_lookup + + +def get_ignore_spec_including_nested_ignores(existing_history, ignore_list, ignore_spec_file=None): + """Get the ignore patterns from nested histories with their respective paths, + so that ignored files from nested histories are also ignored in this session, but are not stored + in the root ascmhl manifest""" + ignore_patterns_cumulated = ignore.default_ignore_list() + # handle non-existent ignores in root history + if existing_history.latest_ignore_patterns() is not None: + for x in existing_history.latest_ignore_patterns(): + if x not in ignore.default_ignore_list(): + ignore_patterns_cumulated.append(x) + + for x in existing_history.latest_ignore_pattern_from_nested_histories(): + ignore_patterns_cumulated.append(x) + + for x in ignore_list: + ignore_patterns_cumulated.append(x) + + patterns_from_file = [] + if ignore_spec_file: + with open(ignore_spec_file, "r") as fh: + patterns_from_file.extend(line.rstrip("\n") for line in fh if line != "\n") + for x in patterns_from_file: + if x not in ignore_patterns_cumulated: + ignore_patterns_cumulated.append(x) + + # we now build the absolute ignore paths for the current session from all nested ignores + # otherwise the post_order_lexicographic() won't ignore these paths + absolute_ignore_paths = [] + path = existing_history.get_root_path() + for pattern in ignore_patterns_cumulated: + if pattern in ignore.default_ignore_list(): + absolute_ignore_paths.append(pattern) + else: + if pattern.find("/") != -1: + if pattern.startswith("/"): + absolute_ignore_paths.append(path + pattern) + elif pattern.startswith("**"): + absolute_ignore_paths.append(pattern) + elif pattern.endswith("/") and pattern[:-1].find("/") == -1: + absolute_ignore_paths.append(pattern) + elif pattern.endswith("/" + "**") and pattern[:-3].find("/") == -1: + absolute_ignore_paths.append(pattern) + else: + absolute_ignore_paths.append(path + os.sep + pattern) + else: + absolute_ignore_paths.append(pattern) + + normalized_paths = [pathspec.util.normalize_file(p) for p in absolute_ignore_paths] + spec = ignore.MHLIgnoreSpec(existing_history.latest_ignore_patterns(), normalized_paths) + return spec diff --git a/ascmhl/generator.py b/ascmhl/generator.py index 170ff8a..d01d248 100644 --- a/ascmhl/generator.py +++ b/ascmhl/generator.py @@ -7,10 +7,12 @@ __email__ = "opensource@pomfort.com" """ +import os from collections import defaultdict +from pathlib import PureWindowsPath from typing import Dict, List -from . import chain_xml_parser +from . import chain_xml_parser, ignore from . import logger from .ignore import MHLIgnoreSpec from .hashlist import MHLHashList, MHLHashEntry, MHLCreatorInfo, MHLProcessInfo @@ -296,9 +298,7 @@ def commit(self, creator_info: MHLCreatorInfo, process_info: MHLProcessInfo): new_hash_list.process_info.root_media_hash = process_info.root_media_hash new_hash_list.process_info.hashlist_custom_basename = process_info.hashlist_custom_basename new_hash_list.process_info.process = process_info.process - new_hash_list.process_info.ignore_spec = MHLIgnoreSpec( - history.latest_ignore_patterns(), self.ignore_spec.get_pattern_list() - ) + new_hash_list.process_info.ignore_spec = self.get_relevant_ignore_pattern(history) history.write_new_generation(new_hash_list) relative_generation_path = self.root_history.get_relative_file_path(new_hash_list.file_path) @@ -307,3 +307,191 @@ def commit(self, creator_info: MHLCreatorInfo, process_info: MHLProcessInfo): referenced_hash_lists[history.parent_history].append(new_hash_list) chain_xml_parser.write_chain(history.chain, new_hash_list) + + def get_relevant_ignore_pattern(self, history) -> MHLIgnoreSpec: + """ + Only store the relevant ignore patterns for an ascmhl-history and ignore others. + This will split the pattern into the relevant bits for the lowest ascmhl-history relative to it + """ + + # get the ignore pattern from the latest history, if there is none, use the default pattern + final_ignores = [] + latest_ignore_patterns = history.latest_ignore_patterns() + if latest_ignore_patterns is None: + final_ignores += ignore.default_ignore_list() + else: + final_ignores += latest_ignore_patterns + + ignore_patterns = self.ignore_spec.get_pattern_list() + + history_path = history.get_root_path() + # get the highest parent history to build the correct relative paths for this generation + if history.parent_history is not None: + parent_history = history.parent_history + while parent_history.parent_history is not None: + parent_history = parent_history.parent_history + + parent_history_path = parent_history.get_root_path() + parent_rel_path = os.path.relpath(history_path, parent_history_path) + for pattern in ignore_patterns: + if not pattern in final_ignores: + if "/" not in pattern: + final_ignores.append(pattern) + elif pattern.startswith("/**/"): + final_ignores.append(pattern) + elif belongs_to_child(pattern, history, parent_history_path): + # if child is ignored itself, we need to append the ignore pattern to the next parent + for child in history.walk_child_histories(history): + child_root: str + if os.name == "nt": + child_root = PureWindowsPath(child.get_root_path()).as_posix() + else: + child_root = child.get_root_path() + if child_root.endswith(pattern): + pattern = extract_ignore_pattern(pattern, history_path) + final_ignores.append(pattern) + continue + elif belongs_to_parent_or_neighbour(pattern, parent_rel_path): + continue + else: + pattern = extract_ignore_pattern(pattern, parent_rel_path) + final_ignores.append(pattern) + else: + continue + else: + for pattern in ignore_patterns: + if not pattern in final_ignores: + if ( + not belongs_to_child(pattern, history, history_path) + and not pattern in ignore.default_ignore_list() + ): + if pattern.startswith("/"): + final_ignores.append(pattern) + elif pattern.find("/") != -1: + final_ignores.append(extract_ignore_pattern(pattern)) + else: + final_ignores.append(pattern) + else: + for child in history.walk_child_histories(history): + child_root: str + if os.name == "nt": + child_root = PureWindowsPath(child.get_root_path()).as_posix() + else: + child_root = child.get_root_path() + if child_root.endswith(pattern): + if history == child.parent_history: + pattern = extract_ignore_pattern(pattern) + final_ignores.append(pattern) + return MHLIgnoreSpec(final_ignores, latest_ignore_patterns) + + +def belongs_to_child(pattern, history, parent_history_path, ignore_child=None) -> bool: + if pattern.startswith("/"): + pattern = pattern[1:] + for child in history.child_histories: + if ignore_child == child: + continue + child_path = child.get_root_path() + parent_rel_path = os.path.relpath(child_path, parent_history_path) + if os.name == "nt": + parent_rel_path = parent_rel_path.replace("\\", "/") + if pattern.startswith(parent_rel_path): + return True + + return False + + +def belongs_to_parent_or_neighbour(pattern, parent_rel_path) -> bool: + if "/" not in pattern: + return False + if pattern.startswith("/") and "/" not in pattern[1:]: + return True + if pattern.endswith("/") and "/" not in pattern[:-1]: + return False + + if pattern.startswith("**/"): + return False + + pattern_parts = pattern.strip("/").split("/") + parent_parts = parent_rel_path.strip(os.sep).split(os.sep) + i = 0 + while i < min(len(pattern_parts), len(parent_parts)): + if pattern_parts[i] != parent_parts[i]: + if i > 0 and i < len(pattern_parts) - 1 and pattern_parts[i] == "**": + return False + return True + else: + i += 1 + if i == len(pattern_parts): + return True + + return False + + +def extract_ignore_pattern(pattern: str, parent_rel_path=None) -> str: + if pattern.startswith("/"): + if "/" in pattern[1:]: + pattern = pattern[1:] + else: + return pattern + + pattern_rel_path = _extract_pattern_relative_to_history(pattern, parent_rel_path) + + if pattern_rel_path is not None: + if pattern.endswith("/"): + if "/" in pattern[:-1]: + return "/" + (pattern if pattern.startswith("**/") else pattern_rel_path) + return pattern + + if pattern.endswith("/**"): + if pattern.startswith("**/"): + return pattern + if "/" in pattern[:-3]: + return pattern_rel_path if pattern.startswith("/") else "/" + pattern_rel_path + return pattern + + if "/" in pattern[:-1]: + return ( + "/" + pattern + if pattern.startswith("**/") + else pattern_rel_path if pattern_rel_path.startswith("/") else "/" + pattern_rel_path + ) + + return pattern + + +def _extract_pattern_relative_to_history(pattern: str, history_path=None) -> str: + if pattern.startswith("**"): + return pattern + if history_path is None: + return pattern + + pattern_parts = pattern.lstrip("/").split("/") + history_path_parts = history_path.lstrip(os.sep).split(os.sep) + + i = j = k = 0 + + while i < len(history_path_parts): + if history_path_parts[i] == pattern_parts[0]: + break + else: + i += 1 + + while j < len(pattern_parts) and i + j < len(history_path_parts): + if history_path_parts[i + j] == pattern_parts[j]: + j += 1 + else: + break + + result = "" + + while j < (len(pattern_parts)): + if k == 0: + result += pattern_parts[j] + j += 1 + k += 1 + else: + result += "/" + pattern_parts[j] + j += 1 + if result != "": + return result diff --git a/ascmhl/history.py b/ascmhl/history.py index e368292..1c8fa1b 100644 --- a/ascmhl/history.py +++ b/ascmhl/history.py @@ -12,7 +12,7 @@ import re from datetime import datetime, date, time -from . import hasher +from . import hasher, ignore from .__version__ import ascmhl_folder_name, ascmhl_file_extension, ascmhl_chainfile_name, ascmhl_collectionfile_name from . import hashlist_xml_parser, chain_xml_parser from .utils import datetime_now_filename_string @@ -98,6 +98,25 @@ def latest_ignore_patterns(self) -> Optional[List[str]]: return None return hash_list.process_info.ignore_spec.get_pattern_list() + def latest_ignore_pattern_from_nested_histories(self) -> Optional[List[str]]: + parent_path = self.get_root_path() + cumulated_ignores = [] + for path, history in self.child_history_mappings.items(): + for pattern in history.latest_ignore_patterns(): + # don't add the default pattern + child_path = history.get_root_path() + path = os.path.relpath(child_path, parent_path) + if pattern in ignore.default_ignore_list(): + continue + else: + # return the directory of the history with the pattern appended + if pattern.find("/") != -1: + cumulated_ignores.append(path + pattern) + else: + cumulated_ignores.append(path + "/**/" + pattern) + + return cumulated_ignores + # methods to query and compare hashes def find_original_hash_entry_for_path(self, relative_path: str) -> Optional[MHLHashEntry]: """Searches the history for the first (original) hash of a file diff --git a/ascmhl/utils.py b/ascmhl/utils.py index 351bc38..c783a89 100644 --- a/ascmhl/utils.py +++ b/ascmhl/utils.py @@ -60,3 +60,10 @@ def convert_posix_to_local_path(path: str) -> str: if os.name == "nt": return str(PureWindowsPath(PurePosixPath(path))) return path + + +def check_path_is_absolute_to_history(base_path, relative_path) -> bool: + base_abs = os.path.abspath(base_path) + relative_abs = os.path.abspath(relative_path) + + return relative_abs.startswith(base_abs) diff --git a/tests/conftest.py b/tests/conftest.py index 49422de..774f223 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -90,3 +90,95 @@ def simple_mhl_folder(fs): # create a simple folder structure with two files fs.create_file("/root/Stuff.txt", contents="stuff\n") fs.create_file("/root/A/A1.txt", contents="A1\n") + + +@pytest.fixture +@freeze_time("2020-01-15 13:00:00") +def post_house_file_structure(fs): + runner = CliRunner() + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/A1.txt", contents="A1") + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/A2.txt", contents="A2") + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/A3.txt", contents="A3") + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/Sidecar.txt", contents="Sidecar1") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1/CameraMedia/A"), "-h", "xxh64"]) + assert result.exit_code == 0 + + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/B1.txt", contents="B1") + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/B2.txt", contents="B2") + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/B3.txt", contents="B3") + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/Sidecar.txt", contents="Sidecar2") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1/CameraMedia/B"), "-h", "xxh64"]) + assert result.exit_code == 0 + + fs.create_file("/root/ShootingDay1/CameraMedia/Report.pdf", contents="A1-3, B1-3") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1/CameraMedia"), "-h", "xxh64"]) + assert result.exit_code == 0 + + fs.create_file("/root/ShootingDay1/Sound/Takes/Sound.txt", contents="Sound") + fs.create_file("/root/ShootingDay1/Sound/Sidecar.txt", contents="Sound Sidecar") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1/Sound"), "-h", "xxh64"]) + assert result.exit_code == 0 + + fs.create_file("/root/ShootingDay1/Report.pdf", contents="A1-3, B1-3, Sound") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1"), "-h", "xxh64"]) + assert result.exit_code == 0 + + # these are particularly relevant for ignore pattern testing + fs.create_file("/root/ShootingDay1/CameraMedia/A/Proxy/A001/A1_p.txt", contents="A1 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/A/Proxy/A001/A2_p.txt", contents="A2 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/A/Proxy/A001/A3_p.txt", contents="A3 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/A/Proxy/A001/A001.ale", contents="A001 ALE Proxy") + + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/B001/A1_p.txt", contents="A1 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/B001/A2_p.txt", contents="A2 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/B001/A3_p.txt", contents="A3 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/B001/B001.ale", contents="B001 ALE Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/Proxy", contents="Proxy") + + +@pytest.fixture +@freeze_time("2020-01-15 13:00:00") +def post_house_file_structure_with_range(fs): + runner = CliRunner() + for i in range(1, 5): + for j in range(1, 5): + fs.create_file(f"/root/ShootingDay1/CameraMedia/A/A00{i}/A00{i}C00{j}.mov", contents=f"A00{i}C00{j}") + + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/A1.txt", contents="A1") + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/A2.txt", contents="A2") + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/A3.txt", contents="A3") + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/Sidecar.txt", contents="Sidecar1") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1/CameraMedia/A"), "-h", "xxh64"]) + assert result.exit_code == 0 + + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/B1.txt", contents="B1") + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/B2.txt", contents="B2") + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/B3.txt", contents="B3") + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/Sidecar.txt", contents="Sidecar2") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1/CameraMedia/B"), "-h", "xxh64"]) + assert result.exit_code == 0 + + fs.create_file("/root/ShootingDay1/CameraMedia/Report.pdf", contents="A1-3, B1-3") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1/CameraMedia"), "-h", "xxh64"]) + assert result.exit_code == 0 + + fs.create_file("/root/ShootingDay1/Sound/Takes/Sound.txt", contents="Sound") + fs.create_file("/root/ShootingDay1/Sound/Sidecar.txt", contents="Sound Sidecar") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1/Sound"), "-h", "xxh64"]) + assert result.exit_code == 0 + + fs.create_file("/root/ShootingDay1/Report.pdf", contents="A1-3, B1-3, Sound") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1"), "-h", "xxh64"]) + assert result.exit_code == 0 + + # these are particularly relevant for ignore pattern testing + fs.create_file("/root/ShootingDay1/CameraMedia/A/Proxy/A001/A1_p.txt", contents="A1 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/A/Proxy/A001/A2_p.txt", contents="A2 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/A/Proxy/A001/A3_p.txt", contents="A3 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/A/Proxy/A001/A001.ale", contents="A001 ALE Proxy") + + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/B001/A1_p.txt", contents="A1 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/B001/A2_p.txt", contents="A2 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/B001/A3_p.txt", contents="A3 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/B001/B001.ale", contents="B001 ALE Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/Proxy", contents="Proxy") diff --git a/tests/test_ignore.py b/tests/test_ignore.py index 6b66663..bdf1caf 100644 --- a/tests/test_ignore.py +++ b/tests/test_ignore.py @@ -7,6 +7,8 @@ """ import os +from pathlib import Path + from .conftest import path_conversion_tests import pytest @@ -80,6 +82,23 @@ def assert_mhl_file_has_exact_ignore_patterns(mhl_file: str, patterns_to_check: assert patterns_in_file == DEFAULT_IGNORE_SET | patterns_to_check, "mhl file has incorrect ignore patterns" +def assert_pattern_ignored_in_result(pattern: [str], result, negate=False): + if negate: + for pattern in pattern: + if os.name == "posix": + assert f"ignoring filepath {pattern}" not in result.output + else: + pattern = Path(pattern).resolve() + assert f"ignoring filepath {pattern}" not in result.output + else: + for pattern in pattern: + if os.name == "posix": + assert f"ignoring filepath {pattern}" in result.output + else: + pattern = Path(pattern).resolve() + assert f"ignoring filepath {pattern}" in result.output + + def mhl_file_for_gen(mhl_dir: str, mhl_gen: int): """ returns the mhl file associated with a generation number. diff --git a/tests/test_ignore_extended.py b/tests/test_ignore_extended.py new file mode 100644 index 0000000..0e62924 --- /dev/null +++ b/tests/test_ignore_extended.py @@ -0,0 +1,1200 @@ +""" +__author__ = "David Frank" +__copyright__ = "Copyright 2025, Pomfort GmbH" +__license__ = "MIT" +__maintainer__ = "Patrick Renner, Alexander Sahm" +__email__ = "opensource@pomfort.com" +""" + +import os +from os.path import abspath +from pathlib import Path + +from click.testing import CliRunner +from freezegun import freeze_time +import ascmhl.commands +from ascmhl.generator import _extract_pattern_relative_to_history +from ascmhl.history import MHLHistory +from tests.conftest import abspath_conversion_tests +from tests.test_ignore import assert_mhl_file_has_exact_ignore_patterns, assert_pattern_ignored_in_result +from ascmhl.generator import belongs_to_parent_or_neighbour + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_relative_path_pattern_1(fs, simple_mhl_history): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-i", "A/A1.txt", "-v"] + ) + assert_mhl_file_has_exact_ignore_patterns("root/ascmhl/0002_root_2020-01-16_091500Z.mhl", {"/A/A1.txt"}) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/A1.txt"], result) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_relative_path_pattern_2(fs, simple_mhl_history): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-i", "/A/A1.txt", "-v"] + ) + assert_mhl_file_has_exact_ignore_patterns("root/ascmhl/0002_root_2020-01-16_091500Z.mhl", {"/A/A1.txt"}) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/A1.txt"], result) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_relative_path_pattern_3(fs, simple_mhl_history): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-i", "**/A/A1.txt", "-v"] + ) + assert_mhl_file_has_exact_ignore_patterns("root/ascmhl/0002_root_2020-01-16_091500Z.mhl", {"/**/A/A1.txt"}) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/A1.txt"], result) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_nested_path_pattern_1(fs, nested_mhl_histories): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root/A"), "-h", "xxh64", "-i", "/AA/AA1.txt", "-v"] + ) + print(result.output) + assert_mhl_file_has_exact_ignore_patterns("root/A/AA/ascmhl/0002_AA_2020-01-16_091500Z.mhl", {"/AA1.txt"}) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/A")).latest_ignore_patterns() + assert "/AA/AA1.txt" not in latest_ignores + + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v"]) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + latest_ignores = MHLHistory.load_from_path(abspath("/root")).latest_ignore_patterns() + assert "/AA/AA1.txt" not in latest_ignores + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_nested_path_pattern_2(fs, nested_mhl_histories): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root/A"), "-h", "xxh64", "-i", "AA/AA1.txt", "-v"] + ) + print(result.output) + assert_mhl_file_has_exact_ignore_patterns("root/A/AA/ascmhl/0002_AA_2020-01-16_091500Z.mhl", {"/AA1.txt"}) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/A")).latest_ignore_patterns() + assert "AA/AA1.txt" not in latest_ignores + + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v"]) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + latest_ignores = MHLHistory.load_from_path(abspath("/root")).latest_ignore_patterns() + assert "AA/AA1.txt" not in latest_ignores + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_nested_path_pattern_3(fs, nested_mhl_histories): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root/A"), "-h", "xxh64", "-i", "**/AA1.txt", "-v"] + ) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + assert_mhl_file_has_exact_ignore_patterns("root/A/ascmhl/0001_A_2020-01-16_091500Z.mhl", {"/**/AA1.txt"}) + assert_mhl_file_has_exact_ignore_patterns("root/A/AA/ascmhl/0002_AA_2020-01-16_091500Z.mhl", {"/**/AA1.txt"}) + + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v"]) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_nested_path_pattern_4(fs, nested_mhl_histories): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-i", "AA1.txt", "-v"] + ) + print(result.output) + assert_mhl_file_has_exact_ignore_patterns("root/ascmhl/0002_root_2020-01-16_091500Z.mhl", {"AA1.txt"}) + assert_mhl_file_has_exact_ignore_patterns("root/A/AA/ascmhl/0002_AA_2020-01-16_091500Z.mhl", {"AA1.txt"}) + assert_mhl_file_has_exact_ignore_patterns("root/B/BB/ascmhl/0002_BB_2020-01-16_091500Z.mhl", {"AA1.txt"}) + assert_mhl_file_has_exact_ignore_patterns("root/B/ascmhl/0002_B_2020-01-16_091500Z.mhl", {"AA1.txt"}) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_nested_path_pattern_5(fs, post_house_file_structure): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root/ShootingDay1"), "-h", "xxh64", "-i", "**/A/A001", "-v"] + ) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/ShootingDay1/CameraMedia/A/A001"], result) + assert_mhl_file_has_exact_ignore_patterns( + "root/ShootingDay1/ascmhl/0002_ShootingDay1_2020-01-16_091500Z.mhl", {"/**/A/A001"} + ) + assert_mhl_file_has_exact_ignore_patterns( + "root/ShootingDay1/CameraMedia/A/ascmhl/0004_A_2020-01-16_091500Z.mhl", {"/**/A/A001"} + ) + assert_mhl_file_has_exact_ignore_patterns( + "root/ShootingDay1/CameraMedia/B/ascmhl/0004_B_2020-01-16_091500Z.mhl", {"/**/A/A001"} + ) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_nested_path_pattern_6(fs, post_house_file_structure): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-h", "xxh64", "-i", "**/?/Proxy/", "-v"], + ) + print(result.output) + assert result.exit_code == 0 + pattern = ["/root/ShootingDay1/CameraMedia/A/Proxy", "/root/ShootingDay1/CameraMedia/B/Proxy"] + assert_pattern_ignored_in_result(pattern, result) + assert_mhl_file_has_exact_ignore_patterns( + "root/ShootingDay1/ascmhl/0002_ShootingDay1_2020-01-16_091500Z.mhl", {"/**/?/Proxy/"} + ) + assert_mhl_file_has_exact_ignore_patterns( + "root/ShootingDay1/CameraMedia/A/ascmhl/0004_A_2020-01-16_091500Z.mhl", {"/**/?/Proxy/"} + ) + assert_mhl_file_has_exact_ignore_patterns( + "root/ShootingDay1/CameraMedia/B/ascmhl/0004_B_2020-01-16_091500Z.mhl", {"/**/?/Proxy/"} + ) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_future_files(fs, simple_mhl_history): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-i", "A/A2.txt", "-v"] + ) + assert result.exit_code == 0 + + fs.create_file("/root/A/A2.txt") + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v"]) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/A2.txt"], result) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_existing_files(fs, simple_mhl_history): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-i", "A/A1.txt", "-v"] + ) + assert_mhl_file_has_exact_ignore_patterns("root/ascmhl/0002_root_2020-01-16_091500Z.mhl", {"/A/A1.txt"}) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/A1.txt"], result) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_deleted_files(fs, simple_mhl_history): + runner = CliRunner() + os.remove("/root/A/A1.txt") + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-i", "A/A1.txt", "-v"] + ) + assert_mhl_file_has_exact_ignore_patterns("root/ascmhl/0002_root_2020-01-16_091500Z.mhl", {"/A/A1.txt"}) + print(result.output) + assert result.exit_code == 0 + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_changing_files(fs, simple_mhl_history): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-i", "A/A1.txt", "-v"] + ) + assert_mhl_file_has_exact_ignore_patterns("root/ascmhl/0002_root_2020-01-16_091500Z.mhl", {"/A/A1.txt"}) + print(result.output) + + os.remove("/root/A/A1.txt") + fs.create_file("/root/A/A1.txt", contents="1234567890") + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v"]) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/A1.txt"], result) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_in_first_generation(fs): + runner = CliRunner() + + fs.create_file("/root/A/A1.txt", contents="1234567890") + fs.create_file("/root/A/A1.RMD", contents="1234567890") + fs.create_file("/root/A/A2.txt", contents="1234567890") + fs.create_file("/root/A/A2.RMD", contents="1234567890") + fs.create_file("/root/A/ignore.ign", contents="1234567890") + fs.create_file("/root/report.pdf", contents="1234567890") + fs.create_file("/root/report.txt", contents="1234567890") + fs.create_file("/root/ignore.ign", contents="1234567890") + + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root"), "-h", "xxh64", "-v", "-i", "*.txt", "-i", "ignore.ign"], + ) + print(result.output) + assert result.exit_code == 0 + pattern = ["/root/A/A1.txt", "/root/A/A2.txt", "/root/A/ignore.ign", "/root/ignore.ign", "/root/report.txt"] + assert_pattern_ignored_in_result(pattern, result) + assert_mhl_file_has_exact_ignore_patterns("root/ascmhl/0001_root_2020-01-16_091500Z.mhl", {"*.txt", "ignore.ign"}) + + +def test_ignore_paths_are_handled_correctly(fs, nested_mhl_histories): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root/A"), "-h", "xxh64", "-v", "-i", "/AB/AB1.txt"] + ) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AB/AB1.txt"], result) + + fs.create_file("/root/AB/AB1.txt") + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v", "-i", "Stuff.txt"] + ) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AB/AB1.txt", "/root/Stuff.txt"], result) + + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v", "-i", "A/AA/AA1.txt"] + ) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v"]) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + + os.remove("/root/A/AA/AA1.txt") + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v"]) + assert result.exit_code == 0 + + os.remove("/root/A/AB/AB1.txt") + os.remove("/root/Stuff.txt") + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v"]) + assert result.exit_code == 0 + + os.remove("/root/B/B1.txt") + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v"]) + assert result.exit_code == 10 + + +def test_nested_histories_absolute_ignore_patterns(fs, nested_mhl_histories): + runner = CliRunner() + + # existing histories: /root, /root/A/AA, /root/B, /root/B/BB (1st gen) + # create a history with an ignore pattern + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root/A/AA"), "-h", "xxh64", "-i", "/test.txt"] + ) + assert result.exit_code == 0 + + # create two test files + fs.create_file("/root/A/AA/test.txt", contents="testAtSubHistoryRoot") + fs.create_file("/root/A/AA/Subfolder/test.txt", contents="testInSubHistorySubfolder") + + # run the create once in the sub history and see if the ignore pattern works + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root/A/AA"), "-h", "xxh64", "-v"]) + assert result.exit_code == 0 + hash_list = MHLHistory.load_from_path(abspath("/root/A/AA")).hash_lists[-1] + + # the ignore pattern /test.txt should ignore this file + assert hash_list.find_media_hash_for_path("test.txt") is None + + # the ignore pattern /test.txt should not ignore this file + assert hash_list.find_media_hash_for_path(f"{Path('Subfolder/test.txt')}") is not None + + # same ignores should be applied when running it on the root history, so we run it again on the root hsitory + result = runner.invoke(ascmhl.commands.create, [abspath("/root"), "-h", "xxh64", "-v"]) + assert result.exit_code == 0 + hash_list = MHLHistory.load_from_path(abspath("/root/A/AA")).hash_lists[-1] + + # the ignore pattern /test.txt from the sub history should ignore this file also when verifying from the root + assert hash_list.find_media_hash_for_path("test.txt") is None + + # the ignore pattern /test.txt from the sub history should not ignore this file + assert hash_list.find_media_hash_for_path(f"{Path('Subfolder/test.txt')}") is not None + + +def test_nested_histories_ignoring_from_root(fs, nested_mhl_histories): + runner = CliRunner() + + # existing histories: /root, /root/A/AA, /root/B, /root/B/BB (1st gen) + # create two test files + fs.create_file("/root/A/AA/test.txt", contents="testAtSubHistoryRoot") + fs.create_file("/root/A/AA/Subfolder/test.txt", contents="testInSubHistorySubfolder") + + # ignore a file in the subhistory from the root hsitory + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-i", "/A/AA/test.txt"] + ) + assert result.exit_code == 0 + + hash_list = MHLHistory.load_from_path(abspath("/root/A/AA")).hash_lists[-1] + + # the ignore pattern should ignore this file + assert hash_list.find_media_hash_for_path("test.txt") is None + + # the ignore pattern should also ignore this file + assert hash_list.find_media_hash_for_path(f"{Path('Subfolder/test.txt')}") is not None + + # the ignore pattern inside the sub history should contain the pattern /test.txt not the path we passed into the root history + latest_ignores = MHLHistory.load_from_path(abspath("/root/A/AA")).latest_ignore_patterns() + assert "/test.txt" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root")).latest_ignore_patterns() + assert "/test.txt" not in latest_ignores + + # a second verification should not fail + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64"]) + assert result.exit_code == 0 + + +def test_ignore_multilevel_histories(fs, nested_mhl_histories): + runner = CliRunner() + + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root/A"), "-h", "xxh64"]) + assert result.exit_code == 0 + + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root/A/AB"), "-v", "-h", "xxh64", "-i", "AB1.txt"] + ) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AB/AB1.txt"], result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/A/AB")).latest_ignore_patterns() + assert "AB1.txt" in latest_ignores + + assert result.exit_code == 0 + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root/B/BA"), "-h", "xxh64"]) + assert result.exit_code == 0 + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64"]) + assert result.exit_code == 0 + + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-v", "-h", "xxh64", "-i", "/A/AA/AA1.txt"] + ) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/A/AA")).latest_ignore_patterns() + assert "/AA1.txt" in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/A")).latest_ignore_patterns() + assert "/AA/AA1.txt" not in latest_ignores + assert "/AA1.txt" not in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/B")).latest_ignore_patterns() + assert "/AA/AA1.txt" not in latest_ignores + assert "/AA1.txt" not in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root")).latest_ignore_patterns() + assert "/A/AA/AA1.txt" not in latest_ignores + assert "/AA/AA1.txt" not in latest_ignores + assert "/AA1.txt" not in latest_ignores + + +def test_ignore_files_in_nested_structures(fs, post_house_file_structure): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-i", "Sidecar.txt", "-h", "xxh64"], + ) + assert result.exit_code == 0 + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/Sound")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + + pattern = [ + "/root/ShootingDay1/CameraMedia/A/A001/Sidecar.txt", + "/root/ShootingDay1/CameraMedia/B/B002/Sidecar.txt", + "/root/ShootingDay1/Sound/Sidecar.txt", + ] + assert_pattern_ignored_in_result(pattern, result) + + +def test_ignore_file_type_in_nested_histories(fs, post_house_file_structure): + """ + This should test the '*' functionality + """ + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root"), "-v", "-i", "*.pdf", "-h", "xxh64"], + ) + assert result.exit_code == 0 + + assert_pattern_ignored_in_result( + ["/root/ShootingDay1/CameraMedia/Report.pdf", "/root/ShootingDay1/Report.pdf"], result + ) + + latest_ignores = MHLHistory.load_from_path(abspath("/root")).latest_ignore_patterns() + assert "*.pdf" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "*.pdf" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/Sound")).latest_ignore_patterns() + assert "*.pdf" in latest_ignores + + fs.create_file("/root/ShootingDay1/Sound/Notes.pdf") + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root"), "-v", "-h", "xxh64"], + ) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/ShootingDay1/Sound/Notes.pdf"], result) + + +def test_ignore_file_type_in_path(fs, post_house_file_structure_with_range): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-i", "CameraMedia/A/A002/*.mov", "-h", "xxh64"], + ) + assert result.exit_code == 0 + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "/A002/*.mov" in latest_ignores + pattern = [ + "/root/ShootingDay1/CameraMedia/A/A002/A002C001.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C002.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C003.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C004.mov", + ] + assert_pattern_ignored_in_result(pattern, result) + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "/CameraMedia/A/A002/*.mov" not in latest_ignores + assert "CameraMedia/A/A002/*.mov" not in latest_ignores + + # verify it again without providing the ignore pattern + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-h", "xxh64"], + ) + assert result.exit_code == 0 + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "/A002/*.mov" in latest_ignores + assert_pattern_ignored_in_result(pattern, result) + + +def test_ignore_directories_in_nested_structures_pattern_1(fs, post_house_file_structure): + """ + This pattern should match "Proxy" anywhere in the filepath + """ + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1/CameraMedia"), "-v", "-i", "Proxy", "-h", "xxh64"], + ) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "Proxy" in latest_ignores + pattern = [ + "/root/ShootingDay1/CameraMedia/A/Proxy", + "/root/ShootingDay1/CameraMedia/B/Proxy", + "/root/ShootingDay1/CameraMedia/Proxy", + ] + assert_pattern_ignored_in_result(pattern, result) + assert result.exit_code == 0 + + +def test_ignore_directories_in_nested_structures_pattern_2(fs, post_house_file_structure): + """ + This pattern should only match directories named "Proxy" anywhere in the directory tree, but not files with that name + """ + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1/CameraMedia"), "-v", "-i", "Proxy/", "-h", "xxh64"], + ) + pattern = ["/root/ShootingDay1/CameraMedia/A/Proxy", "/root/ShootingDay1/CameraMedia/B/Proxy"] + assert_pattern_ignored_in_result(pattern, result) + assert_pattern_ignored_in_result(["/root/ShootingDay1/CameraMedia/Proxy"], result, negate=True) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "Proxy/" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "Proxy/" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/B")).latest_ignore_patterns() + assert "Proxy/" in latest_ignores + assert result.exit_code == 0 + + +def test_ignore_directories_in_nested_structures_pattern_3(fs, post_house_file_structure): + """ + This pattern should only match /A/Proxy/ relative to CameraMedia and not be applied to other directories + """ + runner = CliRunner() + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/A/Proxy/B001/A3_p.txt", contents="A3 Proxy") + + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1/CameraMedia"), "-v", "-i", "A/Proxy/", "-h", "xxh64"], + ) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "/Proxy/" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/B")).latest_ignore_patterns() + assert "A/Proxy/" not in latest_ignores + assert "/A/Proxy/" not in latest_ignores + assert "//A/Proxy/" not in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "A/Proxy/" not in latest_ignores + assert "/A/Proxy/" not in latest_ignores + assert_pattern_ignored_in_result(["/root/ShootingDay1/CameraMedia/A/Proxy"], result) + assert_pattern_ignored_in_result( + ["/root/ShootingDay1/CameraMedia/B/Proxy", "/root/ShootingDay1/CameraMedia/Proxy"], result, negate=True + ) + assert f"created original hash for {Path('B/Proxy/A/Proxy/B001/A3_p.txt')}" in result.output + assert result.exit_code == 0 + + +def test_ignore_directories_in_nested_structures_pattern_3_wrong_directory(fs, post_house_file_structure): + """ + This pattern should only match /A/Proxy/ relative to the ascmhl and not be applied to other directories + (i.e. have no effect when called on root/ShootingDay1, since it does not contain a directory 'A/Proxy/') + """ + runner = CliRunner() + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/A/Proxy/B001/A3_p.txt", contents="A3 Proxy") + + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-i", "A/Proxy/", "-h", "xxh64"], + ) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "/Proxy/" not in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/B")).latest_ignore_patterns() + assert "/A/Proxy/" not in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "/A/Proxy/" not in latest_ignores + pattern = [ + "/root/ShootingDay1/CameraMedia/A/Proxy", + "/root/ShootingDay1/CameraMedia/B/Proxy", + "/root/ShootingDay1/CameraMedia/Proxy", + ] + assert_pattern_ignored_in_result(pattern, result, negate=True) + assert f"created original hash for {Path('CameraMedia/B/Proxy/A/Proxy/B001/A3_p.txt')}" in result.output + assert result.exit_code == 0 + + +def test_ignore__directories_in_nested_structures_pattern_4(fs, post_house_file_structure): + """ + This pattern should ignore any occurence of Proxy in the file- or directoryname below root/ShootingDay1 + """ + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-i", "**/Proxy", "-h", "xxh64"], + ) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "/**/Proxy" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "/**/Proxy" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "/**/Proxy" in latest_ignores + pattern = [ + "/root/ShootingDay1/CameraMedia/A/Proxy", + "/root/ShootingDay1/CameraMedia/B/Proxy", + "/root/ShootingDay1/CameraMedia/Proxy", + ] + assert_pattern_ignored_in_result(pattern, result) + assert result.exit_code == 0 + + +def test_ignore__directories_in_nested_structures_pattern_5(fs, post_house_file_structure): + """ + This pattern should ignore any directory 'Proxy', but not a file named 'Proxy' + """ + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-i", "**/Proxy/", "-h", "xxh64"], + ) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "/**/Proxy/" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "/**/Proxy/" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "/**/Proxy/" in latest_ignores + pattern = ["/root/ShootingDay1/CameraMedia/A/Proxy", "/root/ShootingDay1/CameraMedia/B/Proxy"] + assert_pattern_ignored_in_result(pattern, result) + assert_pattern_ignored_in_result(["/root/ShootingDay1/CameraMedia/Proxy"], result, negate=True) + assert result.exit_code == 0 + + +def test_ignore__directories_in_nested_structures_pattern_6(fs, post_house_file_structure): + """ + This pattern should ignore any files in any directory 'Proxy', but not a file named 'Proxy' + """ + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-i", "**/Proxy/**", "-h", "xxh64"], + ) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "**/Proxy/**" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "**/Proxy/**" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "**/Proxy/**" in latest_ignores + pattern = ["/root/ShootingDay1/CameraMedia/A/Proxy", "/root/ShootingDay1/CameraMedia/B/Proxy"] + assert_pattern_ignored_in_result(pattern, result) + assert_pattern_ignored_in_result(["/root/ShootingDay1/CameraMedia/Proxy"], result, negate=True) + assert result.exit_code == 0 + + +def test_ignore_single_character(fs, post_house_file_structure_with_range): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1/CameraMedia/A"), "-v", "-i", "A002C00?.mov", "-h", "xxh64"], + ) + assert result.exit_code == 0 + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "A002C00?.mov" in latest_ignores + pattern = [ + "/root/ShootingDay1/CameraMedia/A/A002/A002C001.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C002.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C003.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C004.mov", + ] + assert_pattern_ignored_in_result(pattern, result) + + +def test_ignore_single_character_in_path(fs, post_house_file_structure_with_range): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-i", "CameraMedia/A/A002/A002C00?.mov", "-h", "xxh64"], + ) + assert result.exit_code == 0 + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "/A002/A002C00?.mov" in latest_ignores + pattern = [ + "/root/ShootingDay1/CameraMedia/A/A002/A002C001.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C002.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C003.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C004.mov", + ] + assert_pattern_ignored_in_result(pattern, result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "CameraMedia/A/A002C00?.mov" not in latest_ignores + + +def test_ignore_multiple_individual_characters_in_path(fs, post_house_file_structure): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [ + abspath_conversion_tests("/root/ShootingDay1"), + "-v", + "-i", + "CameraMedia/?/Proxy/?001/?001.ale", + "-h", + "xxh64", + ], + ) + assert result.exit_code == 0 + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "/?/Proxy/?001/?001.ale" in latest_ignores + pattern = [ + "/root/ShootingDay1/CameraMedia/A/Proxy/A001/A001.ale", + "/root/ShootingDay1/CameraMedia/B/Proxy/B001/B001.ale", + ] + assert_pattern_ignored_in_result(pattern, result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "CameraMedia/?/Proxy/?001/?001.ale" not in latest_ignores + + +def test_ignore_range_of_characters(fs, post_house_file_structure_with_range): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [ + abspath_conversion_tests("/root/ShootingDay1"), + "-v", + "-i", + "CameraMedia/A/A00[1-5]/A00?C00[1-9].mov", + "-h", + "xxh64", + ], + ) + assert result.exit_code == 0 + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "/A00[1-5]/A00?C00[1-9].mov" in latest_ignores + pattern = [ + "/root/ShootingDay1/CameraMedia/A/A002/A002C001.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C002.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C003.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C004.mov", + ] + assert_pattern_ignored_in_result(pattern, result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "CameraMedia/A/A00[1-5]/A00?C00[1-9].mov" not in latest_ignores + + +def test_ignore_negate_pattern(fs, post_house_file_structure): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-i", "Sidecar.txt", "-h", "xxh64"], + ) + assert result.exit_code == 0 + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/Sound")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + pattern = [ + "/root/ShootingDay1/CameraMedia/A/A001/Sidecar.txt", + "/root/ShootingDay1/CameraMedia/B/B002/Sidecar.txt", + "/root/ShootingDay1/Sound/Sidecar.txt", + ] + assert_pattern_ignored_in_result(pattern, result) + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1/Sound"), "-v", "-i", "!Sidecar.txt", "-h", "xxh64"], + ) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/ShootingDay1/Sound/Sidecar.txt"], result, negate=True) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_old_deleted_files_in_histories(fs, nested_mhl_histories): + runner = CliRunner() + + # existing histories: /root, /root/A/AA, /root/B, /root/B/BB (1st gen) + + fs.create_file("/root/A/AA/AA1.RMD", contents="Lorem ipsum dolor") + fs.create_file("/root/A/AB/AB1.RMD", contents="sit amet con vota") + fs.create_file("/root/B/BA/B1.RMD", contents="lirum alamru aexti") + fs.create_file("/root/A/AA/AA2.txt", contents="Lorem ipsum dolor") + + result = runner.invoke(ascmhl.commands.create, [abspath("/root"), "-h", "xxh64"]) + assert result.exit_code == 0 + result = runner.invoke(ascmhl.commands.create, [abspath("/root/A/AA"), "-h", "xxh64"]) + assert result.exit_code == 0 + result = runner.invoke(ascmhl.commands.create, [abspath("/root/B"), "-h", "xxh64"]) + assert result.exit_code == 0 + result = runner.invoke(ascmhl.commands.create, [abspath("/root/B/BB"), "-h", "xxh64"]) + assert result.exit_code == 0 + + # existing histories: /root, /root/A/AA, /root/B, /root/B/BB (2nd gen) + + print("First session, no ignores yet\n") + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root/A"), "-h", "xxh64", "-v"]) + print(result.output) + assert result.exit_code == 0 + + print("Second Session, should ignore single existing file AB1.txt in root/A/AB/AB1.txt\n") + + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root/A"), "-v", "-h", "xxh64", "-i", "AB/AB1.txt"] + ) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AB/AB1.txt"], result) + assert_mhl_file_has_exact_ignore_patterns("root/A/ascmhl/0002_A_2020-01-16_091500Z.mhl", {"/AB/AB1.txt"}) + + print('Third Session, ignore "*.txt" and "/root/A/AB/AB1.RMD "\nRemove root/A/AA/AA1.txt and /root/A/AB/AB1.txt\n') + os.remove("/root/A/AA/AA1.txt") + os.remove("/root/A/AB/AB1.txt") + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/A"), "-v", "-h", "xxh64", "-i", "*.txt", "-i", "AB/AB1.RMD"], + ) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA2.txt", "/root/A/AB/AB1.RMD"], result) + assert_mhl_file_has_exact_ignore_patterns( + "root/A/ascmhl/0003_A_2020-01-16_091500Z.mhl", {"*.txt", "/AB/AB1.RMD", "/AB/AB1.txt"} + ) + + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-v", "-h", "xxh64"]) + print(result.output) + assert result.exit_code == 0 + + os.remove("/root/B/BB/BB1.txt") + + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-v", "-h", "xxh64"]) + print(result.output) + assert result.exit_code == 10 + + +def test_ignore_multiple_pattern_in_single_command(fs, post_house_file_structure): + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/Report.pdf", contents="A1") + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/ClipList.pdf", contents="A1") + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [ + abspath_conversion_tests("/root/ShootingDay1"), + "-v", + "-h", + "xxh64", + "-i", + "Sidecar.txt", + "-i", + "Proxy/", + "-i", + "CameraMedia/**/*.pdf", + "-i", + "/CameraMedia/B/B002/B3.txt", + ], + ) + assert result.exit_code == 0 + + pattern = [ + "/root/ShootingDay1/CameraMedia/Report.pdf", + "/root/ShootingDay1/CameraMedia/A/A001/Report.pdf", + "/root/ShootingDay1/CameraMedia/B/B002/ClipList.pdf", + "/root/ShootingDay1/CameraMedia/B/B002/Sidecar.txt", + "/root/ShootingDay1/Sound/Sidecar.txt", + "/root/ShootingDay1/CameraMedia/A/Proxy", + "/root/ShootingDay1/CameraMedia/B/Proxy", + ] + assert_pattern_ignored_in_result(pattern, result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + assert "Proxy/" in latest_ignores + assert "/CameraMedia/**/*.pdf" not in latest_ignores + assert "/CameraMedia/B/B002/B3.txt" not in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/Sound")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + assert "Proxy/" in latest_ignores + assert "CameraMedia/**/*.pdf" not in latest_ignores + assert "/CameraMedia/B/B002/B3.txt" not in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + assert "Proxy/" in latest_ignores + assert "/**/*.pdf" in latest_ignores + assert "/CameraMedia/B/B002/B3.txt" not in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + assert "Proxy/" in latest_ignores + assert "/**/*.pdf" in latest_ignores + assert "/B002/B3.txt" not in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/B")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + assert "Proxy/" in latest_ignores + assert "/**/*.pdf" in latest_ignores + assert "/B002/B3.txt" in latest_ignores + + +def test_ignore_diff_nested_multiple_pattern(fs, post_house_file_structure): + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/Report.pdf", contents="A1") + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/ClipList.pdf", contents="A1") + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.diff, + [ + abspath_conversion_tests("/root/ShootingDay1"), + "-v", + "-i", + "Sidecar.txt", + "-i", + "Proxy/", + "-i", + "CameraMedia/**/*.pdf", + "-i", + "/CameraMedia/B/B002/B3.txt", + "-i", + "/CameraMedia/Proxy", + ], + ) + assert result.exit_code == 0 + + pattern = [ + "/root/ShootingDay1/CameraMedia/Report.pdf", + "/root/ShootingDay1/CameraMedia/A/A001/Report.pdf", + "/root/ShootingDay1/CameraMedia/B/B002/ClipList.pdf", + "/root/ShootingDay1/CameraMedia/B/B002/Sidecar.txt", + "/root/ShootingDay1/Sound/Sidecar.txt", + "/root/ShootingDay1/CameraMedia/A/Proxy", + "/root/ShootingDay1/CameraMedia/B/Proxy", + ] + assert_pattern_ignored_in_result(pattern, result) + + +def test_ignore_from_file(fs, post_house_file_structure): + fs.create_file( + "/user/ignore.txt", contents="Sidecar.txt\n/CameraMedia/**/Proxy/**\nCameraMedia/A/A001/A1.txt\n/**/temp\n/dir" + ) + + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-h", "xxh64", "-ii", "/user/ignore.txt"], + ) + assert result.exit_code == 0 + + pattern = [ + "/root/ShootingDay1/CameraMedia/A/A001/A1.txt", + "/root/ShootingDay1/CameraMedia/A/A001/Sidecar.txt", + "/root/ShootingDay1/CameraMedia/B/B002/Sidecar.txt", + "/root/ShootingDay1/CameraMedia/A/Proxy/A001", + "/root/ShootingDay1/CameraMedia/B/Proxy/B001", + "/root/ShootingDay1/Sound/Sidecar.txt", + ] + assert_pattern_ignored_in_result(pattern, result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + assert "/A001/A1.txt" in latest_ignores + assert "/**/temp" in latest_ignores + assert "/dir" not in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/B")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + assert "/A001/A1.txt" not in latest_ignores + assert "/**/temp" in latest_ignores + assert "/dir" not in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "/**/Proxy/**" in latest_ignores + assert "/A001/A1.txt" not in latest_ignores + assert "/**/temp" in latest_ignores + assert "/dir" not in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/Sound")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + assert "/CameraMedia/**/Proxy/**" not in latest_ignores + assert "/CameraMedia/A001/A1.txt" not in latest_ignores + assert "/**/temp" in latest_ignores + assert "/dir" not in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + assert "/CameraMedia/**/Proxy/**" not in latest_ignores + assert "/CameraMedia/A001/A1.txt" not in latest_ignores + assert "/**/temp" in latest_ignores + assert "/dir" in latest_ignores + + +def test_ignore_history_in_ignored_directory(fs): + runner = CliRunner() + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/A1.txt", contents="A1") + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/A2.txt", contents="A2") + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/A3.txt", contents="A3") + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/Sidecar.txt", contents="Sidecar1") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1"), "-h", "xxh64"]) + assert result.exit_code == 0 + + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1/CameraMedia"), "-h", "xxh64"]) + assert result.exit_code == 0 + + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1/CameraMedia/A/A001"), "-v", "-h", "xxh64"], + ) + assert result.exit_code == 0 + + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-i", "/CameraMedia/A/A001", "-h", "xxh64"], + ) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/ShootingDay1/CameraMedia/A/A001"], result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "/A/A001" in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "/CameraMedia/A/A001" not in latest_ignores + + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-h", "xxh64"], + ) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/ShootingDay1/CameraMedia/A/A001"], result) + + +def test_ignore_only_one_child_history(fs): + fs.create_file("/root/A/A001/A001C001.txt") + fs.create_file("/root/A/A001/A001C002.txt") + fs.create_file("/root/A/A001/A001C003.txt") + + fs.create_file("/root/A/A002/A002C001.txt") + fs.create_file("/root/A/A002/A002C002.txt") + fs.create_file("/root/A/A002/A002C003.txt") + + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/A/A002"), "-v", "-h", "xxh64"], + ) + assert result.exit_code == 0 + + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/A"), "-v", "-i", "/A001/A001C001.txt", "-h", "xxh64"], + ) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/A001/A001C001.txt"], result) + + latest_ignores = MHLHistory.load_from_path(abspath("/root/A")).latest_ignore_patterns() + assert "/A001/A001C001.txt" in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/A/A002")).latest_ignore_patterns() + assert "/A001/A001C001.txt" not in latest_ignores + + +def test_ignore_unit_belongs_to_parent_or_neighbour(): + pattern = "*.txt" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == False + + pattern = "A.txt" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == False + + pattern = "A/" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == False + + pattern = "/A" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "/A/" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "/A/AA" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "/A/AA/" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "/A/AA/AAA" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "/A/AA/AAA.txt" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "/A/AA/AAA/A.txt" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == False + + pattern = "A/**/AA" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == False + + pattern = "A/AA/**" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "**/AA/" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == False + + pattern = "/**/AA" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "/**/AA/" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "A/B/AAA" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "A/AA/AAB" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "A/AA/AAB/AA" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "B/AA/AAA" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "/B/AA/AAA" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + +def test_ignore_unit_extract_relpath(): + base = f"{Path('/home/user/docs')}" + target = "/home/user/docs/file.txt" + expected = "file.txt" + result = ascmhl.generator._extract_pattern_relative_to_history(target, base) + assert result == expected + + base = f"{Path('/home/user')}" + target = "/home/user/projects/code" + expected = "projects/code" + result = ascmhl.generator._extract_pattern_relative_to_history(target, base) + assert result == expected + + base = f"{Path('/home/user/docs')}" + target = "/home/user" + expected = None + result = ascmhl.generator._extract_pattern_relative_to_history(target, base) + assert result == expected + + base = f"{Path('/home/user/docs')}" + target = "/home/user/photos/image.jpg" + expected = "photos/image.jpg" + result = ascmhl.generator._extract_pattern_relative_to_history(target, base) + assert result == expected + + base = f"{Path('/home/user/docs')}" + target = "/home/user/docs" + expected = None + result = ascmhl.generator._extract_pattern_relative_to_history(target, base) + assert result == expected + + base = f"{Path('/')}" + target = "/var/log" + expected = "var/log" + result = ascmhl.generator._extract_pattern_relative_to_history(target, base) + assert result == expected + + base = f"{Path('/home/user')}" + target = "/home/user/docs/" + expected = "docs/" + result = ascmhl.generator._extract_pattern_relative_to_history(target, base) + assert result == expected From 1455bede70668c91343615c73e2dce9c658ee99d Mon Sep 17 00:00:00 2001 From: Alexander Sahm Date: Tue, 20 May 2025 09:28:13 +0200 Subject: [PATCH 2/6] Improving performance when verifying 1000s of files (e.g. with create) There is no need to collect the relative paths again as the not_found_paths already contains all those paths --- ascmhl/commands.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/ascmhl/commands.py b/ascmhl/commands.py index 824ec07..4cc1a1d 100644 --- a/ascmhl/commands.py +++ b/ascmhl/commands.py @@ -249,13 +249,11 @@ def create_for_folder_subcommand( dir_hash_context_lookup[hash_format] = DirectoryHashContext(hash_format) for item_name, is_dir in children: file_path = os.path.join(folder_path, item_name) - not_found_paths.discard(file_path) - for hash_list in existing_history.hash_lists: - for media_hash in hash_list.media_hashes: - if media_hash.path == existing_history.get_relative_file_path(file_path): - break - else: - new_paths.add(file_path) + if file_path in not_found_paths: + not_found_paths.discard(file_path) + else: + new_paths.add(file_path) + if is_dir: if not no_directory_hashes: path_content_hash_lookup = dir_content_hash_mapping_lookup.pop(file_path) From be1e6a975deb0ad869146924e4af45f6bb0b049e Mon Sep 17 00:00:00 2001 From: Alexander Sahm Date: Wed, 21 May 2025 09:10:38 +0200 Subject: [PATCH 3/6] Adding recursive parsing to flatten command The flatten command only flattened the root history, but not child histories. Including the hashes from the child histories was missing in the implementation and is added with this pull request. --- ascmhl/commands.py | 34 +++++++++++++++++++++++++--------- ascmhl/generator.py | 38 +++++++++++++++++++++++++++++++++++--- tests/test_flatten.py | 27 +++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 12 deletions(-) diff --git a/ascmhl/commands.py b/ascmhl/commands.py index 824ec07..c2cc02f 100644 --- a/ascmhl/commands.py +++ b/ascmhl/commands.py @@ -34,6 +34,7 @@ from .traverse import post_order_lexicographic from typing import Dict from collections import namedtuple +from .utils import convert_local_path_to_posix @click.command() @@ -1195,18 +1196,30 @@ def flatten_history( if len(existing_history.hash_lists) == 0: raise errors.NoMHLHistoryException(root_path) - for hash_list in existing_history.hash_lists: + flatten_child_histories(existing_history, session, root_path) + + commit_session_for_collection( + session, root_path, author_name, author_email, author_phone, author_role, location, comment + ) + + +def flatten_child_histories(history, session, roothistorypath, pathprefix=""): + for hash_list in history.hash_lists: for media_hash in hash_list.media_hashes: if not media_hash.is_directory: for hash_entry in media_hash.hash_entries: if hash_entry.action != "failed": + # add prefix to media path if subhistory + media_path = media_hash.path + if pathprefix != "": + media_path = convert_local_path_to_posix(pathprefix) + "/" + media_hash.path # check if this entry is newer than the one already in there to avoid duplicate entries - found_media_hash = session.new_hash_lists[collection_history].find_media_hash_for_path( - media_hash.path + found_media_hash = session.new_hash_lists[session.root_history].find_media_hash_for_path( + media_path ) if found_media_hash == None: session.append_file_hash( - media_hash.path, + media_path, media_hash.file_size, media_hash.last_modification_date, hash_entry.hash_format, @@ -1222,7 +1235,7 @@ def flatten_history( if not hashformat_is_already_there: # assuming that hash_entry of same type also has same hash_value .. session.append_file_hash( - media_hash.path, + media_path, media_hash.file_size, media_hash.last_modification_date, hash_entry.hash_format, @@ -1231,9 +1244,12 @@ def flatten_history( hash_date=hash_entry.hash_date, ) - commit_session_for_collection( - session, root_path, author_name, author_email, author_phone, author_role, location, comment - ) + for child_history in history.child_histories: + childpath = child_history.get_root_path() + childrelativepath = os.path.relpath(childpath, roothistorypath) + + logger.info(f"\nChild History at {childrelativepath}:") + flatten_child_histories(child_history, session, roothistorypath, childrelativepath) @click.command() @@ -1471,7 +1487,7 @@ def commit_session_for_collection( process_info.root_media_hash = root_hash process_info.hashlist_custom_basename = "packinglist_" + os.path.basename(root_path) - session.commit(creator_info, process_info) + session.commit(creator_info, process_info, writeChain=False) """ diff --git a/ascmhl/generator.py b/ascmhl/generator.py index 170ff8a..2e266bd 100644 --- a/ascmhl/generator.py +++ b/ascmhl/generator.py @@ -7,6 +7,9 @@ __email__ = "opensource@pomfort.com" """ +import os +import shutil + from collections import defaultdict from typing import Dict, List @@ -15,6 +18,7 @@ from .ignore import MHLIgnoreSpec from .hashlist import MHLHashList, MHLHashEntry, MHLCreatorInfo, MHLProcessInfo from .history import MHLHistory +from .utils import convert_posix_to_local_path class MHLGenerationCreationSession: @@ -131,7 +135,13 @@ def append_file_hash( hash_entry = MHLHashEntry(hash_format, hash_string, hash_date=hash_date) if original_hash_entry is None: hash_entry.action = "original" - logger.verbose(f" created original hash for {relative_path} {hash_format}: {hash_string}") + if relative_path is not None: + logger.verbose(f" created original hash for {relative_path} {hash_format}: {hash_string}") + else: + # flattening works a bit different, because we don't add to individual (nested) histories + logger.verbose( + f" created original hash for {convert_posix_to_local_path(file_path)} {hash_format}: {hash_string}" + ) else: existing_hash_entry = history.find_first_hash_entry_for_path(history_relative_path, hash_format) if existing_hash_entry is not None: @@ -272,7 +282,7 @@ def append_directory_hashes( hash_entry.structure_hash_string = structure_hash_string parent_media_hash.append_hash_entry(hash_entry) - def commit(self, creator_info: MHLCreatorInfo, process_info: MHLProcessInfo): + def commit(self, creator_info: MHLCreatorInfo, process_info: MHLProcessInfo, writeChain=True): """ this method needs to create the generations of the children bottom up # so each history can reference the children correctly and can get the actual hash of the file @@ -306,4 +316,26 @@ def commit(self, creator_info: MHLCreatorInfo, process_info: MHLProcessInfo): if history.parent_history is not None: referenced_hash_lists[history.parent_history].append(new_hash_list) - chain_xml_parser.write_chain(history.chain, new_hash_list) + if writeChain: + # regular history .... + chain_xml_parser.write_chain(history.chain, new_hash_list) + else: + # ... or flattened history manifest + root_path = os.path.dirname(new_hash_list.file_path) + if not os.path.exists(root_path): + logger.error(f"ERROR: folder {root_path} with flattened manifest does not exist") + return + + parent_folder = os.path.dirname(root_path) + + for file_name in os.listdir(root_path): + if file_name.endswith(".mhl"): + src_path = os.path.join(root_path, file_name) + dst_path = os.path.join(parent_folder, file_name) + shutil.move(src_path, dst_path) + + # Remove the folder if empty + if not os.listdir(root_path): + os.rmdir(root_path) + else: + logger.error(f"ERROR: temp folder not empty, did not remove {root_path}") diff --git a/tests/test_flatten.py b/tests/test_flatten.py index 7783386..47c5ae9 100644 --- a/tests/test_flatten.py +++ b/tests/test_flatten.py @@ -10,6 +10,7 @@ import os from click.testing import CliRunner from freezegun import freeze_time +from .conftest import path_conversion_tests from .conftest import abspath_conversion_tests import ascmhl.commands @@ -51,3 +52,29 @@ def test_simple_two_hashformats(fs, simple_mhl_history): ascmhl.commands.flatten, [abspath_conversion_tests("/root"), abspath_conversion_tests("/out")] ) assert result.exit_code == 0 + + +@freeze_time("2020-01-16 09:15:00") +def test_nested(fs, nested_mhl_histories): + runner = CliRunner() + + result = runner.invoke( + ascmhl.commands.flatten, ["-v", abspath_conversion_tests("/root"), abspath_conversion_tests("/out")] + ) + assert result.exit_code == 0 + + # check for files in root and sub histories + assert ( + result.output == f"Flattening folder at path: {abspath_conversion_tests('/root')} ...\n" + f" created original hash for Stuff.txt xxh64: 94c399c2a9a21f9a\n" + f"\n" + f"Child History at {path_conversion_tests('A/AA')}:\n" + f" created original hash for {path_conversion_tests('A/AA/AA1.txt')} xxh64: ab6bec9ec04704f6\n" + f"\n" + f"Child History at B:\n" + f" created original hash for {path_conversion_tests('B/B1.txt')} xxh64: 51fb8fb099e92821\n" + f"\n" + f"Child History at {path_conversion_tests('B/BB')}:\n" + f" created original hash for {path_conversion_tests('B/BB/BB1.txt')} xxh64: 5c14eac4f4ad7501\n" + f"Created new generation {path_conversion_tests('collection_2020-01-16/packinglist_root_2020-01-16_091500Z.mhl')}\n" + ) From fbf549194a276361d553a3ed3c8725e031daad77 Mon Sep 17 00:00:00 2001 From: pfn-djf Date: Wed, 21 May 2025 15:01:53 +0200 Subject: [PATCH 4/6] Make pathspec work against directories in traversal --- ascmhl/traverse.py | 8 +++++--- tests/test_ignore_extended.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ascmhl/traverse.py b/ascmhl/traverse.py index 1d978e8..e4a9d8f 100644 --- a/ascmhl/traverse.py +++ b/ascmhl/traverse.py @@ -31,12 +31,14 @@ def post_order_lexicographic(top: str, ignore_pathspec: pathspec.PathSpec = None children = [] for name in names: file_path = os.path.join(top, name) + is_directory = isdir(file_path) + if is_directory: + file_path = file_path + "/" if ignore_pathspec and ignore_pathspec.match_file(file_path): if os.path.basename(os.path.normpath(file_path)) != ascmhl_folder_name: - logger.verbose(f"ignoring filepath {file_path}") + logger.verbose(f"ignoring filepath {file_path.rstrip('/')}") continue - path = join(top, name) - children.append((name, isdir(path))) + children.append((name, is_directory)) # if directory, yield children recursively in post order until exhausted. for name, is_dir in children: diff --git a/tests/test_ignore_extended.py b/tests/test_ignore_extended.py index 0e62924..2988b84 100644 --- a/tests/test_ignore_extended.py +++ b/tests/test_ignore_extended.py @@ -946,8 +946,8 @@ def test_ignore_from_file(fs, post_house_file_structure): "/root/ShootingDay1/CameraMedia/A/A001/A1.txt", "/root/ShootingDay1/CameraMedia/A/A001/Sidecar.txt", "/root/ShootingDay1/CameraMedia/B/B002/Sidecar.txt", - "/root/ShootingDay1/CameraMedia/A/Proxy/A001", - "/root/ShootingDay1/CameraMedia/B/Proxy/B001", + "/root/ShootingDay1/CameraMedia/A/Proxy", + "/root/ShootingDay1/CameraMedia/B/Proxy", "/root/ShootingDay1/Sound/Sidecar.txt", ] assert_pattern_ignored_in_result(pattern, result) From a19db6488e482ba8d28c1b14facb05caf1dcfd27 Mon Sep 17 00:00:00 2001 From: Alexander Sahm Date: Wed, 21 May 2025 15:07:48 +0200 Subject: [PATCH 5/6] Run github test action on every push on any branch --- .github/workflows/build+test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build+test.yml b/.github/workflows/build+test.yml index 86fbb94..835c228 100644 --- a/.github/workflows/build+test.yml +++ b/.github/workflows/build+test.yml @@ -5,7 +5,6 @@ name: ascmhl-build-test on: push: - branches: [ master, ignore_pattern_in_nested_histories ] pull_request: branches: [ master ] From 921508a1df3212b5e81e530d82c4dbb980f5ca7c Mon Sep 17 00:00:00 2001 From: Alexander Sahm Date: Fri, 4 Jul 2025 09:59:18 +0200 Subject: [PATCH 6/6] Apply the ignore pattern when checking for existing mhl references so that we as well can ignore a completely missing sub history --- ascmhl/commands.py | 5 ++++- tests/conftest.py | 30 ++++++++++++++++++++++++++++++ tests/test_ignore_extended.py | 17 ++++++++++++++++- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/ascmhl/commands.py b/ascmhl/commands.py index 329897e..f792c07 100644 --- a/ascmhl/commands.py +++ b/ascmhl/commands.py @@ -321,7 +321,10 @@ def create_for_folder_subcommand( referenced_asc_folder = os.path.join( os.path.dirname(existing_history.asc_mhl_path), os.path.dirname(ref.path) ) - if not os.path.exists(referenced_asc_folder): + # as we also ignore the ascmhl folder itself check the ignore patterns for the enclosing folder + if not os.path.exists(referenced_asc_folder) and not ignore_spec.get_path_spec().match_file( + os.path.dirname(referenced_asc_folder) + ): missing_asc_mhl_folder.add(os.path.dirname(referenced_asc_folder)) if detect_renaming: diff --git a/tests/conftest.py b/tests/conftest.py index 774f223..92f8139 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,6 +28,36 @@ def path_conversion_tests(path): return Path(path) +# The shutil.rmtree will not work with the fake filesystem on Python 3.12 (because it will not take symlinks), +# so here is an implementation for removing entire directories that will work with the fake fs +def remove_tree(path): + """Recursively delete a directory tree, correctly handling symbolic links.""" + for root, dirs, files in os.walk(path, topdown=False): + for name in files: + file_path = os.path.join(root, name) + try: + os.remove(file_path) # Remove files + except OSError as e: + print(f"Error removing file {file_path}: {e}") + + for name in dirs: + dir_path = os.path.join(root, name) + try: + # If it's a symbolic link, just remove the link itself + if os.path.islink(dir_path): + os.remove(dir_path) + else: + os.rmdir(dir_path) # Remove empty directories + except OSError as e: + print(f"Error removing directory {dir_path}: {e}") + + # Finally remove the root directory (if not a symlink) + if os.path.islink(path): + os.remove(path) + else: + os.rmdir(path) + + @pytest.fixture(scope="session", autouse=True) def set_timezone(): """Fakes the host timezone to UTC so we don't get different mhl files if the tests run on different time zones diff --git a/tests/test_ignore_extended.py b/tests/test_ignore_extended.py index 2988b84..34ceff9 100644 --- a/tests/test_ignore_extended.py +++ b/tests/test_ignore_extended.py @@ -15,7 +15,7 @@ import ascmhl.commands from ascmhl.generator import _extract_pattern_relative_to_history from ascmhl.history import MHLHistory -from tests.conftest import abspath_conversion_tests +from tests.conftest import abspath_conversion_tests, remove_tree from tests.test_ignore import assert_mhl_file_has_exact_ignore_patterns, assert_pattern_ignored_in_result from ascmhl.generator import belongs_to_parent_or_neighbour @@ -1198,3 +1198,18 @@ def test_ignore_unit_extract_relpath(): expected = "docs/" result = ascmhl.generator._extract_pattern_relative_to_history(target, base) assert result == expected + + +def test_ignore_missing_subhistory(fs, post_house_file_structure): + """ + test that creating a new generation doesn't fail if we ignore a missing subhistory + """ + + # delete the whole subhistory, it is referenced from the history in the CameraMedia folder but as we ignore it + # the create command doesn't fail + remove_tree("/root/ShootingDay1/CameraMedia/A") + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root/ShootingDay1/CameraMedia"), "-i", "/A"] + ) + assert result.exit_code == 0