Adding recursive parsing to flatten command

ascmhl/commands.py

@@ -34,6 +34,7 @@
 from .traverse import post_order_lexicographic
 from typing import Dict
 from collections import namedtuple
+from .utils import convert_local_path_to_posix
 
 
 @click.command()
@@ -1195,18 +1196,30 @@ def flatten_history(
     if len(existing_history.hash_lists) == 0:
         raise errors.NoMHLHistoryException(root_path)
 
-    for hash_list in existing_history.hash_lists:
+    flatten_child_histories(existing_history, session, root_path)
+
+    commit_session_for_collection(
+        session, root_path, author_name, author_email, author_phone, author_role, location, comment
+    )
+
+
+def flatten_child_histories(history, session, roothistorypath, pathprefix=""):
+    for hash_list in history.hash_lists:
         for media_hash in hash_list.media_hashes:
             if not media_hash.is_directory:
                 for hash_entry in media_hash.hash_entries:
                     if hash_entry.action != "failed":
+                        # add prefix to media path if subhistory
+                        media_path = media_hash.path
+                        if pathprefix != "":
+                            media_path = convert_local_path_to_posix(pathprefix) + "/" + media_hash.path
                         # check if this entry is newer than the one already in there to avoid duplicate entries
-                        found_media_hash = session.new_hash_lists[collection_history].find_media_hash_for_path(
-                            media_hash.path
+                        found_media_hash = session.new_hash_lists[session.root_history].find_media_hash_for_path(
+                            media_path
                         )
                         if found_media_hash == None:
                             session.append_file_hash(
-                                media_hash.path,
+                                media_path,
                                 media_hash.file_size,
                                 media_hash.last_modification_date,
                                 hash_entry.hash_format,
@@ -1222,7 +1235,7 @@ def flatten_history(
                             if not hashformat_is_already_there:
                                 # assuming that hash_entry of same type also has same hash_value ..
                                 session.append_file_hash(
-                                    media_hash.path,
+                                    media_path,
                                     media_hash.file_size,
                                     media_hash.last_modification_date,
                                     hash_entry.hash_format,
@@ -1231,9 +1244,14 @@ def flatten_history(
                                     hash_date=hash_entry.hash_date,
                                 )
 
-    commit_session_for_collection(
-        session, root_path, author_name, author_email, author_phone, author_role, location, comment
-    )
+    for child_history in history.child_histories:
+        childpath = child_history.get_root_path()
+
+        # if os.path.isabs(file_path):
+        childrelativepath = os.path.relpath(childpath, roothistorypath)
+
+        logger.info(f"\nChild History at {childrelativepath}:")
+        flatten_child_histories(child_history, session, roothistorypath, childrelativepath)
 
 
 @click.command()
@@ -1471,7 +1489,7 @@ def commit_session_for_collection(
     process_info.root_media_hash = root_hash
     process_info.hashlist_custom_basename = "packinglist_" + os.path.basename(root_path)
 
-    session.commit(creator_info, process_info)
+    session.commit(creator_info, process_info, writeChain=False)
 
 
 """

ascmhl/generator.py

@@ -7,6 +7,9 @@
 __email__ = "opensource@pomfort.com"
 """
 
+import os
+import shutil
+
 from collections import defaultdict
 from typing import Dict, List
 
@@ -15,6 +18,7 @@
 from .ignore import MHLIgnoreSpec
 from .hashlist import MHLHashList, MHLHashEntry, MHLCreatorInfo, MHLProcessInfo
 from .history import MHLHistory
+from .utils import convert_posix_to_local_path
 
 
 class MHLGenerationCreationSession:
@@ -131,7 +135,13 @@ def append_file_hash(
         hash_entry = MHLHashEntry(hash_format, hash_string, hash_date=hash_date)
         if original_hash_entry is None:
             hash_entry.action = "original"
-            logger.verbose(f"  created original hash for     {relative_path}  {hash_format}: {hash_string}")
+            if relative_path != None:
+                logger.verbose(f"  created original hash for     {relative_path}  {hash_format}: {hash_string}")
+            else:
+                # flattening works a bit different, because we don't add to individual (nested) histories
+                logger.verbose(
+                    f"  created original hash for     {convert_posix_to_local_path(file_path)}  {hash_format}: {hash_string}"
+                )
         else:
             existing_hash_entry = history.find_first_hash_entry_for_path(history_relative_path, hash_format)
             if existing_hash_entry is not None:
@@ -272,7 +282,7 @@ def append_directory_hashes(
                 hash_entry.structure_hash_string = structure_hash_string
                 parent_media_hash.append_hash_entry(hash_entry)
 
-    def commit(self, creator_info: MHLCreatorInfo, process_info: MHLProcessInfo):
+    def commit(self, creator_info: MHLCreatorInfo, process_info: MHLProcessInfo, writeChain=True):
         """
         this method needs to create the generations of the children bottom up
         # so each history can reference the children correctly and can get the actual hash of the file
@@ -306,4 +316,26 @@ def commit(self, creator_info: MHLCreatorInfo, process_info: MHLProcessInfo):
             if history.parent_history is not None:
                 referenced_hash_lists[history.parent_history].append(new_hash_list)
 
-            chain_xml_parser.write_chain(history.chain, new_hash_list)
+            if writeChain:
+                # regular history ....
+                chain_xml_parser.write_chain(history.chain, new_hash_list)
+            else:
+                # ... or flattened history manifest
+                root_path = os.path.dirname(new_hash_list.file_path)
+                if not os.path.exists(root_path):
+                    print(f"ERR: folder {root_path} with flattened manifest does not exist")
+                    return
+
+                parent_folder = os.path.dirname(root_path)
+
+                for file_name in os.listdir(root_path):
+                    if file_name.endswith(".mhl"):
+                        src_path = os.path.join(root_path, file_name)
+                        dst_path = os.path.join(parent_folder, file_name)
+                        shutil.move(src_path, dst_path)
+
+                # Remove the folder if empty
+                if not os.listdir(root_path):
+                    os.rmdir(root_path)
+                else:
+                    print(f"ERR: temp folder not empty, did not remove {root_path}")

tests/test_flatten.py

@@ -10,6 +10,7 @@
 import os
 from click.testing import CliRunner
 from freezegun import freeze_time
+from .conftest import path_conversion_tests
 from .conftest import abspath_conversion_tests
 
 import ascmhl.commands
@@ -51,3 +52,29 @@ def test_simple_two_hashformats(fs, simple_mhl_history):
         ascmhl.commands.flatten, [abspath_conversion_tests("/root"), abspath_conversion_tests("/out")]
     )
     assert result.exit_code == 0
+
+
+@freeze_time("2020-01-16 09:15:00")
+def test_nested(fs, nested_mhl_histories):
+    runner = CliRunner()
+
+    result = runner.invoke(
+        ascmhl.commands.flatten, ["-v", abspath_conversion_tests("/root"), abspath_conversion_tests("/out")]
+    )
+    assert result.exit_code == 0
+
+    # check for files in root and sub histories
+    assert (
+        result.output == f"Flattening folder at path: {abspath_conversion_tests('/root')} ...\n"
+        f"  created original hash for     Stuff.txt  xxh64: 94c399c2a9a21f9a\n"
+        f"\n"
+        f"Child History at {path_conversion_tests('A/AA')}:\n"
+        f"  created original hash for     {path_conversion_tests('A/AA/AA1.txt')}  xxh64: ab6bec9ec04704f6\n"
+        f"\n"
+        f"Child History at B:\n"
+        f"  created original hash for     {path_conversion_tests('B/B1.txt')}  xxh64: 51fb8fb099e92821\n"
+        f"\n"
+        f"Child History at {path_conversion_tests('B/BB')}:\n"
+        f"  created original hash for     {path_conversion_tests('B/BB/BB1.txt')}  xxh64: 5c14eac4f4ad7501\n"
+        f"Created new generation {path_conversion_tests('collection_2020-01-16/packinglist_root_2020-01-16_091500Z.mhl')}\n"
+    )