Register a converted pack's colors for its own species

Python/convert_sidekick_pack.py

@@ -47,6 +47,33 @@
 # The color atlas parameter on M_Default_Sidekick, set per scheme material instance.
 COLOR_TEXTURE_PARAM = "texture"
 
+# A part's file name ends in the pack's species code, e.g. SK_FANT_SKTN_01_10TORS_SN01. The
+# mesh suffix uses HU for the Human species whose database sk_species.code is HN; the rest match.
+SUFFIX_TO_SPECIES_CODE = {"HU": "HN", "HN": "HN", "GO": "GO", "SN": "SN", "EV": "EV", "ZB": "ZB", "RO": "RO"}
+
+
+def species_code_of_part(name):
+    """The sk_species.code a part belongs to, read from its file-name suffix, or None."""
+    match = re.search(r"_([A-Za-z]+)\d+$", name or "")
+    if not match:
+        return None
+    return SUFFIX_TO_SPECIES_CODE.get(match.group(1).upper())
+
+
+def dominant_species_code(part_names):
+    """The pack's species from its parts' name suffixes. A pack's own species parts carry its
+    code, while shared base parts read as Human, so prefer the most common non-Human code; with
+    none (a plain Human pack), fall back to Human."""
+    counts = {}
+    for name in part_names:
+        code = species_code_of_part(name)
+        if code:
+            counts[code] = counts.get(code, 0) + 1
+    non_human = {code: n for code, n in counts.items() if code != "HN"}
+    if non_human:
+        return max(non_human, key=non_human.get)
+    return "HN"
+
 
 def read_job():
     """job.txt is KEY=VALUE lines. PACKAGE may repeat (a batch of packs); SKELETON/
@@ -186,10 +213,11 @@ def scheme_name(colormap_asset):
     return ("%s %s" % (spaced, index)).strip()
 
 
-def extract_color_maps(unitypackage_path, pack_names):
-    """Pull the pack's ColorMap atlases (under Characters/<pack>/) into a scratch folder.
-    Returns (png path, asset name, scheme name, pack) per atlas. Only atlases whose pack
-    matches a converted outfit pack are taken, so the species' base skin maps are left alone."""
+def extract_color_maps(unitypackage_path, species_code):
+    """Pull the pack's ColorMap atlases (under Characters/<set>/) into a scratch folder. Returns
+    (png path, asset name, scheme name, character set, species code) per atlas. A pack's
+    .unitypackage carries only its own atlases, so every Characters/ atlas (the outfit theme sets
+    and the species/skin sets alike) belongs to this pack and is tagged with the pack's species."""
     archive = tarfile.open(unitypackage_path, "r:gz")
 
     guid_to_path = {}
@@ -207,16 +235,14 @@ def extract_color_maps(unitypackage_path, pack_names):
         segments = unity_path.split("/")
         if "Characters" not in segments:
             continue
-        pack = segments[segments.index("Characters") + 1]
-        if pack not in pack_names:
-            continue
+        character_set = segments[segments.index("Characters") + 1]
 
         asset_name = os.path.splitext(os.path.basename(unity_path))[0]
-        png_path = os.path.join(COLORMAP_TEMP, pack, os.path.basename(unity_path))
+        png_path = os.path.join(COLORMAP_TEMP, character_set, os.path.basename(unity_path))
         os.makedirs(os.path.dirname(png_path), exist_ok=True)
         with open(png_path, "wb") as out_file:
             out_file.write(archive.extractfile(member).read())
-        maps.append((png_path, asset_name, scheme_name(asset_name), pack))
+        maps.append((png_path, asset_name, scheme_name(asset_name), character_set, species_code))
 
     archive.close()
     return maps
@@ -269,7 +295,7 @@ def build_pack_schemes(color_maps, base_material):
     """Import each atlas and build its scheme material instance. Returns a default scheme
     material per pack so parts come in colored."""
     pack_default = {}
-    for png_path, asset_name, name, pack in color_maps:
+    for png_path, asset_name, name, pack, _species_code in color_maps:
         texture = import_color_map_texture(png_path, pack, asset_name)
         if not texture:
             continue
@@ -282,14 +308,14 @@ def build_pack_schemes(color_maps, base_material):
 def write_scheme_manifest(manifest):
     """Leave the schemes for register_color_schemes.py, which the plugin's module runs at the
     next editor startup to write them into the toolkit database, before the toolkit opens it.
-    The manifest is scheme_name<TAB>colormap.png lines."""
+    The manifest is scheme_name<TAB>colormap.png<TAB>species_code lines."""
     if not manifest:
         return
     if not os.path.isdir(JOB_DIR):
         os.makedirs(JOB_DIR)
     with open(SCHEME_MANIFEST, "w") as handle:
-        for name, png_path in manifest:
-            handle.write("%s\t%s\n" % (name, png_path))
+        for name, png_path, species_code in manifest:
+            handle.write("%s\t%s\t%s\n" % (name, png_path, species_code))
 
 
 def run():
@@ -308,24 +334,26 @@ def run():
         report_progress(0, 0, "DONE")
         return
 
-    # Extract every pack first so the progress total covers the whole batch.
+    # Extract every pack first so the progress total covers the whole batch. Each pack's species
+    # is read from its parts and tags that pack's color atlases, so a batch may mix species.
     report_progress(0, 0, "extracting")
     parts = []
+    color_maps = []
     for package in packages:
         extracted = extract_part_meshes(package)
-        log("extracted %d part meshes from %s" % (len(extracted), os.path.basename(package)))
+        species_code = dominant_species_code([name for _disk, _game_dir, name in extracted])
+        log("extracted %d part meshes from %s (species %s)" % (
+            len(extracted), os.path.basename(package), species_code))
         parts += extracted
+        color_maps += extract_color_maps(package, species_code)
 
-    pack_names = {pack_of_part(game_dir) for _, game_dir, _ in parts}
-    color_maps = []
-    for package in packages:
-        color_maps += extract_color_maps(package, pack_names)
-
-    manifest = [(name, png_path) for png_path, _asset, name, _pack in color_maps]
+    manifest = [(name, png_path, species_code)
+                for png_path, _asset, name, _set, species_code in color_maps]
     write_scheme_manifest(manifest)
 
     pack_default_material = build_pack_schemes(color_maps, shared_material)
-    log("found %d color scheme(s) across %d pack(s)" % (len(manifest), len(pack_names)))
+    log("found %d color scheme(s) across %d character set(s)" % (
+        len(manifest), len({entry[3] for entry in color_maps})))
 
     total = len(parts)
     report_progress(0, total, "starting")

Python/register_color_schemes.py

@@ -5,13 +5,21 @@
 after a conversion with the toolkit closed, or at editor startup before the toolkit opens the
 file. It retries with a fresh connection to ride out timing.
 
-For each scheme it writes one Outfits preset and one Attachments preset for the Human species,
-each with a row per swatch property colored from the scheme's ColorMap atlas. Idempotent: a
-re-run replaces a scheme rather than duplicating it.
+From each scheme's ColorMap atlas it writes the Species (skin / robot plating) swatches under the
+pack's own species - except Human, whose Species list is left as the base skin tones (a human pack
+only contributes Outfit/Attachment colors) - and the Outfit/Attachment swatches under the Human
+species (the shared pool) - whichever of those three groups the atlas actually fills. Materials and Elements are never written
+from an atlas; they are a base-curated set, and pack colors landing there would mis-file e.g. robot
+plating as an Element. To match the Unity tool, where the shared pools show for every species, each
+non-Human species the batch converts is then given Human's Outfit/Attachment/Material pools - keyed
+off the species being present, not off what its atlas fills, so a species that only ships skin
+colors (a zombie wearing human outfits) still gets the full menu. Elements stays Human-only.
+Idempotent: a re-run replaces a scheme rather than duplicating it.
 
   python register_color_schemes.py <manifest> <database> [attempts] [delay_seconds]
 
-The manifest is one "scheme name<TAB>colormap.png" line per scheme.
+The manifest is one "scheme name<TAB>colormap.png<TAB>species_code" line per scheme. The species
+code is the sk_species.code of the pack (HN, SN, GO, EV, ZB, RO); a missing third column means HN.
 """
 
 import os
@@ -23,7 +31,22 @@
 import traceback
 import zlib
 
-OUTFIT_COLOR_GROUPS = (2, 3)   # the toolkit's EColorGroup Outfits and Attachments
+# EColorGroup: 1 Species, 2 Outfits, 3 Attachments, 4 Materials, 5 Elements. Species is per-species
+# (human skin / bone properties); the rest are shared/global pools the Unity tool shows for everyone.
+SPECIES_COLOR_GROUP = 1               # written under the pack's own species (skin / robot plating)
+# What we write from a pack's atlases: Species (under the pack's species) + Outfits/Attachments
+# (under Human, the shared pool). Materials and Elements are NOT written from atlases - they are a
+# base-curated set ("Materials 01-10", "Elements 01") and writing pack colors into them mis-files
+# e.g. robot plating as an Element. They reach other species only through replication below.
+REGISTER_GROUPS = (1, 2, 3)
+# Shared with every non-Human species so its color menu matches the Unity tool: Outfits, Attachments,
+# Materials. Elements is left to Human only (no non-Human official pack shows it), avoiding a stray
+# inert category.
+REPLICATE_GROUPS = (2, 3, 4)
+GLOBAL_COLOR_GROUPS = (2, 3, 4, 5)    # groups cleared of stale replicas before re-replicating
+HUMAN_SPECIES_CODE = "HN"
+SENTINEL = "FF0000"                   # an unused swatch cell; toolkit treats it as "no color"
+FILL_FRACTION = 0.5                   # a scheme registers in a group when >= half its swatches are real
 LOG_PATH = None
 
 
@@ -115,37 +138,110 @@ def read_manifest(path):
     schemes = []
     with open(path, "r") as handle:
         for line in handle:
-            name, sep, png = line.rstrip("\n").partition("\t")
-            if sep and os.path.isfile(png):
-                schemes.append((name, png))
+            fields = line.rstrip("\n").split("\t")
+            if len(fields) < 2 or not os.path.isfile(fields[1]):
+                continue
+            name, png = fields[0], fields[1]
+            species_code = fields[2] if len(fields) > 2 and fields[2] else HUMAN_SPECIES_CODE
+            schemes.append((name, png, species_code))
     return schemes
 
 
-def write_schemes(connection, schemes):
-    species = connection.execute("SELECT id FROM sk_species WHERE code='HN'").fetchone()
-    if not species:
-        raise RuntimeError("Human species not found in database")
-    species_id = species[0]
-    properties = {
-        group: connection.execute(
-            "SELECT id, u, v FROM sk_color_property WHERE color_group=?", (group,)).fetchall()
-        for group in OUTFIT_COLOR_GROUPS
-    }
-    for name, pixels in schemes:
-        for group in OUTFIT_COLOR_GROUPS:
+def species_id_for(connection, code, cache):
+    """sk_species.id for a species code, cached. None if the code is unknown."""
+    if code not in cache:
+        row = connection.execute("SELECT id FROM sk_species WHERE code=?", (code,)).fetchone()
+        cache[code] = row[0] if row else None
+    return cache[code]
+
+
+def replace_preset(connection, species_id, group, name, prop_colors):
+    """Write one color preset (replacing any same name/species/group), one row per property."""
+    for (existing_id,) in connection.execute(
+            "SELECT id FROM sk_color_preset WHERE name=? AND ptr_species=? AND color_group=?",
+            (name, species_id, group)).fetchall():
+        connection.execute("DELETE FROM sk_color_preset_row WHERE ptr_color_preset=?", (existing_id,))
+        connection.execute("DELETE FROM sk_color_preset WHERE id=?", (existing_id,))
+    preset_id = connection.execute(
+        "INSERT INTO sk_color_preset (ptr_species, color_group, name) VALUES (?, ?, ?)",
+        (species_id, group, name)).lastrowid
+    connection.executemany(
+        "INSERT INTO sk_color_preset_row "
+        "(ptr_color_preset, ptr_color_property, color, metallic, smoothness, reflection, emission, opacity) "
+        "VALUES (?, ?, ?, 'FF0000', 'FF0000', 'FF0000', 'FF0000', 'FF0000')",
+        [(preset_id, prop_id, color) for prop_id, color in prop_colors])
+
+
+def replicate_global_pools(connection, human_id, species_id, fill_groups):
+    """Give a non-Human species the same shared Outfit/Attachment/Material/Element pools as Human,
+    but only for the groups the pack's own atlases actually use (fill_groups), so the species shows
+    exactly the color categories it needs — like the official tool, where a Skeleton gets Outfits/
+    Attachments/Materials but not Elements. Replicas that share a Human name are refreshed; any stale
+    replica in a group the pack no longer fills is removed; the species' own unique presets (e.g.
+    Goblin's) are left untouched. Idempotent."""
+    for group in GLOBAL_COLOR_GROUPS:
+        human_presets = connection.execute(
+            "SELECT id, name FROM sk_color_preset WHERE ptr_species=? AND color_group=?",
+            (human_id, group)).fetchall()
+        # Drop any prior replica of a Human-named preset first, so a group that is no longer in
+        # fill_groups (e.g. Elements) is cleared out rather than left stale.
+        for _src_id, name in human_presets:
             for (existing_id,) in connection.execute(
                     "SELECT id FROM sk_color_preset WHERE name=? AND ptr_species=? AND color_group=?",
                     (name, species_id, group)).fetchall():
                 connection.execute("DELETE FROM sk_color_preset_row WHERE ptr_color_preset=?", (existing_id,))
                 connection.execute("DELETE FROM sk_color_preset WHERE id=?", (existing_id,))
-            preset_id = connection.execute(
+        if group not in fill_groups:
+            continue
+        for src_id, name in human_presets:
+            new_id = connection.execute(
                 "INSERT INTO sk_color_preset (ptr_species, color_group, name) VALUES (?, ?, ?)",
                 (species_id, group, name)).lastrowid
-            connection.executemany(
+            connection.execute(
                 "INSERT INTO sk_color_preset_row "
                 "(ptr_color_preset, ptr_color_property, color, metallic, smoothness, reflection, emission, opacity) "
-                "VALUES (?, ?, ?, 'FF0000', 'FF0000', 'FF0000', 'FF0000', 'FF0000')",
-                [(preset_id, prop_id, swatch_hex(pixels, u, v)) for prop_id, u, v in properties[group]])
+                "SELECT ?, ptr_color_property, color, metallic, smoothness, reflection, emission, opacity "
+                "FROM sk_color_preset_row WHERE ptr_color_preset=?",
+                (new_id, src_id))
+
+
+def write_schemes(connection, schemes):
+    human_id = species_id_for(connection, HUMAN_SPECIES_CODE, {})
+    if not human_id:
+        raise RuntimeError("Human species not found in database")
+    species_cache = {HUMAN_SPECIES_CODE: human_id}
+    properties = {
+        group: connection.execute(
+            "SELECT id, u, v FROM sk_color_property WHERE color_group=?", (group,)).fetchall()
+        for group in REGISTER_GROUPS
+    }
+    non_human_species = set()
+    for name, pixels, species_code in schemes:
+        own_id = species_id_for(connection, species_code, species_cache) or human_id
+        if own_id != human_id:
+            non_human_species.add(own_id)
+        for group in REGISTER_GROUPS:
+            # Synty keeps the Human Species list as just the base skin tones; a human pack only adds
+            # Outfit/Attachment colors, so never write the Species group under Human (it would only
+            # duplicate the base skin tones). Non-Human species still get their own species colors.
+            if group == SPECIES_COLOR_GROUP and own_id == human_id:
+                continue
+            props = properties[group]
+            if not props:
+                continue
+            colors = [(prop_id, swatch_hex(pixels, u, v)) for prop_id, u, v in props]
+            real = sum(1 for _, hex_color in colors if hex_color != SENTINEL)
+            if real < FILL_FRACTION * len(props):
+                continue  # this atlas doesn't fill this group (e.g. a skeleton has no skin colors)
+            # Species (skin / plating) colors go under the pack's own species; the shared outfit
+            # pools go under Human. Materials/Elements are base-curated and never written here.
+            dest_id = own_id if group == SPECIES_COLOR_GROUP else human_id
+            replace_preset(connection, dest_id, group, name, colors)
+    # Give every non-Human species the shared Outfit/Attachment/Material pools, like the Unity tool
+    # shows for all species. Keyed off the species being present (not what its atlas fills), so a
+    # species that only ships skin colors (zombies wear human outfits) still gets the full menu.
+    for species_id in non_human_species:
+        replicate_global_pools(connection, human_id, species_id, REPLICATE_GROUPS)
 
 
 def register(db_path, schemes, attempts=90, delay_seconds=2):
@@ -203,10 +299,10 @@ def main():
             log("database not found, leaving schemes for a later startup: %s" % db_path)
             return 1
         decoded = []
-        for name, png_path in schemes:
+        for name, png_path, species_code in schemes:
             with open(png_path, "rb") as handle:
                 _, _, pixels = decode_png(handle.read())
-            decoded.append((name, pixels))
+            decoded.append((name, pixels, species_code))
         return 0 if register(db_path, decoded, attempts, delay_seconds) else 1
     except Exception as error:
         log("ERROR: " + repr(error))