feat: sync serialization with TypeScript — strip redundant fields, thread raga context

Strip redundant/duplicate fields from to_json() to match TypeScript PR #887:
- Pitch: remove ratios and fundamental (inherited from raga)
- Trajectory: remove name, instrumentation, and tags (derived/defaulted)
- Phrase: remove raga (redundant with piece-level raga)

Thread ratios/fundamental from piece-level Raga down through the
deserialization chain (Piece → Phrase → Trajectory → Pitch) with
fallback to embedded values for backward compatibility with old data.

Also adds clone_transcription() and delete_transcription() to SwaraClient,
fixes Group/Phrase to_json() serialization bugs, and hardens Piece.from_json()
against unknown server fields.

Verified 68.3% size reduction on a 6.9MB transcription with all 6,770
pitch frequencies preserved exactly through full clone/save/reload cycle.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: jon-myers

idtap/classes/group.py

@@ -106,11 +106,12 @@ def add_traj(self, traj: Trajectory) -> None:
 
     def to_json(self) -> Dict:
         return {
-            'trajectories': self.trajectories,
+            'trajectories': [t.to_json() if hasattr(t, 'to_json') else t for t in self.trajectories],
             'id': self.id,
         }
 
     @staticmethod
     def from_json(obj: Dict) -> 'Group':
-        trajs = obj.get('trajectories', [])
+        trajs = [t if isinstance(t, Trajectory) else Trajectory.from_json(t)
+                 for t in obj.get('trajectories', [])]
         return Group({'trajectories': trajs, 'id': obj.get('id')})

idtap/classes/phrase.py

@@ -359,11 +359,13 @@ def realign_pitches(self) -> None:
         if not self.raga:
             return
         ratios = self.raga.stratified_ratios
+        fundamental = self.raga.fundamental
         for traj in self.trajectories:
             new_pitches = []
             for p in traj.pitches:
                 opts = p.to_json()
                 opts['ratios'] = ratios
+                opts['fundamental'] = fundamental
                 new_pitches.append(Pitch(opts))
             traj.pitches = new_pitches
 
@@ -545,29 +547,38 @@ def to_json(self) -> Dict[str, Any]:
             'durTot': self.dur_tot,
             'durArray': self.dur_array,
             'chikaris': {k: c.to_json() for k, c in self.chikaris.items()},
-            'raga': self.raga.to_json() if self.raga else None,
             'startTime': self.start_time,
             'trajectoryGrid': [[t.to_json() for t in row] for row in self.trajectory_grid],
-            'instrumentation': self.instrumentation,
-            'groupsGrid': self.groups_grid,
+            'instrumentation': [i.value if hasattr(i, 'value') else i for i in self.instrumentation],
+            'groupsGrid': [[g.to_json() for g in row] for row in self.groups_grid],
             'categorizationGrid': self.categorization_grid,
             'uniqueId': self.unique_id,
             'adHocCategorizationGrid': self.ad_hoc_categorization_grid,
             'isSectionStart': self.is_section_start,
         }
 
     @staticmethod
-    def from_json(obj: Dict[str, Any]) -> 'Phrase':
+    def from_json(obj: Dict[str, Any], ratios=None, fundamental=None) -> 'Phrase':
         opts = selective_decamelize(obj)
+
+        # If phrase has its own raga (legacy data), use it as fallback context
+        phrase_raga = opts.get('raga')
+        if phrase_raga is not None and not isinstance(phrase_raga, Raga):
+            phrase_raga = Raga.from_json(phrase_raga)
+            opts['raga'] = phrase_raga
+
+        r = ratios if ratios is not None else (phrase_raga.stratified_ratios if phrase_raga else None)
+        f = fundamental if fundamental is not None else (phrase_raga.fundamental if phrase_raga else None)
+
         trajectory_grid = opts.get('trajectory_grid')
         if trajectory_grid is not None:
             tg = []
             for row in trajectory_grid:
-                tg.append([Trajectory.from_json(t) for t in row])
+                tg.append([Trajectory.from_json(t, ratios=r, fundamental=f) for t in row])
             opts['trajectory_grid'] = tg
         trajectories = opts.get('trajectories')
         if trajectories is not None:
-            opts['trajectories'] = [Trajectory.from_json(t) for t in trajectories]
+            opts['trajectories'] = [Trajectory.from_json(t, ratios=r, fundamental=f) for t in trajectories]
         chikaris = opts.get('chikaris')
         if chikaris is not None:
             new_c = {}
@@ -583,9 +594,6 @@ def from_json(obj: Dict[str, Any]) -> 'Phrase':
                     new_obj[str(k)] = Chikari.from_json(v)
                 new_grid.append(new_obj)
             opts['chikari_grid'] = new_grid
-        raga = opts.get('raga')
-        if raga is not None and not isinstance(raga, Raga):
-            opts['raga'] = Raga.from_json(raga)
         return Phrase(opts)
 
     def to_note_view_phrase(self) -> 'NoteViewPhrase':

idtap/classes/piece.py

@@ -371,8 +371,8 @@ def _validate_parameter_types(self, opts: dict) -> None:
         if 'collections' in opts and opts['collections'] is not None:
             if not isinstance(opts['collections'], list):
                 raise TypeError(f"Parameter 'collections' must be a list, got {type(opts['collections']).__name__}")
-            if not all(isinstance(item, str) for item in opts['collections']):
-                raise TypeError("All items in 'collections' must be strings")
+            if not all(isinstance(item, str) or item is None for item in opts['collections']):
+                raise TypeError("All items in 'collections' must be strings or None")
 
         # Validate trackTitles
         if 'trackTitles' in opts and opts['trackTitles'] is not None:
@@ -1451,12 +1451,19 @@ def to_json(self) -> Dict[str, Any]:
     @staticmethod
     def from_json(obj: Dict[str, Any]) -> "Piece":
         new_obj = dict(obj)
+        raga = None
         if "raga" in new_obj:
-            new_obj["raga"] = Raga.from_json(new_obj["raga"])
+            raga = Raga.from_json(new_obj["raga"])
+            new_obj["raga"] = raga
+
+        # Extract raga context for threading down to pitches
+        ratios = raga.stratified_ratios if raga else None
+        fundamental = raga.fundamental if raga else None
+
         if "phraseGrid" in new_obj:
             pg = []
             for row in new_obj["phraseGrid"]:
-                phrase_row = [Phrase.from_json(p) for p in row]
+                phrase_row = [Phrase.from_json(p, ratios=ratios, fundamental=fundamental) for p in row]
                 pg.append(phrase_row)
             # reconstruct groups so they reference existing trajectories
             for row in pg:
@@ -1465,6 +1472,8 @@ def from_json(obj: Dict[str, Any]) -> "Piece":
                     for g_list in phrase.groups_grid:
                         rebuilt: List[Group] = []
                         for g in g_list:
+                            if isinstance(g, str):
+                                continue  # skip bare ID strings
                             data = g if isinstance(g, dict) else g.to_json()
                             trajs = []
                             for t in data.get("trajectories", []):
@@ -1494,6 +1503,20 @@ def from_json(obj: Dict[str, Any]) -> "Piece":
                 dm = dm["$date"]
             new_obj["dateModified"] = datetime.fromisoformat(str(dm).replace('Z',''))
 
+        # Strip keys not recognized by the constructor (e.g. server-only
+        # fields like 'userId' that are not part of the data model).
+        allowed = {
+            'raga', 'instrumentation', 'phraseGrid', 'phrases', 'title',
+            'dateCreated', 'dateModified', 'location', '_id', 'audioID',
+            'audio_DB_ID', 'userID', 'name', 'family_name', 'given_name',
+            'permissions', 'soloist', 'soloInstrument', 'explicitPermissions',
+            'meters', 'sectionStartsGrid', 'sectionStarts', 'sectionCatGrid',
+            'sectionCategorization', 'adHocSectionCatGrid', 'excerptRange',
+            'assemblageDescriptors', 'collections', 'durTot', 'durArrayGrid',
+            'durArray', 'trackTitles',
+        }
+        new_obj = {k: v for k, v in new_obj.items() if k in allowed}
+
         piece = Piece(new_obj)
 
         # reconnect groups to actual trajectories

idtap/classes/pitch.py

@@ -203,8 +203,6 @@ def to_json(self):
             'swara': self.swara,
             'raised': self.raised,
             'oct': self.oct,
-            'ratios': self.ratios,
-            'fundamental': self.fundamental,
             'logOffset': self.log_offset,
         } 
 
@@ -391,5 +389,10 @@ def same_as(self, other: "Pitch") -> bool:
         return self.swara == other.swara and self.oct == other.oct and self.raised == other.raised
 
     @classmethod
-    def from_json(cls, obj: dict) -> "Pitch":
-        return cls(obj)
+    def from_json(cls, obj: dict, ratios=None, fundamental=None) -> "Pitch":
+        opts = dict(obj)
+        if ratios is not None:
+            opts['ratios'] = ratios
+        if fundamental is not None:
+            opts['fundamental'] = fundamental
+        return cls(opts)

idtap/classes/trajectory.py

@@ -911,10 +911,8 @@ def to_json(self) -> Dict:
             'articulations': {k: a.to_json() for k, a in self.articulations.items()},
             'startTime': self.start_time,
             'num': self.num,
-            'name': self.name,
             'fundID12': self.fund_id12,
             'vibObj': self.vib_obj,
-            'instrumentation': self.instrumentation.value if isinstance(self.instrumentation, Instrument) else self.instrumentation,
             'vowel': self.vowel,
             'startConsonant': self.start_consonant,
             'startConsonantHindi': self.start_consonant_hindi,
@@ -927,15 +925,15 @@ def to_json(self) -> Dict:
             'groupId': self.group_id,
             'automation': self.automation.to_json() if self.automation else None,
             'uniqueId': self.unique_id,
-            'tags': self.tags,
         }
         # drop None values so they serialize as undefined (omitted) rather than null
         return {k: v for k, v in data.items() if v is not None}
 
     @staticmethod
-    def from_json(obj: Dict) -> 'Trajectory':
+    def from_json(obj: Dict, ratios=None, fundamental=None) -> 'Trajectory':
         opts = humps.decamelize(obj)
-        pitches = [Pitch.from_json(p) for p in opts.get('pitches', [])]
+        pitches = [Pitch.from_json(p, ratios=ratios, fundamental=fundamental)
+                   for p in opts.get('pitches', [])]
         arts = {}
         for k,v in opts.get('articulations', {}).items():
             if v is not None:

idtap/client.py

@@ -126,6 +126,15 @@ def _get(self, endpoint: str, params: Optional[Dict[str, Any]] = None) -> Any:
             return response.json()
         return response.content
 
+    def _delete_json(self, endpoint: str, payload: Dict[str, Any]) -> Any:
+        url = self.base_url + endpoint
+        headers = self._auth_headers()
+        response = requests.delete(url, json=payload, headers=headers, timeout=1800)
+        response.raise_for_status()
+        if response.content:
+            return response.json()
+        return None
+
     # ---- API methods ----
     def get_piece(self, piece_id: str, fetch_rule_set: bool = True) -> Any:
         """Return transcription JSON for the given id.
@@ -182,6 +191,76 @@ def insert_new_transcription(self, piece: Dict[str, Any]) -> Any:
         payload["userID"] = self.user_id
         return self._post_json("insertNewTranscription", payload)
 
+    def clone_transcription(
+        self,
+        piece_id: str,
+        title: Optional[str] = None,
+        explicit_permissions: Optional[Dict[str, Any]] = None,
+        soloist: Optional[str] = None,
+        solo_instrument: Optional[str] = None,
+    ) -> Any:
+        """Clone a transcription, creating a new copy owned by the current user.
+
+        The server copies all transcription data (phrases, trajectories, pitches,
+        raga, audio association, etc.) and assigns a new ID, owner, and timestamps.
+
+        Args:
+            piece_id: The ID of the transcription to clone.
+            title: Title for the cloned transcription. Defaults to server behavior.
+            explicit_permissions: Permission object with 'edit', 'view' (user ID
+                lists) and 'publicView' (bool). Defaults to private.
+            soloist: Soloist name for the clone.
+            solo_instrument: Solo instrument for the clone.
+
+        Returns:
+            Server response with ``insertedId`` of the new transcription.
+        """
+        if not self.user_id:
+            raise RuntimeError("Not authenticated: cannot clone transcription")
+        payload: Dict[str, Any] = {
+            "id": piece_id,
+            "newOwner": self.user_id,
+        }
+        if title is not None:
+            payload["title"] = title
+        if self.user:
+            payload["name"] = self.user.get("name", "")
+            payload["family_name"] = self.user.get("family_name", "")
+            payload["given_name"] = self.user.get("given_name", "")
+        if explicit_permissions is not None:
+            payload["explicitPermissions"] = explicit_permissions
+        else:
+            payload["explicitPermissions"] = {
+                "edit": [],
+                "view": [],
+                "publicView": False,
+            }
+        if soloist is not None:
+            payload["soloist"] = soloist
+        if solo_instrument is not None:
+            payload["soloInstrument"] = solo_instrument
+        return self._post_json("cloneTranscription", payload)
+
+    def delete_transcription(self, piece_id: str) -> Any:
+        """Delete a transcription from the server.
+
+        Removes the transcription document and the reference from the user's
+        transcriptions array.
+
+        Args:
+            piece_id: The ID of the transcription to delete.
+
+        Returns:
+            Server response with ``deletedCount``.
+        """
+        if not self.user_id:
+            raise RuntimeError("Not authenticated: cannot delete transcription")
+        payload = {
+            "_id": piece_id,
+            "userID": self.user_id,
+        }
+        return self._delete_json("oneTranscription", payload)
+
     def _prompt_for_waiver_if_needed(self) -> None:
         """Interactively prompt user to agree to waiver if not already agreed."""
         if self.has_agreed_to_waiver():

idtap/tests/pitch_test.py

@@ -38,8 +38,6 @@ def test_default_pitch():
         'swara': 0,
         'raised': True,
         'oct': 0,
-        'ratios': ratios,
-        'fundamental': 261.63,
         'logOffset': 0,
     }