Optimize the diff algorithm, part 3

.gitignore

@@ -1,4 +1,5 @@
 .idea/
 .cache
 *.pyc
-*.egg-info
+*.egg-info
+.DS_Store

WikiWho/utils.py

@@ -42,7 +42,10 @@ def split_into_paragraphs(text):
     text = text.replace('<table>', '\n\n<table>').replace('</table>', '</table>\n\n')
     text = text.replace('<tr>', '\n\n<tr>').replace('</tr>', '</tr>\n\n')
     # wp table syntax
-    text = text.replace('{|', '\n\n{|').replace('|}', '|}\n\n')
+    text = text.replace('{|', '\n\n{|')
+    # Treat a line-start `|}}` as a template ending. It is ambiguous with a
+    # table close followed by a literal `}`, which this fast path cannot parse.
+    text = re.sub(r'(?m)^([ \t]*\|\})(?!\})', r'\1\n\n', text)
     text = text.replace('|-\n', '\n\n|-\n')
     return text.split('\n\n')
 

tests/authorship_cases.json

@@ -293,5 +293,101 @@
         }
       }
     ]
+  },
+  {
+    "title": "The Tell-Tale Brain",
+    "fixture": "fixtures/the_tell-tale_brain_revisions.json",
+    "history_through_revid": 466775504,
+    "snapshot_revid": 466775504,
+    "cases": [
+      {
+        "id": "tell_tale_brain_partial_historical_sentence_restore",
+        "context": "When VS Ramachandran, one of the",
+        "focus": ["When VS Ramachandran, one of the"],
+        "expected": {
+          "values": ["when", "vs", "ramachandran", ",", "one", "of", "the"],
+          "origin_rev_ids": [460801847, 460730548, 460730548, 466775504, 460730548, 466775504, 460730548],
+          "last_rev_ids": [466775504, 466775504, 466775504, 466775504, 466775504, 466775504, 466775504],
+          "inbound": [[466775504], [466775504], [466775504], [], [466775504], [], [466775504]],
+          "outbound": [[463754831], [463754831], [463754831], [], [463754831], [], [463754831]]
+        }
+      }
+    ]
+  },
+  {
+    "title": "Luis Donaldo Colosio Riojas",
+    "fixture": "fixtures/luis_donaldo_colosio_riojas_revisions.json",
+    "history_through_revid": 1222367802,
+    "snapshot_revid": 1222367802,
+    "cases": [
+      {
+        "id": "colosio_citation_title_move_across_punctuation",
+        "context": "|title=Colosio, Vizcaíno, Marina y Cerqueda, las nuevas caras de la política en México |url=",
+        "focus": ["Colosio, Vizcaíno, Marina y Cerqueda, las nuevas caras de la política en México"],
+        "expected": {
+          "values": ["colosio", ",", "vizcaíno", ",", "marina", "y", "cerqueda", ",", "las", "nuevas", "caras", "de", "la", "política", "en", "méxico"],
+          "origin_rev_ids": [1028074078, 1028074078, 1028074078, 1028074078, 1028074078, 1028074078, 1028074078, 1028074078, 1028074078, 1028074078, 1028074078, 1028074078, 1028074078, 1028074078, 1028074078, 1028074078],
+          "last_rev_ids": [1222367802, 1222367802, 1222367802, 1222367802, 1222367802, 1222367802, 1222367802, 1222367802, 1222367802, 1222367802, 1222367802, 1222367802, 1222367802, 1222367802, 1222367802, 1222367802]
+        }
+      },
+      {
+        "id": "colosio_citation_date_survives_field_reorder",
+        "context": "{{cite news |date=8 June 2021 |title=Colosio, Vizcaíno, Marina y Cerqueda",
+        "focus": ["8 June 2021"],
+        "expected": {
+          "values": ["8", "june", "2021"],
+          "origin_rev_ids": [1028074078, 1028074078, 1028074078],
+          "last_rev_ids": [1222367802, 1222367802, 1222367802]
+        }
+      },
+      {
+        "id": "colosio_second_citation_date_survives_field_reorder",
+        "context": "|last=Pacheco |first=Gustavo |date=23 March 2019 |title=Esto ha pasado",
+        "focus": ["23 March 2019"],
+        "expected": {
+          "values": ["23", "march", "2019"],
+          "origin_rev_ids": [1028090673, 1028090673, 1028090673],
+          "last_rev_ids": [1222367802, 1222367802, 1222367802]
+        }
+      }
+    ]
+  },
+  {
+    "title": "Luis Donaldo Colosio Riojas",
+    "fixture": "fixtures/luis_donaldo_colosio_riojas_revisions.json",
+    "history_through_revid": 1347221443,
+    "snapshot_revid": 1347221443,
+    "cases": [
+      {
+        "id": "colosio_term_start_survives_infobox_restructure",
+        "context": "|term_start2=4 June 2024 |term_end2=22 August 2024",
+        "focus": ["4 June 2024"],
+        "expected": {
+          "values": ["4", "june", "2024"],
+          "origin_rev_ids": [1227424767, 1227424767, 1227424767],
+          "last_rev_ids": [1347221443, 1347221443, 1347221443]
+        }
+      }
+    ]
+  },
+  {
+    "title": "One Thing Leads 2 Another",
+    "fixture": "fixtures/one_thing_leads_2_another_revisions.json",
+    "history_through_revid": 1153115618,
+    "snapshot_revid": 1153115618,
+    "cases": [
+      {
+        "id": "one_thing_leads_2_another_template_spacing_preserves_field",
+        "context": "{{single chart|Switzerland|62|artist=Vanessa Amorosi|song=One Thing Leads 2 Another|accessdate",
+        "focus": ["Switzerland"],
+        "expected": {
+          "values": ["switzerland"],
+          "origin_rev_ids": [925279062],
+          "last_rev_ids": [1153115618],
+          "inbound": [[]],
+          "outbound": [[]]
+        }
+      }
+    ]
   }
 ]