diff --git a/README.md b/README.md index da8b08a..42deae1 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,16 @@ See source code for examples * CI: https://github.com/stefankoegl/python-json-patch/actions * Coveralls: https://coveralls.io/github/stefankoegl/python-json-patch +String append extension +----------------------- +`JsonPatchWithAppend` is a drop-in replacement for `JsonPatch` with a +non-RFC string `append` operation. Its `from_diff()` method sends only the +new suffix when an object member's string grows. `JsonPatch`, `make_patch()` +and `apply_patch()` retain their standard RFC 6902 behavior. + +See the [tutorial](doc/tutorial.rst#string-append-extension) for examples. +Both sender and receiver must support the extension. + Type hints ---------- jsonpatch ships type stubs (in the `jsonpatch-stubs` directory, as diff --git a/doc/tutorial.rst b/doc/tutorial.rst index 879155a..d39b6f4 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -365,6 +365,7 @@ support for Python's ``Decimal`` type, can be used to create a custom >>> patch.apply(doc) {'foo': 1, 'bar': Decimal('1.10')} + Instead of subclassing it is also possible to pass a dump function to ``from_diff``: @@ -391,3 +392,44 @@ and load function to ``from_string``: >>> doc = {'foo': 1} >>> patch.apply(doc) {'foo': 1, 'bar': Decimal('1.10')} + + +String append extension +----------------------- + +``JsonPatchWithAppend`` accepts the standard operations and an additional +``append`` operation for existing strings. This extension is not part of +RFC 6902, so both the producer and consumer of the patch must support it. +The standard ``JsonPatch`` class and module-level helpers remain unchanged. + +.. code-block:: python + + >>> src = {'message': 'Hello'} + >>> dst = {'message': 'Hello World'} + >>> patch = jsonpatch.JsonPatchWithAppend.from_diff(src, dst) + >>> patch.patch + [{'op': 'append', 'path': '/message', 'value': ' World'}] + >>> patch.apply(src) + {'message': 'Hello World'} + >>> jsonpatch.JsonPatchWithAppend.from_string(patch.to_string()).apply(src) == dst + True + +Every append operation requires ``op``, ``path`` and a string ``value``. +Its target must be an existing string in an object member or array element; +appending to the root document is not supported. Operations can also be +constructed directly: + +.. code-block:: python + + >>> patch = jsonpatch.JsonPatchWithAppend([ + ... {'op': 'append', 'path': '/message', 'value': '!'}, + ... ]) + >>> patch.apply({'message': 'Hello'}) + {'message': 'Hello!'} + +Automatic suffix detection applies to object members, including objects +nested in arrays. Root strings and scalar array elements retain the standard +diff behavior. ``optimization=False`` disables moves but retains suffix +detection. ``generate_test_ops=True`` also tests the original string before +appending. As with custom operations, ``in_place=True, atomic=True`` uses +the existing copy-based rollback fallback. diff --git a/jsonpatch-stubs/__init__.pyi b/jsonpatch-stubs/__init__.pyi index 5245be6..851ac36 100644 --- a/jsonpatch-stubs/__init__.pyi +++ b/jsonpatch-stubs/__init__.pyi @@ -88,6 +88,7 @@ class MoveOperation(PatchOperation): class TestOperation(PatchOperation): ... class CopyOperation(PatchOperation): ... +class AppendOperation(PatchOperation): ... class JsonPatch: json_dumper: ClassVar[Callable[..., str]] @@ -122,6 +123,8 @@ class JsonPatch: def to_string(self, dumps: Callable[[Any], str] | None = None) -> str: ... def apply(self, obj: Any, in_place: bool = False, atomic: bool = False) -> Any: ... +class JsonPatchWithAppend(JsonPatch): ... + # Computes the operations for JsonPatch.from_diff; mostly an implementation # detail, so its internal methods are typed loosely. class DiffBuilder: diff --git a/jsonpatch.py b/jsonpatch.py index a90a753..c5f0368 100644 --- a/jsonpatch.py +++ b/jsonpatch.py @@ -672,6 +672,39 @@ def _apply(self, obj, journal): return obj +class AppendOperation(PatchOperation): + """Appends a string suffix to an existing string (not part of RFC 6902).""" + + def _apply(self, obj, journal): + try: + value = self.operation['value'] + except KeyError: + raise InvalidJsonPatch("The operation does not contain a 'value' member", op=self) + if not isinstance(value, str): + raise JsonPatchConflict("Cannot append a non-string value", op=self) + + try: + subobj, part = self.pointer.to_last(obj) + except JsonPointerException as ex: + raise JsonPatchPathNotFound(str(ex), op=self) + + if part is None: + raise JsonPatchConflict("Cannot append to root document", op=self) + if isinstance(subobj, MutableSequence): + if not isinstance(part, int) or part < 0 or part >= len(subobj): + raise JsonPatchPathNotFound("Cannot append outside of list", op=self) + elif isinstance(subobj, MutableMapping): + if part not in subobj: + raise JsonPatchPathNotFound("Cannot append to a non-existent member", op=self) + else: + raise JsonPatchConflict("Cannot append to a non-string value", op=self) + + if not isinstance(subobj[part], str): + raise JsonPatchConflict("Cannot append to a non-string value", op=self) + journal.setitem(subobj, part, subobj[part] + value) + return obj + + def _restore(obj, saved): """Restores the contents of a document from a copy of it. @@ -1004,6 +1037,35 @@ def _get_operation(self, operation): return cls(operation, pointer_cls=self.pointer_cls) +class JsonPatchWithAppend(JsonPatch): + """A JsonPatch with an optional, non-RFC string append operation. + + Each append operation requires full ``op``, ``path`` and ``value`` members. + It appends a string suffix to an existing object member or array element. + The standard JsonPatch class and module-level helpers remain RFC 6902 only. + """ + + operations = MappingProxyType(dict(JsonPatch.operations, append=AppendOperation)) + + @classmethod + def from_diff(cls, src, dst, optimization=True, dumps=None, + pointer_cls=JsonPointer, generate_test_ops=False): + """Creates a patch, using append for string growth in object members. + + Root strings and scalar array elements retain standard diff behavior. + Options follow JsonPatch.from_diff; optimization=False disables moves, + while append remains enabled. With generate_test_ops=True, an append + is preceded by a test of the original string. + """ + json_dumper = dumps or cls.json_dumper + builder = _AppendDiffBuilder(src, dst, json_dumper, pointer_cls=pointer_cls, + optimization=optimization, + generate_test_ops=generate_test_ops) + builder._compare_values((), None, src, dst) + ops = copy.deepcopy(list(builder.execute())) + return cls(ops, pointer_cls=pointer_cls) + + class DiffBuilder(object): def __init__(self, src_doc, dst_doc, dumps=json.dumps, pointer_cls=JsonPointer, @@ -1357,6 +1419,20 @@ def _compare_values(self, path, key, src, dst): self._item_replaced(path, key, dst, src) +class _AppendDiffBuilder(DiffBuilder): + + def _item_replaced(self, path, key, item, old_item): + if (path or key is not None) and isinstance(old_item, str) and \ + isinstance(item, str) and len(item) > len(old_item) and \ + item.startswith(old_item): + location = self._location(_path_join(path, key)) + self._insert_test(location, old_item) + self.insert({'op': 'append', 'path': location, + 'value': item[len(old_item):]}) + else: + super()._item_replaced(path, key, item, old_item) + + def _positional_cost(old, new): """ Estimates how many operations change old into new, if the items of arrays are compared by position """ diff --git a/property_tests.py b/property_tests.py index ac0c5c3..6d19078 100644 --- a/property_tests.py +++ b/property_tests.py @@ -694,6 +694,26 @@ def patch_gives_same_result(self): self.doc) +class AppendDiffProperties(unittest.TestCase): + + @given(key=json_keys, prefix=st.text(max_size=10), + suffix=st.text(min_size=1, max_size=10), generate_tests=st.booleans()) + def test_string_growth_uses_suffix(self, key, prefix, suffix, generate_tests): + src, dst = {key: prefix}, {key: prefix + suffix} + patch = jsonpatch.JsonPatchWithAppend.from_diff( + src, dst, generate_test_ops=generate_tests) + self.assertEqual(patch.patch[-1]['op'], 'append') + self.assertEqual(patch.patch[-1]['value'], suffix) + self.assertEqual(patch.apply(src), dst) + + @given(pair=doc_pairs) + def test_generated_patch_round_trip(self, pair): + src, dst = pair + patch = jsonpatch.JsonPatchWithAppend.from_diff(src, dst) + restored = jsonpatch.JsonPatchWithAppend.from_string(patch.to_string()) + assert_json_equal(restored.apply(src), dst) + + PatchOperationTests = PatchOperationMachine.TestCase diff --git a/tests.py b/tests.py old mode 100755 new mode 100644 index 8f27ae5..e78a196 --- a/tests.py +++ b/tests.py @@ -3283,6 +3283,259 @@ def test_commandline(self): self.assertEqual(res.stdout, expected) +class AppendOperationTests(unittest.TestCase): + + def test_appends_at_existing_string_locations(self): + cases = [ + ({'message': 'Hello'}, '/message', ' World', {'message': 'Hello World'}), + ({'a': {'b': ''}}, '/a/b', '\ud55c\uae00', {'a': {'b': '\ud55c\uae00'}}), + (['Hello'], '/0', '!', ['Hello!']), + ({'a/b~c': 'Hello'}, '/a~1b~0c', '!', {'a/b~c': 'Hello!'}), + ({'message': 'Hello'}, '/message', '', {'message': 'Hello'}), + ({'-': 'a'}, '/-', 'b', {'-': 'ab'}), + ] + for doc, path, value, expected in cases: + with self.subTest(path=path, value=value): + original = copy.deepcopy(doc) + operations = [{'op': 'append', 'path': path, 'value': value}] + patch = jsonpatch.JsonPatchWithAppend(operations) + self.assertEqual(patch.apply(doc), expected) + self.assertEqual(doc, original) + self.assertEqual(patch.apply(doc, in_place=True), expected) + self.assertEqual(operations, [{'op': 'append', 'path': path, 'value': value}]) + + def test_rejects_non_string_values_and_targets(self): + for value in (None, 1, True, [], {}, b'bytes'): + with self.subTest(value=value): + patch = jsonpatch.JsonPatchWithAppend([ + {'op': 'append', 'path': '/message', 'value': value}, + ]) + with self.assertRaises(jsonpatch.JsonPatchConflict) as raised: + patch.apply({'message': 'Hello'}) + self.assertIsInstance(raised.exception.op, jsonpatch.AppendOperation) + for target in (None, 1, True, [], {}, b'bytes'): + with self.subTest(target=target): + patch = jsonpatch.JsonPatchWithAppend([ + {'op': 'append', 'path': '/message', 'value': '!'}, + ]) + with self.assertRaises(jsonpatch.JsonPatchConflict): + patch.apply({'message': target}) + + def test_reports_missing_value(self): + patch = jsonpatch.JsonPatchWithAppend([{'op': 'append', 'path': '/message'}]) + with self.assertRaises(jsonpatch.InvalidJsonPatch) as raised: + patch.apply({'message': 'Hello'}) + self.assertIsInstance(raised.exception.op, jsonpatch.AppendOperation) + + def test_rejects_root_append(self): + patch = jsonpatch.JsonPatchWithAppend([ + {'op': 'append', 'path': '', 'value': '!'}, + ]) + with self.assertRaises(jsonpatch.JsonPatchConflict): + patch.apply('Hello') + + def test_reports_missing_paths(self): + for doc, path in (({}, '/missing'), ({}, '/missing/child'), + (['a'], '/1'), (['a'], '/-1'), (['a'], '/-')): + with self.subTest(path=path): + patch = jsonpatch.JsonPatchWithAppend([ + {'op': 'append', 'path': path, 'value': '!'}, + ]) + with self.assertRaises(jsonpatch.JsonPatchPathNotFound) as raised: + patch.apply(doc) + self.assertIsInstance(raised.exception.op, jsonpatch.AppendOperation) + + +class AppendInterfaceTests(unittest.TestCase): + + def test_preserves_standard_operations(self): + operations = [ + {'op': 'replace', 'path': '/message', 'value': 'Hello', 'v': 'ignored'}, + {'op': 'append', 'path': '/message', 'value': '!'}, + {'op': 'test', 'path': '/message', 'value': 'Hello!'}, + ] + self.assertEqual(jsonpatch.JsonPatchWithAppend(operations).apply({'message': ''}), + {'message': 'Hello!'}) + with self.assertRaises(jsonpatch.InvalidJsonPatch): + jsonpatch.JsonPatch([{'op': 'append', 'path': '/message', 'value': '!'}]) + + def test_rejects_nonstandard_notation(self): + first = {'op': 'append', 'path': '/message', 'value': 'Hello'} + for operations in ([{'o': 'append', 'p': '/message', 'v': '!'}], + [first, {'v': '!'}], [first, {'value': '!'}]): + with self.subTest(operations=operations): + with self.assertRaises(jsonpatch.InvalidJsonPatch): + jsonpatch.JsonPatchWithAppend(operations) + + def test_inherited_serialization_and_generator_reuse(self): + operations = [{'op': 'append', 'path': '/message', 'value': '!'}] + patch = jsonpatch.JsonPatchWithAppend(op for op in operations) + restored = jsonpatch.JsonPatchWithAppend.from_string(patch.to_string()) + self.assertEqual(list(patch), operations) + self.assertEqual(json.loads(patch.to_string()), operations) + self.assertEqual(patch, restored) + self.assertEqual(hash(patch), hash(restored)) + for _ in range(2): + self.assertEqual(patch.apply({'message': 'Hello'}), {'message': 'Hello!'}) + + def test_custom_loader_pointer_and_subclass(self): + calls = [] + + def loads(text): + calls.append(text) + return json.loads(text) + + class CustomPatch(jsonpatch.JsonPatchWithAppend): + json_loader = staticmethod(loads) + + text = '[{"op":"append","path":"/message","value":"!"}]' + for kwargs in ({}, {'loads': loads}): + patch = CustomPatch.from_string(text, pointer_cls=CustomJsonPointer, **kwargs) + self.assertIsInstance(patch, CustomPatch) + self.assertIsInstance(patch._ops[0].pointer, CustomJsonPointer) + self.assertEqual(patch.apply({'message': 'Hello'}), {'message': 'Hello!'}) + self.assertEqual(calls, [text, text]) + + +class AppendAtomicTests(unittest.TestCase): + + def test_atomic_failure_restores_mixed_operations(self): + for doc, path in (({'message': 'Hello'}, '/message'), (['Hello'], '/0')): + for failure in ({'op': 'test', 'path': path, 'value': 'wrong'}, + {'op': 'remove', 'path': '/99' if isinstance(doc, list) else '/missing'}): + with self.subTest(doc=doc, failure=failure): + original = copy.deepcopy(doc) + patch = jsonpatch.JsonPatchWithAppend([ + {'op': 'append', 'path': path, 'value': '!'}, failure, + ]) + with self.assertRaises(jsonpatch.JsonPatchException) as raised: + patch.apply(doc, in_place=True, atomic=True) + self.assertEqual(doc, original) + self.assertEqual(raised.exception.op.operation, failure) + + def test_non_atomic_failure_keeps_previous_changes(self): + doc = {'message': 'Hello'} + patch = jsonpatch.JsonPatchWithAppend([ + {'op': 'append', 'path': '/message', 'value': '!'}, + {'op': 'test', 'path': '/message', 'value': 'wrong'}, + ]) + with self.assertRaises(jsonpatch.JsonPatchTestFailed): + patch.apply(doc, in_place=True) + self.assertEqual(doc, {'message': 'Hello!'}) + + def test_atomic_success_and_append_failure(self): + doc = {'message': 'Hello'} + patch = jsonpatch.JsonPatchWithAppend([ + {'op': 'append', 'path': '/message', 'value': '!'}, + ]) + self.assertIs(patch.apply(doc, in_place=True, atomic=True), doc) + self.assertEqual(doc, {'message': 'Hello!'}) + patch = jsonpatch.JsonPatchWithAppend([ + {'op': 'replace', 'path': '/message', 'value': 1}, + {'op': 'append', 'path': '/message', 'value': '!'}, + ]) + with self.assertRaises(jsonpatch.JsonPatchConflict): + patch.apply(doc, in_place=True, atomic=True) + self.assertEqual(doc, {'message': 'Hello!'}) + + +class AppendDiffTests(unittest.TestCase): + + def test_generates_only_the_string_suffix(self): + for optimization in (True, False): + patch = jsonpatch.JsonPatchWithAppend.from_diff( + {'message': 'Hello'}, {'message': 'Hello World'}, optimization=optimization) + self.assertEqual(patch.patch, [ + {'op': 'append', 'path': '/message', 'value': ' World'}, + ]) + self.assertEqual(patch.apply({'message': 'Hello'}), {'message': 'Hello World'}) + + def test_preserves_root_array_and_non_append_changes(self): + cases = [ + ('Hello', 'Hello World', [{'op': 'replace', 'path': '', 'value': 'Hello World'}]), + (['Hello'], ['Hello World'], [{'op': 'replace', 'path': '/0', 'value': 'Hello World'}]), + ({'m': 'Hello'}, {'m': 'Hello'}, []), + ({'m': 'Hello'}, {'m': 'He'}, [{'op': 'replace', 'path': '/m', 'value': 'He'}]), + ({'m': 'Hello'}, {'m': 'Hallo'}, [{'op': 'replace', 'path': '/m', 'value': 'Hallo'}]), + ({'m': 'Hello'}, {'m': 1}, [{'op': 'replace', 'path': '/m', 'value': 1}]), + ({}, {'m': 'Hello'}, [{'op': 'add', 'path': '/m', 'value': 'Hello'}]), + ] + for src, dst, expected in cases: + with self.subTest(src=src, dst=dst): + patch = jsonpatch.JsonPatchWithAppend.from_diff(src, dst) + self.assertEqual(patch.patch, expected) + self.assertEqual(patch.apply(src), dst) + + def test_nested_and_escaped_locations(self): + for src, dst, path in ( + ({'a/b~c': ''}, {'a/b~c': '\ud55c\uae00'}, '/a~1b~0c'), + ([{'message': 'Hello'}], [{'message': 'Hello!'}], '/0/message'), + ({'a': {'m': 'Hello'}}, {'a': {'m': 'Hello!'}}, '/a/m')): + with self.subTest(path=path): + patch = jsonpatch.JsonPatchWithAppend.from_diff(src, dst) + self.assertEqual(patch.patch[0]['op'], 'append') + self.assertEqual(patch.patch[0]['path'], path) + self.assertEqual(patch.apply(src), dst) + + def test_generated_tests_guard_the_original_text(self): + patch = jsonpatch.JsonPatchWithAppend.from_diff( + {'m': 'Hello'}, {'m': 'Hello!'}, generate_test_ops=True) + self.assertEqual(patch.patch, [ + {'op': 'test', 'path': '/m', 'value': 'Hello'}, + {'op': 'append', 'path': '/m', 'value': '!'}, + ]) + self.assertEqual(patch.apply({'m': 'Hello'}), {'m': 'Hello!'}) + with self.assertRaises(jsonpatch.JsonPatchTestFailed): + patch.apply({'m': 'Other'}) + + def test_preserves_options_subclasses_and_copied_values(self): + calls = [] + + def dumps(value): + calls.append(value) + return json.dumps(value) + + class CustomPatch(jsonpatch.JsonPatchWithAppend): + json_dumper = staticmethod(dumps) + + for kwargs in ({}, {'dumps': dumps}): + for optimization in (True, False): + src = {'m': 'Hello', 'old': 1} + dst = {'m': 'Hello!', 'new': 1, 'items': [2]} + patch = CustomPatch.from_diff(src, dst, optimization=optimization, + pointer_cls=CustomJsonPointer, **kwargs) + self.assertIsInstance(patch, CustomPatch) + self.assertEqual(patch.apply(src), dst) + self.assertEqual(any(op['op'] == 'move' for op in patch), optimization) + self.assertTrue(all(isinstance(op.pointer, CustomJsonPointer) for op in patch._ops)) + dst['items'].append(3) + self.assertEqual(patch.apply(src)['items'], [2]) + self.assertTrue(calls) + + +class StandardAppendIsolationTests(unittest.TestCase): + + def test_standard_entrypoints_reject_append(self): + operations = [{'op': 'append', 'path': '/message', 'value': ' World'}] + for create in (lambda: jsonpatch.JsonPatch(operations), + lambda: jsonpatch.JsonPatch.from_string(json.dumps(operations)), + lambda: jsonpatch.apply_patch({'message': 'Hello'}, operations)): + with self.assertRaises(jsonpatch.InvalidJsonPatch): + create() + + def test_standard_diff_replaces_strings(self): + patch = jsonpatch.make_patch({'message': 'Hello'}, {'message': 'Hello World'}) + self.assertEqual(patch.patch, [ + {'op': 'replace', 'path': '/message', 'value': 'Hello World'}, + ]) + + def test_additional_members_are_not_aliases(self): + operations = [{'op': 'replace', 'path': '/message', + 'value': 'canonical', 'v': 'ignored'}] + self.assertEqual(jsonpatch.JsonPatch(operations).apply({'message': 'Hello'}), + {'message': 'canonical'}) + + if __name__ == '__main__': modules = ['jsonpatch'] @@ -3311,6 +3564,11 @@ def get_suite(): suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(PathNotFoundTests)) suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(CommandLineTests)) suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(DocumentationTests)) + suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(StandardAppendIsolationTests)) + suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(AppendOperationTests)) + suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(AppendInterfaceTests)) + suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(AppendAtomicTests)) + suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(AppendDiffTests)) return suite diff --git a/typetests/usage.py b/typetests/usage.py index f6a3698..88371ed 100644 --- a/typetests/usage.py +++ b/typetests/usage.py @@ -61,6 +61,24 @@ class MyPatch(JsonPatch): assert_type(MyPatch.from_string('[]'), MyPatch) assert_type(MyPatch.from_diff(doc, {}), MyPatch) +# Inherited APIs and factory return types for the append extension +append_patch = jsonpatch.JsonPatchWithAppend([ + {'op': 'append', 'path': '/foo', 'value': '!'}, +]) +assert_type(append_patch.apply(doc, in_place=True, atomic=True), Any) +assert_type(append_patch.to_string(), str) +assert_type(jsonpatch.JsonPatchWithAppend.from_diff(doc, {}), jsonpatch.JsonPatchWithAppend) +assert_type(jsonpatch.JsonPatchWithAppend.from_string('[]'), jsonpatch.JsonPatchWithAppend) +assert_type(jsonpatch.AppendOperation({'op': 'append', 'path': '/foo', 'value': '!'}).apply(doc), Any) + + +class MyAppendPatch(jsonpatch.JsonPatchWithAppend): + pass + + +assert_type(MyAppendPatch.from_diff(doc, {}), MyAppendPatch) +assert_type(MyAppendPatch.from_string('[]'), MyAppendPatch) + # operations operation = jsonpatch.AddOperation({'op': 'add', 'path': '/a/0', 'value': 1}) assert_type(operation.apply({'a': []}), Any)