Support dynamic encodings

rlp/codec.py

@@ -17,7 +17,7 @@
 
 try:
     import rusty_rlp
-except ImportError as e:
+except ImportError:
     import logging
     from rlp.atomic import (
         Atomic,
@@ -108,8 +108,8 @@ def encode(obj, sedes=None, infer_serializer=True, cache=True):
             return cached_rlp
         else:
             really_cache = (
-                cache and
-                sedes is None
+                cache
+                and sedes is None
             )
     else:
         really_cache = False
@@ -242,7 +242,7 @@ def consume_item(rlp, start):
 
 
 def decode(rlp, sedes=None, strict=True, recursive_cache=False, **kwargs):
-    """Decode an RLP encoded object.
+    r"""Decode an RLP encoded object.
 
     If the deserialized result `obj` has an attribute :attr:`_cached_rlp` (e.g. if `sedes` is a
     subclass of :class:`rlp.Serializable`) it will be set to `rlp`, which will improve performance
@@ -252,7 +252,7 @@ def decode(rlp, sedes=None, strict=True, recursive_cache=False, **kwargs):
 
     :param sedes: an object implementing a function ``deserialize(code)`` which will be applied
                   after decoding, or ``None`` if no deserialization should be performed
-    :param \*\*kwargs: additional keyword arguments that will be passed to the deserializer
+    :param **kwargs: additional keyword arguments that will be passed to the deserializer
     :param strict: if false inputs that are longer than necessary don't cause an exception
     :returns: the decoded and maybe deserialized Python object
     :raises: :exc:`rlp.DecodingError` if the input string does not end after the root item and

rlp/lazy.py

@@ -6,7 +6,7 @@
 
 
 def decode_lazy(rlp, sedes=None, **sedes_kwargs):
-    """Decode an RLP encoded object in a lazy fashion.
+    r"""Decode an RLP encoded object in a lazy fashion.
 
     If the encoded object is a bytestring, this function acts similar to
     :func:`rlp.decode`. If it is a list however, a :class:`LazyList` is
@@ -23,7 +23,7 @@ def decode_lazy(rlp, sedes=None, **sedes_kwargs):
     :param sedes: an object implementing a method ``deserialize(code)`` which
                   is used as described above, or ``None`` if no
                   deserialization should be performed
-    :param \*\*sedes_kwargs: additional keyword arguments that will be passed
+    :param **sedes_kwargs: additional keyword arguments that will be passed
                              to the deserializers
     :returns: either the already decoded and deserialized object (if encoded as
               a string) or an instance of :class:`rlp.LazyList`
@@ -63,7 +63,7 @@ def consume_item_lazy(rlp, start):
 
 
 class LazyList(Sequence):
-    """A RLP encoded list which decodes itself when necessary.
+    r"""A RLP encoded list which decodes itself when necessary.
 
     Both indexing with positive indices and iterating are supported.
     Getting the length with :func:`len` is possible as well but requires full
@@ -74,7 +74,7 @@ class LazyList(Sequence):
     :param end: the position of the last payload byte of the encoded list
     :param sedes: a sedes object which deserializes each element of the list,
                   or ``None`` for no deserialization
-    :param \*\*sedes_kwargs: keyword arguments which will be passed on to the
+    :param **sedes_kwargs: keyword arguments which will be passed on to the
                              deserializer
     """
 

rlp/sedes/big_endian_int.py

@@ -9,19 +9,19 @@
 class BigEndianInt(object):
     """A sedes for big endian integers.
 
-    :param l: the size of the serialized representation in bytes or `None` to
+    :param length: the size of the serialized representation in bytes or `None` to
               use the shortest possible one
     """
 
-    def __init__(self, l=None):
-        self.l = l
+    def __init__(self, length=None):
+        self.length = length
 
     def serialize(self, obj):
         if isinstance(obj, bool) or not isinstance(obj, int):
             raise SerializationError('Can only serialize integers', obj)
-        if self.l is not None and obj >= 256**self.l:
+        if self.length is not None and obj >= 256**self.length:
             raise SerializationError('Integer too large (does not fit in {} '
-                                     'bytes)'.format(self.l), obj)
+                                     'bytes)'.format(self.length), obj)
         if obj < 0:
             raise SerializationError('Cannot serialize negative integers', obj)
 
@@ -30,16 +30,16 @@ def serialize(self, obj):
         else:
             s = int_to_big_endian(obj)
 
-        if self.l is not None:
-            return b'\x00' * max(0, self.l - len(s)) + s
+        if self.length is not None:
+            return b'\x00' * max(0, self.length - len(s)) + s
         else:
             return s
 
     def deserialize(self, serial):
-        if self.l is not None and len(serial) != self.l:
+        if self.length is not None and len(serial) != self.length:
             raise DeserializationError('Invalid serialization (wrong size)',
                                        serial)
-        if self.l is None and len(serial) > 0 and serial[0:1] == b'\x00':
+        if self.length is None and len(serial) > 0 and serial[0:1] == b'\x00':
             raise DeserializationError('Invalid serialization (not minimal '
                                        'length)', serial)
 

rlp/sedes/binary.py

@@ -28,9 +28,9 @@ def fixed_length(cls, l, allow_empty=False):
     def is_valid_type(cls, obj):
         return isinstance(obj, (bytes, bytearray))
 
-    def is_valid_length(self, l):
-        return any((self.min_length <= l <= self.max_length,
-                    self.allow_empty and l == 0))
+    def is_valid_length(self, length):
+        return any((self.min_length <= length <= self.max_length,
+                    self.allow_empty and length == 0))
 
     def serialize(self, obj):
         if not Binary.is_valid_type(obj):

rlp/sedes/dynamic.py

@@ -0,0 +1,30 @@
+from abc import ABC, abstractmethod
+from typing import Any, List
+
+from rlp.exceptions import (
+    ObjectDeserializationError,
+    ObjectSerializationError,
+)
+
+
+class DynamicSerializable(ABC):
+    @property
+    @abstractmethod
+    def sedes_options(self) -> List:
+        ...
+
+    @classmethod
+    def deserialize(cls, obj: Any) -> bytes:
+        for sedes in cls.sedes_options:
+            try:
+                return sedes.deserialize(obj)
+            except ObjectDeserializationError:
+                pass
+
+    @classmethod
+    def serialize(cls, obj: Any) -> bytes:
+        for sedes in cls.sedes_options:
+            try:
+                return sedes.serialize(obj)
+            except ObjectSerializationError:
+                pass

rlp/sedes/serializable.py

@@ -482,8 +482,8 @@ def __new__(cls, name, bases, attrs):
             name,
             bases,
             dict(
-                field_props +
-                tuple(attrs.items())
+                field_props
+                + tuple(attrs.items())
             ),
         )
 

rlp/sedes/text.py

@@ -29,10 +29,10 @@ def fixed_length(cls, l, allow_empty=False):
     def is_valid_type(cls, obj):
         return isinstance(obj, str)
 
-    def is_valid_length(self, l):
+    def is_valid_length(self, length):
         return any((
-            self.min_length <= l <= self.max_length,
-            self.allow_empty and l == 0
+            self.min_length <= length <= self.max_length,
+            self.allow_empty and length == 0
         ))
 
     def serialize(self, obj):

setup.py

@@ -12,7 +12,7 @@
         "hypothesis==5.19.0",
     ],
     'lint': [
-        "flake8==3.4.1",
+        "flake8==3.7.9",
     ],
     'doc': [
         "Sphinx>=1.6.5,<2",

tests/test_codec.py → tests/core/test_codec.py

@@ -58,8 +58,7 @@ def test_consume_item():
     rlp = encode(obj)
     item, per_item_rlp, end = consume_item(rlp, 0)
     assert per_item_rlp == [
-        (b'\xf8y' b'f' + b'\x83bar' + b'\xb8d' + b'a' * 100 + b'i' +
-         b'\xcc\x86nested\x84list'),
+        (b'\xf8y' b'f' + b'\x83bar' + b'\xb8d' + b'a' * 100 + b'i' + b'\xcc\x86nested\x84list'),
         [b'f'],
         [b'\x83bar'],
         [b'\xb8d' + b'a' * 100],

tests/core/test_dynamic_entries.py

@@ -0,0 +1,91 @@
+# Sometimes elements of a Serializable might be arbitrarily byte-strings or lists.
+# See: https://github.com/ethereum/pyrlp/issues/112#issuecomment-785326962
+
+import rlp
+from rlp.sedes import big_endian_int, binary, CountableList
+from rlp.sedes.dynamic import DynamicSerializable
+from rlp.sedes.serializable import Serializable
+
+
+class LegacyTransaction(Serializable):
+    fields = [
+        ('nonce', big_endian_int),
+        ('gasprice', big_endian_int),
+        # ...snip
+    ]
+
+
+class TypedTransaction(DynamicSerializable):
+    sedes_options = [
+        LegacyTransaction,
+        binary,  # New typed transactions are opaque bytes
+    ]
+
+
+class Block(Serializable):
+    fields = [
+        ('transaction_list', CountableList(TypedTransaction)),
+    ]
+
+
+class LegacyBlock(Serializable):
+    fields = [
+        ('transaction_list', CountableList(LegacyTransaction)),
+    ]
+
+
+def test_dynamic_internal_legacy():
+    # Just establish that the basic test setup is working
+    block = LegacyBlock([LegacyTransaction(1, 2)])
+    assert block.transaction_list[0].nonce == 1
+    assert block.transaction_list[0].gasprice == 2
+
+    serialized = rlp.encode(block)
+    assert serialized == b'\xc4\xc3\xc2\x01\x02'
+
+    decoded_block = rlp.decode(serialized, sedes=LegacyBlock)
+    assert decoded_block == block
+    assert decoded_block.transaction_list[0].nonce == 1
+    assert decoded_block.transaction_list[0].gasprice == 2
+
+
+def test_dynamic_internal_list():
+    block = Block([LegacyTransaction(1, 2)])
+    assert block.transaction_list[0].nonce == 1
+    assert block.transaction_list[0].gasprice == 2
+
+    serialized = rlp.encode(block)
+    assert serialized == b'\xc4\xc3\xc2\x01\x02'
+
+    decoded_block = rlp.decode(serialized, sedes=Block)
+    assert decoded_block == block
+    assert decoded_block.transaction_list[0].nonce == 1
+    assert decoded_block.transaction_list[0].gasprice == 2
+
+
+def test_dynamic_internal_bytes():
+    block = Block([b'\x01new-type'])
+    assert block.transaction_list[0] == b'\x01new-type'
+
+    serialized = rlp.encode(block)
+    assert serialized == b'\xcb\xca\x89\x01new-type'
+
+    decoded_block = rlp.decode(serialized, sedes=Block)
+    assert decoded_block == block
+    assert decoded_block.transaction_list[0] == b'\x01new-type'
+
+
+def test_dynamic_internal_combined():
+    block = Block([LegacyTransaction(1, 2), b'\x01new-type'])
+    assert block.transaction_list[0].nonce == 1
+    assert block.transaction_list[0].gasprice == 2
+    assert block.transaction_list[1] == b'\x01new-type'
+
+    serialized = rlp.encode(block)
+    assert serialized == b'\xce\xcd\xc2\x01\x02\x89\x01new-type'
+
+    decoded_block = rlp.decode(serialized, sedes=Block)
+    assert decoded_block == block
+    assert decoded_block.transaction_list[0].nonce == 1
+    assert decoded_block.transaction_list[0].gasprice == 2
+    assert decoded_block.transaction_list[1] == b'\x01new-type'

tests/test_json.py → tests/core/test_json.py

@@ -44,7 +44,7 @@ def compare_nested(got, expected):
             return False
 
 
-with open('tests/rlptest.json') as rlptest_file:
+with open('tests/core/rlptest.json') as rlptest_file:
     test_data = json.load(rlptest_file)
     test_pieces = [
         (

tests/test_lazy.py → tests/core/test_lazy.py

@@ -46,33 +46,33 @@ def dec():
 
 
 def test_list_getitem():
-    l = rlp.decode_lazy(rlp.encode([1, 2, 3]), big_endian_int)
-    assert isinstance(l, rlp.lazy.LazyList)
-    assert l[0] == 1
-    assert l[1] == 2
-    assert l[2] == 3
-    assert l[-1] == 3
-    assert l[-2] == 2
-    assert l[-3] == 1
-    assert l[0:3] == [1, 2, 3]
-    assert l[0:2] == [1, 2]
-    assert l[0:1] == [1]
-    assert l[1:2] == [2]
-    assert l[1:] == [2, 3]
-    assert l[1:-1] == [2]
-    assert l[-2:] == [2, 3]
-    assert l[:2] == [1, 2]
+    decoded = rlp.decode_lazy(rlp.encode([1, 2, 3]), big_endian_int)
+    assert isinstance(decoded, rlp.lazy.LazyList)
+    assert decoded[0] == 1
+    assert decoded[1] == 2
+    assert decoded[2] == 3
+    assert decoded[-1] == 3
+    assert decoded[-2] == 2
+    assert decoded[-3] == 1
+    assert decoded[0:3] == [1, 2, 3]
+    assert decoded[0:2] == [1, 2]
+    assert decoded[0:1] == [1]
+    assert decoded[1:2] == [2]
+    assert decoded[1:] == [2, 3]
+    assert decoded[1:-1] == [2]
+    assert decoded[-2:] == [2, 3]
+    assert decoded[:2] == [1, 2]
 
 
 def test_nested_list():
-    l = ((), (b'a'), (b'b', b'c', b'd'))
+    nested = ((), (b'a'), (b'b', b'c', b'd'))
 
     def dec():
-        return rlp.decode_lazy(rlp.encode(l))
+        return rlp.decode_lazy(rlp.encode(nested))
 
     assert isinstance(dec(), Sequence)
-    assert len(dec()) == len(l)
-    assert evaluate(dec()) == l
+    assert len(dec()) == len(nested)
+    assert evaluate(dec()) == nested
     with pytest.raises(IndexError):
         dec()[0][0]
     with pytest.raises(IndexError):

tox.ini

@@ -8,7 +8,7 @@ envlist =
 [flake8]
 max-line-length= 100
 exclude= venv*,.tox,docs,build
-ignore=
+ignore=W503
 
 [testenv]
 usedevelop=True