diff --git a/rlp/codec.py b/rlp/codec.py index efd6547..c53c4e9 100644 --- a/rlp/codec.py +++ b/rlp/codec.py @@ -17,7 +17,7 @@ try: import rusty_rlp -except ImportError as e: +except ImportError: import logging from rlp.atomic import ( Atomic, @@ -108,8 +108,8 @@ def encode(obj, sedes=None, infer_serializer=True, cache=True): return cached_rlp else: really_cache = ( - cache and - sedes is None + cache + and sedes is None ) else: really_cache = False @@ -242,7 +242,7 @@ def consume_item(rlp, start): def decode(rlp, sedes=None, strict=True, recursive_cache=False, **kwargs): - """Decode an RLP encoded object. + r"""Decode an RLP encoded object. If the deserialized result `obj` has an attribute :attr:`_cached_rlp` (e.g. if `sedes` is a subclass of :class:`rlp.Serializable`) it will be set to `rlp`, which will improve performance @@ -252,7 +252,7 @@ def decode(rlp, sedes=None, strict=True, recursive_cache=False, **kwargs): :param sedes: an object implementing a function ``deserialize(code)`` which will be applied after decoding, or ``None`` if no deserialization should be performed - :param \*\*kwargs: additional keyword arguments that will be passed to the deserializer + :param **kwargs: additional keyword arguments that will be passed to the deserializer :param strict: if false inputs that are longer than necessary don't cause an exception :returns: the decoded and maybe deserialized Python object :raises: :exc:`rlp.DecodingError` if the input string does not end after the root item and diff --git a/rlp/lazy.py b/rlp/lazy.py index d8d6250..f4c972e 100644 --- a/rlp/lazy.py +++ b/rlp/lazy.py @@ -6,7 +6,7 @@ def decode_lazy(rlp, sedes=None, **sedes_kwargs): - """Decode an RLP encoded object in a lazy fashion. + r"""Decode an RLP encoded object in a lazy fashion. If the encoded object is a bytestring, this function acts similar to :func:`rlp.decode`. If it is a list however, a :class:`LazyList` is @@ -23,7 +23,7 @@ def decode_lazy(rlp, sedes=None, **sedes_kwargs): :param sedes: an object implementing a method ``deserialize(code)`` which is used as described above, or ``None`` if no deserialization should be performed - :param \*\*sedes_kwargs: additional keyword arguments that will be passed + :param **sedes_kwargs: additional keyword arguments that will be passed to the deserializers :returns: either the already decoded and deserialized object (if encoded as a string) or an instance of :class:`rlp.LazyList` @@ -63,7 +63,7 @@ def consume_item_lazy(rlp, start): class LazyList(Sequence): - """A RLP encoded list which decodes itself when necessary. + r"""A RLP encoded list which decodes itself when necessary. Both indexing with positive indices and iterating are supported. Getting the length with :func:`len` is possible as well but requires full @@ -74,7 +74,7 @@ class LazyList(Sequence): :param end: the position of the last payload byte of the encoded list :param sedes: a sedes object which deserializes each element of the list, or ``None`` for no deserialization - :param \*\*sedes_kwargs: keyword arguments which will be passed on to the + :param **sedes_kwargs: keyword arguments which will be passed on to the deserializer """ diff --git a/rlp/sedes/big_endian_int.py b/rlp/sedes/big_endian_int.py index e90501c..363ef1f 100644 --- a/rlp/sedes/big_endian_int.py +++ b/rlp/sedes/big_endian_int.py @@ -9,19 +9,19 @@ class BigEndianInt(object): """A sedes for big endian integers. - :param l: the size of the serialized representation in bytes or `None` to + :param length: the size of the serialized representation in bytes or `None` to use the shortest possible one """ - def __init__(self, l=None): - self.l = l + def __init__(self, length=None): + self.length = length def serialize(self, obj): if isinstance(obj, bool) or not isinstance(obj, int): raise SerializationError('Can only serialize integers', obj) - if self.l is not None and obj >= 256**self.l: + if self.length is not None and obj >= 256**self.length: raise SerializationError('Integer too large (does not fit in {} ' - 'bytes)'.format(self.l), obj) + 'bytes)'.format(self.length), obj) if obj < 0: raise SerializationError('Cannot serialize negative integers', obj) @@ -30,16 +30,16 @@ def serialize(self, obj): else: s = int_to_big_endian(obj) - if self.l is not None: - return b'\x00' * max(0, self.l - len(s)) + s + if self.length is not None: + return b'\x00' * max(0, self.length - len(s)) + s else: return s def deserialize(self, serial): - if self.l is not None and len(serial) != self.l: + if self.length is not None and len(serial) != self.length: raise DeserializationError('Invalid serialization (wrong size)', serial) - if self.l is None and len(serial) > 0 and serial[0:1] == b'\x00': + if self.length is None and len(serial) > 0 and serial[0:1] == b'\x00': raise DeserializationError('Invalid serialization (not minimal ' 'length)', serial) diff --git a/rlp/sedes/binary.py b/rlp/sedes/binary.py index 14a8005..76eb2a6 100644 --- a/rlp/sedes/binary.py +++ b/rlp/sedes/binary.py @@ -28,9 +28,9 @@ def fixed_length(cls, l, allow_empty=False): def is_valid_type(cls, obj): return isinstance(obj, (bytes, bytearray)) - def is_valid_length(self, l): - return any((self.min_length <= l <= self.max_length, - self.allow_empty and l == 0)) + def is_valid_length(self, length): + return any((self.min_length <= length <= self.max_length, + self.allow_empty and length == 0)) def serialize(self, obj): if not Binary.is_valid_type(obj): diff --git a/rlp/sedes/dynamic.py b/rlp/sedes/dynamic.py new file mode 100644 index 0000000..ddbab87 --- /dev/null +++ b/rlp/sedes/dynamic.py @@ -0,0 +1,30 @@ +from abc import ABC, abstractmethod +from typing import Any, List + +from rlp.exceptions import ( + ObjectDeserializationError, + ObjectSerializationError, +) + + +class DynamicSerializable(ABC): + @property + @abstractmethod + def sedes_options(self) -> List: + ... + + @classmethod + def deserialize(cls, obj: Any) -> bytes: + for sedes in cls.sedes_options: + try: + return sedes.deserialize(obj) + except ObjectDeserializationError: + pass + + @classmethod + def serialize(cls, obj: Any) -> bytes: + for sedes in cls.sedes_options: + try: + return sedes.serialize(obj) + except ObjectSerializationError: + pass diff --git a/rlp/sedes/serializable.py b/rlp/sedes/serializable.py index dee7ece..cbdcd72 100644 --- a/rlp/sedes/serializable.py +++ b/rlp/sedes/serializable.py @@ -482,8 +482,8 @@ def __new__(cls, name, bases, attrs): name, bases, dict( - field_props + - tuple(attrs.items()) + field_props + + tuple(attrs.items()) ), ) diff --git a/rlp/sedes/text.py b/rlp/sedes/text.py index b59fc9c..fb9ceb2 100644 --- a/rlp/sedes/text.py +++ b/rlp/sedes/text.py @@ -29,10 +29,10 @@ def fixed_length(cls, l, allow_empty=False): def is_valid_type(cls, obj): return isinstance(obj, str) - def is_valid_length(self, l): + def is_valid_length(self, length): return any(( - self.min_length <= l <= self.max_length, - self.allow_empty and l == 0 + self.min_length <= length <= self.max_length, + self.allow_empty and length == 0 )) def serialize(self, obj): diff --git a/setup.py b/setup.py index adf304a..4d09764 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ "hypothesis==5.19.0", ], 'lint': [ - "flake8==3.4.1", + "flake8==3.7.9", ], 'doc': [ "Sphinx>=1.6.5,<2", diff --git a/tests/test_benchmark.py b/tests/benchmark/test_benchmark.py similarity index 100% rename from tests/test_benchmark.py rename to tests/benchmark/test_benchmark.py diff --git a/tests/rlptest.json b/tests/core/rlptest.json similarity index 100% rename from tests/rlptest.json rename to tests/core/rlptest.json diff --git a/tests/speed.py b/tests/core/speed.py similarity index 100% rename from tests/speed.py rename to tests/core/speed.py diff --git a/tests/test_big_endian.py b/tests/core/test_big_endian.py similarity index 100% rename from tests/test_big_endian.py rename to tests/core/test_big_endian.py diff --git a/tests/test_binary_sedes.py b/tests/core/test_binary_sedes.py similarity index 100% rename from tests/test_binary_sedes.py rename to tests/core/test_binary_sedes.py diff --git a/tests/test_boolean_serializer.py b/tests/core/test_boolean_serializer.py similarity index 100% rename from tests/test_boolean_serializer.py rename to tests/core/test_boolean_serializer.py diff --git a/tests/test_bytearray.py b/tests/core/test_bytearray.py similarity index 100% rename from tests/test_bytearray.py rename to tests/core/test_bytearray.py diff --git a/tests/test_codec.py b/tests/core/test_codec.py similarity index 97% rename from tests/test_codec.py rename to tests/core/test_codec.py index a4661e3..a9e9424 100644 --- a/tests/test_codec.py +++ b/tests/core/test_codec.py @@ -58,8 +58,7 @@ def test_consume_item(): rlp = encode(obj) item, per_item_rlp, end = consume_item(rlp, 0) assert per_item_rlp == [ - (b'\xf8y' b'f' + b'\x83bar' + b'\xb8d' + b'a' * 100 + b'i' + - b'\xcc\x86nested\x84list'), + (b'\xf8y' b'f' + b'\x83bar' + b'\xb8d' + b'a' * 100 + b'i' + b'\xcc\x86nested\x84list'), [b'f'], [b'\x83bar'], [b'\xb8d' + b'a' * 100], diff --git a/tests/test_countablelist.py b/tests/core/test_countablelist.py similarity index 100% rename from tests/test_countablelist.py rename to tests/core/test_countablelist.py diff --git a/tests/core/test_dynamic_entries.py b/tests/core/test_dynamic_entries.py new file mode 100644 index 0000000..130afb1 --- /dev/null +++ b/tests/core/test_dynamic_entries.py @@ -0,0 +1,91 @@ +# Sometimes elements of a Serializable might be arbitrarily byte-strings or lists. +# See: https://github.com/ethereum/pyrlp/issues/112#issuecomment-785326962 + +import rlp +from rlp.sedes import big_endian_int, binary, CountableList +from rlp.sedes.dynamic import DynamicSerializable +from rlp.sedes.serializable import Serializable + + +class LegacyTransaction(Serializable): + fields = [ + ('nonce', big_endian_int), + ('gasprice', big_endian_int), + # ...snip + ] + + +class TypedTransaction(DynamicSerializable): + sedes_options = [ + LegacyTransaction, + binary, # New typed transactions are opaque bytes + ] + + +class Block(Serializable): + fields = [ + ('transaction_list', CountableList(TypedTransaction)), + ] + + +class LegacyBlock(Serializable): + fields = [ + ('transaction_list', CountableList(LegacyTransaction)), + ] + + +def test_dynamic_internal_legacy(): + # Just establish that the basic test setup is working + block = LegacyBlock([LegacyTransaction(1, 2)]) + assert block.transaction_list[0].nonce == 1 + assert block.transaction_list[0].gasprice == 2 + + serialized = rlp.encode(block) + assert serialized == b'\xc4\xc3\xc2\x01\x02' + + decoded_block = rlp.decode(serialized, sedes=LegacyBlock) + assert decoded_block == block + assert decoded_block.transaction_list[0].nonce == 1 + assert decoded_block.transaction_list[0].gasprice == 2 + + +def test_dynamic_internal_list(): + block = Block([LegacyTransaction(1, 2)]) + assert block.transaction_list[0].nonce == 1 + assert block.transaction_list[0].gasprice == 2 + + serialized = rlp.encode(block) + assert serialized == b'\xc4\xc3\xc2\x01\x02' + + decoded_block = rlp.decode(serialized, sedes=Block) + assert decoded_block == block + assert decoded_block.transaction_list[0].nonce == 1 + assert decoded_block.transaction_list[0].gasprice == 2 + + +def test_dynamic_internal_bytes(): + block = Block([b'\x01new-type']) + assert block.transaction_list[0] == b'\x01new-type' + + serialized = rlp.encode(block) + assert serialized == b'\xcb\xca\x89\x01new-type' + + decoded_block = rlp.decode(serialized, sedes=Block) + assert decoded_block == block + assert decoded_block.transaction_list[0] == b'\x01new-type' + + +def test_dynamic_internal_combined(): + block = Block([LegacyTransaction(1, 2), b'\x01new-type']) + assert block.transaction_list[0].nonce == 1 + assert block.transaction_list[0].gasprice == 2 + assert block.transaction_list[1] == b'\x01new-type' + + serialized = rlp.encode(block) + assert serialized == b'\xce\xcd\xc2\x01\x02\x89\x01new-type' + + decoded_block = rlp.decode(serialized, sedes=Block) + assert decoded_block == block + assert decoded_block.transaction_list[0].nonce == 1 + assert decoded_block.transaction_list[0].gasprice == 2 + assert decoded_block.transaction_list[1] == b'\x01new-type' diff --git a/tests/test_invalid.py b/tests/core/test_invalid.py similarity index 100% rename from tests/test_invalid.py rename to tests/core/test_invalid.py diff --git a/tests/test_json.py b/tests/core/test_json.py similarity index 97% rename from tests/test_json.py rename to tests/core/test_json.py index ed4f952..1ab0b5e 100644 --- a/tests/test_json.py +++ b/tests/core/test_json.py @@ -44,7 +44,7 @@ def compare_nested(got, expected): return False -with open('tests/rlptest.json') as rlptest_file: +with open('tests/core/rlptest.json') as rlptest_file: test_data = json.load(rlptest_file) test_pieces = [ ( diff --git a/tests/test_lazy.py b/tests/core/test_lazy.py similarity index 76% rename from tests/test_lazy.py rename to tests/core/test_lazy.py index bbbbb37..a034345 100644 --- a/tests/test_lazy.py +++ b/tests/core/test_lazy.py @@ -46,33 +46,33 @@ def dec(): def test_list_getitem(): - l = rlp.decode_lazy(rlp.encode([1, 2, 3]), big_endian_int) - assert isinstance(l, rlp.lazy.LazyList) - assert l[0] == 1 - assert l[1] == 2 - assert l[2] == 3 - assert l[-1] == 3 - assert l[-2] == 2 - assert l[-3] == 1 - assert l[0:3] == [1, 2, 3] - assert l[0:2] == [1, 2] - assert l[0:1] == [1] - assert l[1:2] == [2] - assert l[1:] == [2, 3] - assert l[1:-1] == [2] - assert l[-2:] == [2, 3] - assert l[:2] == [1, 2] + decoded = rlp.decode_lazy(rlp.encode([1, 2, 3]), big_endian_int) + assert isinstance(decoded, rlp.lazy.LazyList) + assert decoded[0] == 1 + assert decoded[1] == 2 + assert decoded[2] == 3 + assert decoded[-1] == 3 + assert decoded[-2] == 2 + assert decoded[-3] == 1 + assert decoded[0:3] == [1, 2, 3] + assert decoded[0:2] == [1, 2] + assert decoded[0:1] == [1] + assert decoded[1:2] == [2] + assert decoded[1:] == [2, 3] + assert decoded[1:-1] == [2] + assert decoded[-2:] == [2, 3] + assert decoded[:2] == [1, 2] def test_nested_list(): - l = ((), (b'a'), (b'b', b'c', b'd')) + nested = ((), (b'a'), (b'b', b'c', b'd')) def dec(): - return rlp.decode_lazy(rlp.encode(l)) + return rlp.decode_lazy(rlp.encode(nested)) assert isinstance(dec(), Sequence) - assert len(dec()) == len(l) - assert evaluate(dec()) == l + assert len(dec()) == len(nested) + assert evaluate(dec()) == nested with pytest.raises(IndexError): dec()[0][0] with pytest.raises(IndexError): diff --git a/tests/test_raw_sedes.py b/tests/core/test_raw_sedes.py similarity index 100% rename from tests/test_raw_sedes.py rename to tests/core/test_raw_sedes.py diff --git a/tests/test_sedes.py b/tests/core/test_sedes.py similarity index 100% rename from tests/test_sedes.py rename to tests/core/test_sedes.py diff --git a/tests/test_serializable.py b/tests/core/test_serializable.py similarity index 100% rename from tests/test_serializable.py rename to tests/core/test_serializable.py diff --git a/tests/test_text_sedes.py b/tests/core/test_text_sedes.py similarity index 100% rename from tests/test_text_sedes.py rename to tests/core/test_text_sedes.py diff --git a/tox.ini b/tox.ini index 1f9770b..f1f9c9a 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ envlist = [flake8] max-line-length= 100 exclude= venv*,.tox,docs,build -ignore= +ignore=W503 [testenv] usedevelop=True