From b687ab2d895e7ce386307b58109079998d60c4de Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 28 Sep 2015 11:48:29 -0500 Subject: [PATCH 01/19] Fix indenting. --- distarray/localapi/tests/paralleltest_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distarray/localapi/tests/paralleltest_io.py b/distarray/localapi/tests/paralleltest_io.py index 03a91e71..b826fa40 100644 --- a/distarray/localapi/tests/paralleltest_io.py +++ b/distarray/localapi/tests/paralleltest_io.py @@ -12,7 +12,7 @@ from distarray.testing import ParallelTestCase, import_or_skip, temp_filepath from distarray.localapi import LocalArray, ndenumerate from distarray.localapi import (save_dnpy, load_dnpy, save_hdf5, load_hdf5, - load_npy) + load_npy) from distarray.localapi.maps import Distribution From 8780cd3b16cf7e3f338e1093cb72c9854b9aff49 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 28 Sep 2015 12:27:19 -0500 Subject: [PATCH 02/19] Add a "Notes" section title. --- distarray/localapi/format.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/distarray/localapi/format.py b/distarray/localapi/format.py index c08c31a5..04c3e13b 100644 --- a/distarray/localapi/format.py +++ b/distarray/localapi/format.py @@ -56,6 +56,9 @@ the magic number for ``.npy`` files, followed by the ``.npy`` header and array data. +Notes +----- + The ``.npy`` format, including reasons for creating it and a comparison of alternatives, is described fully in the "npy-format" NEP and in the module docstring for ``numpy.lib.format``. From fbca3e1eb52eb35fa4f1bdce30ed744ebd376c0b Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 28 Sep 2015 12:27:33 -0500 Subject: [PATCH 03/19] Add tests for format.py. --- distarray/localapi/tests/test_format.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 distarray/localapi/tests/test_format.py diff --git a/distarray/localapi/tests/test_format.py b/distarray/localapi/tests/test_format.py new file mode 100644 index 00000000..83cc1e00 --- /dev/null +++ b/distarray/localapi/tests/test_format.py @@ -0,0 +1,24 @@ +# encoding: utf-8 +# --------------------------------------------------------------------------- +# Copyright (C) 2008-2014, IPython Development Team and Enthought, Inc. +# Distributed under the terms of the BSD License. See COPYING.rst. +# --------------------------------------------------------------------------- + + +import unittest +import six + +from distarray.localapi import format as fmt + + +class TestMagic(unittest.TestCase): + + def test_magic_0(self): + expected = six.b('\x93DARRY\x03\x02') + + prefix = six.b('\x93DARRY') + major = 3 + minor = 2 + + result = fmt.magic(major=major, minor=minor, prefix=prefix) + self.assertEqual(result, expected) From 8ea513899ec75867d6d4d75c98b1f37de55401c7 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 28 Sep 2015 12:29:04 -0500 Subject: [PATCH 04/19] Simplify read_magic. --- distarray/localapi/format.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/distarray/localapi/format.py b/distarray/localapi/format.py index 04c3e13b..faa1bb30 100644 --- a/distarray/localapi/format.py +++ b/distarray/localapi/format.py @@ -104,12 +104,8 @@ def magic(major, minor, prefix=MAGIC_PREFIX): raise ValueError("Major version must be 0 <= major < 256.") if minor < 0 or minor > 255: raise ValueError("Minor version must be 0 <= minor < 256.") - if six.PY2: - return prefix + chr(major) + chr(minor) - elif six.PY3: - return prefix + bytes([major, minor]) - else: - raise _raise_nie() + + return prefix + six.int2byte(major) + six.int2byte(minor) def write_localarray(fp, arr, version=(1, 0)): From 799847227dcd0b5937ec81ec5b099b3809c0ad58 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 28 Sep 2015 12:31:10 -0500 Subject: [PATCH 05/19] Add another test. --- distarray/localapi/tests/test_format.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/distarray/localapi/tests/test_format.py b/distarray/localapi/tests/test_format.py index 83cc1e00..239685f6 100644 --- a/distarray/localapi/tests/test_format.py +++ b/distarray/localapi/tests/test_format.py @@ -22,3 +22,13 @@ def test_magic_0(self): result = fmt.magic(major=major, minor=minor, prefix=prefix) self.assertEqual(result, expected) + + def test_magic_1(self): + expected = six.b('\x93NUMPY\x01\x00') + + prefix = six.b('\x93NUMPY') + major = 1 + minor = 0 + + result = fmt.magic(major=major, minor=minor, prefix=prefix) + self.assertEqual(result, expected) From 72fe2376f79299ad15ce11b7e8a3b2c54f8cdc74 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 28 Sep 2015 12:35:09 -0500 Subject: [PATCH 06/19] Reorder to match numpy/lib/format. --- distarray/localapi/format.py | 50 ++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/distarray/localapi/format.py b/distarray/localapi/format.py index faa1bb30..c7eed5e3 100644 --- a/distarray/localapi/format.py +++ b/distarray/localapi/format.py @@ -108,6 +108,31 @@ def magic(major, minor, prefix=MAGIC_PREFIX): return prefix + six.int2byte(major) + six.int2byte(minor) +def read_magic(fp): + """Read the magic string to get the version of the file format. + + Parameters + ---------- + fp : filelike object + + Returns + ------- + major : int + minor : int + """ + magic_str = _read_bytes(fp, MAGIC_LEN, "magic string") + if magic_str[:-2] != MAGIC_PREFIX: + msg = "the magic string is not correct; expected %r, got %r" + raise ValueError(msg % (MAGIC_PREFIX, magic_str[:-2])) + if six.PY2: + major, minor = map(ord, magic_str[-2:]) + elif six.PY3: + major, minor = magic_str[-2:] + else: + raise _raise_nie() + return major, minor + + def write_localarray(fp, arr, version=(1, 0)): """ Write a LocalArray to a .dnpy file, including a header. @@ -151,31 +176,6 @@ def write_localarray(fp, arr, version=(1, 0)): np.save(fp, distbuffer['buffer']) -def read_magic(fp): - """Read the magic string to get the version of the file format. - - Parameters - ---------- - fp : filelike object - - Returns - ------- - major : int - minor : int - """ - magic_str = _read_bytes(fp, MAGIC_LEN, "magic string") - if magic_str[:-2] != MAGIC_PREFIX: - msg = "the magic string is not correct; expected %r, got %r" - raise ValueError(msg % (MAGIC_PREFIX, magic_str[:-2])) - if six.PY2: - major, minor = map(ord, magic_str[-2:]) - elif six.PY3: - major, minor = magic_str[-2:] - else: - raise _raise_nie() - return major, minor - - def read_array_header_1_0(fp): """ Read an array header from a filelike object using the 1.0 file format From 38c2556f2be7d80a008f733973ab79b3311c340b Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 28 Sep 2015 12:48:56 -0500 Subject: [PATCH 07/19] Fix up magic's docstring. --- distarray/localapi/format.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/distarray/localapi/format.py b/distarray/localapi/format.py index c7eed5e3..fead2033 100644 --- a/distarray/localapi/format.py +++ b/distarray/localapi/format.py @@ -89,10 +89,12 @@ def magic(major, minor, prefix=MAGIC_PREFIX): ---------- major : int in [0, 255] minor : int in [0, 255] + prefix : bytes + The magic prefix to concatenate with version number Returns ------- - magic : str + magic : bytes Raises ------ From 5fc4af3039caec0f245e04e5a219451dfb73fb9c Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 28 Sep 2015 12:49:06 -0500 Subject: [PATCH 08/19] Add a test for read_magic. --- distarray/localapi/tests/test_format.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/distarray/localapi/tests/test_format.py b/distarray/localapi/tests/test_format.py index 239685f6..d713b810 100644 --- a/distarray/localapi/tests/test_format.py +++ b/distarray/localapi/tests/test_format.py @@ -32,3 +32,16 @@ def test_magic_1(self): result = fmt.magic(major=major, minor=minor, prefix=prefix) self.assertEqual(result, expected) + + +class TestReadMagic(unittest.TestCase): + + def test_read_magic(self): + prefix = six.b('\x93DARRY') + prefix_len = 8 + fp = six.BytesIO(six.b('\x93DARRY\x03\x02')) + + major, minor = fmt.read_magic(fp, prefix=prefix, prefix_len=prefix_len) + + expected = (3, 2) + self.assertEqual((major, minor), expected) From 03c5232518c1cbae4361107dca16f3d2caa69f29 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 28 Sep 2015 12:53:42 -0500 Subject: [PATCH 09/19] Add another test for read_magic. --- distarray/localapi/tests/test_format.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/distarray/localapi/tests/test_format.py b/distarray/localapi/tests/test_format.py index d713b810..e7d7e852 100644 --- a/distarray/localapi/tests/test_format.py +++ b/distarray/localapi/tests/test_format.py @@ -36,7 +36,7 @@ def test_magic_1(self): class TestReadMagic(unittest.TestCase): - def test_read_magic(self): + def test_read_magic_0(self): prefix = six.b('\x93DARRY') prefix_len = 8 fp = six.BytesIO(six.b('\x93DARRY\x03\x02')) @@ -45,3 +45,13 @@ def test_read_magic(self): expected = (3, 2) self.assertEqual((major, minor), expected) + + def test_read_magic_1(self): + prefix = six.b('\x93NUMPY') + prefix_len = 8 + fp = six.BytesIO(six.b('\x93NUMPY\x01\x01')) + + major, minor = fmt.read_magic(fp, prefix=prefix, prefix_len=prefix_len) + + expected = (1, 1) + self.assertEqual((major, minor), expected) From e7e80b97a1010c583edd465201bbf63a09bc2b46 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 28 Sep 2015 12:53:53 -0500 Subject: [PATCH 10/19] Simplify and document read_magic. --- distarray/localapi/format.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/distarray/localapi/format.py b/distarray/localapi/format.py index fead2033..b209fdb3 100644 --- a/distarray/localapi/format.py +++ b/distarray/localapi/format.py @@ -110,28 +110,28 @@ def magic(major, minor, prefix=MAGIC_PREFIX): return prefix + six.int2byte(major) + six.int2byte(minor) -def read_magic(fp): +def read_magic(fp, prefix=MAGIC_PREFIX, prefix_len=MAGIC_LEN): """Read the magic string to get the version of the file format. Parameters ---------- fp : filelike object + prefix : bytes + Magic prefix to look for + prefix_len : int + Number of bytes in `prefix` Returns ------- major : int minor : int """ - magic_str = _read_bytes(fp, MAGIC_LEN, "magic string") - if magic_str[:-2] != MAGIC_PREFIX: + magic_str = _read_bytes(fp, prefix_len, "magic string") + if magic_str[:-2] != prefix: msg = "the magic string is not correct; expected %r, got %r" - raise ValueError(msg % (MAGIC_PREFIX, magic_str[:-2])) - if six.PY2: - major, minor = map(ord, magic_str[-2:]) - elif six.PY3: - major, minor = magic_str[-2:] - else: - raise _raise_nie() + raise ValueError(msg % (prefix, magic_str[:-2])) + + major, minor = map(six.byte2int, magic_str[-2:]) return major, minor From 2c2ff86e80b7fb5e0586254b3c6584d6f698fd2e Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 28 Sep 2015 12:56:07 -0500 Subject: [PATCH 11/19] Remove unnecessary import. --- distarray/localapi/format.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/distarray/localapi/format.py b/distarray/localapi/format.py index b209fdb3..57a9bf07 100644 --- a/distarray/localapi/format.py +++ b/distarray/localapi/format.py @@ -73,8 +73,6 @@ from numpy.lib.utils import safe_eval from numpy.compat import asbytes -from distarray.utils import _raise_nie - MAGIC_PREFIX = asbytes('\x93DARRY') MAGIC_LEN = len(MAGIC_PREFIX) + 2 From 117ef770ca5c3a3b6e13e1f54a91cda27b9f6612 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 28 Sep 2015 14:06:03 -0500 Subject: [PATCH 12/19] Remove unnecessary import. --- distarray/testing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/distarray/testing.py b/distarray/testing.py index e25f4986..201682d4 100644 --- a/distarray/testing.py +++ b/distarray/testing.py @@ -20,7 +20,6 @@ from distarray.externals import six from distarray.externals import protocol_validator from distarray.globalapi.context import Context, ContextCreationError -from distarray.globalapi.ipython_utils import IPythonClient from distarray.error import InvalidCommSizeError from distarray.localapi.mpiutils import MPI, create_comm_of_size From 00e89ebb52ab89c108cd539d696bdebb38a34982 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 28 Sep 2015 14:06:27 -0500 Subject: [PATCH 13/19] Allow smaller comm_size. --- distarray/localapi/tests/paralleltest_io.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/distarray/localapi/tests/paralleltest_io.py b/distarray/localapi/tests/paralleltest_io.py index b826fa40..d04eabb0 100644 --- a/distarray/localapi/tests/paralleltest_io.py +++ b/distarray/localapi/tests/paralleltest_io.py @@ -18,6 +18,8 @@ class TestDnpyFileIO(ParallelTestCase): + comm_size = 1 + def setUp(self): d = Distribution.from_shape(comm=self.comm, shape=(7,)) self.larr0 = LocalArray(d) From c4994f9bdde9ccc63167129039a18a76a05074fe Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 28 Sep 2015 14:07:41 -0500 Subject: [PATCH 14/19] Copy in some other functions and simplify. So we no longer have a dependence on numpy/lib/format.py --- distarray/localapi/format.py | 75 +++++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 10 deletions(-) diff --git a/distarray/localapi/format.py b/distarray/localapi/format.py index 57a9bf07..b4cabfb3 100644 --- a/distarray/localapi/format.py +++ b/distarray/localapi/format.py @@ -69,7 +69,6 @@ from distarray.externals import six import numpy as np -from numpy.lib.format import write_array_header_1_0 from numpy.lib.utils import safe_eval from numpy.compat import asbytes @@ -133,7 +132,59 @@ def read_magic(fp, prefix=MAGIC_PREFIX, prefix_len=MAGIC_LEN): return major, minor -def write_localarray(fp, arr, version=(1, 0)): +# mostly copied from numpy/lib/format +# dependance on _filter_header removed, since we don't care about npz-style +# headers +def write_localarray_header(fp, d, version=None): + """Write the header for a localarray and return the version used + + Parameters + ---------- + fp : filelike object + d : dict + This has the appropriate entries for writing its string representation + to the header of the file. + version: tuple or None + None means use oldest that works + explicit version will raise a ValueError if the format does not + allow saving this data. Default: None + + Returns + ------- + version : tuple of int + the file version which needs to be used to store the data + """ + import struct + header = ["{"] + for key, value in sorted(d.items()): + # Need to use repr here, since we eval these when reading + header.append("'%s': %s, " % (key, repr(value))) + header.append("}") + header = "".join(header) + # Pad the header with spaces and a final newline such that the magic + # string, the header-length short and the header are aligned on a + # 16-byte boundary. Hopefully, some system, possibly memory-mapping, + # can take advantage of our premature optimization. + current_header_len = MAGIC_LEN + 2 + len(header) + 1 # 1 for the newline + topad = 16 - (current_header_len % 16) + header = header + ' '*topad + '\n' + header = asbytes(header) + + hlen = len(header) + if hlen < 256*256 and version in (None, (1, 0)): + version = (1, 0) + header_prefix = magic(1, 0) + struct.pack(' Date: Mon, 28 Sep 2015 14:08:09 -0500 Subject: [PATCH 15/19] Fix the dnpy bug. Magic string was being written twice. --- distarray/localapi/format.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/distarray/localapi/format.py b/distarray/localapi/format.py index b4cabfb3..50adb2c5 100644 --- a/distarray/localapi/format.py +++ b/distarray/localapi/format.py @@ -216,8 +216,6 @@ def write_localarray(fp, larr, version=(1, 0)): msg = "Only version (1, 0) is supported, not %s." raise ValueError(msg % (version,)) - fp.write(magic(*version)) - distbuffer = larr.__distarray__() metadata = {'__version__': distbuffer['__version__'], 'dim_data': distbuffer['dim_data'], From 79f258fee5a053280d0b22ce33a588dea08915d2 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 28 Sep 2015 14:08:18 -0500 Subject: [PATCH 16/19] Unskip the relevant tests. --- distarray/globalapi/tests/test_distributed_io.py | 2 -- distarray/localapi/tests/paralleltest_io.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/distarray/globalapi/tests/test_distributed_io.py b/distarray/globalapi/tests/test_distributed_io.py index b4a117e5..c95abdcf 100644 --- a/distarray/globalapi/tests/test_distributed_io.py +++ b/distarray/globalapi/tests/test_distributed_io.py @@ -45,7 +45,6 @@ def setUpClass(cls): cls.da = cls.context.empty(cls.distribution) cls.output_paths = cls.context.apply(engine_temp_path) - @unittest.skip("FIXME") def test_save_load_with_filenames(self): try: @@ -57,7 +56,6 @@ def test_save_load_with_filenames(self): for filepath, target in zip(self.output_paths, self.context.targets): self.context.apply(cleanup_file, (filepath,), targets=(target,)) - @unittest.skip("FIXME") def test_save_load_with_prefix(self): output_path = self.output_paths[0] diff --git a/distarray/localapi/tests/paralleltest_io.py b/distarray/localapi/tests/paralleltest_io.py index d04eabb0..8ce34fc6 100644 --- a/distarray/localapi/tests/paralleltest_io.py +++ b/distarray/localapi/tests/paralleltest_io.py @@ -48,14 +48,12 @@ def test_flat_file_save_with_file_object(self): self.assertTrue(magic == b'\x93DARRY') - @unittest.skip("FIXME") def test_flat_file_save_load_with_filename(self): save_dnpy(self.output_path, self.larr0) larr1 = load_dnpy(comm=self.comm, file=self.output_path) self.assertTrue(isinstance(larr1, LocalArray)) assert_allclose(self.larr0, larr1) - @unittest.skip("FIXME") def test_flat_file_save_load_with_file_object(self): save_dnpy(self.output_path, self.larr0) with open(self.output_path, 'rb') as fp: From 839c8e6015a32863dd594d47408f72fb2d2e438d Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 28 Sep 2015 14:17:00 -0500 Subject: [PATCH 17/19] Remove unneeded import. --- distarray/localapi/tests/paralleltest_io.py | 1 - 1 file changed, 1 deletion(-) diff --git a/distarray/localapi/tests/paralleltest_io.py b/distarray/localapi/tests/paralleltest_io.py index 8ce34fc6..479a21bd 100644 --- a/distarray/localapi/tests/paralleltest_io.py +++ b/distarray/localapi/tests/paralleltest_io.py @@ -6,7 +6,6 @@ import os import numpy -import unittest from numpy.testing import assert_allclose, assert_equal from distarray.testing import ParallelTestCase, import_or_skip, temp_filepath From 64d5b79a9096ab0fff86673f11e70b4cd7c5c26f Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 28 Sep 2015 15:55:13 -0500 Subject: [PATCH 18/19] Fix read_magic under Python3. --- distarray/localapi/format.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/distarray/localapi/format.py b/distarray/localapi/format.py index 50adb2c5..387ededd 100644 --- a/distarray/localapi/format.py +++ b/distarray/localapi/format.py @@ -128,7 +128,10 @@ def read_magic(fp, prefix=MAGIC_PREFIX, prefix_len=MAGIC_LEN): msg = "the magic string is not correct; expected %r, got %r" raise ValueError(msg % (prefix, magic_str[:-2])) - major, minor = map(six.byte2int, magic_str[-2:]) + if six.PY2: + major, minor = map(ord, magic_str[-2:]) + if six.PY3: + major, minor = magic_str[-2:] return major, minor From 97f435f9bd37ebac2310f492c5a9e36003573c32 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 28 Sep 2015 16:43:05 -0500 Subject: [PATCH 19/19] Require sphinx 1.3. --- quickstart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quickstart b/quickstart index f7a6c7ef..dcd24513 100755 --- a/quickstart +++ b/quickstart @@ -88,7 +88,7 @@ resolve_conda() { } install_osx() { - conda create -n $CONDA_ENV python=$PY_VER numpy ipyparallel notebook cython sphinx mock matplotlib + conda create -n $CONDA_ENV python=$PY_VER numpy ipyparallel notebook cython sphinx=1.3 mock matplotlib source activate $CONDA_ENV pip install mpi4py pip install sphinxcontrib-programoutput @@ -98,7 +98,7 @@ install_osx() { } install_linux() { - conda create -n $CONDA_ENV python=$PY_VER mpi4py numpy ipyparallel notebook cython sphinx mock matplotlib + conda create -n $CONDA_ENV python=$PY_VER mpi4py numpy ipyparallel notebook cython sphinx=1.3 mock matplotlib source activate $CONDA_ENV pip install sphinxcontrib-programoutput python setup.py $INSTALL_MODE