diff --git a/examples/features.ipynb b/examples/features.ipynb index b4c99691..75721580 100644 --- a/examples/features.ipynb +++ b/examples/features.ipynb @@ -1,7 +1,7 @@ { "metadata": { "name": "", - "signature": "sha256:d0eac20bb684ec8096f777561418fc4f2b5788d763a92b7bd6acc0462c990053" + "signature": "sha256:676cb478085c537769827ff0cb26c98b3d609edac014cb6b459fbcd8108eaf3b" }, "nbformat": 3, "nbformat_minor": 0, @@ -19,7 +19,7 @@ "DistArray: Distributed Arrays for Python\n", "========================================\n", "\n", - "[github.com/enthought/distarray](https://github.com/enthought/distarray)" + "[docs.enthought.com/distarray](http://docs.enthought.com/distarray)" ] }, { @@ -30,9 +30,9 @@ } }, "source": [ - "Start a cluster\n", - "---------------\n", - "These examples require an `IPython.parallel` cluster to be running.\n", + "Setup\n", + "-----\n", + "Much of this notebook requires an `IPython.parallel` cluster to be running.\n", "Outside the notebook, run\n", "```\n", "dacluster start -n4\n", @@ -76,20 +76,16 @@ }, "source": [ "NumPy Arrays\n", - "------------" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "DistArray is heavily inspired by NumPy." + "------------\n", + "\n", + "DistArray is built on NumPy and provides a NumPy-array-like interface. First, let's generate a NumPy array and examine some of its attributes." ] }, { "cell_type": "code", "collapsed": false, "input": [ + "# a 4-row 5-column NumPy array with random contents\n", "nparr = numpy.random.random((4, 5))\n", "nparr" ], @@ -105,10 +101,10 @@ "output_type": "pyout", "prompt_number": 2, "text": [ - "array([[ 0.48, 0.75, 0. , 0.71, 0.06],\n", - " [ 0.21, 0.74, 0.79, 0.23, 0.54],\n", - " [ 0.84, 0.33, 0.71, 0.62, 0.98],\n", - " [ 0.46, 0.13, 0.12, 0.61, 0.35]])" + "array([[ 0.44, 0.87, 0.87, 0.22, 0.23],\n", + " [ 0.15, 0.05, 0.7 , 0.04, 0.8 ],\n", + " [ 0.68, 0.94, 0.51, 0.01, 0.2 ],\n", + " [ 0.84, 0.81, 0.99, 0.45, 0.02]])" ] } ], @@ -157,17 +153,20 @@ }, "source": [ "DistArrays\n", - "----------" + "----------\n", + "\n", + "We'll make our first `DistArray` out of the NumPy array created above." ] }, { "cell_type": "code", "collapsed": false, "input": [ + "# First we need a `Context` object. More on this later.\n", + "# For now, think of this object like the `NumPy` module.\n", + "# `Context`s manage the worker engines for us.\n", "from distarray.globalapi import Context\n", - "context = Context()\n", - "darr = context.fromarray(nparr)\n", - "darr" + "context = Context()" ], "language": "python", "metadata": { @@ -175,23 +174,38 @@ "slide_type": "subslide" } }, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# Make a DistArray from a NumPy array.\n", + "# This will push sections of the original NumPy array out\n", + "# to the engines.\n", + "darr = context.fromarray(nparr)\n", + "darr" + ], + "language": "python", + "metadata": {}, "outputs": [ { "metadata": {}, "output_type": "pyout", - "prompt_number": 4, + "prompt_number": 5, "text": [ "" ] } ], - "prompt_number": 4 + "prompt_number": 5 }, { "cell_type": "code", "collapsed": false, "input": [ - "# parts of the array are stored on each engine\n", + "# Print the array section stored on each engine\n", "for i, a in enumerate(darr.get_localarrays()):\n", " print(i, a)" ], @@ -206,20 +220,20 @@ "output_type": "stream", "stream": "stdout", "text": [ - "0 [[ 0.48 0.75 0. 0.71 0.06]]\n", - "1 [[ 0.21 0.74 0.79 0.23 0.54]]\n", - "2 [[ 0.84 0.33 0.71 0.62 0.98]]\n", - "3 [[ 0.46 0.13 0.12 0.61 0.35]]\n" + "0 [[ 0.44 0.87 0.87 0.22 0.23]]\n", + "1 [[ 0.15 0.05 0.7 0.04 0.8 ]]\n", + "2 [[ 0.68 0.94 0.51 0.01 0.2 ]]\n", + "3 [[ 0.84 0.81 0.99 0.45 0.02]]\n" ] } ], - "prompt_number": 5 + "prompt_number": 6 }, { "cell_type": "code", "collapsed": false, "input": [ - "# DistArray attributes\n", + "# DistArrays have similar attributes to NumPy arrays,\n", "print(\"type:\", type(darr))\n", "print(\"dtype:\", darr.dtype)\n", "print(\"ndim:\", darr.ndim)\n", @@ -247,13 +261,13 @@ ] } ], - "prompt_number": 6 + "prompt_number": 7 }, { "cell_type": "code", "collapsed": false, "input": [ - "# with some extra...\n", + "# and some additional attributes.\n", "print(\"targets:\", darr.targets)\n", "print(\"context:\", darr.context)\n", "print(\"distribution:\", darr.distribution)" @@ -270,12 +284,12 @@ "stream": "stdout", "text": [ "targets: [0, 1, 2, 3]\n", - "context: \n", - "distribution: \n" + "context: \n", + "distribution: \n" ] } ], - "prompt_number": 7 + "prompt_number": 8 }, { "cell_type": "markdown", @@ -293,7 +307,8 @@ "cell_type": "code", "collapsed": false, "input": [ - "# NumPy\n", + "# NumPy provides `ufuncs`, or Universal Functions, that operate\n", + "# elementwise over NumPy arrays.\n", "numpy.sin(nparr)" ], "language": "python", @@ -306,22 +321,22 @@ { "metadata": {}, "output_type": "pyout", - "prompt_number": 8, + "prompt_number": 9, "text": [ - "array([[ 0.46, 0.69, 0. , 0.65, 0.06],\n", - " [ 0.21, 0.67, 0.71, 0.22, 0.51],\n", - " [ 0.75, 0.32, 0.65, 0.58, 0.83],\n", - " [ 0.44, 0.13, 0.12, 0.57, 0.35]])" + "array([[ 0.42, 0.76, 0.76, 0.22, 0.23],\n", + " [ 0.15, 0.05, 0.64, 0.04, 0.72],\n", + " [ 0.63, 0.81, 0.49, 0.01, 0.2 ],\n", + " [ 0.74, 0.72, 0.84, 0.43, 0.02]])" ] } ], - "prompt_number": 8 + "prompt_number": 9 }, { "cell_type": "code", "collapsed": false, "input": [ - "# DistArray\n", + "# DistArray provides ufuncs as well, for `DistArray`s.\n", "import distarray.globalapi as da\n", "da.sin(darr)" ], @@ -335,18 +350,21 @@ { "metadata": {}, "output_type": "pyout", - "prompt_number": 9, + "prompt_number": 10, "text": [ "" ] } ], - "prompt_number": 9 + "prompt_number": 10 }, { "cell_type": "code", "collapsed": false, "input": [ + "# `toarray` makes a NumPy array out of a DistArray, pulling all of the\n", + "# pieces back to the client. We do this to display the contents of the\n", + "# DistArray.\n", "da.sin(darr).toarray()" ], "language": "python", @@ -359,22 +377,22 @@ { "metadata": {}, "output_type": "pyout", - "prompt_number": 10, + "prompt_number": 11, "text": [ - "array([[ 0.46, 0.69, 0. , 0.65, 0.06],\n", - " [ 0.21, 0.67, 0.71, 0.22, 0.51],\n", - " [ 0.75, 0.32, 0.65, 0.58, 0.83],\n", - " [ 0.44, 0.13, 0.12, 0.57, 0.35]])" + "array([[ 0.42, 0.76, 0.76, 0.22, 0.23],\n", + " [ 0.15, 0.05, 0.64, 0.04, 0.72],\n", + " [ 0.63, 0.81, 0.49, 0.01, 0.2 ],\n", + " [ 0.74, 0.72, 0.84, 0.43, 0.02]])" ] } ], - "prompt_number": 10 + "prompt_number": 11 }, { "cell_type": "code", "collapsed": false, "input": [ - "# NumPy\n", + "# A NumPy binary ufunc.\n", "nparr + nparr" ], "language": "python", @@ -387,22 +405,24 @@ { "metadata": {}, "output_type": "pyout", - "prompt_number": 11, + "prompt_number": 12, "text": [ - "array([[ 0.96, 1.51, 0.01, 1.43, 0.12],\n", - " [ 0.43, 1.48, 1.57, 0.45, 1.08],\n", - " [ 1.69, 0.66, 1.41, 1.25, 1.96],\n", - " [ 0.92, 0.26, 0.24, 1.22, 0.71]])" + "array([[ 0.87, 1.74, 1.73, 0.44, 0.46],\n", + " [ 0.3 , 0.1 , 1.4 , 0.07, 1.6 ],\n", + " [ 1.36, 1.88, 1.02, 0.02, 0.4 ],\n", + " [ 1.68, 1.61, 1.98, 0.9 , 0.05]])" ] } ], - "prompt_number": 11 + "prompt_number": 12 }, { "cell_type": "code", "collapsed": false, "input": [ - "# DistArray\n", + "# The equivalent DistArray ufunc.\n", + "# Notice that a new DistArray is created without\n", + "# pulling data back to the client.\n", "darr + darr" ], "language": "python", @@ -415,18 +435,19 @@ { "metadata": {}, "output_type": "pyout", - "prompt_number": 12, + "prompt_number": 13, "text": [ "" ] } ], - "prompt_number": 12 + "prompt_number": 13 }, { "cell_type": "code", "collapsed": false, "input": [ + "# Contents of the resulting DistArray.\n", "(darr + darr).toarray()" ], "language": "python", @@ -439,16 +460,16 @@ { "metadata": {}, "output_type": "pyout", - "prompt_number": 13, + "prompt_number": 14, "text": [ - "array([[ 0.96, 1.51, 0.01, 1.43, 0.12],\n", - " [ 0.43, 1.48, 1.57, 0.45, 1.08],\n", - " [ 1.69, 0.66, 1.41, 1.25, 1.96],\n", - " [ 0.92, 0.26, 0.24, 1.22, 0.71]])" + "array([[ 0.87, 1.74, 1.73, 0.44, 0.46],\n", + " [ 0.3 , 0.1 , 1.4 , 0.07, 1.6 ],\n", + " [ 1.36, 1.88, 1.02, 0.02, 0.4 ],\n", + " [ 1.68, 1.61, 1.98, 0.9 , 0.05]])" ] } ], - "prompt_number": 13 + "prompt_number": 14 }, { "cell_type": "markdown", @@ -458,16 +479,19 @@ } }, "source": [ - "Distributions\n", - "-------------" + "Reductions\n", + "----------\n", + "\n", + "Functions like `sum`, `mean`, `min`, and `max` are known as *reductions*, since they take an array and produce a smaller array or a scalar. In NumPy and DistArray, some of these functions can be applied over a specific ``axis``." ] }, { "cell_type": "code", "collapsed": false, "input": [ - "# Distributions control which processes own which (global) indices\n", - "distribution = darr.distribution" + "# NumPy sum\n", + "print(\"sum:\", nparr.sum())\n", + "print(\"sum over an axis:\", nparr.sum(axis=1))" ], "language": "python", "metadata": { @@ -475,15 +499,25 @@ "slide_type": "subslide" } }, - "outputs": [], - "prompt_number": 14 + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "sum: 9.79905043187\n", + "sum over an axis: [ 2.62 1.73 2.33 3.11]\n" + ] + } + ], + "prompt_number": 15 }, { "cell_type": "code", "collapsed": false, "input": [ - "# this is a 2D distribution\n", - "pprint(distribution.maps)" + "# DistArray sum\n", + "print(\"sum:\", darr.sum(), darr.sum().toarray())\n", + "print(\"sum over an axis:\", darr.sum(axis=1), darr.sum(axis=1).toarray())" ], "language": "python", "metadata": { @@ -496,35 +530,40 @@ "output_type": "stream", "stream": "stdout", "text": [ - "[,\n", - " ]\n" + "sum: 9.79905043187\n", + "sum over an axis:" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " [ 2.62 1.73 2.33 3.11]\n" ] } ], - "prompt_number": 15 + "prompt_number": 16 }, { - "cell_type": "code", - "collapsed": false, - "input": [ - "# setup\n", - "from distarray.plotting import plot_array_distribution\n", - "process_coords = [(0, 0), (1, 0), (2, 0), (3, 0)]" - ], - "language": "python", + "cell_type": "markdown", "metadata": { "slideshow": { - "slide_type": "subslide" + "slide_type": "slide" } }, - "outputs": [], - "prompt_number": 16 + "source": [ + "Indexing and Slicing\n", + "--------------------\n", + "\n", + "DistArrays support standard NumPy Indexing and distributed slicing, including slices with a step. Slicing is currently only supported for Block (and undistributed) DistArrays." + ] }, { "cell_type": "code", "collapsed": false, "input": [ - "plot_array_distribution(darr, process_coords, cell_label=False, legend=True)" + "# Our example array, as a reminder:\n", + "darr.toarray()" ], "language": "python", "metadata": { @@ -538,15 +577,10 @@ "output_type": "pyout", "prompt_number": 17, "text": [ - "" - ] - }, - { - "metadata": {}, - "output_type": "display_data", - "png": "iVBORw0KGgoAAAANSUhEUgAAAbUAAAEzCAYAAAC2Q50YAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAD69JREFUeJzt3X2MZXV9x/H3Z3eh0hLTWkwU2Dq2FV1AolYJirG7TWw2\nptGmqdKNtMY/+iiBmqbSB5vCX/7TB/6gJiYuSDUFW4ymSFbS4I7F0Cxi2JZ9wGLiNDxEbCrSIjHZ\nlW//mDs6PLj37s6dOYfvvF/Jzd57Z37nfDMh8+acc+feVBWSJHWwZegBJEmaF6MmSWrDqEmS2jBq\nkqQ2jJokqQ2jJklqo03UkuxO8kCSB5NcPfQ8Y5DkhiSPJbl/6FnGIsn2JPuTHE5yKMmVQ880Bkle\nlORAkoNJjiT5yNAzjUWSrUnuS3Lb0LNouhZRS7IVuB7YDZwP7EmyY9ipRuFGln8m+qFjwAer6gLg\nEuAD/rcCVfU9YFdVvQ64CNiV5K0DjzUWVwFHAP+o9wWgRdSAi4GvV9VSVR0DbgHeNfBMg6uqu4DH\nh55jTKrqm1V1cHL/SeAocPawU41DVT01uXs6sBX49oDjjEKSc4F3AB8HMvA4mkGXqJ0DPLTq8cOT\n56QfKckC8HrgwLCTjEOSLUkOAo8B+6vqyNAzjcDfAn8MPD30IJpNl6h5WkAnJcmZwK3AVZMjtk2v\nqp6enH48F3hbkp0DjzSoJL8CfKuq7sOjtBeMLlF7BNi+6vF2lo/WpOdIchrwGeBTVfW5oecZm6p6\nArgdeOPQswzsLcA7k3wDuBn4pSR/P/BMmqJL1O4FXpVkIcnpwGXAPw88k0YoSYC9wJGqum7oecYi\nyVlJfnJy/wzg7cB9w041rKr6s6raXlWvBH4D+GJV/dbQc+nEWkStqo4DVwB3sPwqpU9X1dFhpxpe\nkpuBu4HzkjyU5P1DzzQClwKXs/zqvvsmN18hCi8Hvji5pnYAuK2q7hx4prHxMscLQPzoGUlSFy2O\n1CRJAqMmSWrEqEmS2jBqkqQ2jJokqY1ta91AEl8+KUkbqKo27B1O1vI7fiPnXLHmqC37y/lsZs0W\ngZ0DzzBGi/hzeT6L+HN5tkX8mTyfRcbzc7l2w/d4zQatmQdPP0qS2jBqkqQ2mkVtYegBRmph6AFG\namHoAUZoYegBRmph6AE0I6O2KSwMPcBILQw9wAgtDD3ASC0MPYBm1CxqkqTNzKhJktowapKkNoya\nJKkNoyZJasOoSZLaMGqSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktowapKkNoya\nJKkNoyZJasOoSZLaMGqSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqY2pUUuyO8kDSR5McvVGDCVJ\n6iXJi5IcSHIwyZEkH1mP/Zwwakm2AtcDu4HzgT1JdqzHIJKkvqrqe8CuqnodcBGwK8lb572faUdq\nFwNfr6qlqjoG3AK8a95DSJL6q6qnJndPB7YC3573PqZF7RzgoVWPH548J0nSSUmyJclB4DFgf1Ud\nmfc+pkWt5r1DSdLmVFVPT04/ngu8LcnOee9j25SvPwJsX/V4O8tHa8+yuOr+wuQmSVq7pclt3L7B\n7FNW1RNJbgfeyDMDsmbTonYv8KokC8CjwGXAnud+2855ziRJ+oEFnnmg8KVhxpjilZPbimdPmeQs\n4HhVfSfJGcDbgWvnPccJo1ZVx5NcAdzB8kW9vVV1dN5DSJLaezlwU5ItLF/6+mRV3TnvnUw7UqOq\n9gH75r1jSdLmUVX3A29Y7/34jiKSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktow\napKkNoyaJKkNoyZJasOoSZLaMGqSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktow\napKkNoyaJKkNoyZJasOoSZLaMGqSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqY1tc9nKoWvmshlJ\n0hQXXjv0BKOWqlrbBpLi0Nq2IUma0YWhqrJRu0tS9egprDubDZ1zhacfJUltGDVJUhtGTZLUhlGT\nJLVh1CRJbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGT\nJLVh1CRJbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGT\nJLVh1CRJbUyNWpIbkjyW5P6NGEiS1E+S7Un2Jzmc5FCSK9djP7Mcqd0I7F6PnUuSNo1jwAer6gLg\nEuADSXbMeydTo1ZVdwGPz3vHkqTNo6q+WVUHJ/efBI4CZ897P15TkyRtqCQLwOuBA/Pe9ra5bOXv\nrvnh/TfthIt3zmWzkrTp3bMIX1kceoqpFu9evk2T5EzgVuCqyRHbXKWqZhliAbitql77PF8rDk3f\nhiRpDi4MVZWN2l2SqkdPYd3ZPGfOJKcBnwf2VdV185nwmTz9KElad0kC7AWOrFfQYLaX9N8M3A2c\nl+ShJO9fr2EkSW1dClwO7Epy3+Q291fWT72mVlV75r1TSdLmUlVfZgPODnr6UZLUhlGTJLVh1CRJ\nbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNkjQaSbYl+dqprjdqkqTRqKrjwANJXnEq6+fz0TOS\nJM3PS4DDSe4Bvjt5rqrqndMWGjVJ0tj8xeTflc81y6r7J2TUJEmjUlWLSV4GvInlmN1TVd+aZa3X\n1CRJo5LkPcAB4N3Ae4B7krx7lrUeqUmSxubDwJtWjs6SvBS4E/inaQs9UpMkjU2A/171+H8mz03l\nkZokaWy+ANyR5B9YjtllwL5ZFho1SdLYfAj4NeDSyeOPVdVnZ1lo1CRJY/PjwOeq6jNJXg28Jslp\nVXVs2kKvqUmSxuYu4MeSnAPcAVwOfGKWhUZNkjQ2qaqnWD4F+dGqejdw4SwLjZokaXSSvBl4L3D7\n5KmZemXUJElj84fAnwKfrarDSX4O2D/LwlTN9HZaP3oDSXFobduQJM3owlBVM/3N1jwkqXr0FNad\nzVzmTLIFOLOq/neW7/dITZI0KkluTvLiJD8BHAKOJvnQLGuNmiRpbM6fHJn9Kst/dL0A/OYsC42a\nJGlstiU5jeWo3Tb5+7SZrnMZNUnS2HwMWALOBP41yQLwxCwLfaGIJL2QbLIXikxmCLC1qo5P+16P\n1CRJo5LkZUn2JvnC5KkdwPtmWTuX936sn9qw/2mQpE1tk/y2/QRwI/Dnk8cPAv8I7J220Dc0liSd\nUB4/lUtMa8rvWVX16SR/AlBVx5JMPfUInn6UJI3Pk0l+euVBkkuY8YUiHqlJksbmj4DbgJ9Ncjfw\nUuDXZ1lo1CRJo1JVX03yi8CrWT6P+cAsn6UGnn6UJI1MkitYfr/HQ1V1P3Bmkj+YZa1RkySNzW9X\n1eMrDyb3f2eWhUZNkjQ2Wybvzg9Akq3AabMs9JqaJGls7gBuSfIxlq+p/S7whRMvWWbUJEljczXL\npxt/f/L4X4CPz7LQqEmSRqWqvp/kBuDLk6ceqKrvz7LWqEmSRiXJTuAm4L8mT/1MkvdV1ZemrTVq\nkqSx+Rvgl6vqawBJzgNuAd4wbaGvfpQkjc22laABVNV/MuNBmEdqkqSx+WqSjwOfYvnVj+8F7p1l\noVGTJI3N7wFXAFdOHt8FfHSWhUZNkjQaSbYB/15VrwH++mTXe01NkjQaVXUc+FqSV5zKeo/UJElj\n8xLgcJJ7gO9Onquqeue0hUZNkjQ2H578u/rjs2f6+G2jJkkahSRnsPwikZ8H/gO4YdbPUVvhNTVJ\n0ljcBPwCy0F7B/BXJ7sBj9QkSWOxo6peC5BkL/CVk92AR2qSpLE4vnJn8irIk+aRmiRpLC5K8n+r\nHp+x6nFV1YunbcCoSZJGoaq2rnUbnn6UJLVh1CRJbRg1SVIbRk2S1IZRkyS1MTVqSbYn2Z/kcJJD\nSa6ctkaSpNWS3JDksST3r+d+ZjlSOwZ8sKouAC4BPpBkx3oOJUlq50Zg93rvZGrUquqbVXVwcv9J\n4Chw9noPJknqo6ruAh5f7/2c1DW1JAvA64ED6zGMJElrMfM7iiQ5E7gVuGpyxPYD16z6wO2db4ad\nb5nXeJK0uS3eDYv/NvQUM7hnEb6yOPQUpGr6564lOQ34PLCvqq571teqHlmn6SRJz5BzoKoy/Tvn\ntL+kODzT53M+0wV5zpyTs323rbwT/3qY5dWPAfYCR54dNEmSxmSWa2qXApcDu5LcN7mt+ytYJEl9\nJLkZuBs4L8lDSd6/HvuZek2tqr6Mf6QtSVqDqtqzEfsxVpKkNoyaJKkNoyZJasOoSZLaMGqSpDaM\nmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktowapKkNoyaJKkNoyZJasOoSZLaMGqSpDaM\nmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktowapKkNoyaJKkNoyZJasOoSZLaMGqSpDaM\nmiSpjW3z2Mg158xjK5Ikrc1coiZJauyCa4eeYGaefpQktWHUJEltGDVJUhtGTZLUhlGTJLVh1CRJ\nbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGTJLVh1CRJ\nbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUxgmjluRFSQ4k\nOZjkSJKPbNRgkqRekuxO8kCSB5NcvR77OGHUqup7wK6qeh1wEbAryVvXYxBJUl9JtgLXA7uB84E9\nSXbMez9TTz9W1VOTu6cDW4Fvz3sISVJ7FwNfr6qlqjoG3AK8a947mRq1JFuSHAQeA/ZX1ZF5DyFJ\nau8c4KFVjx+ePDdXsxypPT05/Xgu8LYkO+c9hCSpvdqInWyb9Rur6okktwNvBBZXf231g4XJTZK0\ndkuT2/h9gymTPgJsX/V4O8tHa3N1wqglOQs4XlXfSXIG8Hbg2md/3855TyVJAp57oPClYcaYwSsn\ntxXPmfRe4FVJFoBHgcuAPfOeYtqR2suBm5JsYflU5Ser6s55DyFJ6q2qjie5AriD5Rcd7q2qo/Pe\nzwmjVlX3A2+Y904lSZtPVe0D9q3nPnxHEUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGTJLVh\n1CRJbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGTJLVh\n1CRJbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktdEqaktDDzBSS0MPMFJLQw8wQktD\nDzBSS0MPoJkZtU1gaegBRmpp6AFGaGnoAUZqaegBNLNWUZMkbW5GTZLURqpqbRtI1rYBSdJJqaps\n1L7W8jt+I+dcseaoSZI0Fp5+lCS1YdQkSW0YNUlSG0ZNktSGUZMktfH/OydIpv9Xa0UAAAAASUVO\nRK5CYII=\n", - "text": [ - "" + "array([[ 0.44, 0.87, 0.87, 0.22, 0.23],\n", + " [ 0.15, 0.05, 0.7 , 0.04, 0.8 ],\n", + " [ 0.68, 0.94, 0.51, 0.01, 0.2 ],\n", + " [ 0.84, 0.81, 0.99, 0.45, 0.02]])" ] } ], @@ -556,7 +590,8 @@ "cell_type": "code", "collapsed": false, "input": [ - "distribution.maps[0].bounds" + "# The shapes of the local sections of our DistArray\n", + "darr.localshapes()" ], "language": "python", "metadata": { @@ -570,7 +605,7 @@ "output_type": "pyout", "prompt_number": 18, "text": [ - "[(0, 1), (1, 2), (2, 3), (3, 4)]" + "[(1, 5), (1, 5), (1, 5), (1, 5)]" ] } ], @@ -580,58 +615,31 @@ "cell_type": "code", "collapsed": false, "input": [ - "# the above is the default, but you can make more complex distributions\n", - "from distarray.globalapi import Distribution\n", - "distribution = Distribution(context, (64, 64), dist=('b', 'c'))\n", - "a = context.zeros(distribution, dtype='int32')\n", - "plot_array_distribution(a, process_coords, cell_label=False, legend=True)" + "# Return the value of a single element\n", + "darr[0, 2]" ], "language": "python", - "metadata": { - "slideshow": { - "slide_type": "subslide" - } - }, + "metadata": {}, "outputs": [ { "metadata": {}, "output_type": "pyout", "prompt_number": 19, "text": [ - "" - ] - }, - { - "metadata": {}, - "output_type": "display_data", - "png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEzCAYAAAAlyyi6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFLdJREFUeJzt3X+MZXd93vH3410TDDRYC2jXGJdx2ywxENc2ZSGGhDGy\nkRNFG6sqJlaKVgjRtDWFpoiypGkz/qMCquKEKjZCie2unNSEmnplixh7cTwuyJXXhl38c72h9Ua2\nYcdWcRx+CNWGT/+4Z9jJenbnzsy9c8989/2Sruacc8/3nmet2cdnv/fcc1NVSJLacdKkA0iSRsti\nl6TGWOyS1BiLXZIaY7FLUmMsdklqzFiLPcnFSQ4k+cskHxvnsVYiybVJ5pI8sGDbpiR7khxMcnuS\nUyeZcaEkZyS5M8lDSR5M8qFue+8yJ3lxknuS7E/ycJJP9DXrQkk2JNmX5JZuvZd5kxxKcn+XdW+3\nrZdZAZKcmuTGJI90vw9v6XPe9W5sxZ5kA/CHwMXA64HLkpw1ruOt0HUM8i20E9hTVVuBO7r1vngO\n+O2qegPwVuDy7r9p7zJX1Y+AC6rqHOBs4IIkb6eHWY/yYeBhYP4DHn3NW8B0VZ1bVdu6bX3NCvAZ\n4M+r6iwGvw8H6Hfe9a2qxvIAfhH48oL1ncDOcR1vFTmngAcWrB8ANnfLW4ADk854nOy7gQv7nhl4\nCXAv8IY+ZwVeA3wFuAC4pc+/D8BjwCuO2tbXrC8H/s8i23uZt4XHOKdiTgceX7D+RLet7zZX1Vy3\nPAdsnmSYY0kyBZwL3ENPMyc5Kcl+BpnurKqH6GnWzu8DHwV+smBbX/MW8JUk9yX5QLetr1nPBJ5O\ncl2SbyT5oyQvpb95171xFvu6v1dBDU4levfnSPIy4IvAh6vqewuf61PmqvpJDaZiXgP8cpILjnq+\nN1mT/BrwVFXtA7LYPn3KC7ytqs4FfoXBlNwvLXyyZ1k3AucBV1fVecAPOGrapWd5171xFvuTwBkL\n1s9gcNbed3NJtgAkOQ14asJ5/pYkJzMo9eurane3udeZq+pZ4EvAm+hv1vOB7UkeA24A3pnkenqa\nt6q+0/18GrgJ2EZPszL4e/9EVd3brd/IoOgP9zTvujfOYr8P+LkkU0leBLwHuHmMxxuVm4Ed3fIO\nBvPYvZAkwDXAw1X1Bwue6l3mJK+cv8ohySnARcA+epgVoKp+p6rOqKozgd8A/qKq3ksP8yZ5SZK/\n0y2/FHgX8AA9zApQVYeBx5Ns7TZdCDwE3EIP8zZhnBP4DP6Z+CjwLeDjk35DYZF8NwDfBv4fg/cD\n3gdsYvAG2kHgduDUSedckPftDOZ/9zMoyX0MrurpXWbgF4BvdFnvBz7abe9d1kWyvwO4ua95GcxZ\n7+8eD87/3epj1gWZ/yGDN9C/CfwPBm+o9jbven+k+48uSWqEnzyVpMZY7JLUmFUVe99vGSBJJ6IV\nz7F3twx4lME73E8yeGPksqp6ZHTxJEnLtZoz9m3At6rqUFU9B3we+PXRxJIkrdTGVYxd7JYBb1m4\nQxIvuZE0FlW16CeEx2E1XbaWOeetptiH/IP+HjALTMODMy98+o1HbTt6n6OfH9U+x80yy6J5J5Jl\nqdeYBaZ7kmWYfWYZOu/Ef19mOWbWNc9ynOd/us8sI/1dWFWWYV5jlhXnfeOadyUzS+4xmjGjsJpi\nH/KWAbPAocHPvbOwbXoVh5R0YjoEV81MOsS6sZpi/+ktAxh8evM9wGUv3G2an/6f2VKXtCJTcPnM\nkdXPXjGpIOvCiou9qp5P8kHgNmADcM2xr4iZWulhJmRq0gGWYWrSAZZpatIBlmFq0gGWaWrSAZZp\natIBmrWaM3aq6lbg1qX3nFrNYSZgatIBlmFq0gGWaWrSAZZhatIBlmlq0gGWaWrSAZrlJ08lqTEW\nuyQ1xmKXpMZY7JLUGItdkhpjsUtSYyx2SWqMxS5JjbHYJakxFrskNcZil6TGWOyS1BiLXZIaY7FL\nUmMsdklqjMUuSY2x2CWpMRa7JDXGYpekxljsktQYi12SGmOxS1JjLHZJaozFLkmNsdglaY0keXGS\ne5LsT/Jwkk+M4zgbx/GikqQXqqofJbmgqn6YZCPwtSRvr6qvjfI4nrFL0hqqqh92iy8CNgDfHfUx\nLHZJWkNJTkqyH5gD7qyqh0d9DItdktZQVf2kqs4BXgP8cpLpUR/DOXZJGpHHgEND7ltVzyb5EvCP\ngNlR5rDYJWlEzuwe8+466vkkrwSer6q/TnIKcBFwxahzWOyStHZOA3YlOYnBVPj1VXXHqA9isUvS\nGqmqB4Dzxn0c3zyVpMZY7JLUmCWLPcm1SeaSPLBg26Yke5IcTHJ7klPHG1OSNKxhztivAy4+attO\nYE9VbQXu6NYlST2wZLFX1VeBZ47avB3Y1S3vAi4ZcS5J0gqtdI59c1XNdctzwOYR5ZEkrdKq3zyt\nqgJqBFkkSSOw0uvY55JsqarDSU4Dnjr2rrNHFvfOwrbpFR5S0onrEFw1M+kQ68ZKi/1mYAfwqe7n\n7mPvOn1k0VKXtCJTcPnMkdXPjvxT+E0Z5nLHG4C7gdcleTzJ+4BPAhclOQi8s1uXJPXAkmfsVXXZ\nMZ66cMRZJEkj4CdPJakxFrskNcZil6TGWOyS1BiLXZIaY7FLUmMsdklqjMUuSY2x2CWpMRa7JDXG\nYpekxljsktQYi12SGmOxS1JjLHZJaozFLkmNsdglqTEWuyQ1xmKXpMZY7JLUmCW/zFqSBL/37eWP\nmXn16HMMwzN2SWqMxS5JjbHYJakxFrskNcZil6TGWOyS1BiLXZIaY7FLUmMsdklqjMUuSY2x2CWp\nMRa7JDXGYpekxljsktSYJYs9yRlJ7kzyUJIHk3yo274pyZ4kB5PcnuTU8ceVJC1lmDP254Dfrqo3\nAG8FLk9yFrAT2FNVW4E7unVJ0oQtWexVdbiq9nfL3wceAU4HtgO7ut12AZeMK6QkaXjLmmNPMgWc\nC9wDbK6que6pOWDzSJNJklZk6K/GS/Iy4IvAh6vqe0l++lxVVZJafOTskcW9s7BteiU5JZ3QDsFV\nM5MOsW4MVexJTmZQ6tdX1e5u81ySLVV1OMlpwFOLj54+smipS1qRKbh85sjqZ6+YVJB1YZirYgJc\nAzxcVX+w4KmbgR3d8g5g99FjJUlrb5gz9rcB/xS4P8m+btvHgU8CX0jyfuAQcOlYEkqSlmXJYq+q\nr3HsM/sLRxtHkrRafvJUkhpjsUtSYyx2SWqMxS5JjbHYJakxFrskNcZil6Q1cqzboI/a0PeKkSSt\n2vxt0Pd399/6epI9VfXIKA/iGbskrZFj3Ab91aM+jsUuSRNw1G3QR8qpGEkakdm7B4+ldNMwNzK4\nDfr3R53DYpekEZk+f/CYd8WVL9xnwW3Q/2TBbdBHyqkYSVojx7kN+khZ7JK0duZvg35Bkn3d4+JR\nH8SpGElaI0vcBn1kPGOXpMZY7JLUGItdkhpjsUtSYyx2SWqMxS5JjbHYJakxFrskNcZil6SeSbIx\nyaMrHW+xS1LPVNXzwIEkr13JeG8pIEn9tAl4KMle4Afdtqqq7UsNtNglqZ/+ffezup9ZsHxcFrsk\n9VBVzSbZAryZQaHvraqnhhnrHLsk9VCSSxl8bd67gUuBvUnePcxYz9glqZ9+F3jz/Fl6klcBdwD/\nfamBnrFLUj8FeHrB+v/tti3JM3ZJ6qcvA7cl+W8MCv09wK3DDLTYJamf/i3wjxl8nR7A56rqpmEG\nWuyS1E8vAXZX1ReTvA74+SQnV9VzSw087hx7khcnuSfJ/iQPJ/lEt31Tkj1JDia5Pcmpo/lzSJI6\nXwV+JsnpwG0MvgT7vw4z8LjFXlU/Ai6oqnOAsxl8s/bbgZ3AnqrayuBd2p0rzy5JWkSq6ocMpmOu\nrqp3A28cZuCSV8V0LwzwImAD8AywHdjVbd8FXLLcxJKk40vyi8BvAl/qNg11JeOSOyU5Kcl+YA64\ns6oeAjZX1Vy3yxywefmRJUnH8a+BjwM3VdVDSf4+cOcwA5d887SqfgKck+TlDC69ueCo5yvJUPcv\nkCQNp6ruAu6CwQk28HRVfWiYsUNfFVNVzyb5EvAmYC7Jlqo6nOQ04Dj3L5g9srh3FrZND3tISeoc\ngqtmJh1iTSW5Afgt4MfAvcDLk3ymqv7TUmOXuirmlfNXvCQ5BbgI2AfcDOzodtsB7D72q0wfeVjq\nklZkCi6fOfI4Mby+qv6GwXuYtwJTwHuHGbjUGftpwK7unwEnAddX1R1J9gFfSPJ+4BCDG9RIkkZn\nY5KTGRT7VVX13LDT3sct9qp6ADhvke3fBS5cSVJJ0lA+x+DE+X7gfyaZAp4dZqA3AZOkHqqq/1JV\np1fVr3QXsfwVcMFS48Bil6ReSrIlyTVJvtxtOosj720ef2zV+K5UTFL15JH1mdNfuM/Mk0etn378\n50e1j1nMYpb1m+UKoKqGuoXtKCSp+vYKxr165Tm7Qr8O+HdVdXY3376vqpb89Kk3AZOkIeSZlZwE\nr+r/Pa+sqj9LshOge/P0+WEGOhUjSf30/SSvmF9J8laGfPPUM3ZJ6qePALcAfy/J3cCrgH8yzECL\nXZJ6qKq+nuQdwOsYzOkcGOZe7OBUjCT1UpIPAi+rqge7zxS9LMm/HGasxS5J/fSBqnpmfqVb/mfD\nDLTYJamfTupu5wJAkg3AycMMdI5dkvrpNuDzST7HYI79t4AvH3/IgMUuSf30MQZTL/+iW98D/PEw\nAy12SeqhqvpxkmuBr3WbDlTVj4cZa7FLUg8lmWbwndJ/1W36u0l2dN+sdFwWuyT105XAu6rqUYAk\nW4HPs8it1I/mVTGS1E8b50sdoKoOMuTJuGfsktRPX0/yx8CfMLgq5jeB+4YZaLFLUj/9c+CDwIe6\n9a8CVw8z0GKXpJ5JshH4ZlX9PPDp5Y53jl2SeqaqngceTfLalYz3jF2S+mkT8FCSvcAPum1VVduX\nGmixS1I//W73c+HXMA31NU4WuyT1SJJTGLxx+g+A+4Frh70P+zzn2CWpX3YBb2JQ6r8K/OflvoBn\n7JLUL2dV1S8AJLkGuHe5L+AZuyT1y/PzC93VMcvmGbsk9cvZSb63YP2UBetVVT+71AtY7JLUI1W1\nYbWv4VSMJDXGYpekxljsktQYi12SGmOxS1JjLHZJWiNJrk0yl+SBcR7HYpektXMdcPG4DzJUsSfZ\nkGRfklu69U1J9iQ5mOT2JKeON6YkrX9V9VXgmXEfZ9gz9g8DD3PklpE7gT1VtRW4o1uXJPXAkp88\nTfIaBncY+4/Av+k2bwfe0S3vAmax3CWd6PbOwr2zk04x1C0Ffh/4KLDw/gSbq2quW54DNo86mCSt\nO9umB495V18xkRjHLfYkvwY8VVX7kkwvtk9VVZJjfqvHzIKvYT0ETK0gpKQT2+zdg2kBDWepM/bz\nge1JfhV4MfCzSa4H5pJsqarDSU4DnjrWC8x8ZMHylasPLOnEM30+TC9Yv2tSQVYpyQ0MprFfkeRx\n4D9U1XWjPs5x3zytqt+pqjOq6kzgN4C/qKr3AjcDO7rddgC7Rx1MklpTVZdV1aur6me6bh15qcPy\nr2Ofn3L5JHBRkoPAO7t1SVIPDH0/9qq6i+5fQFX1XeDCcYWSJK2cnzyVpMZY7JLUGItdkhpjsUtS\nYyx2SWqMxS5JjbHYJakxFrskNcZil6TGWOyS1BiLXZIaY7FLUmMsdklqjMUuSY2x2CWpMRa7JDXG\nYpekxljsktQYi12SGmOxS1JjLHZJaozFLkmNsdglqTEWuyQ1xmKXpMZY7JLUGItdkhpjsUtSYyx2\nSWrMxkkHkKR14Q1XTDrB0Dxjl6TGWOyS1BiLXZIaY7FLUmOGevM0ySHgb4AfA89V1bYkm4A/A14L\nHAIuraq/HlNOSdKQhj1jL2C6qs6tqm3dtp3AnqraCtzRrUuSJmw5UzE5an07sKtb3gVcMpJEkqRV\nWc4Z+1eS3JfkA922zVU11y3PAZtHnk6StGzDfkDpbVX1nSSvAvYkObDwyaqqJDX6eJKk5Rqq2Kvq\nO93Pp5PcBGwD5pJsqarDSU4Dnlps7MynjywfAqZWl1fSCWj2bpiddIh1ZMliT/ISYENVfS/JS4F3\nAVcANwM7gE91P3cvNn7mIwuWr1x9YEknnunzYXrB+l2TCrJODHPGvhm4Kcn8/n9aVbcnuQ/4QpL3\n013uOLaUkqShLVnsVfUYcM4i278LXDiOUJKklfOTp5LUGItdkhpjsUtSYyx2SWqMxS5JjbHYJakx\nFrskNcZil6TGWOyS1BiLXZIaY7FLUmMsdklqjMUuSY2x2CWpMRa7JDXGYpekNZTk4iQHkvxlko+N\n4xgWuyStkSQbgD8ELgZeD1yW5KxRH8dil6S1sw34VlUdqqrngM8Dvz7qg1jskrR2TgceX7D+RLdt\npCx2SVo7tRYHWfLLrCVJw3oMOHS8HZ4EzliwfgaDs/aRstglaWTO7B7z7jp6h/uAn0syBXwbeA9w\n2ahTWOyStEaq6vkkHwRuAzYA11TVI6M+jsUuSWuoqm4Fbh3nMXzzVJIaY7FLUmMsdklqjMUuSY2x\n2CWpMRa7JDXGYpekxljsktQYi12SGmOxS1JjLHZJasxQxZ7k1CQ3JnkkycNJ3pJkU5I9SQ4muT3J\nqeMOK0la2rBn7J8B/ryqzgLOBg4AO4E9VbUVuKNblyRN2JLFnuTlwC9V1bUwuO1kVT0LbAd2dbvt\nAi4ZW0pJ0tCGOWM/E3g6yXVJvpHkj5K8FNhcVXPdPnPA5rGllCQNbZhi3wicB1xdVecBP+CoaZeq\nKtbou/wkScc3zBdtPAE8UVX3dus3Ah8HDifZUlWHk5wGPLXY4JlPH1k+BEytIqykE9Ps3TA76RDr\nyJLF3hX340m2VtVB4ELgoe6xA/hU93P3YuNnPrJg+coRJJZ0wpk+H6YXrL/gm0T1twz71Xj/CvjT\nJC8C/jfwPgbf1/eFJO9ncDJ+6VgSSpKWZahir6pvAm9e5KkLRxtHkrRafvJUkhpjsUtSYyx2SWqM\nxS5JjbHYJakxFrskNcZil6TGrEmxz969FkcZnfWUdz1lhfWVdz1lBfPqiLUp9v+1FkcZnfWUdz1l\nhfWVdz1lBfPqCKdiJKkxFrskNSaDW6mP6cUT79EuaSyqKmt1rNV02VrmnDfWYpckrT2nYiSpMRa7\nJDXGYpekxljsktQYi12SGvP/AdzJez1KiumSAAAAAElFTkSuQmCC\n", - "text": [ - "" + "0.86608017321365971" ] } ], "prompt_number": 19 }, - { - "cell_type": "markdown", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "Contexts\n", - "--------" - ] - }, { "cell_type": "code", "collapsed": false, "input": [ - "# Context objects manage the setup and communication of the worker processes\n", - "# for DistArray objects. \n", - "print(\"targets:\", context.targets)\n", - "print(\"comm:\", context.comm)" + "# Take a column slice\n", + "darr_view = darr[:, 3] # all rows, third column\n", + "print(darr_view)\n", + "print(darr_view.toarray())" ], "language": "python", "metadata": { @@ -644,34 +652,26 @@ "output_type": "stream", "stream": "stdout", "text": [ - "targets: [0, 1, 2, 3]\n", - "comm: \n" + "\n", + "[ 0.22 0.04 0.01 0.45]\n" ] } ], "prompt_number": 20 }, - { - "cell_type": "markdown", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "IO Support\n", - "-----------------" - ] - }, { "cell_type": "code", "collapsed": false, "input": [ - "# load .npy files in parallel\n", - "numpy.save(\"/tmp/outfile.npy\", nparr)\n", - "distribution = Distribution(context, nparr.shape) \n", - "new_darr = context.load_npy(\"/tmp/outfile.npy\", distribution)\n", - "new_darr" + "# Slices return a new DistArray that is a view on the\n", + "# original, just like in NumPy.\n", + "# Changes in the view change the original array.\n", + "darr_view[3] = -0.99\n", + "print(\"view:\")\n", + "print(darr_view.toarray())\n", + "\n", + "print(\"original:\")\n", + "print(darr.toarray())" ], "language": "python", "metadata": { @@ -681,11 +681,16 @@ }, "outputs": [ { - "metadata": {}, - "output_type": "pyout", - "prompt_number": 21, + "output_type": "stream", + "stream": "stdout", "text": [ - "" + "view:\n", + "[ 0.22 0.04 0.01 -0.99]\n", + "original:\n", + "[[ 0.44 0.87 0.87 0.22 0.23]\n", + " [ 0.15 0.05 0.7 0.04 0.8 ]\n", + " [ 0.68 0.94 0.51 0.01 0.2 ]\n", + " [ 0.84 0.81 0.99 -0.99 0.02]]\n" ] } ], @@ -693,10 +698,11 @@ }, { "cell_type": "code", - "collapsed": false, + "collapsed": true, "input": [ - "# save DistArrays to .hdf5 files in parallel\n", - "context.save_hdf5(\"/tmp/outfile.hdf5\", darr, mode='w')" + "# A more complex slice, with negative indices and a step.\n", + "print(darr[:, 2::2])\n", + "print(darr[:-1, 2::2].toarray())" ], "language": "python", "metadata": { @@ -704,40 +710,62 @@ "slide_type": "subslide" } }, - "outputs": [], + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "[[ 0.87 0.23]\n", + " [ 0.7 0.8 ]\n", + " [ 0.51 0.2 ]]\n" + ] + } + ], "prompt_number": 22 }, { "cell_type": "code", - "collapsed": true, + "collapsed": false, "input": [ - "# load DistArrays from .hdf5 files in parallel (using h5py)\n", - "context.load_hdf5(\"/tmp/outfile.hdf5\", distribution)" + "# Incomplete indexing\n", + "# Grab the first row\n", + "darr[0]" ], "language": "python", - "metadata": { - "slideshow": { - "slide_type": "subslide" - } - }, + "metadata": {}, "outputs": [ { "metadata": {}, "output_type": "pyout", "prompt_number": 23, "text": [ - "" + "" ] } ], "prompt_number": 23 }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Distributions\n", + "-------------\n", + "Above, when we created a DistArray out of a NumPy array, we didn't specify *how* the elements should be distributed among our engines. `Distribution`s give you control over this, if you want it. In other words, `Distribution`s control which processes own which (global) indices." + ] + }, { "cell_type": "code", "collapsed": false, "input": [ - "# save to .dnpy (a built-in flat-file format based on .npy)\n", - "context.save_dnpy(\"/tmp/outfile\", darr)" + "# Let's look at the `Distribution` object that was created for us\n", + "# automatically by `fromarray`.\n", + "distribution = darr.distribution" ], "language": "python", "metadata": { @@ -752,8 +780,9 @@ "cell_type": "code", "collapsed": false, "input": [ - "# load from .dnpy\n", - "context.load_dnpy(\"/tmp/outfile\")" + "# This is a 2D distribution: its 0th dimension is Block-distributed,\n", + "# and it's 1st dimension isn't distributed.\n", + "pprint(distribution.maps)" ], "language": "python", "metadata": { @@ -763,35 +792,25 @@ }, "outputs": [ { - "metadata": {}, - "output_type": "pyout", - "prompt_number": 25, + "output_type": "stream", + "stream": "stdout", "text": [ - "" + "[,\n", + " ]\n" ] } ], "prompt_number": 25 }, - { - "cell_type": "markdown", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "Reductions\n", - "-----------------" - ] - }, { "cell_type": "code", "collapsed": false, "input": [ - "# NumPy\n", - "print(\"sum:\", nparr.sum())\n", - "print(\"sum over an axis:\", nparr.sum(axis=1))" + "# Plot this Distribution, color-coding which process each global index\n", + "# belongs to.\n", + "from distarray.plotting import plot_array_distribution\n", + "process_coords = [(0, 0), (1, 0), (2, 0), (3, 0)]\n", + "plot_array_distribution(darr, process_coords, cell_label=False, legend=True)" ], "language": "python", "metadata": { @@ -801,11 +820,19 @@ }, "outputs": [ { - "output_type": "stream", - "stream": "stdout", + "metadata": {}, + "output_type": "pyout", + "prompt_number": 26, "text": [ - "sum: 9.67697548135\n", - "sum over an axis: [ 2.01 2.51 3.48 1.67]\n" + "" + ] + }, + { + "metadata": {}, + "output_type": "display_data", + "png": "iVBORw0KGgoAAAANSUhEUgAAAbUAAAEzCAYAAAC2Q50YAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAD69JREFUeJzt3X2MZXV9x/H3Z3eh0hLTWkwU2Dq2FV1AolYJirG7TWw2\nptGmqdKNtMY/+iiBmqbSB5vCX/7TB/6gJiYuSDUFW4ymSFbS4I7F0Cxi2JZ9wGLiNDxEbCrSIjHZ\nlW//mDs6PLj37s6dOYfvvF/Jzd57Z37nfDMh8+acc+feVBWSJHWwZegBJEmaF6MmSWrDqEmS2jBq\nkqQ2jJokqQ2jJklqo03UkuxO8kCSB5NcPfQ8Y5DkhiSPJbl/6FnGIsn2JPuTHE5yKMmVQ880Bkle\nlORAkoNJjiT5yNAzjUWSrUnuS3Lb0LNouhZRS7IVuB7YDZwP7EmyY9ipRuFGln8m+qFjwAer6gLg\nEuAD/rcCVfU9YFdVvQ64CNiV5K0DjzUWVwFHAP+o9wWgRdSAi4GvV9VSVR0DbgHeNfBMg6uqu4DH\nh55jTKrqm1V1cHL/SeAocPawU41DVT01uXs6sBX49oDjjEKSc4F3AB8HMvA4mkGXqJ0DPLTq8cOT\n56QfKckC8HrgwLCTjEOSLUkOAo8B+6vqyNAzjcDfAn8MPD30IJpNl6h5WkAnJcmZwK3AVZMjtk2v\nqp6enH48F3hbkp0DjzSoJL8CfKuq7sOjtBeMLlF7BNi+6vF2lo/WpOdIchrwGeBTVfW5oecZm6p6\nArgdeOPQswzsLcA7k3wDuBn4pSR/P/BMmqJL1O4FXpVkIcnpwGXAPw88k0YoSYC9wJGqum7oecYi\nyVlJfnJy/wzg7cB9w041rKr6s6raXlWvBH4D+GJV/dbQc+nEWkStqo4DVwB3sPwqpU9X1dFhpxpe\nkpuBu4HzkjyU5P1DzzQClwKXs/zqvvsmN18hCi8Hvji5pnYAuK2q7hx4prHxMscLQPzoGUlSFy2O\n1CRJAqMmSWrEqEmS2jBqkqQ2jJokqY1ta91AEl8+KUkbqKo27B1O1vI7fiPnXLHmqC37y/lsZs0W\ngZ0DzzBGi/hzeT6L+HN5tkX8mTyfRcbzc7l2w/d4zQatmQdPP0qS2jBqkqQ2mkVtYegBRmph6AFG\namHoAUZoYegBRmph6AE0I6O2KSwMPcBILQw9wAgtDD3ASC0MPYBm1CxqkqTNzKhJktowapKkNoya\nJKkNoyZJasOoSZLaMGqSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktowapKkNoya\nJKkNoyZJasOoSZLaMGqSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqY2pUUuyO8kDSR5McvVGDCVJ\n6iXJi5IcSHIwyZEkH1mP/Zwwakm2AtcDu4HzgT1JdqzHIJKkvqrqe8CuqnodcBGwK8lb572faUdq\nFwNfr6qlqjoG3AK8a95DSJL6q6qnJndPB7YC3573PqZF7RzgoVWPH548J0nSSUmyJclB4DFgf1Ud\nmfc+pkWt5r1DSdLmVFVPT04/ngu8LcnOee9j25SvPwJsX/V4O8tHa8+yuOr+wuQmSVq7pclt3L7B\n7FNW1RNJbgfeyDMDsmbTonYv8KokC8CjwGXAnud+2855ziRJ+oEFnnmg8KVhxpjilZPbimdPmeQs\n4HhVfSfJGcDbgWvnPccJo1ZVx5NcAdzB8kW9vVV1dN5DSJLaezlwU5ItLF/6+mRV3TnvnUw7UqOq\n9gH75r1jSdLmUVX3A29Y7/34jiKSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktow\napKkNoyaJKkNoyZJasOoSZLaMGqSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktow\napKkNoyaJKkNoyZJasOoSZLaMGqSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqY1tc9nKoWvmshlJ\n0hQXXjv0BKOWqlrbBpLi0Nq2IUma0YWhqrJRu0tS9egprDubDZ1zhacfJUltGDVJUhtGTZLUhlGT\nJLVh1CRJbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGT\nJLVh1CRJbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGT\nJLVh1CRJbUyNWpIbkjyW5P6NGEiS1E+S7Un2Jzmc5FCSK9djP7Mcqd0I7F6PnUuSNo1jwAer6gLg\nEuADSXbMeydTo1ZVdwGPz3vHkqTNo6q+WVUHJ/efBI4CZ897P15TkyRtqCQLwOuBA/Pe9ra5bOXv\nrvnh/TfthIt3zmWzkrTp3bMIX1kceoqpFu9evk2T5EzgVuCqyRHbXKWqZhliAbitql77PF8rDk3f\nhiRpDi4MVZWN2l2SqkdPYd3ZPGfOJKcBnwf2VdV185nwmTz9KElad0kC7AWOrFfQYLaX9N8M3A2c\nl+ShJO9fr2EkSW1dClwO7Epy3+Q291fWT72mVlV75r1TSdLmUlVfZgPODnr6UZLUhlGTJLVh1CRJ\nbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNkjQaSbYl+dqprjdqkqTRqKrjwANJXnEq6+fz0TOS\nJM3PS4DDSe4Bvjt5rqrqndMWGjVJ0tj8xeTflc81y6r7J2TUJEmjUlWLSV4GvInlmN1TVd+aZa3X\n1CRJo5LkPcAB4N3Ae4B7krx7lrUeqUmSxubDwJtWjs6SvBS4E/inaQs9UpMkjU2A/171+H8mz03l\nkZokaWy+ANyR5B9YjtllwL5ZFho1SdLYfAj4NeDSyeOPVdVnZ1lo1CRJY/PjwOeq6jNJXg28Jslp\nVXVs2kKvqUmSxuYu4MeSnAPcAVwOfGKWhUZNkjQ2qaqnWD4F+dGqejdw4SwLjZokaXSSvBl4L3D7\n5KmZemXUJElj84fAnwKfrarDSX4O2D/LwlTN9HZaP3oDSXFobduQJM3owlBVM/3N1jwkqXr0FNad\nzVzmTLIFOLOq/neW7/dITZI0KkluTvLiJD8BHAKOJvnQLGuNmiRpbM6fHJn9Kst/dL0A/OYsC42a\nJGlstiU5jeWo3Tb5+7SZrnMZNUnS2HwMWALOBP41yQLwxCwLfaGIJL2QbLIXikxmCLC1qo5P+16P\n1CRJo5LkZUn2JvnC5KkdwPtmWTuX936sn9qw/2mQpE1tk/y2/QRwI/Dnk8cPAv8I7J220Dc0liSd\nUB4/lUtMa8rvWVX16SR/AlBVx5JMPfUInn6UJI3Pk0l+euVBkkuY8YUiHqlJksbmj4DbgJ9Ncjfw\nUuDXZ1lo1CRJo1JVX03yi8CrWT6P+cAsn6UGnn6UJI1MkitYfr/HQ1V1P3Bmkj+YZa1RkySNzW9X\n1eMrDyb3f2eWhUZNkjQ2Wybvzg9Akq3AabMs9JqaJGls7gBuSfIxlq+p/S7whRMvWWbUJEljczXL\npxt/f/L4X4CPz7LQqEmSRqWqvp/kBuDLk6ceqKrvz7LWqEmSRiXJTuAm4L8mT/1MkvdV1ZemrTVq\nkqSx+Rvgl6vqawBJzgNuAd4wbaGvfpQkjc22laABVNV/MuNBmEdqkqSx+WqSjwOfYvnVj+8F7p1l\noVGTJI3N7wFXAFdOHt8FfHSWhUZNkjQaSbYB/15VrwH++mTXe01NkjQaVXUc+FqSV5zKeo/UJElj\n8xLgcJJ7gO9Onquqeue0hUZNkjQ2H578u/rjs2f6+G2jJkkahSRnsPwikZ8H/gO4YdbPUVvhNTVJ\n0ljcBPwCy0F7B/BXJ7sBj9QkSWOxo6peC5BkL/CVk92AR2qSpLE4vnJn8irIk+aRmiRpLC5K8n+r\nHp+x6nFV1YunbcCoSZJGoaq2rnUbnn6UJLVh1CRJbRg1SVIbRk2S1IZRkyS1MTVqSbYn2Z/kcJJD\nSa6ctkaSpNWS3JDksST3r+d+ZjlSOwZ8sKouAC4BPpBkx3oOJUlq50Zg93rvZGrUquqbVXVwcv9J\n4Chw9noPJknqo6ruAh5f7/2c1DW1JAvA64ED6zGMJElrMfM7iiQ5E7gVuGpyxPYD16z6wO2db4ad\nb5nXeJK0uS3eDYv/NvQUM7hnEb6yOPQUpGr6564lOQ34PLCvqq571teqHlmn6SRJz5BzoKoy/Tvn\ntL+kODzT53M+0wV5zpyTs323rbwT/3qY5dWPAfYCR54dNEmSxmSWa2qXApcDu5LcN7mt+ytYJEl9\nJLkZuBs4L8lDSd6/HvuZek2tqr6Mf6QtSVqDqtqzEfsxVpKkNoyaJKkNoyZJasOoSZLaMGqSpDaM\nmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktowapKkNoyaJKkNoyZJasOoSZLaMGqSpDaM\nmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktowapKkNoyaJKkNoyZJasOoSZLaMGqSpDaM\nmiSpjW3z2Mg158xjK5Ikrc1coiZJauyCa4eeYGaefpQktWHUJEltGDVJUhtGTZLUhlGTJLVh1CRJ\nbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGTJLVh1CRJ\nbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUxgmjluRFSQ4k\nOZjkSJKPbNRgkqRekuxO8kCSB5NcvR77OGHUqup7wK6qeh1wEbAryVvXYxBJUl9JtgLXA7uB84E9\nSXbMez9TTz9W1VOTu6cDW4Fvz3sISVJ7FwNfr6qlqjoG3AK8a947mRq1JFuSHAQeA/ZX1ZF5DyFJ\nau8c4KFVjx+ePDdXsxypPT05/Xgu8LYkO+c9hCSpvdqInWyb9Rur6okktwNvBBZXf231g4XJTZK0\ndkuT2/h9gymTPgJsX/V4O8tHa3N1wqglOQs4XlXfSXIG8Hbg2md/3855TyVJAp57oPClYcaYwSsn\ntxXPmfRe4FVJFoBHgcuAPfOeYtqR2suBm5JsYflU5Ser6s55DyFJ6q2qjie5AriD5Rcd7q2qo/Pe\nzwmjVlX3A2+Y904lSZtPVe0D9q3nPnxHEUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGTJLVh\n1CRJbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGTJLVh\n1CRJbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktdEqaktDDzBSS0MPMFJLQw8wQktD\nDzBSS0MPoJkZtU1gaegBRmpp6AFGaGnoAUZqaegBNLNWUZMkbW5GTZLURqpqbRtI1rYBSdJJqaps\n1L7W8jt+I+dcseaoSZI0Fp5+lCS1YdQkSW0YNUlSG0ZNktSGUZMktfH/OydIpv9Xa0UAAAAASUVO\nRK5CYII=\n", + "text": [ + "" ] } ], @@ -815,9 +842,9 @@ "cell_type": "code", "collapsed": false, "input": [ - "# DistArray\n", - "print(\"sum:\", darr.sum(), darr.sum().toarray())\n", - "print(\"sum over an axis:\", darr.sum(axis=1), darr.sum(axis=1).toarray())" + "# Check out which sections of this array's 0th dimension are on\n", + "# each process.\n", + "distribution.maps[0].bounds" ], "language": "python", "metadata": { @@ -827,18 +854,11 @@ }, "outputs": [ { - "output_type": "stream", - "stream": "stdout", + "metadata": {}, + "output_type": "pyout", + "prompt_number": 27, "text": [ - "sum: 9.67697548135\n", - "sum over an axis:" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " [ 2.01 2.51 3.48 1.67]\n" + "[(0, 1), (1, 2), (2, 3), (3, 4)]" ] } ], @@ -846,26 +866,28 @@ }, { "cell_type": "markdown", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, + "metadata": {}, "source": [ - "Context.apply\n", - "--------------------\n", - "Global view, local control" + "The Distribution above was created for us by `fromarray`,\n", + "but DistArray lets us specify more complex distributions.\n", + "\n", + "Here, we specify that the 0th dimension has a Block distribution ('b')\n", + "and the 1st dimension has a Cyclic distribution.\n", + "\n", + "DistArray supports Block, Cyclic, Block-Cyclic, Unstructured,\n", + "and No-distribution dimensions. See the\n", + "[ScaLAPACK Documentation](http://netlib.org/scalapack/slug/node75.html) for more information about Distribution types." ] }, { "cell_type": "code", "collapsed": false, "input": [ - "def get_local_random():\n", - " import numpy\n", - " return numpy.random.randint(10)\n", + "from distarray.globalapi import Distribution\n", "\n", - "context.apply(get_local_random)" + "distribution = Distribution(context, shape=(64, 64), dist=('b', 'c'))\n", + "a = context.zeros(distribution, dtype='int32')\n", + "plot_array_distribution(a, process_coords, cell_label=False, legend=True)" ], "language": "python", "metadata": { @@ -879,20 +901,168 @@ "output_type": "pyout", "prompt_number": 28, "text": [ - "[9, 7, 1, 8]" + "" + ] + }, + { + "metadata": {}, + "output_type": "display_data", + "png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEzCAYAAAAlyyi6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFLdJREFUeJzt3X+MZXd93vH3410TDDRYC2jXGJdx2ywxENc2ZSGGhDGy\nkRNFG6sqJlaKVgjRtDWFpoiypGkz/qMCquKEKjZCie2unNSEmnplixh7cTwuyJXXhl38c72h9Ua2\nYcdWcRx+CNWGT/+4Z9jJenbnzsy9c8989/2Sruacc8/3nmet2cdnv/fcc1NVSJLacdKkA0iSRsti\nl6TGWOyS1BiLXZIaY7FLUmMsdklqzFiLPcnFSQ4k+cskHxvnsVYiybVJ5pI8sGDbpiR7khxMcnuS\nUyeZcaEkZyS5M8lDSR5M8qFue+8yJ3lxknuS7E/ycJJP9DXrQkk2JNmX5JZuvZd5kxxKcn+XdW+3\nrZdZAZKcmuTGJI90vw9v6XPe9W5sxZ5kA/CHwMXA64HLkpw1ruOt0HUM8i20E9hTVVuBO7r1vngO\n+O2qegPwVuDy7r9p7zJX1Y+AC6rqHOBs4IIkb6eHWY/yYeBhYP4DHn3NW8B0VZ1bVdu6bX3NCvAZ\n4M+r6iwGvw8H6Hfe9a2qxvIAfhH48oL1ncDOcR1vFTmngAcWrB8ANnfLW4ADk854nOy7gQv7nhl4\nCXAv8IY+ZwVeA3wFuAC4pc+/D8BjwCuO2tbXrC8H/s8i23uZt4XHOKdiTgceX7D+RLet7zZX1Vy3\nPAdsnmSYY0kyBZwL3ENPMyc5Kcl+BpnurKqH6GnWzu8DHwV+smBbX/MW8JUk9yX5QLetr1nPBJ5O\ncl2SbyT5oyQvpb95171xFvu6v1dBDU4levfnSPIy4IvAh6vqewuf61PmqvpJDaZiXgP8cpILjnq+\nN1mT/BrwVFXtA7LYPn3KC7ytqs4FfoXBlNwvLXyyZ1k3AucBV1fVecAPOGrapWd5171xFvuTwBkL\n1s9gcNbed3NJtgAkOQ14asJ5/pYkJzMo9eurane3udeZq+pZ4EvAm+hv1vOB7UkeA24A3pnkenqa\nt6q+0/18GrgJ2EZPszL4e/9EVd3brd/IoOgP9zTvujfOYr8P+LkkU0leBLwHuHmMxxuVm4Ed3fIO\nBvPYvZAkwDXAw1X1Bwue6l3mJK+cv8ohySnARcA+epgVoKp+p6rOqKozgd8A/qKq3ksP8yZ5SZK/\n0y2/FHgX8AA9zApQVYeBx5Ns7TZdCDwE3EIP8zZhnBP4DP6Z+CjwLeDjk35DYZF8NwDfBv4fg/cD\n3gdsYvAG2kHgduDUSedckPftDOZ/9zMoyX0MrurpXWbgF4BvdFnvBz7abe9d1kWyvwO4ua95GcxZ\n7+8eD87/3epj1gWZ/yGDN9C/CfwPBm+o9jbven+k+48uSWqEnzyVpMZY7JLUmFUVe99vGSBJJ6IV\nz7F3twx4lME73E8yeGPksqp6ZHTxJEnLtZoz9m3At6rqUFU9B3we+PXRxJIkrdTGVYxd7JYBb1m4\nQxIvuZE0FlW16CeEx2E1XbaWOeetptiH/IP+HjALTMODMy98+o1HbTt6n6OfH9U+x80yy6J5J5Jl\nqdeYBaZ7kmWYfWYZOu/Ef19mOWbWNc9ynOd/us8sI/1dWFWWYV5jlhXnfeOadyUzS+4xmjGjsJpi\nH/KWAbPAocHPvbOwbXoVh5R0YjoEV81MOsS6sZpi/+ktAxh8evM9wGUv3G2an/6f2VKXtCJTcPnM\nkdXPXjGpIOvCiou9qp5P8kHgNmADcM2xr4iZWulhJmRq0gGWYWrSAZZpatIBlmFq0gGWaWrSAZZp\natIBmrWaM3aq6lbg1qX3nFrNYSZgatIBlmFq0gGWaWrSAZZhatIBlmlq0gGWaWrSAZrlJ08lqTEW\nuyQ1xmKXpMZY7JLUGItdkhpjsUtSYyx2SWqMxS5JjbHYJakxFrskNcZil6TGWOyS1BiLXZIaY7FL\nUmMsdklqjMUuSY2x2CWpMRa7JDXGYpekxljsktQYi12SGmOxS1JjLHZJaozFLkmNsdglaY0keXGS\ne5LsT/Jwkk+M4zgbx/GikqQXqqofJbmgqn6YZCPwtSRvr6qvjfI4nrFL0hqqqh92iy8CNgDfHfUx\nLHZJWkNJTkqyH5gD7qyqh0d9DItdktZQVf2kqs4BXgP8cpLpUR/DOXZJGpHHgEND7ltVzyb5EvCP\ngNlR5rDYJWlEzuwe8+466vkkrwSer6q/TnIKcBFwxahzWOyStHZOA3YlOYnBVPj1VXXHqA9isUvS\nGqmqB4Dzxn0c3zyVpMZY7JLUmCWLPcm1SeaSPLBg26Yke5IcTHJ7klPHG1OSNKxhztivAy4+attO\nYE9VbQXu6NYlST2wZLFX1VeBZ47avB3Y1S3vAi4ZcS5J0gqtdI59c1XNdctzwOYR5ZEkrdKq3zyt\nqgJqBFkkSSOw0uvY55JsqarDSU4Dnjr2rrNHFvfOwrbpFR5S0onrEFw1M+kQ68ZKi/1mYAfwqe7n\n7mPvOn1k0VKXtCJTcPnMkdXPjvxT+E0Z5nLHG4C7gdcleTzJ+4BPAhclOQi8s1uXJPXAkmfsVXXZ\nMZ66cMRZJEkj4CdPJakxFrskNcZil6TGWOyS1BiLXZIaY7FLUmMsdklqjMUuSY2x2CWpMRa7JDXG\nYpekxljsktQYi12SGmOxS1JjLHZJaozFLkmNsdglqTEWuyQ1xmKXpMZY7JLUmCW/zFqSBL/37eWP\nmXn16HMMwzN2SWqMxS5JjbHYJakxFrskNcZil6TGWOyS1BiLXZIaY7FLUmMsdklqjMUuSY2x2CWp\nMRa7JDXGYpekxljsktSYJYs9yRlJ7kzyUJIHk3yo274pyZ4kB5PcnuTU8ceVJC1lmDP254Dfrqo3\nAG8FLk9yFrAT2FNVW4E7unVJ0oQtWexVdbiq9nfL3wceAU4HtgO7ut12AZeMK6QkaXjLmmNPMgWc\nC9wDbK6que6pOWDzSJNJklZk6K/GS/Iy4IvAh6vqe0l++lxVVZJafOTskcW9s7BteiU5JZ3QDsFV\nM5MOsW4MVexJTmZQ6tdX1e5u81ySLVV1OMlpwFOLj54+smipS1qRKbh85sjqZ6+YVJB1YZirYgJc\nAzxcVX+w4KmbgR3d8g5g99FjJUlrb5gz9rcB/xS4P8m+btvHgU8CX0jyfuAQcOlYEkqSlmXJYq+q\nr3HsM/sLRxtHkrRafvJUkhpjsUtSYyx2SWqMxS5JjbHYJakxFrskNcZil6Q1cqzboI/a0PeKkSSt\n2vxt0Pd399/6epI9VfXIKA/iGbskrZFj3Ab91aM+jsUuSRNw1G3QR8qpGEkakdm7B4+ldNMwNzK4\nDfr3R53DYpekEZk+f/CYd8WVL9xnwW3Q/2TBbdBHyqkYSVojx7kN+khZ7JK0duZvg35Bkn3d4+JR\nH8SpGElaI0vcBn1kPGOXpMZY7JLUGItdkhpjsUtSYyx2SWqMxS5JjbHYJakxFrskNcZil6SeSbIx\nyaMrHW+xS1LPVNXzwIEkr13JeG8pIEn9tAl4KMle4Afdtqqq7UsNtNglqZ/+ffezup9ZsHxcFrsk\n9VBVzSbZAryZQaHvraqnhhnrHLsk9VCSSxl8bd67gUuBvUnePcxYz9glqZ9+F3jz/Fl6klcBdwD/\nfamBnrFLUj8FeHrB+v/tti3JM3ZJ6qcvA7cl+W8MCv09wK3DDLTYJamf/i3wjxl8nR7A56rqpmEG\nWuyS1E8vAXZX1ReTvA74+SQnV9VzSw087hx7khcnuSfJ/iQPJ/lEt31Tkj1JDia5Pcmpo/lzSJI6\nXwV+JsnpwG0MvgT7vw4z8LjFXlU/Ai6oqnOAsxl8s/bbgZ3AnqrayuBd2p0rzy5JWkSq6ocMpmOu\nrqp3A28cZuCSV8V0LwzwImAD8AywHdjVbd8FXLLcxJKk40vyi8BvAl/qNg11JeOSOyU5Kcl+YA64\ns6oeAjZX1Vy3yxywefmRJUnH8a+BjwM3VdVDSf4+cOcwA5d887SqfgKck+TlDC69ueCo5yvJUPcv\nkCQNp6ruAu6CwQk28HRVfWiYsUNfFVNVzyb5EvAmYC7Jlqo6nOQ04Dj3L5g9srh3FrZND3tISeoc\ngqtmJh1iTSW5Afgt4MfAvcDLk3ymqv7TUmOXuirmlfNXvCQ5BbgI2AfcDOzodtsB7D72q0wfeVjq\nklZkCi6fOfI4Mby+qv6GwXuYtwJTwHuHGbjUGftpwK7unwEnAddX1R1J9gFfSPJ+4BCDG9RIkkZn\nY5KTGRT7VVX13LDT3sct9qp6ADhvke3fBS5cSVJJ0lA+x+DE+X7gfyaZAp4dZqA3AZOkHqqq/1JV\np1fVr3QXsfwVcMFS48Bil6ReSrIlyTVJvtxtOosj720ef2zV+K5UTFL15JH1mdNfuM/Mk0etn378\n50e1j1nMYpb1m+UKoKqGuoXtKCSp+vYKxr165Tm7Qr8O+HdVdXY3376vqpb89Kk3AZOkIeSZlZwE\nr+r/Pa+sqj9LshOge/P0+WEGOhUjSf30/SSvmF9J8laGfPPUM3ZJ6qePALcAfy/J3cCrgH8yzECL\nXZJ6qKq+nuQdwOsYzOkcGOZe7OBUjCT1UpIPAi+rqge7zxS9LMm/HGasxS5J/fSBqnpmfqVb/mfD\nDLTYJamfTupu5wJAkg3AycMMdI5dkvrpNuDzST7HYI79t4AvH3/IgMUuSf30MQZTL/+iW98D/PEw\nAy12SeqhqvpxkmuBr3WbDlTVj4cZa7FLUg8lmWbwndJ/1W36u0l2dN+sdFwWuyT105XAu6rqUYAk\nW4HPs8it1I/mVTGS1E8b50sdoKoOMuTJuGfsktRPX0/yx8CfMLgq5jeB+4YZaLFLUj/9c+CDwIe6\n9a8CVw8z0GKXpJ5JshH4ZlX9PPDp5Y53jl2SeqaqngceTfLalYz3jF2S+mkT8FCSvcAPum1VVduX\nGmixS1I//W73c+HXMA31NU4WuyT1SJJTGLxx+g+A+4Frh70P+zzn2CWpX3YBb2JQ6r8K/OflvoBn\n7JLUL2dV1S8AJLkGuHe5L+AZuyT1y/PzC93VMcvmGbsk9cvZSb63YP2UBetVVT+71AtY7JLUI1W1\nYbWv4VSMJDXGYpekxljsktQYi12SGmOxS1JjLHZJWiNJrk0yl+SBcR7HYpektXMdcPG4DzJUsSfZ\nkGRfklu69U1J9iQ5mOT2JKeON6YkrX9V9VXgmXEfZ9gz9g8DD3PklpE7gT1VtRW4o1uXJPXAkp88\nTfIaBncY+4/Av+k2bwfe0S3vAmax3CWd6PbOwr2zk04x1C0Ffh/4KLDw/gSbq2quW54DNo86mCSt\nO9umB495V18xkRjHLfYkvwY8VVX7kkwvtk9VVZJjfqvHzIKvYT0ETK0gpKQT2+zdg2kBDWepM/bz\nge1JfhV4MfCzSa4H5pJsqarDSU4DnjrWC8x8ZMHylasPLOnEM30+TC9Yv2tSQVYpyQ0MprFfkeRx\n4D9U1XWjPs5x3zytqt+pqjOq6kzgN4C/qKr3AjcDO7rddgC7Rx1MklpTVZdV1aur6me6bh15qcPy\nr2Ofn3L5JHBRkoPAO7t1SVIPDH0/9qq6i+5fQFX1XeDCcYWSJK2cnzyVpMZY7JLUGItdkhpjsUtS\nYyx2SWqMxS5JjbHYJakxFrskNcZil6TGWOyS1BiLXZIaY7FLUmMsdklqjMUuSY2x2CWpMRa7JDXG\nYpekxljsktQYi12SGmOxS1JjLHZJaozFLkmNsdglqTEWuyQ1xmKXpMZY7JLUGItdkhpjsUtSYyx2\nSWrMxkkHkKR14Q1XTDrB0Dxjl6TGWOyS1BiLXZIaY7FLUmOGevM0ySHgb4AfA89V1bYkm4A/A14L\nHAIuraq/HlNOSdKQhj1jL2C6qs6tqm3dtp3AnqraCtzRrUuSJmw5UzE5an07sKtb3gVcMpJEkqRV\nWc4Z+1eS3JfkA922zVU11y3PAZtHnk6StGzDfkDpbVX1nSSvAvYkObDwyaqqJDX6eJKk5Rqq2Kvq\nO93Pp5PcBGwD5pJsqarDSU4Dnlps7MynjywfAqZWl1fSCWj2bpiddIh1ZMliT/ISYENVfS/JS4F3\nAVcANwM7gE91P3cvNn7mIwuWr1x9YEknnunzYXrB+l2TCrJODHPGvhm4Kcn8/n9aVbcnuQ/4QpL3\n013uOLaUkqShLVnsVfUYcM4i278LXDiOUJKklfOTp5LUGItdkhpjsUtSYyx2SWqMxS5JjbHYJakx\nFrskNcZil6TGWOyS1BiLXZIaY7FLUmMsdklqjMUuSY2x2CWpMRa7JDXGYpekNZTk4iQHkvxlko+N\n4xgWuyStkSQbgD8ELgZeD1yW5KxRH8dil6S1sw34VlUdqqrngM8Dvz7qg1jskrR2TgceX7D+RLdt\npCx2SVo7tRYHWfLLrCVJw3oMOHS8HZ4EzliwfgaDs/aRstglaWTO7B7z7jp6h/uAn0syBXwbeA9w\n2ahTWOyStEaq6vkkHwRuAzYA11TVI6M+jsUuSWuoqm4Fbh3nMXzzVJIaY7FLUmMsdklqjMUuSY2x\n2CWpMRa7JDXGYpekxljsktQYi12SGmOxS1JjLHZJasxQxZ7k1CQ3JnkkycNJ3pJkU5I9SQ4muT3J\nqeMOK0la2rBn7J8B/ryqzgLOBg4AO4E9VbUVuKNblyRN2JLFnuTlwC9V1bUwuO1kVT0LbAd2dbvt\nAi4ZW0pJ0tCGOWM/E3g6yXVJvpHkj5K8FNhcVXPdPnPA5rGllCQNbZhi3wicB1xdVecBP+CoaZeq\nKtbou/wkScc3zBdtPAE8UVX3dus3Ah8HDifZUlWHk5wGPLXY4JlPH1k+BEytIqykE9Ps3TA76RDr\nyJLF3hX340m2VtVB4ELgoe6xA/hU93P3YuNnPrJg+coRJJZ0wpk+H6YXrL/gm0T1twz71Xj/CvjT\nJC8C/jfwPgbf1/eFJO9ncDJ+6VgSSpKWZahir6pvAm9e5KkLRxtHkrRafvJUkhpjsUtSYyx2SWqM\nxS5JjbHYJakxFrskNcZil6TGrEmxz969FkcZnfWUdz1lhfWVdz1lBfPqiLUp9v+1FkcZnfWUdz1l\nhfWVdz1lBfPqCKdiJKkxFrskNSaDW6mP6cUT79EuaSyqKmt1rNV02VrmnDfWYpckrT2nYiSpMRa7\nJDXGYpekxljsktQYi12SGvP/AdzJez1KiumSAAAAAElFTkSuQmCC\n", + "text": [ + "" ] } ], "prompt_number": 28 }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Redistribution\n", + "--------------\n", + "\n", + "Since `DistArray`s are distributed, the equivalent to NumPy's `reshape` (`distribute_as`) can be a more complex and costly operation. For convenience, you can supply either a `shape` or a full `Distribution` object. Only Block distributions (and No-dist) are currently redistributable." + ] + }, { "cell_type": "code", "collapsed": false, "input": [ - "def get_local_var(darr):\n", - " return darr.ndarray.var()\n", - "\n", - "context.apply(get_local_var, args=(darr.key,))" + "darr" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 29, + "text": [ + "" + ] + } + ], + "prompt_number": 29 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "darr.toarray()" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 30, + "text": [ + "array([[ 0.44, 0.87, 0.87, 0.22, 0.23],\n", + " [ 0.15, 0.05, 0.7 , 0.04, 0.8 ],\n", + " [ 0.68, 0.94, 0.51, 0.01, 0.2 ],\n", + " [ 0.84, 0.81, 0.99, -0.99, 0.02]])" + ] + } + ], + "prompt_number": 30 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# simple reshaping\n", + "reshaped = darr.distribute_as((10, 2))\n", + "reshaped" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 31, + "text": [ + "" + ] + } + ], + "prompt_number": 31 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "reshaped.toarray()" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 32, + "text": [ + "array([[ 0.44, 0.87],\n", + " [ 0.87, 0.22],\n", + " [ 0.23, 0.15],\n", + " [ 0.05, 0.7 ],\n", + " [ 0.04, 0.8 ],\n", + " [ 0.68, 0.94],\n", + " [ 0.51, 0.01],\n", + " [ 0.2 , 0.84],\n", + " [ 0.81, 0.99],\n", + " [-0.99, 0.02]])" + ] + } + ], + "prompt_number": 32 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# A more complex resdistribution,\n", + "# changing shape, dist, and targets\n", + "dist = Distribution(context, shape=(5, 4),\n", + " dist=('b', 'b'), targets=(1, 3))\n", + "darr.distribute_as(dist)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 33, + "text": [ + "" + ] + } + ], + "prompt_number": 33 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Contexts\n", + "--------\n", + "Context objects manage the setup of and communication to the worker processes for DistArray objects. They also act as the namespace to which\n", + "DistArray creation functions are attached." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "print(\"targets:\", context.targets)\n", + "print(\"comm:\", context.comm)" ], "language": "python", "metadata": { @@ -900,20 +1070,57 @@ "slide_type": "subslide" } }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "targets: [0, 1, 2, 3]\n", + "comm: \n" + ] + } + ], + "prompt_number": 34 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "context.zeros((5, 3))" + ], + "language": "python", + "metadata": {}, "outputs": [ { "metadata": {}, "output_type": "pyout", - "prompt_number": 29, + "prompt_number": 35, "text": [ - "[0.10035742766264857,\n", - " 0.059686240460491848,\n", - " 0.048523006620087086,\n", - " 0.036220776799519065]" + "" ] } ], - "prompt_number": 29 + "prompt_number": 35 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "context.ones((20, 20))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 36, + "text": [ + "" + ] + } + ], + "prompt_number": 36 }, { "cell_type": "markdown", @@ -923,16 +1130,21 @@ } }, "source": [ - "Distributed Slicing\n", - "--------------------------" + "Parallel IO\n", + "-----------\n", + "\n", + "DistArray has support for reading NumPy `.npy` files in parallel, for reading *and* writing `.dnpy` files in parallel (our own flat-file format), and reading and writing HDF5 files in parallel (if you have a parallel build of `h5py`)." ] }, { "cell_type": "code", "collapsed": false, "input": [ - "# as a reminder\n", - "darr.toarray()" + "# load .npy files in parallel\n", + "numpy.save(\"/tmp/outfile.npy\", nparr)\n", + "distribution = Distribution(context, nparr.shape) \n", + "new_darr = context.load_npy(\"/tmp/outfile.npy\", distribution)\n", + "new_darr" ], "language": "python", "metadata": { @@ -944,22 +1156,36 @@ { "metadata": {}, "output_type": "pyout", - "prompt_number": 30, + "prompt_number": 37, "text": [ - "array([[ 0.48, 0.75, 0. , 0.71, 0.06],\n", - " [ 0.21, 0.74, 0.79, 0.23, 0.54],\n", - " [ 0.84, 0.33, 0.71, 0.62, 0.98],\n", - " [ 0.46, 0.13, 0.12, 0.61, 0.35]])" + "" ] } ], - "prompt_number": 30 + "prompt_number": 37 }, { "cell_type": "code", "collapsed": false, "input": [ - "darr.localshapes()" + "# save to .dnpy (a built-in flat-file format based on .npy)\n", + "context.save_dnpy(\"/tmp/outfile\", darr)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [], + "prompt_number": 38 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# load from .dnpy\n", + "context.load_dnpy(\"/tmp/outfile\")" ], "language": "python", "metadata": { @@ -971,22 +1197,36 @@ { "metadata": {}, "output_type": "pyout", - "prompt_number": 31, + "prompt_number": 39, "text": [ - "[(1, 5), (1, 5), (1, 5), (1, 5)]" + "" ] } ], - "prompt_number": 31 + "prompt_number": 39 }, { "cell_type": "code", "collapsed": false, "input": [ - "# take a column slice\n", - "darr_view = darr[:, 3]\n", - "print(darr_view)\n", - "print(darr_view.toarray())" + "# save DistArrays to .hdf5 files in parallel\n", + "context.save_hdf5(\"/tmp/outfile.hdf5\", darr, mode='w')" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [], + "prompt_number": 40 + }, + { + "cell_type": "code", + "collapsed": true, + "input": [ + "# load DistArrays from .hdf5 files in parallel (using h5py)\n", + "context.load_hdf5(\"/tmp/outfile.hdf5\", distribution)" ], "language": "python", "metadata": { @@ -996,27 +1236,40 @@ }, "outputs": [ { - "output_type": "stream", - "stream": "stdout", + "metadata": {}, + "output_type": "pyout", + "prompt_number": 41, "text": [ - "\n", - "[ 0.71 0.23 0.62 0.61]\n" + "" ] } ], - "prompt_number": 32 + "prompt_number": 41 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Context.apply\n", + "-------------\n", + "\n", + "Global view, local control. The `apply` method on a `Context` allows you to write functions that are applied *locally* (that is, on the engines) to each section of a DistArray. This allows you to push your computation close to your data, avoiding communication round-trips and possibly speeding up your computations." + ] }, { "cell_type": "code", "collapsed": false, "input": [ - "# changes in the view change the original\n", - "darr_view[3] = -0.99\n", - "print(\"view:\")\n", - "print(darr_view.toarray())\n", + "def get_local_random():\n", + " \"\"\"Function to be applied locally.\"\"\"\n", + " import numpy\n", + " return numpy.random.randint(10)\n", "\n", - "print(\"original:\")\n", - "print(darr.toarray())" + "context.apply(get_local_random)" ], "language": "python", "metadata": { @@ -1026,28 +1279,25 @@ }, "outputs": [ { - "output_type": "stream", - "stream": "stdout", + "metadata": {}, + "output_type": "pyout", + "prompt_number": 42, "text": [ - "view:\n", - "[ 0.71 0.23 0.62 -0.99]\n", - "original:\n", - "[[ 0.48 0.75 0. 0.71 0.06]\n", - " [ 0.21 0.74 0.79 0.23 0.54]\n", - " [ 0.84 0.33 0.71 0.62 0.98]\n", - " [ 0.46 0.13 0.12 -0.99 0.35]]\n" + "[0, 4, 7, 4]" ] } ], - "prompt_number": 33 + "prompt_number": 42 }, { "cell_type": "code", - "collapsed": true, + "collapsed": false, "input": [ - "# a more complex slice\n", - "print(darr[:, 2::2])\n", - "print(darr[:-1, 2::2].toarray())" + "def get_local_var(darr):\n", + " \"\"\"Another local computation.\"\"\"\n", + " return darr.ndarray.var()\n", + "\n", + "context.apply(get_local_var, args=(darr.key,))" ], "language": "python", "metadata": { @@ -1057,24 +1307,27 @@ }, "outputs": [ { - "output_type": "stream", - "stream": "stdout", + "metadata": {}, + "output_type": "pyout", + "prompt_number": 43, "text": [ - "\n", - "[[ 0. 0.06]\n", - " [ 0.79 0.54]\n", - " [ 0.71 0.98]]\n" + "[0.084390695325227721,\n", + " 0.11007623996111866,\n", + " 0.11016943924066562,\n", + " 0.5519645107967377]" ] } ], - "prompt_number": 34 + "prompt_number": 43 }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Registering Functions in a Context\n", - "-----------------------------------------" + "Context.register\n", + "----------------\n", + "\n", + "`Context.register` is similar to `Context.apply`, but it allows you to *register* your function with a `Context` up front, and then call it repeatedly, with a nice syntax." ] }, { @@ -1089,7 +1342,7 @@ "language": "python", "metadata": {}, "outputs": [], - "prompt_number": 35 + "prompt_number": 44 }, { "cell_type": "code", @@ -1103,16 +1356,16 @@ { "metadata": {}, "output_type": "pyout", - "prompt_number": 36, + "prompt_number": 45, "text": [ - "[array([[ 0.08, 0.35, -0.4 , 0.31, -0.34]]),\n", - " array([[-0.29, 0.24, 0.29, -0.28, 0.04]]),\n", - " array([[ 0.15, -0.37, 0.01, -0.07, 0.28]]),\n", - " array([[ 0.44, 0.11, 0.11, -1. , 0.34]])]" + "[array([[-0.09, 0.34, 0.34, -0.3 , -0.3 ]]),\n", + " array([[-0.2 , -0.3 , 0.35, -0.31, 0.45]]),\n", + " array([[ 0.21, 0.47, 0.04, -0.46, -0.27]]),\n", + " array([[ 0.51, 0.47, 0.66, -1.32, -0.31]])]" ] } ], - "prompt_number": 36 + "prompt_number": 45 }, { "cell_type": "markdown", @@ -1126,13 +1379,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Instead of using an IPython client (which uses ZeroMQ to communicate to the engines), you can now run in MPI-only mode (using an extra MPI process for the client). This can be more performant." + "Instead of using an IPython client (which uses ZeroMQ to communicate to the engines), you can run your DistArray code in MPI-only mode (using an extra MPI process for the client). This can be more performant." ] }, { "cell_type": "code", "collapsed": false, "input": [ + "# an example script to run in MPI-only mode\n", "%cd julia_set\n", "!python benchmark_julia.py -h" ], @@ -1150,12 +1404,14 @@ "output_type": "stream", "stream": "stdout", "text": [ - "usage: benchmark_julia.py [-h] [-r REPEAT_COUNT] [-o OUTPUT_FILENAME] [-k {fancy,numpy,cython}] [-s {strong,weak}] N [N ...]\r\n", + "usage: benchmark_julia.py [-h] [-r REPEAT_COUNT] [-o OUTPUT_FILENAME] [-k {fancy,numpy,cython}] [-s {strong,weak}]\r\n", + " N [N ...]\r\n", "\r\n", - "Calculate some Julia sets using DistArray and measure the performance. The Julia set, for a given complex number $c$, is the set of points $z$ such\r\n", - "that $|z_{i}|$ remains bounded where $z_{i+1} = z_{i}^2 + c$. This can be plotted by counting how many iterations are required for $|z_{i}|$ to exceed\r\n", - "a cutoff. Depending on the value of $c$, the Julia set may be connected and contain a lot of points, or it could be disconnected and contain fewer\r\n", - "points. The points in the set will require the maximum iteration count, so the connected sets will usually take longer to compute.\r\n", + "Calculate some Julia sets using DistArray and measure the performance. The Julia set, for a given complex number $c$, is\r\n", + "the set of points $z$ such that $|z_{i}|$ remains bounded where $z_{i+1} = z_{i}^2 + c$. This can be plotted by counting\r\n", + "how many iterations are required for $|z_{i}|$ to exceed a cutoff. Depending on the value of $c$, the Julia set may be\r\n", + "connected and contain a lot of points, or it could be disconnected and contain fewer points. The points in the set will\r\n", + "require the maximum iteration count, so the connected sets will usually take longer to compute.\r\n", "\r\n", "positional arguments:\r\n", " N resolutions of the Julia set to benchmark (NxN)\r\n", @@ -1173,13 +1429,13 @@ ] } ], - "prompt_number": 37 + "prompt_number": 46 }, { "cell_type": "code", "collapsed": false, "input": [ - "# Note that we're using 5 MPI processes:\n", + "# Run the benchmarking script with 5 MPI processes:\n", "# 4 worker processes and 1 client process\n", "!mpiexec -np 5 python benchmark_julia.py --kernel=cython -r1 1024" ], @@ -1197,123 +1453,123 @@ "output_type": "stream", "stream": "stdout", "text": [ - "(1/17: 0.181s) (1408556620.436587, 1408556620.617281, 'numpy', 1024, '(-0.045+0.45j)', 1, [32763832L])\r\n" + "(1/17: 0.177s) (1410237104.372746, 1410237104.549673, 'numpy', 1024, '(-0.045+0.45j)', 1, [32763832L])\r\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ - "(2/17: 0.183s) (1408556620.627409, 1408556620.810083, 'b-n', 1024, '(-0.045+0.45j)', 1, [32763832L])\r\n" + "(2/17: 0.178s) (1410237104.55856, 1410237104.736667, 'b-n', 1024, '(-0.045+0.45j)', 1, [32763832L])\r\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ - "(3/17: 0.180s) (1408556620.818441, 1408556620.998368, 'c-n', 1024, '(-0.045+0.45j)', 1, [32763832L])\r\n" + "(3/17: 0.175s) (1410237104.743266, 1410237104.918744, 'c-n', 1024, '(-0.045+0.45j)', 1, [32763832L])\r\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ - "(4/17: 0.178s) (1408556621.006173, 1408556621.184294, 'b-b', 1024, '(-0.045+0.45j)', 1, [32763832L])\r\n" + "(4/17: 0.176s) (1410237104.926051, 1410237105.101706, 'b-b', 1024, '(-0.045+0.45j)', 1, [32763832L])\r\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ - "(5/17: 0.175s) (1408556621.195951, 1408556621.370637, 'c-c', 1024, '(-0.045+0.45j)', 1, [32763832L])\r\n" + "(5/17: 0.176s) (1410237105.109474, 1410237105.285465, 'c-c', 1024, '(-0.045+0.45j)', 1, [32763832L])\r\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ - "(6/17: 0.090s) (1408556621.379416, 1408556621.469484, 'b-n', 1024, '(-0.045+0.45j)', 2, [16345977L, 16417855L])\r\n" + "(6/17: 0.093s) (1410237105.29489, 1410237105.38807, 'b-n', 1024, '(-0.045+0.45j)', 2, [16345977L, 16417855L])\r\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ - "(7/17: 0.091s) (1408556621.476928, 1408556621.567663, 'c-n', 1024, '(-0.045+0.45j)', 2, [16382826L, 16381006L])\r\n" + "(7/17: 0.089s) (1410237105.395181, 1410237105.483749, 'c-n', 1024, '(-0.045+0.45j)', 2, [16382826L, 16381006L])\r\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ - "(8/17: 0.091s) (1408556621.573445, 1408556621.663993, 'b-b', 1024, '(-0.045+0.45j)', 2, [16354361L, 16409471L])\r\n" + "(8/17: 0.089s) (1410237105.489446, 1410237105.57825, 'b-b', 1024, '(-0.045+0.45j)', 2, [16354361L, 16409471L])\r\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ - "(9/17: 0.094s) (1408556621.66925, 1408556621.763122, 'c-c', 1024, '(-0.045+0.45j)', 2, [16384932L, 16378900L])\r\n" + "(9/17: 0.091s) (1410237105.584189, 1410237105.675209, 'c-c', 1024, '(-0.045+0.45j)', 2, [16384932L, 16378900L])\r\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ - "(10/17: 0.114s) (1408556621.772253, 1408556621.886436, 'b-n', 1024, '(-0.045+0.45j)', 3, [6243924L, 20326746L, 6193162L])\r\n" + "(10/17: 0.111s) (1410237105.683329, 1410237105.794306, 'b-n', 1024, '(-0.045+0.45j)', 3, [6243924L, 20326746L, 6193162L])\r\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ - "(11/17: 0.066s) (1408556621.896535, 1408556621.962862, 'c-n', 1024, '(-0.045+0.45j)', 3, [10922548L, 10921645L, 10919639L])\r\n" + "(11/17: 0.062s) (1410237105.800505, 1410237105.862825, 'c-n', 1024, '(-0.045+0.45j)', 3, [10922548L, 10921645L, 10919639L])\r\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ - "(12/17: 0.103s) (1408556621.969013, 1408556622.071996, 'b-b', 1024, '(-0.045+0.45j)', 3, [6779920L, 19264315L, 6719597L])\r\n" + "(12/17: 0.106s) (1410237105.868804, 1410237105.974531, 'b-b', 1024, '(-0.045+0.45j)', 3, [6779920L, 19264315L, 6719597L])\r\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ - "(13/17: 0.063s) (1408556622.076449, 1408556622.139083, 'c-c', 1024, '(-0.045+0.45j)', 3, [10921355L, 10920452L, 10922025L])\r\n" + "(13/17: 0.061s) (1410237105.979521, 1410237106.040812, 'c-c', 1024, '(-0.045+0.45j)', 3, [10921355L, 10920452L, 10922025L])\r\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ - "(14/17: 0.074s) (1408556622.145831, 1408556622.219845, 'b-n', 1024, '(-0.045+0.45j)', 4, [2725843L, 13620134L, 13659620L, 2758235L])\r\n" + "(14/17: 0.077s) (1410237106.04839, 1410237106.125411, 'b-n', 1024, '(-0.045+0.45j)', 4, [2725843L, 13620134L, 13659620L, 2758235L])\r\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ - "(15/17: 0.048s) (1408556622.226353, 1408556622.274515, 'c-n', 1024, '(-0.045+0.45j)', 4, [8190582L, 8190503L, 8192244L, 8190503L])\r\n" + "(15/17: 0.049s) (1410237106.131263, 1410237106.179974, 'c-n', 1024, '(-0.045+0.45j)', 4, [8190582L, 8190503L, 8192244L, 8190503L])\r\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ - "(16/17: 0.058s) (1408556622.280794, 1408556622.339083, 'b-b', 1024, '(-0.045+0.45j)', 4, [5859333L, 10486644L, 10495028L, 5922827L])\r\n" + "(16/17: 0.059s) (1410237106.186044, 1410237106.244741, 'b-b', 1024, '(-0.045+0.45j)', 4, [5859333L, 10486644L, 10495028L, 5922827L])\r\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ - "(17/17: 0.047s) (1408556622.343801, 1408556622.390988, 'c-c', 1024, '(-0.045+0.45j)', 4, [8193330L, 8189496L, 8191602L, 8189404L])\r\n" + "(17/17: 0.049s) (1410237106.249857, 1410237106.29855, 'c-c', 1024, '(-0.045+0.45j)', 4, [8193330L, 8189496L, 8191602L, 8189404L])\r\n" ] } ], - "prompt_number": 38 + "prompt_number": 47 }, { "cell_type": "markdown", @@ -1324,7 +1580,10 @@ }, "source": [ "Distributed Array Protocol\n", - "---------------------------------\n", + "--------------------------\n", + "\n", + "Already have a library with its own distributed arrays? Use the Distributed Array Protocol to work with DistArray.\n", + "\n", "The Distributed Array Protocol (DAP) is a process-local protocol that allows two subscribers, called the \"producer\" and the \"consumer\" or the \"exporter\" and the \"importer\", to communicate the essential data and metadata necessary to share a distributed-memory array between them. This allows two independently developed components to access, modify, and update a distributed array without copying. The protocol formalizes the metadata and buffers involved in the transfer, allowing several distributed array projects to collaborate, facilitating interoperability. By not copying the underlying array data, the protocol allows for efficient sharing of array data.\n", "\n", "http://distributed-array-protocol.readthedocs.org/en/rel-0.9.0/" @@ -1349,10 +1608,10 @@ { "metadata": {}, "output_type": "pyout", - "prompt_number": 39, + "prompt_number": 48, "text": [ "[{'__version__': '0.10.0',\n", - " 'buffer': array([[ 0.48, 0.75, 0. , 0.71, 0.06]]),\n", + " 'buffer': array([[ 0.44, 0.87, 0.87, 0.22, 0.23]]),\n", " 'dim_data': ({'dist_type': 'b',\n", " 'proc_grid_rank': 0,\n", " 'proc_grid_size': 4,\n", @@ -1366,7 +1625,7 @@ " 'start': 0,\n", " 'stop': 5})},\n", " {'__version__': '0.10.0',\n", - " 'buffer': array([[ 0.21, 0.74, 0.79, 0.23, 0.54]]),\n", + " 'buffer': array([[ 0.15, 0.05, 0.7 , 0.04, 0.8 ]]),\n", " 'dim_data': ({'dist_type': 'b',\n", " 'proc_grid_rank': 1,\n", " 'proc_grid_size': 4,\n", @@ -1380,7 +1639,7 @@ " 'start': 0,\n", " 'stop': 5})},\n", " {'__version__': '0.10.0',\n", - " 'buffer': array([[ 0.84, 0.33, 0.71, 0.62, 0.98]]),\n", + " 'buffer': array([[ 0.68, 0.94, 0.51, 0.01, 0.2 ]]),\n", " 'dim_data': ({'dist_type': 'b',\n", " 'proc_grid_rank': 2,\n", " 'proc_grid_size': 4,\n", @@ -1394,7 +1653,7 @@ " 'start': 0,\n", " 'stop': 5})},\n", " {'__version__': '0.10.0',\n", - " 'buffer': array([[ 0.46, 0.13, 0.12, -0.99, 0.35]]),\n", + " 'buffer': array([[ 0.84, 0.81, 0.99, -0.99, 0.02]]),\n", " 'dim_data': ({'dist_type': 'b',\n", " 'proc_grid_rank': 3,\n", " 'proc_grid_size': 4,\n", @@ -1410,7 +1669,7 @@ ] } ], - "prompt_number": 39 + "prompt_number": 48 }, { "cell_type": "markdown",