diff --git a/examples/features.ipynb b/examples/features.ipynb new file mode 100644 index 00000000..f811a045 --- /dev/null +++ b/examples/features.ipynb @@ -0,0 +1,1165 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:f5bdf8fd3c36062461714583e9943cde0b315a8e25cbe8b7f4417dcca161fe9a" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "DistArray: Distributed Arrays for Python\n", + "========================================\n", + "\n", + "[github.com/enthought/distarray](https://github.com/enthought/distarray)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "source": [ + "Start a cluster\n", + "---------------\n", + "These examples require an `IPython.parallel` cluster to be running.\n", + "Outside the notebook, run\n", + "```\n", + "dacluster start -n4\n", + "```" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# some utility imports\n", + "from __future__ import print_function\n", + "from pprint import pprint\n", + "from matplotlib import pyplot as plt\n", + "\n", + "# main imports\n", + "import numpy\n", + "import distarray\n", + "\n", + "numpy.set_printoptions(precision=2) # display formatting" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [], + "prompt_number": 1 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "NumPy Arrays\n", + "------------" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "nparr = numpy.random.random((4, 5))\n", + "nparr" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 2, + "text": [ + "array([[ 0.77, 0.06, 0.37, 0.19, 0.31],\n", + " [ 0.58, 0.67, 0.41, 0.02, 0.41],\n", + " [ 0.1 , 0.56, 0.08, 0.56, 0.72],\n", + " [ 0.91, 0.13, 0.48, 0.14, 0.13]])" + ] + } + ], + "prompt_number": 2 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# NumPy array attributes\n", + "print(\"type:\", type(nparr))\n", + "print(\"dtype:\", nparr.dtype)\n", + "print(\"ndim:\", nparr.ndim)\n", + "print(\"shape:\", nparr.shape)\n", + "print(\"itemsize:\", nparr.itemsize)\n", + "print(\"nbytes:\", nparr.nbytes)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "type: \n", + "dtype: float64\n", + "ndim: 2\n", + "shape: (4, 5)\n", + "itemsize: 8\n", + "nbytes: 160\n" + ] + } + ], + "prompt_number": 3 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "DistArrays\n", + "----------" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "from distarray.dist import Context\n", + "context = Context()\n", + "darr = context.fromarray(nparr)\n", + "darr" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 4, + "text": [ + "" + ] + } + ], + "prompt_number": 4 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# parts of the array are stored on each engine\n", + "for i, a in enumerate(darr.get_localarrays()):\n", + " print(i, a)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "0 [[ 0.77 0.06 0.37 0.19 0.31]]\n", + "1 [[ 0.58 0.67 0.41 0.02 0.41]]\n", + "2 [[ 0.1 0.56 0.08 0.56 0.72]]\n", + "3 [[ 0.91 0.13 0.48 0.14 0.13]]\n" + ] + } + ], + "prompt_number": 5 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# DistArray attributes\n", + "print(\"type:\", type(darr))\n", + "print(\"dtype:\", darr.dtype)\n", + "print(\"ndim:\", darr.ndim)\n", + "print(\"shape:\", darr.shape)\n", + "print(\"itemsize:\", darr.itemsize)\n", + "print(\"nbytes:\", darr.nbytes)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "type: \n", + "dtype: float64\n", + "ndim: 2\n", + "shape: (4, 5)\n", + "itemsize: 8\n", + "nbytes: 160\n" + ] + } + ], + "prompt_number": 6 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# with some extra...\n", + "print(\"targets:\", darr.targets)\n", + "print(\"context:\", darr.context)\n", + "print(\"distribution:\", darr.distribution)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "targets: [0, 1, 2, 3]\n", + "context: \n", + "distribution: \n" + ] + } + ], + "prompt_number": 7 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Universal Functions (ufuncs)\n", + "----------------------------" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "## NumPy ##\n", + "numpy.sin(nparr)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 8, + "text": [ + "array([[ 0.7 , 0.06, 0.36, 0.19, 0.31],\n", + " [ 0.55, 0.62, 0.4 , 0.02, 0.4 ],\n", + " [ 0.1 , 0.53, 0.08, 0.53, 0.66],\n", + " [ 0.79, 0.13, 0.46, 0.14, 0.13]])" + ] + } + ], + "prompt_number": 8 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "## DistArray ##\n", + "import distarray.dist as da\n", + "da.sin(darr)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 9, + "text": [ + "" + ] + } + ], + "prompt_number": 9 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "da.sin(darr).toarray()" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 10, + "text": [ + "array([[ 0.7 , 0.06, 0.36, 0.19, 0.31],\n", + " [ 0.55, 0.62, 0.4 , 0.02, 0.4 ],\n", + " [ 0.1 , 0.53, 0.08, 0.53, 0.66],\n", + " [ 0.79, 0.13, 0.46, 0.14, 0.13]])" + ] + } + ], + "prompt_number": 10 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "## NumPy ##\n", + "nparr + nparr" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 11, + "text": [ + "array([[ 1.54, 0.12, 0.74, 0.37, 0.62],\n", + " [ 1.16, 1.34, 0.81, 0.03, 0.83],\n", + " [ 0.19, 1.11, 0.16, 1.13, 1.44],\n", + " [ 1.82, 0.26, 0.96, 0.28, 0.25]])" + ] + } + ], + "prompt_number": 11 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "## DistArray ##\n", + "darr + darr" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 12, + "text": [ + "" + ] + } + ], + "prompt_number": 12 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "(darr + darr).toarray()" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 13, + "text": [ + "array([[ 1.54, 0.12, 0.74, 0.37, 0.62],\n", + " [ 1.16, 1.34, 0.81, 0.03, 0.83],\n", + " [ 0.19, 1.11, 0.16, 1.13, 1.44],\n", + " [ 1.82, 0.26, 0.96, 0.28, 0.25]])" + ] + } + ], + "prompt_number": 13 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Distributions\n", + "-------------" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# Distributions control which processes own which (global) indices\n", + "distribution = darr.distribution" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [], + "prompt_number": 14 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# this is a 2D distribution\n", + "pprint(distribution.maps)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "[,\n", + " ]\n" + ] + } + ], + "prompt_number": 15 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# setup\n", + "from distarray.plotting import plot_array_distribution\n", + "process_coords = [(0, 0), (1, 0), (2, 0), (3, 0)]" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [], + "prompt_number": 16 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "plot_array_distribution(darr, process_coords, cell_label=False, legend=True)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 17, + "text": [ + "" + ] + }, + { + "metadata": {}, + "output_type": "display_data", + "png": "iVBORw0KGgoAAAANSUhEUgAAAbUAAAEzCAYAAAC2Q50YAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAD69JREFUeJzt3X2MZXV9x/H3Z3eh0hLTWkwU2Dq2FV1AolYJirG7TWw2\nptGmqdKNtMY/+iiBmqbSB5vCX/7TB/6gJiYuSDUFW4ymSFbS4I7F0Cxi2JZ9wGLiNDxEbCrSIjHZ\nlW//mDs6PLj37s6dOYfvvF/Jzd57Z37nfDMh8+acc+feVBWSJHWwZegBJEmaF6MmSWrDqEmS2jBq\nkqQ2jJokqQ2jJklqo03UkuxO8kCSB5NcPfQ8Y5DkhiSPJbl/6FnGIsn2JPuTHE5yKMmVQ880Bkle\nlORAkoNJjiT5yNAzjUWSrUnuS3Lb0LNouhZRS7IVuB7YDZwP7EmyY9ipRuFGln8m+qFjwAer6gLg\nEuAD/rcCVfU9YFdVvQ64CNiV5K0DjzUWVwFHAP+o9wWgRdSAi4GvV9VSVR0DbgHeNfBMg6uqu4DH\nh55jTKrqm1V1cHL/SeAocPawU41DVT01uXs6sBX49oDjjEKSc4F3AB8HMvA4mkGXqJ0DPLTq8cOT\n56QfKckC8HrgwLCTjEOSLUkOAo8B+6vqyNAzjcDfAn8MPD30IJpNl6h5WkAnJcmZwK3AVZMjtk2v\nqp6enH48F3hbkp0DjzSoJL8CfKuq7sOjtBeMLlF7BNi+6vF2lo/WpOdIchrwGeBTVfW5oecZm6p6\nArgdeOPQswzsLcA7k3wDuBn4pSR/P/BMmqJL1O4FXpVkIcnpwGXAPw88k0YoSYC9wJGqum7oecYi\nyVlJfnJy/wzg7cB9w041rKr6s6raXlWvBH4D+GJV/dbQc+nEWkStqo4DVwB3sPwqpU9X1dFhpxpe\nkpuBu4HzkjyU5P1DzzQClwKXs/zqvvsmN18hCi8Hvji5pnYAuK2q7hx4prHxMscLQPzoGUlSFy2O\n1CRJAqMmSWrEqEmS2jBqkqQ2jJokqY1ta91AEl8+KUkbqKo27B1O1vI7fiPnXLHmqC37y/lsZs0W\ngZ0DzzBGi/hzeT6L+HN5tkX8mTyfRcbzc7l2w/d4zQatmQdPP0qS2jBqkqQ2mkVtYegBRmph6AFG\namHoAUZoYegBRmph6AE0I6O2KSwMPcBILQw9wAgtDD3ASC0MPYBm1CxqkqTNzKhJktowapKkNoya\nJKkNoyZJasOoSZLaMGqSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktowapKkNoya\nJKkNoyZJasOoSZLaMGqSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqY2pUUuyO8kDSR5McvVGDCVJ\n6iXJi5IcSHIwyZEkH1mP/Zwwakm2AtcDu4HzgT1JdqzHIJKkvqrqe8CuqnodcBGwK8lb572faUdq\nFwNfr6qlqjoG3AK8a95DSJL6q6qnJndPB7YC3573PqZF7RzgoVWPH548J0nSSUmyJclB4DFgf1Ud\nmfc+pkWt5r1DSdLmVFVPT04/ngu8LcnOee9j25SvPwJsX/V4O8tHa8+yuOr+wuQmSVq7pclt3L7B\n7FNW1RNJbgfeyDMDsmbTonYv8KokC8CjwGXAnud+2855ziRJ+oEFnnmg8KVhxpjilZPbimdPmeQs\n4HhVfSfJGcDbgWvnPccJo1ZVx5NcAdzB8kW9vVV1dN5DSJLaezlwU5ItLF/6+mRV3TnvnUw7UqOq\n9gH75r1jSdLmUVX3A29Y7/34jiKSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktow\napKkNoyaJKkNoyZJasOoSZLaMGqSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktow\napKkNoyaJKkNoyZJasOoSZLaMGqSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqY1tc9nKoWvmshlJ\n0hQXXjv0BKOWqlrbBpLi0Nq2IUma0YWhqrJRu0tS9egprDubDZ1zhacfJUltGDVJUhtGTZLUhlGT\nJLVh1CRJbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGT\nJLVh1CRJbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGT\nJLVh1CRJbUyNWpIbkjyW5P6NGEiS1E+S7Un2Jzmc5FCSK9djP7Mcqd0I7F6PnUuSNo1jwAer6gLg\nEuADSXbMeydTo1ZVdwGPz3vHkqTNo6q+WVUHJ/efBI4CZ897P15TkyRtqCQLwOuBA/Pe9ra5bOXv\nrvnh/TfthIt3zmWzkrTp3bMIX1kceoqpFu9evk2T5EzgVuCqyRHbXKWqZhliAbitql77PF8rDk3f\nhiRpDi4MVZWN2l2SqkdPYd3ZPGfOJKcBnwf2VdV185nwmTz9KElad0kC7AWOrFfQYLaX9N8M3A2c\nl+ShJO9fr2EkSW1dClwO7Epy3+Q291fWT72mVlV75r1TSdLmUlVfZgPODnr6UZLUhlGTJLVh1CRJ\nbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNkjQaSbYl+dqprjdqkqTRqKrjwANJXnEq6+fz0TOS\nJM3PS4DDSe4Bvjt5rqrqndMWGjVJ0tj8xeTflc81y6r7J2TUJEmjUlWLSV4GvInlmN1TVd+aZa3X\n1CRJo5LkPcAB4N3Ae4B7krx7lrUeqUmSxubDwJtWjs6SvBS4E/inaQs9UpMkjU2A/171+H8mz03l\nkZokaWy+ANyR5B9YjtllwL5ZFho1SdLYfAj4NeDSyeOPVdVnZ1lo1CRJY/PjwOeq6jNJXg28Jslp\nVXVs2kKvqUmSxuYu4MeSnAPcAVwOfGKWhUZNkjQ2qaqnWD4F+dGqejdw4SwLjZokaXSSvBl4L3D7\n5KmZemXUJElj84fAnwKfrarDSX4O2D/LwlTN9HZaP3oDSXFobduQJM3owlBVM/3N1jwkqXr0FNad\nzVzmTLIFOLOq/neW7/dITZI0KkluTvLiJD8BHAKOJvnQLGuNmiRpbM6fHJn9Kst/dL0A/OYsC42a\nJGlstiU5jeWo3Tb5+7SZrnMZNUnS2HwMWALOBP41yQLwxCwLfaGIJL2QbLIXikxmCLC1qo5P+16P\n1CRJo5LkZUn2JvnC5KkdwPtmWTuX936sn9qw/2mQpE1tk/y2/QRwI/Dnk8cPAv8I7J220Dc0liSd\nUB4/lUtMa8rvWVX16SR/AlBVx5JMPfUInn6UJI3Pk0l+euVBkkuY8YUiHqlJksbmj4DbgJ9Ncjfw\nUuDXZ1lo1CRJo1JVX03yi8CrWT6P+cAsn6UGnn6UJI1MkitYfr/HQ1V1P3Bmkj+YZa1RkySNzW9X\n1eMrDyb3f2eWhUZNkjQ2Wybvzg9Akq3AabMs9JqaJGls7gBuSfIxlq+p/S7whRMvWWbUJEljczXL\npxt/f/L4X4CPz7LQqEmSRqWqvp/kBuDLk6ceqKrvz7LWqEmSRiXJTuAm4L8mT/1MkvdV1ZemrTVq\nkqSx+Rvgl6vqawBJzgNuAd4wbaGvfpQkjc22laABVNV/MuNBmEdqkqSx+WqSjwOfYvnVj+8F7p1l\noVGTJI3N7wFXAFdOHt8FfHSWhUZNkjQaSbYB/15VrwH++mTXe01NkjQaVXUc+FqSV5zKeo/UJElj\n8xLgcJJ7gO9Onquqeue0hUZNkjQ2H578u/rjs2f6+G2jJkkahSRnsPwikZ8H/gO4YdbPUVvhNTVJ\n0ljcBPwCy0F7B/BXJ7sBj9QkSWOxo6peC5BkL/CVk92AR2qSpLE4vnJn8irIk+aRmiRpLC5K8n+r\nHp+x6nFV1YunbcCoSZJGoaq2rnUbnn6UJLVh1CRJbRg1SVIbRk2S1IZRkyS1MTVqSbYn2Z/kcJJD\nSa6ctkaSpNWS3JDksST3r+d+ZjlSOwZ8sKouAC4BPpBkx3oOJUlq50Zg93rvZGrUquqbVXVwcv9J\n4Chw9noPJknqo6ruAh5f7/2c1DW1JAvA64ED6zGMJElrMfM7iiQ5E7gVuGpyxPYD16z6wO2db4ad\nb5nXeJK0uS3eDYv/NvQUM7hnEb6yOPQUpGr6564lOQ34PLCvqq571teqHlmn6SRJz5BzoKoy/Tvn\ntL+kODzT53M+0wV5zpyTs323rbwT/3qY5dWPAfYCR54dNEmSxmSWa2qXApcDu5LcN7mt+ytYJEl9\nJLkZuBs4L8lDSd6/HvuZek2tqr6Mf6QtSVqDqtqzEfsxVpKkNoyaJKkNoyZJasOoSZLaMGqSpDaM\nmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktowapKkNoyaJKkNoyZJasOoSZLaMGqSpDaM\nmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktowapKkNoyaJKkNoyZJasOoSZLaMGqSpDaM\nmiSpjW3z2Mg158xjK5Ikrc1coiZJauyCa4eeYGaefpQktWHUJEltGDVJUhtGTZLUhlGTJLVh1CRJ\nbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGTJLVh1CRJ\nbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUxgmjluRFSQ4k\nOZjkSJKPbNRgkqRekuxO8kCSB5NcvR77OGHUqup7wK6qeh1wEbAryVvXYxBJUl9JtgLXA7uB84E9\nSXbMez9TTz9W1VOTu6cDW4Fvz3sISVJ7FwNfr6qlqjoG3AK8a947mRq1JFuSHAQeA/ZX1ZF5DyFJ\nau8c4KFVjx+ePDdXsxypPT05/Xgu8LYkO+c9hCSpvdqInWyb9Rur6okktwNvBBZXf231g4XJTZK0\ndkuT2/h9gymTPgJsX/V4O8tHa3N1wqglOQs4XlXfSXIG8Hbg2md/3855TyVJAp57oPClYcaYwSsn\ntxXPmfRe4FVJFoBHgcuAPfOeYtqR2suBm5JsYflU5Ser6s55DyFJ6q2qjie5AriD5Rcd7q2qo/Pe\nzwmjVlX3A2+Y904lSZtPVe0D9q3nPnxHEUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGTJLVh\n1CRJbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGTJLVh\n1CRJbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktdEqaktDDzBSS0MPMFJLQw8wQktD\nDzBSS0MPoJkZtU1gaegBRmpp6AFGaGnoAUZqaegBNLNWUZMkbW5GTZLURqpqbRtI1rYBSdJJqaps\n1L7W8jt+I+dcseaoSZI0Fp5+lCS1YdQkSW0YNUlSG0ZNktSGUZMktfH/OydIpv9Xa0UAAAAASUVO\nRK5CYII=\n", + "text": [ + "" + ] + } + ], + "prompt_number": 17 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "distribution.maps[0].bounds" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 18, + "text": [ + "[(0, 1), (1, 2), (2, 3), (3, 4)]" + ] + } + ], + "prompt_number": 18 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# the above is the default, you can make more complex distributions\n", + "from distarray.dist import Distribution\n", + "distribution = Distribution.from_shape(context, (64, 64), dist=('b', 'c'))\n", + "a = context.zeros(distribution, dtype='int32')\n", + "plot_array_distribution(a, process_coords, cell_label=False, legend=True)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 19, + "text": [ + "" + ] + }, + { + "metadata": {}, + "output_type": "display_data", + "png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEzCAYAAAAlyyi6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFLdJREFUeJzt3X+MZXd93vH3410TDDRYC2jXGJdx2ywxENc2ZSGGhDGy\nkRNFG6sqJlaKVgjRtDWFpoiypGkz/qMCquKEKjZCie2unNSEmnplixh7cTwuyJXXhl38c72h9Ua2\nYcdWcRx+CNWGT/+4Z9jJenbnzsy9c8989/2Sruacc8/3nmet2cdnv/fcc1NVSJLacdKkA0iSRsti\nl6TGWOyS1BiLXZIaY7FLUmMsdklqzFiLPcnFSQ4k+cskHxvnsVYiybVJ5pI8sGDbpiR7khxMcnuS\nUyeZcaEkZyS5M8lDSR5M8qFue+8yJ3lxknuS7E/ycJJP9DXrQkk2JNmX5JZuvZd5kxxKcn+XdW+3\nrZdZAZKcmuTGJI90vw9v6XPe9W5sxZ5kA/CHwMXA64HLkpw1ruOt0HUM8i20E9hTVVuBO7r1vngO\n+O2qegPwVuDy7r9p7zJX1Y+AC6rqHOBs4IIkb6eHWY/yYeBhYP4DHn3NW8B0VZ1bVdu6bX3NCvAZ\n4M+r6iwGvw8H6Hfe9a2qxvIAfhH48oL1ncDOcR1vFTmngAcWrB8ANnfLW4ADk854nOy7gQv7nhl4\nCXAv8IY+ZwVeA3wFuAC4pc+/D8BjwCuO2tbXrC8H/s8i23uZt4XHOKdiTgceX7D+RLet7zZX1Vy3\nPAdsnmSYY0kyBZwL3ENPMyc5Kcl+BpnurKqH6GnWzu8DHwV+smBbX/MW8JUk9yX5QLetr1nPBJ5O\ncl2SbyT5oyQvpb95171xFvu6v1dBDU4levfnSPIy4IvAh6vqewuf61PmqvpJDaZiXgP8cpILjnq+\nN1mT/BrwVFXtA7LYPn3KC7ytqs4FfoXBlNwvLXyyZ1k3AucBV1fVecAPOGrapWd5171xFvuTwBkL\n1s9gcNbed3NJtgAkOQ14asJ5/pYkJzMo9eurane3udeZq+pZ4EvAm+hv1vOB7UkeA24A3pnkenqa\nt6q+0/18GrgJ2EZPszL4e/9EVd3brd/IoOgP9zTvujfOYr8P+LkkU0leBLwHuHmMxxuVm4Ed3fIO\nBvPYvZAkwDXAw1X1Bwue6l3mJK+cv8ohySnARcA+epgVoKp+p6rOqKozgd8A/qKq3ksP8yZ5SZK/\n0y2/FHgX8AA9zApQVYeBx5Ns7TZdCDwE3EIP8zZhnBP4DP6Z+CjwLeDjk35DYZF8NwDfBv4fg/cD\n3gdsYvAG2kHgduDUSedckPftDOZ/9zMoyX0MrurpXWbgF4BvdFnvBz7abe9d1kWyvwO4ua95GcxZ\n7+8eD87/3epj1gWZ/yGDN9C/CfwPBm+o9jbven+k+48uSWqEnzyVpMZY7JLUmFUVe99vGSBJJ6IV\nz7F3twx4lME73E8yeGPksqp6ZHTxJEnLtZoz9m3At6rqUFU9B3we+PXRxJIkrdTGVYxd7JYBb1m4\nQxIvuZE0FlW16CeEx2E1XbaWOeetptiH/IP+HjALTMODMy98+o1HbTt6n6OfH9U+x80yy6J5J5Jl\nqdeYBaZ7kmWYfWYZOu/Ef19mOWbWNc9ynOd/us8sI/1dWFWWYV5jlhXnfeOadyUzS+4xmjGjsJpi\nH/KWAbPAocHPvbOwbXoVh5R0YjoEV81MOsS6sZpi/+ktAxh8evM9wGUv3G2an/6f2VKXtCJTcPnM\nkdXPXjGpIOvCiou9qp5P8kHgNmADcM2xr4iZWulhJmRq0gGWYWrSAZZpatIBlmFq0gGWaWrSAZZp\natIBmrWaM3aq6lbg1qX3nFrNYSZgatIBlmFq0gGWaWrSAZZhatIBlmlq0gGWaWrSAZrlJ08lqTEW\nuyQ1xmKXpMZY7JLUGItdkhpjsUtSYyx2SWqMxS5JjbHYJakxFrskNcZil6TGWOyS1BiLXZIaY7FL\nUmMsdklqjMUuSY2x2CWpMRa7JDXGYpekxljsktQYi12SGmOxS1JjLHZJaozFLkmNsdglaY0keXGS\ne5LsT/Jwkk+M4zgbx/GikqQXqqofJbmgqn6YZCPwtSRvr6qvjfI4nrFL0hqqqh92iy8CNgDfHfUx\nLHZJWkNJTkqyH5gD7qyqh0d9DItdktZQVf2kqs4BXgP8cpLpUR/DOXZJGpHHgEND7ltVzyb5EvCP\ngNlR5rDYJWlEzuwe8+466vkkrwSer6q/TnIKcBFwxahzWOyStHZOA3YlOYnBVPj1VXXHqA9isUvS\nGqmqB4Dzxn0c3zyVpMZY7JLUmCWLPcm1SeaSPLBg26Yke5IcTHJ7klPHG1OSNKxhztivAy4+attO\nYE9VbQXu6NYlST2wZLFX1VeBZ47avB3Y1S3vAi4ZcS5J0gqtdI59c1XNdctzwOYR5ZEkrdKq3zyt\nqgJqBFkkSSOw0uvY55JsqarDSU4Dnjr2rrNHFvfOwrbpFR5S0onrEFw1M+kQ68ZKi/1mYAfwqe7n\n7mPvOn1k0VKXtCJTcPnMkdXPjvxT+E0Z5nLHG4C7gdcleTzJ+4BPAhclOQi8s1uXJPXAkmfsVXXZ\nMZ66cMRZJEkj4CdPJakxFrskNcZil6TGWOyS1BiLXZIaY7FLUmMsdklqjMUuSY2x2CWpMRa7JDXG\nYpekxljsktQYi12SGmOxS1JjLHZJaozFLkmNsdglqTEWuyQ1xmKXpMZY7JLUmCW/zFqSBL/37eWP\nmXn16HMMwzN2SWqMxS5JjbHYJakxFrskNcZil6TGWOyS1BiLXZIaY7FLUmMsdklqjMUuSY2x2CWp\nMRa7JDXGYpekxljsktSYJYs9yRlJ7kzyUJIHk3yo274pyZ4kB5PcnuTU8ceVJC1lmDP254Dfrqo3\nAG8FLk9yFrAT2FNVW4E7unVJ0oQtWexVdbiq9nfL3wceAU4HtgO7ut12AZeMK6QkaXjLmmNPMgWc\nC9wDbK6que6pOWDzSJNJklZk6K/GS/Iy4IvAh6vqe0l++lxVVZJafOTskcW9s7BteiU5JZ3QDsFV\nM5MOsW4MVexJTmZQ6tdX1e5u81ySLVV1OMlpwFOLj54+smipS1qRKbh85sjqZ6+YVJB1YZirYgJc\nAzxcVX+w4KmbgR3d8g5g99FjJUlrb5gz9rcB/xS4P8m+btvHgU8CX0jyfuAQcOlYEkqSlmXJYq+q\nr3HsM/sLRxtHkrRafvJUkhpjsUtSYyx2SWqMxS5JjbHYJakxFrskNcZil6Q1cqzboI/a0PeKkSSt\n2vxt0Pd399/6epI9VfXIKA/iGbskrZFj3Ab91aM+jsUuSRNw1G3QR8qpGEkakdm7B4+ldNMwNzK4\nDfr3R53DYpekEZk+f/CYd8WVL9xnwW3Q/2TBbdBHyqkYSVojx7kN+khZ7JK0duZvg35Bkn3d4+JR\nH8SpGElaI0vcBn1kPGOXpMZY7JLUGItdkhpjsUtSYyx2SWqMxS5JjbHYJakxFrskNcZil6SeSbIx\nyaMrHW+xS1LPVNXzwIEkr13JeG8pIEn9tAl4KMle4Afdtqqq7UsNtNglqZ/+ffezup9ZsHxcFrsk\n9VBVzSbZAryZQaHvraqnhhnrHLsk9VCSSxl8bd67gUuBvUnePcxYz9glqZ9+F3jz/Fl6klcBdwD/\nfamBnrFLUj8FeHrB+v/tti3JM3ZJ6qcvA7cl+W8MCv09wK3DDLTYJamf/i3wjxl8nR7A56rqpmEG\nWuyS1E8vAXZX1ReTvA74+SQnV9VzSw087hx7khcnuSfJ/iQPJ/lEt31Tkj1JDia5Pcmpo/lzSJI6\nXwV+JsnpwG0MvgT7vw4z8LjFXlU/Ai6oqnOAsxl8s/bbgZ3AnqrayuBd2p0rzy5JWkSq6ocMpmOu\nrqp3A28cZuCSV8V0LwzwImAD8AywHdjVbd8FXLLcxJKk40vyi8BvAl/qNg11JeOSOyU5Kcl+YA64\ns6oeAjZX1Vy3yxywefmRJUnH8a+BjwM3VdVDSf4+cOcwA5d887SqfgKck+TlDC69ueCo5yvJUPcv\nkCQNp6ruAu6CwQk28HRVfWiYsUNfFVNVzyb5EvAmYC7Jlqo6nOQ04Dj3L5g9srh3FrZND3tISeoc\ngqtmJh1iTSW5Afgt4MfAvcDLk3ymqv7TUmOXuirmlfNXvCQ5BbgI2AfcDOzodtsB7D72q0wfeVjq\nklZkCi6fOfI4Mby+qv6GwXuYtwJTwHuHGbjUGftpwK7unwEnAddX1R1J9gFfSPJ+4BCDG9RIkkZn\nY5KTGRT7VVX13LDT3sct9qp6ADhvke3fBS5cSVJJ0lA+x+DE+X7gfyaZAp4dZqA3AZOkHqqq/1JV\np1fVr3QXsfwVcMFS48Bil6ReSrIlyTVJvtxtOosj720ef2zV+K5UTFL15JH1mdNfuM/Mk0etn378\n50e1j1nMYpb1m+UKoKqGuoXtKCSp+vYKxr165Tm7Qr8O+HdVdXY3376vqpb89Kk3AZOkIeSZlZwE\nr+r/Pa+sqj9LshOge/P0+WEGOhUjSf30/SSvmF9J8laGfPPUM3ZJ6qePALcAfy/J3cCrgH8yzECL\nXZJ6qKq+nuQdwOsYzOkcGOZe7OBUjCT1UpIPAi+rqge7zxS9LMm/HGasxS5J/fSBqnpmfqVb/mfD\nDLTYJamfTupu5wJAkg3AycMMdI5dkvrpNuDzST7HYI79t4AvH3/IgMUuSf30MQZTL/+iW98D/PEw\nAy12SeqhqvpxkmuBr3WbDlTVj4cZa7FLUg8lmWbwndJ/1W36u0l2dN+sdFwWuyT105XAu6rqUYAk\nW4HPs8it1I/mVTGS1E8b50sdoKoOMuTJuGfsktRPX0/yx8CfMLgq5jeB+4YZaLFLUj/9c+CDwIe6\n9a8CVw8z0GKXpJ5JshH4ZlX9PPDp5Y53jl2SeqaqngceTfLalYz3jF2S+mkT8FCSvcAPum1VVduX\nGmixS1I//W73c+HXMA31NU4WuyT1SJJTGLxx+g+A+4Frh70P+zzn2CWpX3YBb2JQ6r8K/OflvoBn\n7JLUL2dV1S8AJLkGuHe5L+AZuyT1y/PzC93VMcvmGbsk9cvZSb63YP2UBetVVT+71AtY7JLUI1W1\nYbWv4VSMJDXGYpekxljsktQYi12SGmOxS1JjLHZJWiNJrk0yl+SBcR7HYpektXMdcPG4DzJUsSfZ\nkGRfklu69U1J9iQ5mOT2JKeON6YkrX9V9VXgmXEfZ9gz9g8DD3PklpE7gT1VtRW4o1uXJPXAkp88\nTfIaBncY+4/Av+k2bwfe0S3vAmax3CWd6PbOwr2zk04x1C0Ffh/4KLDw/gSbq2quW54DNo86mCSt\nO9umB495V18xkRjHLfYkvwY8VVX7kkwvtk9VVZJjfqvHzIKvYT0ETK0gpKQT2+zdg2kBDWepM/bz\nge1JfhV4MfCzSa4H5pJsqarDSU4DnjrWC8x8ZMHylasPLOnEM30+TC9Yv2tSQVYpyQ0MprFfkeRx\n4D9U1XWjPs5x3zytqt+pqjOq6kzgN4C/qKr3AjcDO7rddgC7Rx1MklpTVZdV1aur6me6bh15qcPy\nr2Ofn3L5JHBRkoPAO7t1SVIPDH0/9qq6i+5fQFX1XeDCcYWSJK2cnzyVpMZY7JLUGItdkhpjsUtS\nYyx2SWqMxS5JjbHYJakxFrskNcZil6TGWOyS1BiLXZIaY7FLUmMsdklqjMUuSY2x2CWpMRa7JDXG\nYpekxljsktQYi12SGmOxS1JjLHZJaozFLkmNsdglqTEWuyQ1xmKXpMZY7JLUGItdkhpjsUtSYyx2\nSWrMxkkHkKR14Q1XTDrB0Dxjl6TGWOyS1BiLXZIaY7FLUmOGevM0ySHgb4AfA89V1bYkm4A/A14L\nHAIuraq/HlNOSdKQhj1jL2C6qs6tqm3dtp3AnqraCtzRrUuSJmw5UzE5an07sKtb3gVcMpJEkqRV\nWc4Z+1eS3JfkA922zVU11y3PAZtHnk6StGzDfkDpbVX1nSSvAvYkObDwyaqqJDX6eJKk5Rqq2Kvq\nO93Pp5PcBGwD5pJsqarDSU4Dnlps7MynjywfAqZWl1fSCWj2bpiddIh1ZMliT/ISYENVfS/JS4F3\nAVcANwM7gE91P3cvNn7mIwuWr1x9YEknnunzYXrB+l2TCrJODHPGvhm4Kcn8/n9aVbcnuQ/4QpL3\n013uOLaUkqShLVnsVfUYcM4i278LXDiOUJKklfOTp5LUGItdkhpjsUtSYyx2SWqMxS5JjbHYJakx\nFrskNcZil6TGWOyS1BiLXZIaY7FLUmMsdklqjMUuSY2x2CWpMRa7JDXGYpekNZTk4iQHkvxlko+N\n4xgWuyStkSQbgD8ELgZeD1yW5KxRH8dil6S1sw34VlUdqqrngM8Dvz7qg1jskrR2TgceX7D+RLdt\npCx2SVo7tRYHWfLLrCVJw3oMOHS8HZ4EzliwfgaDs/aRstglaWTO7B7z7jp6h/uAn0syBXwbeA9w\n2ahTWOyStEaq6vkkHwRuAzYA11TVI6M+jsUuSWuoqm4Fbh3nMXzzVJIaY7FLUmMsdklqjMUuSY2x\n2CWpMRa7JDXGYpekxljsktQYi12SGmOxS1JjLHZJasxQxZ7k1CQ3JnkkycNJ3pJkU5I9SQ4muT3J\nqeMOK0la2rBn7J8B/ryqzgLOBg4AO4E9VbUVuKNblyRN2JLFnuTlwC9V1bUwuO1kVT0LbAd2dbvt\nAi4ZW0pJ0tCGOWM/E3g6yXVJvpHkj5K8FNhcVXPdPnPA5rGllCQNbZhi3wicB1xdVecBP+CoaZeq\nKtbou/wkScc3zBdtPAE8UVX3dus3Ah8HDifZUlWHk5wGPLXY4JlPH1k+BEytIqykE9Ps3TA76RDr\nyJLF3hX340m2VtVB4ELgoe6xA/hU93P3YuNnPrJg+coRJJZ0wpk+H6YXrL/gm0T1twz71Xj/CvjT\nJC8C/jfwPgbf1/eFJO9ncDJ+6VgSSpKWZahir6pvAm9e5KkLRxtHkrRafvJUkhpjsUtSYyx2SWqM\nxS5JjbHYJakxFrskNcZil6TGrEmxz969FkcZnfWUdz1lhfWVdz1lBfPqiLUp9v+1FkcZnfWUdz1l\nhfWVdz1lBfPqCKdiJKkxFrskNSaDW6mP6cUT79EuaSyqKmt1rNV02VrmnDfWYpckrT2nYiSpMRa7\nJDXGYpekxljsktQYi12SGvP/AdzJez1KiumSAAAAAElFTkSuQmCC\n", + "text": [ + "" + ] + } + ], + "prompt_number": 19 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Contexts\n", + "--------" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# Context objects manage the setup and communication of the worker processes\n", + "# for DistArray objects. \n", + "print(\"targets:\", context.targets)\n", + "print(\"comm:\", context.comm)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "targets: [0, 1, 2, 3]\n", + "comm: __distarray__26749f47735b3b2f0\n" + ] + } + ], + "prompt_number": 20 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "IO Support (v0.2)\n", + "-----------------" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# load .npy files in parallel\n", + "numpy.save(\"/tmp/outfile.npy\", nparr)\n", + "distribution = Distribution.from_shape(context, nparr.shape) \n", + "new_darr = context.load_npy(\"/tmp/outfile.npy\", distribution)\n", + "new_darr" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 21, + "text": [ + "" + ] + } + ], + "prompt_number": 21 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# save DistArrays to .hdf5 files in parallel\n", + "context.save_hdf5(\"/tmp/outfile.hdf5\", darr, mode='w')" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [], + "prompt_number": 22 + }, + { + "cell_type": "code", + "collapsed": true, + "input": [ + "# load DistArrays from .hdf5 files in parallel (using h5py)\n", + "context.load_hdf5(\"/tmp/outfile.hdf5\", distribution)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 23, + "text": [ + "" + ] + } + ], + "prompt_number": 23 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# save to .dnpy (a built-in flat-file format based on .npy)\n", + "context.save_dnpy(\"/tmp/outfile\", darr)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [], + "prompt_number": 24 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# load from .dnpy\n", + "context.load_dnpy(\"/tmp/outfile\")" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 25, + "text": [ + "" + ] + } + ], + "prompt_number": 25 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Reductions (v0.3)\n", + "-----------------" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "## NumPy ##\n", + "print(\"sum:\", nparr.sum())\n", + "print(\"sum over an axis:\", nparr.sum(axis=1))" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "sum: 7.59320976333\n", + "sum over an axis: [ 1.7 2.09 2.02 1.79]\n" + ] + } + ], + "prompt_number": 26 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "## DistArray ##\n", + "print(\"sum:\", darr.sum(), darr.sum().toarray())\n", + "print(\"sum over an axis:\", darr.sum(axis=1), darr.sum(axis=1).toarray())" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "sum: 7.59320976333\n", + "sum over an axis: [ 1.7 2.09 2.02 1.79]\n" + ] + } + ], + "prompt_number": 27 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Context.apply (v0.3)\n", + "--------------------\n", + "Global view, local control" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def get_local_random():\n", + " import numpy\n", + " return numpy.random.randint(10)\n", + "\n", + "context.apply(get_local_random)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 28, + "text": [ + "[3, 6, 4, 9]" + ] + } + ], + "prompt_number": 28 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def get_local_var(darr):\n", + " return darr.ndarray.var()\n", + "\n", + "context.apply(get_local_var, args=(darr.key,))" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 29, + "text": [ + "[0.057695824074284786,\n", + " 0.050026935244417328,\n", + " 0.069633636558013801,\n", + " 0.095146790094971609]" + ] + } + ], + "prompt_number": 29 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Distributed Slicing (v0.4)\n", + "--------------------------" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# as a reminder\n", + "darr.toarray()" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 30, + "text": [ + "array([[ 0.77, 0.06, 0.37, 0.19, 0.31],\n", + " [ 0.58, 0.67, 0.41, 0.02, 0.41],\n", + " [ 0.1 , 0.56, 0.08, 0.56, 0.72],\n", + " [ 0.91, 0.13, 0.48, 0.14, 0.13]])" + ] + } + ], + "prompt_number": 30 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "darr.localshapes()" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 31, + "text": [ + "[(1, 5), (1, 5), (1, 5), (1, 5)]" + ] + } + ], + "prompt_number": 31 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# take a column slice\n", + "darr_view = darr[:, 3]\n", + "print(darr_view)\n", + "print(darr_view.toarray())" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "[ 0.19 0.02 0.56 0.14]\n" + ] + } + ], + "prompt_number": 32 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# changes in the view change the original\n", + "darr_view[3] = -0.99\n", + "print(\"view:\")\n", + "print(darr_view.toarray())\n", + "\n", + "print(\"original:\")\n", + "print(darr.toarray())" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "view:\n", + "[ 0.19 0.02 0.56 -0.99]\n", + "original:\n", + "[[ 0.77 0.06 0.37 0.19 0.31]\n", + " [ 0.58 0.67 0.41 0.02 0.41]\n", + " [ 0.1 0.56 0.08 0.56 0.72]\n", + " [ 0.91 0.13 0.48 -0.99 0.13]]\n" + ] + } + ], + "prompt_number": 33 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# a more complex slice\n", + "print(darr[:, 2::2])\n", + "print(darr[:-1, 2::2].toarray())" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "[[ 0.37 0.31]\n", + " [ 0.41 0.41]\n", + " [ 0.08 0.72]]\n" + ] + } + ], + "prompt_number": 34 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Distributed Array Protocol (v0.2)\n", + "---------------------------------\n", + "(exporting and importing distributed arrays)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def return_protocol_structure(darr):\n", + " return darr.__distarray__()\n", + "\n", + "context.apply(return_protocol_structure, args=(darr.key,))" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 35, + "text": [ + "[{'__version__': '0.10.0',\n", + " 'buffer': array([[ 0.77, 0.06, 0.37, 0.19, 0.31]]),\n", + " 'dim_data': ({'dist_type': 'b',\n", + " 'proc_grid_rank': 0,\n", + " 'proc_grid_size': 4,\n", + " 'size': 4,\n", + " 'start': 0,\n", + " 'stop': 1},\n", + " {'dist_type': 'b',\n", + " 'proc_grid_rank': 0,\n", + " 'proc_grid_size': 1,\n", + " 'size': 5,\n", + " 'start': 0,\n", + " 'stop': 5})},\n", + " {'__version__': '0.10.0',\n", + " 'buffer': array([[ 0.58, 0.67, 0.41, 0.02, 0.41]]),\n", + " 'dim_data': ({'dist_type': 'b',\n", + " 'proc_grid_rank': 1,\n", + " 'proc_grid_size': 4,\n", + " 'size': 4,\n", + " 'start': 1,\n", + " 'stop': 2},\n", + " {'dist_type': 'b',\n", + " 'proc_grid_rank': 0,\n", + " 'proc_grid_size': 1,\n", + " 'size': 5,\n", + " 'start': 0,\n", + " 'stop': 5})},\n", + " {'__version__': '0.10.0',\n", + " 'buffer': array([[ 0.1 , 0.56, 0.08, 0.56, 0.72]]),\n", + " 'dim_data': ({'dist_type': 'b',\n", + " 'proc_grid_rank': 2,\n", + " 'proc_grid_size': 4,\n", + " 'size': 4,\n", + " 'start': 2,\n", + " 'stop': 3},\n", + " {'dist_type': 'b',\n", + " 'proc_grid_rank': 0,\n", + " 'proc_grid_size': 1,\n", + " 'size': 5,\n", + " 'start': 0,\n", + " 'stop': 5})},\n", + " {'__version__': '0.10.0',\n", + " 'buffer': array([[ 0.91, 0.13, 0.48, -0.99, 0.13]]),\n", + " 'dim_data': ({'dist_type': 'b',\n", + " 'proc_grid_rank': 3,\n", + " 'proc_grid_size': 4,\n", + " 'size': 4,\n", + " 'start': 3,\n", + " 'stop': 4},\n", + " {'dist_type': 'b',\n", + " 'proc_grid_rank': 0,\n", + " 'proc_grid_size': 1,\n", + " 'size': 5,\n", + " 'start': 0,\n", + " 'stop': 5})}]" + ] + } + ], + "prompt_number": 35 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Acknowledgement and Disclaimer\n", + "------------------------------\n", + "\n", + "This material is based upon work supported by the Department of Energy under Award Number DE-SC0007699.\n", + "\n", + "This report was prepared as an account of work sponsored by an agency of the United States Government. Neither the United States Government nor any agency thereof, nor any of their employees, makes any warranty, express or implied, or assumes any legal liability or responsibility for the accuracy, completeness, or usefulness of any information, apparatus, product, or process disclosed, or represents that its use would not infringe privately owned rights. Reference herein to any specific commercial product, process, or service by trade name, trademark, manufacturer, or otherwise does not necessarily constitute or imply its endorsement, recommendation, or favoring by the United States Government or any agency thereof. The views and opinions of authors expressed herein do not necessarily state or reflect those of the United States Government or any agency thereof. " + ] + } + ], + "metadata": {} + } + ] +} \ No newline at end of file