From 60bea415cd415ea3295c3e712e78ec198747051f Mon Sep 17 00:00:00 2001 From: "Sujee Maniyam @ sujee-mac2" Date: Mon, 11 Dec 2017 14:55:45 -0800 Subject: [PATCH 01/19] adding docker for bigdl --- docker/Dockerfile | 145 +++++++++++++++++++++++++++++++++++++++ docker/README.md | 81 ++++++++++++++++++++++ docker/run-bigdl.sh | 64 +++++++++++++++++ docker/start-notebook.sh | 12 ++++ 4 files changed, 302 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100755 docker/run-bigdl.sh create mode 100755 docker/start-notebook.sh diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..8d0ddc6 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,145 @@ +## pick a specific version, to ensure predictability +FROM jupyter/pyspark-notebook@sha256:64420e4c348ab48fb806f42332109cbc205ae74cda67240c3e0974c5f7e6e969 +## or latest +#FROM jupyter/pyspark-notebook@latest + +MAINTAINER Intel BigDL Project + +## --- CONFIG +ARG MAVEN_VERSION=3.5.2 +ARG SCALA_VERSION=2.11.8 +ARG SPARK_VERSION=2.2.0 +ARG SBT_VERSION=1.0.2 +ARG INSTALL_DIR=/usr/local +ENV BIGDL_HOME ${INSTALL_DIR}/BigDL + +#ARG BIGDL_URL=https://s3.amazonaws.com/elephantscale-public/BigDL/BigDL.zip +## Download BigDL from release page +## https://bigdl-project.github.io/0.3.0/#release-download/ +ARG BIGDL_URL=https://repo1.maven.org/maven2/com/intel/analytics/bigdl/dist-spark-${SPARK_VERSION}-scala-2.11.8-linux64/0.3.0/dist-spark-${SPARK_VERSION}-scala-2.11.8-linux64-0.3.0-dist.zip +## --- end CONFIG + +USER root + +## apt update +RUN apt-get update -yq && \ + apt-get -yq dist-upgrade + +## basic utils + jdk +RUN apt-get install -yq --no-install-recommends \ + atop \ + curl \ + less \ + openjdk-8-jdk-headless \ + rsync \ + unzip \ + wget \ + zip + +## install maven +RUN curl -fsL http://apache.cs.utah.edu/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz | tar xfz - -C ${INSTALL_DIR} + +RUN cd ${INSTALL_DIR} && rm -f maven && ln -s apache-maven-${MAVEN_VERSION} maven + +ENV MAVEN_HOME ${INSTALL_DIR}/maven +ENV PATH=$MAVEN_HOME/bin:$PATH + + +## Scala expects this file +RUN touch /usr/lib/jvm/java-8-openjdk-amd64/release + +## Install Scala +RUN \ + curl -fsL https://downloads.typesafe.com/scala/${SCALA_VERSION}/scala-${SCALA_VERSION}.tgz | tar xfz - -C ${INSTALL_DIR} + +RUN cd ${INSTALL_DIR} && rm -f scala && ln -s scala-${SCALA_VERSION} scala + +ENV PATH=${INSTALL_DIR}/scala/bin:$PATH + +## Install sbt +RUN \ + curl -L -o sbt-${SBT_VERSION}.deb "https://dl.bintray.com/sbt/debian/sbt-${SBT_VERSION}.deb" && \ + dpkg -i sbt-${SBT_VERSION}.deb && \ + rm sbt-${SBT_VERSION}.deb && \ + apt-get update && \ + apt-get install sbt && \ + sbt sbtVersion + +## install spark +RUN \ + curl -fsL "https://d3kbcqa49mib13.cloudfront.net/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz" | tar xfz - -C ${INSTALL_DIR} && \ + cd ${INSTALL_DIR} && rm -f spark && ln -s spark-${SPARK_VERSION}-bin-hadoop2.7 spark + +ENV APACHE_SPARK_VERSION=${SPARK_VERSION} + +## cleanup apt +RUN apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +## ----- install BigDL +RUN \ + mkdir -p ${BIGDL_HOME} && \ + cd ${BIGDL_HOME} && \ + wget --quiet "${BIGDL_URL}" && \ + unzip *.zip && \ + rm -f *.zip + +## disable sudo password +RUN echo "${NB_USER} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + +ENV PATH=${INSTALL_DIR}/spark/bin:$PATH + +# this is where volumes will be mounted +RUN mkdir /work + + +### now as a regular user +USER $NB_USER + +## update conda +# RUN conda update --all +## install NLTK +RUN conda install -y nltk + +## python 3.5 env +RUN conda create -y -n py35 python=3.5 numpy scipy pandas scikit-learn matplotlib seaborn jupyter nltk tensorflow + +## python 2.7 env +RUN conda create -y -n py27 python=2.7 numpy scipy pandas scikit-learn matplotlib seaborn jupyter nltk tensorflow + +# list envs +RUN conda info -e + +## install tensorboard with pip for py27 +RUN /bin/bash -c "source activate py27 && \ + pip install tensorboard && \ + source deactivate" + +## install tensorboard with pip for py35 +RUN /bin/bash -c "source activate py35 && \ + pip install tensorboard && \ + source deactivate" + +## source python 3.5 +RUN echo "source activate py35" >> ~/.bashrc + +## source python 2.7 +## This will be the default python env +RUN echo "source activate py27" >> ~/.bashrc + +# working directory +ENV WORKING_DIR ${HOME}/work +RUN rm -rf ${WORKING_DIR} && ln -s /work ${WORKING_DIR} + +## --- copy files last, so not to bust the cache --- + +COPY run-bigdl.sh $HOME/ + +USER root +RUN mv /usr/local/bin/start-notebook.sh /usr/local/bin/start-notebook-old.sh +COPY start-notebook.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/start-notebook.sh + + +## finally switch back to jovyan to avoid accidental container runs as root +USER $NB_USER diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..4cb191e --- /dev/null +++ b/docker/README.md @@ -0,0 +1,81 @@ +# How to use the Docker image + +## Building + +```bash + $ cd bigdl-tutorials/docker + + # to use default 'Dockerfile' + # do not forget this DOT (.) at the end + $ docker build . + + # to do a different build file + # do not forget this DOT (.) at the end + $ docker build -f Dockerfile-v3 . + + # to add a tag + $ docker build -f Dockerfile-v3 -t bigdl/bigdl . +``` + +To force a build provide `--no-cache ` option +```bash + $ docker build --no-cache . +``` + +To see built images + +```bash + $ docker images + $ docker image inspect +``` + +## Running it + +``` +$ docker images +``` +Will list all images. + +#### Option 1: +The simplest way to run the docker container is to use `run-bigdl-docker.sh` script at the project root directory. It will run the image and also mounts a working directory so all the work is saved automatically. + +``` + $ ./run-bigdl-docker.sh +``` + +#### Option 2: +Running the Jupyter notebook by default (~/run_jupyter.sh) +``` + $ docker run -it -p 8888:8888 IMAGE_ID +``` + +#### Option 3: +Shell access +``` + $ docker run -it -p 8888:8888 IMAGE_ID bash +``` + +In Docker container you can run Jupyter as follows +``` + $ ./run_jupyter.sh +``` + +Go to http://localhost:8888 in browser + + +## Pushing to Dockerhub + +```bash + + ## login + $ docker login + ## enter username, password + + ## tag the image + $ docker tag IMAGE_ID elephantscale/bigdl-sandbox:latest + $ docker images + + ## Pushing + $ docker push elephantscale/bigdl-sandbox:latest + +``` diff --git a/docker/run-bigdl.sh b/docker/run-bigdl.sh new file mode 100755 index 0000000..2305bcd --- /dev/null +++ b/docker/run-bigdl.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +## Usage +# BIGDL_HOME=~/apps/BigDL SPARK_HOME=~/apps/spark ./run-bigdl.sh + + +# Check environment variables +if [ -z "${BIGDL_HOME}" ]; then + echo "Please set BIGDL_HOME environment variable" + exit 1 +fi + +if [ -z "${SPARK_HOME}" ]; then + echo "Please set SPARK_HOME environment variable" + exit 1 +fi + +# activate py35 environment +#source activate py35 +# activate py27 environment +source activate py27 +conda info -e + + +#setup paths +export PYSPARK_PYTHON=$(which python) +export PYSPARK_DRIVER_PYTHON=$(which jupyter) +echo "### PYSPARK_PYTHON=$PYSPARK_PYTHON" +echo "### PYSPARK_DRIVER_PYTHON=$PYSPARK_DRIVER_PYTHON" +# this starts the notebook without a security token +#export PYSPARK_DRIVER_PYTHON_OPTS="notebook --notebook-dir=./ --ip=* --no-browser --NotebookApp.token=''" +export PYSPARK_DRIVER_PYTHON_OPTS="notebook --notebook-dir=./ --ip=* --no-browser" +export BIGDL_JAR_NAME=`ls ${BIGDL_HOME}/lib/ | grep jar-with-dependencies.jar` +export BIGDL_JAR="${BIGDL_HOME}/lib/$BIGDL_JAR_NAME" +export BIGDL_PY_ZIP_NAME=`ls ${BIGDL_HOME}/lib/ | grep python-api.zip` +export BIGDL_PY_ZIP="${BIGDL_HOME}/lib/$BIGDL_PY_ZIP_NAME" +export BIGDL_CONF=${BIGDL_HOME}/conf/spark-bigdl.conf + +# Check files +if [ ! -f ${BIGDL_CONF} ]; then + echo "Cannot find ${BIGDL_CONF}" + exit 1 +fi + +if [ ! -f ${BIGDL_PY_ZIP} ]; then + echo ${BIGDL_PY_ZIP} + echo "Cannot find ${BIGDL_PY_ZIP}" + exit 1 +fi + +if [ ! -f $BIGDL_JAR ]; then + echo "Cannot find $BIGDL_JAR" + exit 1 +fi + +${SPARK_HOME}/bin/pyspark \ + --master local[4] \ + --driver-memory 4g \ + --properties-file ${BIGDL_CONF} \ + --py-files ${BIGDL_PY_ZIP} \ + --jars ${BIGDL_JAR} \ + --conf spark.driver.extraClassPath=${BIGDL_JAR} \ + --conf spark.executor.extraClassPath=${BIGDL_JAR} \ + --conf spark.sql.catalogImplementation='in-memory' diff --git a/docker/start-notebook.sh b/docker/start-notebook.sh new file mode 100755 index 0000000..11b4829 --- /dev/null +++ b/docker/start-notebook.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e + +# activate py35 environment +#source activate py35 +#conda info -e + +/usr/local/bin/start.sh ~/run-bigdl.sh + +# And run bash shell, so the container doesn't exit when Jupyter exits +# This way we can re-run ./run-bigdl.sh if needed to +/bin/bash From 80c1f196bf5a5a4974fe1ea63ed7717534810231 Mon Sep 17 00:00:00 2001 From: "Sujee Maniyam @ sujee-mac2" Date: Mon, 11 Dec 2017 15:19:35 -0800 Subject: [PATCH 02/19] run bigdl in docker container --- run-bigdl-docker.sh | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100755 run-bigdl-docker.sh diff --git a/run-bigdl-docker.sh b/run-bigdl-docker.sh new file mode 100755 index 0000000..6d01f68 --- /dev/null +++ b/run-bigdl-docker.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +## Usage +# ./run-bigdl-docker.sh [optional command] +# +# ./run-bigdl-docker.sh xxx/yyy +# +# or during developing, give a local docker image id +# ./run-bigdl-docker.sh abcd1234 +# +# provide an optional command, here is an example of simple bash +# ./run-bigdl-docker.sh abcd1234 bash + +if [ -z "$1" ] ; then + echo "Usage: $0 [optional command]" + echo "Missing Docker image id. exiting" + exit -1 +fi + +image_id="$1" +cmd="$2" +name="bigdl" + +## remove any previously running containers +docker rm -f "$name" + +# mount the current directory at /work +this="${BASH_SOURCE-$0}" +mydir=$(cd -P -- "$(dirname -- "$this")" && pwd -P) + +docker run -it --name "$name" \ + -p 8888:8888 \ + -p 6006:6006 \ + -v"$mydir:/work" \ + "$image_id" \ + ${cmd} From eaa8fbcf68701f09e77af23ba3dc236c5805c03d Mon Sep 17 00:00:00 2001 From: "Sujee Maniyam @ sujee-mac2" Date: Mon, 11 Dec 2017 15:19:47 -0800 Subject: [PATCH 03/19] script to launch pyspark with bigdl dependencies --- run-bigdl-native.sh | 66 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100755 run-bigdl-native.sh diff --git a/run-bigdl-native.sh b/run-bigdl-native.sh new file mode 100755 index 0000000..11abd84 --- /dev/null +++ b/run-bigdl-native.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +## Runs BigDL natively + +## Usage +# BIGDL_HOME=~/apps/BigDL SPARK_HOME=~/apps/spark ./run-bigdl.sh + + +# Check environment variables +if [ -z "${BIGDL_HOME}" ]; then + echo "Please set BIGDL_HOME environment variable" + exit 1 +fi + +if [ -z "${SPARK_HOME}" ]; then + echo "Please set SPARK_HOME environment variable" + exit 1 +fi + +# activate py35 environment +#source activate py35 +# activate py27 environment +source activate py27 +conda info -e + + +#setup paths +export PYSPARK_PYTHON=$(which python) +export PYSPARK_DRIVER_PYTHON=$(which jupyter) +echo "### PYSPARK_PYTHON=$PYSPARK_PYTHON" +echo "### PYSPARK_DRIVER_PYTHON=$PYSPARK_DRIVER_PYTHON" +# this starts the notebook without a security token +#export PYSPARK_DRIVER_PYTHON_OPTS="notebook --notebook-dir=./ --ip=* --no-browser --NotebookApp.token=''" +export PYSPARK_DRIVER_PYTHON_OPTS="notebook --notebook-dir=./ --ip=* --no-browser" +export BIGDL_JAR_NAME=`ls ${BIGDL_HOME}/lib/ | grep jar-with-dependencies.jar` +export BIGDL_JAR="${BIGDL_HOME}/lib/$BIGDL_JAR_NAME" +export BIGDL_PY_ZIP_NAME=`ls ${BIGDL_HOME}/lib/ | grep python-api.zip` +export BIGDL_PY_ZIP="${BIGDL_HOME}/lib/$BIGDL_PY_ZIP_NAME" +export BIGDL_CONF=${BIGDL_HOME}/conf/spark-bigdl.conf + +# Check files +if [ ! -f ${BIGDL_CONF} ]; then + echo "Cannot find ${BIGDL_CONF}" + exit 1 +fi + +if [ ! -f ${BIGDL_PY_ZIP} ]; then + echo ${BIGDL_PY_ZIP} + echo "Cannot find ${BIGDL_PY_ZIP}" + exit 1 +fi + +if [ ! -f $BIGDL_JAR ]; then + echo "Cannot find $BIGDL_JAR" + exit 1 +fi + +${SPARK_HOME}/bin/pyspark \ + --master local[4] \ + --driver-memory 4g \ + --properties-file ${BIGDL_CONF} \ + --py-files ${BIGDL_PY_ZIP} \ + --jars ${BIGDL_JAR} \ + --conf spark.driver.extraClassPath=${BIGDL_JAR} \ + --conf spark.executor.extraClassPath=${BIGDL_JAR} \ + --conf spark.sql.catalogImplementation='in-memory' From 4130673cefa9c85afa9afa0b239fb153b4d52a70 Mon Sep 17 00:00:00 2001 From: "Sujee Maniyam @ sujee-mac2" Date: Mon, 11 Dec 2017 15:32:36 -0800 Subject: [PATCH 04/19] gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..04b6866 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.ipynb_checkpoints/ + +*.log +*.zip +.DS_Store From 7f1646782e7c6f388eb213273cbb076294dd8ab4 Mon Sep 17 00:00:00 2001 From: "Sujee Maniyam @ sujee-mac2" Date: Mon, 11 Dec 2017 15:33:39 -0800 Subject: [PATCH 05/19] adding Testing123 notebook --- notebooks/Testing-123.ipynb | 229 ++++++++++++++++++++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 notebooks/Testing-123.ipynb diff --git a/notebooks/Testing-123.ipynb b/notebooks/Testing-123.ipynb new file mode 100644 index 0000000..9c10fb3 --- /dev/null +++ b/notebooks/Testing-123.ipynb @@ -0,0 +1,229 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Testing BigDL Installation\n", + "\n", + "Run this notebook to verify BigDL and Spark are setup properly.\n", + "You can select 'Cell --> Run All Cells'.\n", + "And if there are no exceptions, the installation is good!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1 - Spark\n", + "The following cell should print out Spark run time info." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# SparkContext\n", + "sc" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Spark Session (newer API)\n", + "spark" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# dataframe\n", + "a = spark.range(1,10)\n", + "a.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2 - BigDL Imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from bigdl.util.common import *\n", + "from bigdl.nn.layer import *\n", + "import bigdl.version \n", + "\n", + "print(\"Imports are looking good!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3 - BigDL Hello World" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from bigdl.util.common import *\n", + "from bigdl.nn.layer import *\n", + "import bigdl.version \n", + "\n", + "print(\"Hello BigDL world!\")\n", + "init_engine() # prepare the bigdl environment \n", + "print(\"BigDL version : \" , bigdl.version.__version__) # Get the current BigDL version\n", + "linear = Linear(2, 3) # Try to create a Linear layer" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4 - NumPy and Pandas" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# numpy\n", + "a = np.array([1,2,3])\n", + "print(a)\n", + "\n", + "# pandas\n", + "df = pd.DataFrame({'date' : ['2016-01-01', '2016-01-02', '2016-01-03'],\n", + " 'qty': [20, 30, 40]})\n", + "print(df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5 - NLTK" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import nltk\n", + "nltk.download('punkt')\n", + "\n", + "s = \"We are going to need a bigger boat!\"\n", + "tokens = nltk.word_tokenize(s)\n", + "print(tokens)\n", + "print(\"==> NLTK is working\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "## 6 - Graphs, We Need Graphs!\n", + "This will test MatplotLib." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", + "\n", + "plt.plot([1,5,2,4])\n", + "plt.title(\"Do you see me?\")\n", + "plt.show()\n", + "\n", + "print(\"If you see a plot above, all good!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7 - We Need More Graphs (Seaborn)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "\n", + "\n", + "sns.set(color_codes=True)\n", + "x = np.random.normal(size=100)\n", + "sns.distplot(x);\n", + "\n", + "print(\"if you see a nice distribution plot below, we are good!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## That's all Folks\n", + "If you are here and all code above ran without any exceptions... Congrats!\n", + "You are ready to explore BigDL." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 67fbd52762684c58b76679e2d621442aaa3df8b5 Mon Sep 17 00:00:00 2001 From: "Sujee Maniyam @ sujee-mac2" Date: Mon, 11 Dec 2017 15:19:35 -0800 Subject: [PATCH 06/19] run bigdl in docker container --- run-bigdl-docker.sh | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100755 run-bigdl-docker.sh diff --git a/run-bigdl-docker.sh b/run-bigdl-docker.sh new file mode 100755 index 0000000..6d01f68 --- /dev/null +++ b/run-bigdl-docker.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +## Usage +# ./run-bigdl-docker.sh [optional command] +# +# ./run-bigdl-docker.sh xxx/yyy +# +# or during developing, give a local docker image id +# ./run-bigdl-docker.sh abcd1234 +# +# provide an optional command, here is an example of simple bash +# ./run-bigdl-docker.sh abcd1234 bash + +if [ -z "$1" ] ; then + echo "Usage: $0 [optional command]" + echo "Missing Docker image id. exiting" + exit -1 +fi + +image_id="$1" +cmd="$2" +name="bigdl" + +## remove any previously running containers +docker rm -f "$name" + +# mount the current directory at /work +this="${BASH_SOURCE-$0}" +mydir=$(cd -P -- "$(dirname -- "$this")" && pwd -P) + +docker run -it --name "$name" \ + -p 8888:8888 \ + -p 6006:6006 \ + -v"$mydir:/work" \ + "$image_id" \ + ${cmd} From 23bf53978eeffb59e71d9e2842561a24a2b0bef8 Mon Sep 17 00:00:00 2001 From: "Sujee Maniyam @ sujee-mac2" Date: Mon, 11 Dec 2017 15:19:47 -0800 Subject: [PATCH 07/19] script to launch pyspark with bigdl dependencies --- run-bigdl-native.sh | 66 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100755 run-bigdl-native.sh diff --git a/run-bigdl-native.sh b/run-bigdl-native.sh new file mode 100755 index 0000000..11abd84 --- /dev/null +++ b/run-bigdl-native.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +## Runs BigDL natively + +## Usage +# BIGDL_HOME=~/apps/BigDL SPARK_HOME=~/apps/spark ./run-bigdl.sh + + +# Check environment variables +if [ -z "${BIGDL_HOME}" ]; then + echo "Please set BIGDL_HOME environment variable" + exit 1 +fi + +if [ -z "${SPARK_HOME}" ]; then + echo "Please set SPARK_HOME environment variable" + exit 1 +fi + +# activate py35 environment +#source activate py35 +# activate py27 environment +source activate py27 +conda info -e + + +#setup paths +export PYSPARK_PYTHON=$(which python) +export PYSPARK_DRIVER_PYTHON=$(which jupyter) +echo "### PYSPARK_PYTHON=$PYSPARK_PYTHON" +echo "### PYSPARK_DRIVER_PYTHON=$PYSPARK_DRIVER_PYTHON" +# this starts the notebook without a security token +#export PYSPARK_DRIVER_PYTHON_OPTS="notebook --notebook-dir=./ --ip=* --no-browser --NotebookApp.token=''" +export PYSPARK_DRIVER_PYTHON_OPTS="notebook --notebook-dir=./ --ip=* --no-browser" +export BIGDL_JAR_NAME=`ls ${BIGDL_HOME}/lib/ | grep jar-with-dependencies.jar` +export BIGDL_JAR="${BIGDL_HOME}/lib/$BIGDL_JAR_NAME" +export BIGDL_PY_ZIP_NAME=`ls ${BIGDL_HOME}/lib/ | grep python-api.zip` +export BIGDL_PY_ZIP="${BIGDL_HOME}/lib/$BIGDL_PY_ZIP_NAME" +export BIGDL_CONF=${BIGDL_HOME}/conf/spark-bigdl.conf + +# Check files +if [ ! -f ${BIGDL_CONF} ]; then + echo "Cannot find ${BIGDL_CONF}" + exit 1 +fi + +if [ ! -f ${BIGDL_PY_ZIP} ]; then + echo ${BIGDL_PY_ZIP} + echo "Cannot find ${BIGDL_PY_ZIP}" + exit 1 +fi + +if [ ! -f $BIGDL_JAR ]; then + echo "Cannot find $BIGDL_JAR" + exit 1 +fi + +${SPARK_HOME}/bin/pyspark \ + --master local[4] \ + --driver-memory 4g \ + --properties-file ${BIGDL_CONF} \ + --py-files ${BIGDL_PY_ZIP} \ + --jars ${BIGDL_JAR} \ + --conf spark.driver.extraClassPath=${BIGDL_JAR} \ + --conf spark.executor.extraClassPath=${BIGDL_JAR} \ + --conf spark.sql.catalogImplementation='in-memory' From 0e2f2d9a407717976f6752082f68a9574b0a8da2 Mon Sep 17 00:00:00 2001 From: "Sujee Maniyam @ sujee-mac2" Date: Mon, 11 Dec 2017 15:32:36 -0800 Subject: [PATCH 08/19] gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..04b6866 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.ipynb_checkpoints/ + +*.log +*.zip +.DS_Store From 47248814c9a5d29d18712a5a0b1b6c343df40f80 Mon Sep 17 00:00:00 2001 From: "Sujee Maniyam @ sujee-mac2" Date: Fri, 15 Dec 2017 09:51:35 -0800 Subject: [PATCH 09/19] update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 04b6866..09f3c0b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ *.log *.zip .DS_Store +*spark-warehouse* From 56e00e0fddfd08b361ab4a6c127877585266389e Mon Sep 17 00:00:00 2001 From: "Sujee Maniyam @ sujee-mac2" Date: Mon, 11 Dec 2017 15:33:39 -0800 Subject: [PATCH 10/19] adding Testing123 notebook --- notebooks/Testing-123.ipynb | 229 ++++++++++++++++++++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 notebooks/Testing-123.ipynb diff --git a/notebooks/Testing-123.ipynb b/notebooks/Testing-123.ipynb new file mode 100644 index 0000000..9c10fb3 --- /dev/null +++ b/notebooks/Testing-123.ipynb @@ -0,0 +1,229 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Testing BigDL Installation\n", + "\n", + "Run this notebook to verify BigDL and Spark are setup properly.\n", + "You can select 'Cell --> Run All Cells'.\n", + "And if there are no exceptions, the installation is good!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1 - Spark\n", + "The following cell should print out Spark run time info." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# SparkContext\n", + "sc" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Spark Session (newer API)\n", + "spark" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# dataframe\n", + "a = spark.range(1,10)\n", + "a.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2 - BigDL Imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from bigdl.util.common import *\n", + "from bigdl.nn.layer import *\n", + "import bigdl.version \n", + "\n", + "print(\"Imports are looking good!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3 - BigDL Hello World" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from bigdl.util.common import *\n", + "from bigdl.nn.layer import *\n", + "import bigdl.version \n", + "\n", + "print(\"Hello BigDL world!\")\n", + "init_engine() # prepare the bigdl environment \n", + "print(\"BigDL version : \" , bigdl.version.__version__) # Get the current BigDL version\n", + "linear = Linear(2, 3) # Try to create a Linear layer" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4 - NumPy and Pandas" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# numpy\n", + "a = np.array([1,2,3])\n", + "print(a)\n", + "\n", + "# pandas\n", + "df = pd.DataFrame({'date' : ['2016-01-01', '2016-01-02', '2016-01-03'],\n", + " 'qty': [20, 30, 40]})\n", + "print(df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5 - NLTK" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import nltk\n", + "nltk.download('punkt')\n", + "\n", + "s = \"We are going to need a bigger boat!\"\n", + "tokens = nltk.word_tokenize(s)\n", + "print(tokens)\n", + "print(\"==> NLTK is working\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "## 6 - Graphs, We Need Graphs!\n", + "This will test MatplotLib." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", + "\n", + "plt.plot([1,5,2,4])\n", + "plt.title(\"Do you see me?\")\n", + "plt.show()\n", + "\n", + "print(\"If you see a plot above, all good!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7 - We Need More Graphs (Seaborn)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "\n", + "\n", + "sns.set(color_codes=True)\n", + "x = np.random.normal(size=100)\n", + "sns.distplot(x);\n", + "\n", + "print(\"if you see a nice distribution plot below, we are good!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## That's all Folks\n", + "If you are here and all code above ran without any exceptions... Congrats!\n", + "You are ready to explore BigDL." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 5573f581e308aa5d0f0a92e83aee9f433809510a Mon Sep 17 00:00:00 2001 From: "Sujee Maniyam @ sujee-mac2" Date: Fri, 15 Dec 2017 11:35:55 -0800 Subject: [PATCH 11/19] removing redundant env variable --- docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8d0ddc6..9dfc189 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -70,7 +70,6 @@ RUN \ curl -fsL "https://d3kbcqa49mib13.cloudfront.net/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz" | tar xfz - -C ${INSTALL_DIR} && \ cd ${INSTALL_DIR} && rm -f spark && ln -s spark-${SPARK_VERSION}-bin-hadoop2.7 spark -ENV APACHE_SPARK_VERSION=${SPARK_VERSION} ## cleanup apt RUN apt-get clean && \ From 64bdcfdcf36b52fa9ebec6c6904125dcbee2fb7c Mon Sep 17 00:00:00 2001 From: Tim Fox Date: Fri, 19 Jan 2018 14:23:01 -0500 Subject: [PATCH 12/19] Added notebooks for videos --- ...forward-credit-card-default-pipeline.ipynb | 363 +++++++++ .../feedforward-credit-card-default.ipynb | 761 ++++++++++++++++++ ...edforward-credit-card-fraud-pipeline.ipynb | 549 +++++++++++++ notebooks/feedforward-credit-card-fraud.ipynb | 618 ++++++++++++++ notebooks/feedforward-iris-pipeline.ipynb | 525 ++++++++++++ notebooks/feedforward-iris.ipynb | 702 ++++++++++++++++ notebooks/lstm-20news.ipynb | 312 +++++++ notebooks/lstm-stocks.ipynb | 245 ++++++ scripts/imagenet.py | 126 +++ scripts/inception_transfer.py | 660 +++++++++++++++ scripts/run_transfer.sh | 65 ++ scripts/transformer.py | 173 ++++ 12 files changed, 5099 insertions(+) create mode 100644 notebooks/feedforward-credit-card-default-pipeline.ipynb create mode 100644 notebooks/feedforward-credit-card-default.ipynb create mode 100644 notebooks/feedforward-credit-card-fraud-pipeline.ipynb create mode 100644 notebooks/feedforward-credit-card-fraud.ipynb create mode 100644 notebooks/feedforward-iris-pipeline.ipynb create mode 100644 notebooks/feedforward-iris.ipynb create mode 100644 notebooks/lstm-20news.ipynb create mode 100644 notebooks/lstm-stocks.ipynb create mode 100644 scripts/imagenet.py create mode 100644 scripts/inception_transfer.py create mode 100644 scripts/run_transfer.sh create mode 100644 scripts/transformer.py diff --git a/notebooks/feedforward-credit-card-default-pipeline.ipynb b/notebooks/feedforward-credit-card-default-pipeline.ipynb new file mode 100644 index 0000000..721783e --- /dev/null +++ b/notebooks/feedforward-credit-card-default-pipeline.ipynb @@ -0,0 +1,363 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Feedforward Network with Credit Card Default\n", + "\n", + "Let us look at a BigDL example with Credit Card Default. We will train a simple, feedforward neural network with " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import pandas as pd\n", + "import datetime as dt\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from sklearn.metrics import confusion_matrix\n", + "from sklearn.metrics import accuracy_score\n", + "import seaborn as sn\n", + "import pandas as pd\n", + "import random as rd\n", + "import datetime as dt\n", + "\n", + "\n", + "from bigdl.dataset.transformer import *\n", + "from bigdl.dataset.base import *\n", + "from bigdl.nn.layer import *\n", + "from bigdl.nn.criterion import *\n", + "from bigdl.optim.optimizer import *\n", + "from bigdl.util.common import *\n", + "from utils import *\n", + "from bigdl.models.ml_pipeline.dl_classifier import *\n", + "\n", + "\n", + "from pyspark.sql.types import DoubleType\n", + "from pyspark.sql.functions import col, udf\n", + "from pyspark.ml import Pipeline\n", + "from pyspark.ml.feature import VectorAssembler, StandardScaler\n", + "from pyspark.ml.evaluation import BinaryClassificationEvaluator, MulticlassClassificationEvaluator\n", + "\n", + "\n", + "\n", + "\n", + "init_engine()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "learning_rate = 0.1\n", + "training_epochs = 20\n", + "batch_size = 1024\n", + "display_step = 1\n", + "\n", + "# Network Parameters\n", + "n_input = 5\n", + "n_classes = 2\n", + "n_hidden_1 = 3 # 1st layer number of features\n", + "n_hidden_2 = 2 # 1st layer number of features\n", + "\n", + "\n", + "\n", + "filename = \"../data/cc-default/default-simple.csv\"\n", + "\n", + "LABELS = [\"Good\", \"Default\"] " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hidden layer 1 (Guess) : 2.59002006411\n", + "Hidden layer 2 (Guess) : 2.2360679775\n" + ] + } + ], + "source": [ + "# Number of hidden layers\n", + "\n", + "n_hidden_guess = np.sqrt(np.sqrt((n_classes + 2) * n_input) + 2 * np.sqrt(n_input /(n_classes+2.)))\n", + "print(\"Hidden layer 1 (Guess) : \" + str(n_hidden_guess))\n", + "\n", + "n_hidden_guess_2 = n_classes * np.sqrt(n_input / (n_classes + 2.))\n", + "print(\"Hidden layer 2 (Guess) : \" + str(n_hidden_guess_2))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "cc_training = spark.read.csv(filename, header=True, inferSchema=\"true\", mode=\"DROPMALFORMED\")\n", + "cc_training = cc_training.withColumn('label', cc_training.default.cast(\"double\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+---+-------+---+---------+--------+---+-------+-----+\n", + "| id|balance|sex|education|marriage|age|default|label|\n", + "+---+-------+---+---------+--------+---+-------+-----+\n", + "| 1| 20000| 2| 2| 1| 24| 2| 2.0|\n", + "| 2| 120000| 2| 2| 2| 26| 2| 2.0|\n", + "| 3| 90000| 2| 2| 2| 34| 1| 1.0|\n", + "| 4| 50000| 2| 2| 1| 37| 1| 1.0|\n", + "| 5| 50000| 1| 2| 1| 57| 1| 1.0|\n", + "| 6| 50000| 1| 1| 2| 37| 1| 1.0|\n", + "| 7| 500000| 1| 1| 2| 29| 1| 1.0|\n", + "| 8| 100000| 2| 2| 2| 23| 1| 1.0|\n", + "| 9| 140000| 2| 3| 1| 28| 1| 1.0|\n", + "| 10| 20000| 1| 3| 2| 35| 1| 1.0|\n", + "| 11| 200000| 2| 3| 2| 34| 1| 1.0|\n", + "| 12| 260000| 2| 1| 2| 51| 1| 1.0|\n", + "| 13| 630000| 2| 2| 2| 41| 1| 1.0|\n", + "| 14| 70000| 1| 2| 2| 30| 2| 2.0|\n", + "| 15| 250000| 1| 1| 2| 29| 1| 1.0|\n", + "| 16| 50000| 2| 3| 3| 23| 1| 1.0|\n", + "| 17| 20000| 1| 1| 2| 24| 2| 2.0|\n", + "| 18| 320000| 1| 1| 1| 49| 1| 1.0|\n", + "| 19| 360000| 2| 1| 1| 49| 1| 1.0|\n", + "| 20| 180000| 2| 1| 2| 29| 1| 1.0|\n", + "+---+-------+---+---------+--------+---+-------+-----+\n", + "only showing top 20 rows\n", + "\n" + ] + } + ], + "source": [ + "cc_training.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------+------------------+------------------+------------------+------------------+-----------------+------------------+\n", + "|summary| balance| sex| education| marriage| age| default|\n", + "+-------+------------------+------------------+------------------+------------------+-----------------+------------------+\n", + "| count| 30000| 30000| 30000| 30000| 30000| 30000|\n", + "| mean|167484.32266666667|1.6037333333333332|1.8531333333333333|1.5518666666666667| 35.4855| 1.2212|\n", + "| stddev|129747.66156720246|0.4891291960902602|0.7903486597207269|0.5219696006132467|9.217904068090155|0.4150618056909329|\n", + "| min| 10000| 1| 0| 0| 21| 1|\n", + "| max| 1000000| 2| 6| 3| 79| 2|\n", + "+-------+------------------+------------------+------------------+------------------+-----------------+------------------+\n", + "\n" + ] + } + ], + "source": [ + "cc_training.select('balance','sex','education','marriage','age','default').describe().show()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEWCAYAAACufwpNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGlFJREFUeJzt3XuYZVV95vHva4MCIrcAprloo7ZG\nVEKwFROTiJcQ1CCaqBFNRB8RY2CSTMwIOjOBqDFmnDgZ1KjEMIAEBTUqJiA2RCQYBVpFLkZDB1Ea\nEBoauUlE8Dd/7FV4KE51ne5e1UXR38/znKfOWXvttdfedfq8tdbevU+qCkmSenjIfHdAkvTgYahI\nkroxVCRJ3RgqkqRuDBVJUjeGiiSpG0NFm6wk+yW5fCNv8+Qkx2zMbU7b/qok+7Xn/zPJBzu1uyjJ\n7Uke1V533c8kH07y1l7tae4YKppI+8CYevwkyZ0jr1813/2bTZLNklSSJVNlVXVuVT1p/no1v6rq\n7VX1e7PVS3J+ktfM0tY9VbV1VX1vQ/uV5NAk505r/9CqeueGtq25t9l8d0ALQ1VtPfU8yVXAoVV1\n9kz1k2xWVXdvjL5pfvm71ihHKuoiyTuSnJrko0luA34nyS8m+UqSHyS5LsmxSTZv9adGDm9IsjLJ\nzUmOHWnv8UnOS3JLkhuTnDKy7H1tGufWJBcl+aWRZZu1aZ3/aMtXJNkFOK9VubyNrn4ryfNaQE6t\n+6QkX2z9vTTJC0eWndz6f2aS25J8Ockeazkev9r2/ZYkVyf53TF1fibJGUlWt/3/bJJdR5a/LslV\nbXtXJnnFbMdmzDZek+S7rd5RY35nJ7TnWyU5JclNbf8vTLJjkr8EfhH4YDtufz3yu/v9JCuBb40b\nCQI7JTmn9f8LSXZv23pckprWl/NbX58CvA/4lba9G0eO/zEj9X+vvW9uSvLpJItb+VrfV9oIqsqH\nj3V6AFcBz5tW9g7gLuBAhj9WtgSeBuzLMCJ+DPDvwBGt/mZAAZ8BtgWWAGum2gU+DhzZ2toCeObI\ntn4X2KG1cSRwDfCwtuwtwDeApW3dvUfqFrBkpJ3nAVe15w8FvgO8Gdi8LbsdeFxbfjJwI7CsLT8V\nOHmG47MHcBvw8rbdHYG9R9o5pj3fCXhJO1bbAP8AfKIt2wa4BVjaXi8G9pzt2Ezrx1PaPjwTeBhw\nLHA3sN/I7+yE9vxw4NOtL4vafm7dlp0PvGak3alj+Tlg+7bOfY5v289bRrb9fuDctuxxQE3r673b\nAA6dqjuyfPS47Q/c0H63WwB/A/zzJO8rH3P/cKSins6vqs9W1U+q6s6quqiqLqiqu6vqSuA44FnT\n1vmLqrqlqq4CzmX4oAD4McMHwuKq+s+q+tLUClX1kapaU8OUy/9i+AB+XFt8KPDWqrqi9ePiqloz\nQd+fyRAs766qH9cwtXcm8IqROp+oqhVV9WPg70f6Ot3vAJ+rqtPavt9YVRdPr1RVq6vqU+1Y3Qq8\nc9rxKeDJSbaoquuq6puzHZtpXgZ8uqq+VFU/At4KZIa6P2YIv8fVcH5kRVXdPkPdKe+sqpur6s4Z\nln922rZ/dWpEsYFeBXy4/W7/EzgKeFaS3UbqzPS+0hwzVNTT1aMvkvxckn9K8v0ktwJvY/jgGvX9\nkec/BKbO3byJYUSwok1FHTLS7puTfCvJLcDNwMNH2t0d+I/16PsuwPeqanRa5rvAriOvZ+rrdBP1\nIcnDM1zV9L12fP6Zth8tZA5mGEF8P8k/Jnl8W3XGYzNmn+79nbSQmClgTwDOBk5Lck2SdyWZ7Zzr\n1ZMur6pbGEYuu8yyziR2YfjdTLV9K8P7YH1+V+rMUFFP0295/SHgMoa/frcB/pSZ/1K+b0PDX+aH\nVtVihg/W45LskeTZwB8DvwVsxzD9cvtIu1cDj52gb9NdC+yeZLR/j2KYWltXM/VhujczTJU9vR2f\n54wurKozq+p5DFNfKxmO54zHZkz71zEEHABJtmaYCryfqrqrqo6pqicCv8wwLTd1Vd9Mx262Yzq6\n7W0ZpqOuBe5oZVuN1P3ZdWj3WuDRI20/guF9sD6/K3VmqGguPYLhr9M7kjwReMOkKyZ5+chJ6x8w\nfNDc09q8m+H8xubAMQwjlSkfBt6R5LEZ7J1kh6q6B7iJ4dzOOP/a2n1Tks2TPAd4AXDapH0ecTJw\nQLsYYLN2wvvnx9R7BMNf0Tcn+RmG0J3a/8VJDmwfvHcxfBDf05bNdGym+zhwUIYLJh7GcA5l7Ad2\nkuckeXKShwC3MkyHTbV5PTMft7U5cNq2z6+q6xhGEd9nuJhjUZLDGAmJtr3d0i7qGOOjwOuS7NXa\n/gvgX6pq1Xr0UZ0ZKppLbwIOYThp/SGGk9uT2he4KMkdDCewD6/h/0CcwTBNcwXDBQO3MvxFPuXd\nDCecz2nLjmM4mQtwNHBKu7rpN0c31ub9DwQOYgisY4FXVtW/r0Ofp9r6TmvrSIbppq8xnDSf7j0M\nf73fxBBqZ44sWwT8t7ZvNwG/BBzRls10bKb34xLgDxmC8Rp++mE+zi6trVuByxmO8Ufbsr8GDm7H\n7T2z7P6okxnC5EZgL4YLLGhTjK9nOM9yI8P5sAtG1lvO8Pu9Psn9+ltVn2OYSv0Uw/F5FD8dVWme\n5b5TyJIkrT9HKpKkbgwVSVI3hookqRtDRZLUzSZ3Q8kdd9yxlixZMt/dkKQF5atf/eqNVbXTbPU2\nuVBZsmQJK1asmO9uSNKCkuS7s9dy+kuS1JGhIknqxlCRJHVjqEiSujFUJEndGCqSpG4MFUlSN4aK\nJKkbQ0WS1M0m9z/qF4olR/3TfHfhQeOqd71wvrsgbTIcqUiSujFUJEndGCqSpG4MFUlSN4aKJKkb\nQ0WS1I2hIknqxlCRJHVjqEiSujFUJEndGCqSpG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHVjqEiS\nujFUJEndGCqSpG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHVjqEiSupmzUEmye5IvJPm3JJcn+cNW\nvkOS5UmuaD+3b+VJcmySlUkuSbLPSFuHtPpXJDlkpPypSS5t6xybJHO1P5Kk2c3lSOVu4E1V9UTg\nGcDhSfYEjgLOqaqlwDntNcDzgaXtcRjwARhCCDga2Bd4OnD0VBC1OoeNrHfAHO6PJGkWcxYqVXVd\nVX2tPb8N+DdgV+Ag4MRW7UTgxe35QcBJNfgKsF2SxcCvA8urak1V3QwsBw5oy7apqi9XVQEnjbQl\nSZoHG+WcSpIlwC8AFwCPrKrrYAgeYOdWbVfg6pHVVrWytZWvGlMuSZoncx4qSbYGPgn8UVXduraq\nY8pqPcrH9eGwJCuSrFi9evVsXZYkrac5DZUkmzMEyt9X1T+04uvb1BXt5w2tfBWw+8jquwHXzlK+\n25jy+6mq46pqWVUt22mnnTZspyRJM5rLq78C/B3wb1X1npFFpwNTV3AdAnxmpPzV7SqwZwC3tOmx\ns4D9k2zfTtDvD5zVlt2W5BltW68eaUuSNA82m8O2nwn8LnBpkotb2VuBdwGnJXkd8D3gZW3ZGcAL\ngJXAD4HXAlTVmiRvBy5q9d5WVWva8zcCJwBbAme2hyRpnsxZqFTV+Yw/7wHw3DH1Czh8hraOB44f\nU74CePIGdFOS1JH/o16S1I2hIknqxlCRJHVjqEiSujFUJEndGCqSpG4MFUlSN4aKJKkbQ0WS1I2h\nIknqxlCRJHVjqEiSujFUJEndGCqSpG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHVjqEiSujFUJEnd\nGCqSpG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHVjqEiSujFUJEndGCqSpG4MFUlSN4aKJKkbQ0WS\n1I2hIknqxlCRJHVjqEiSupmzUElyfJIbklw2UnZMkmuSXNweLxhZ9pYkK5N8O8mvj5Qf0MpWJjlq\npHyPJBckuSLJqUkeOlf7IkmazFyOVE4ADhhT/n+qau/2OAMgyZ7AK4AntXX+JsmiJIuA9wPPB/YE\nDm51Af6ytbUUuBl43RzuiyRpAnMWKlV1HrBmwuoHAR+rqh9V1XeAlcDT22NlVV1ZVXcBHwMOShLg\nOcAn2vonAi/uugOSpHU2UagkeXLHbR6R5JI2PbZ9K9sVuHqkzqpWNlP5zwA/qKq7p5WPleSwJCuS\nrFi9enWv/ZAkTTPpSOWDSS5M8vtJttuA7X0AeCywN3Ad8FetPGPq1nqUj1VVx1XVsqpattNOO61b\njyVJE5soVKrql4FXAbsDK5KckuTX1nVjVXV9Vd1TVT8B/pZheguGkcbuI1V3A65dS/mNwHZJNptW\nLkmaRxOfU6mqK4D/ARwJPAs4Nsm3kvzmpG0kWTzy8iXA1JVhpwOvSPKwJHsAS4ELgYuApe1Kr4cy\nnMw/vaoK+ALw0rb+IcBnJu2HJGlubDZ7FUiyF/Ba4IXAcuDAqvpakl2ALwP/MGadjwL7ATsmWQUc\nDeyXZG+GqaqrgDcAVNXlSU4DvgncDRxeVfe0do4AzgIWAcdX1eVtE0cCH0vyDuDrwN+t895Lkrqa\nKFSA9zFMV721qu6cKqyqa5P8j3ErVNXBY4pn/OCvqj8H/nxM+RnAGWPKr+Sn02eSpAeASUPlBcCd\nI6OHhwBbVNUPq+ojc9Y7SdKCMuk5lbOBLUdeb9XKJEm616ShskVV3T71oj3fam66JElaqCYNlTuS\n7DP1IslTgTvXUl+StAma9JzKHwEfTzL1f0EWA789N12SJC1UE4VKVV2U5OeAJzD8b/ZvVdWP57Rn\nkqQFZ9KRCsDTgCVtnV9IQlWdNCe9kiQtSJP+58ePMNyz62LgnlZcgKEiSbrXpCOVZcCe7fYokiSN\nNenVX5cBPzuXHZEkLXyTjlR2BL6Z5ELgR1OFVfWiOemVJGlBmjRUjpnLTkiSHhwmvaT4i0keDSyt\nqrOTbMVw12BJku416dcJv57h++A/1Ip2BT49V52SJC1Mk56oPxx4JnAr3PuFXTvPVackSQvTpKHy\no6q6a+pF+xpfLy+WJN3HpKHyxSRvBbZs303/ceCzc9ctSdJCNGmoHAWsBi5l+ArgMxi+r16SpHtN\nevXXTxi+Tvhv57Y7kqSFbNJ7f32HMedQquox3XskSVqw1uXeX1O2AF4G7NC/O5KkhWyicypVddPI\n45qq+mvgOXPcN0nSAjPp9Nc+Iy8fwjByecSc9EiStGBNOv31VyPP7wauAl7evTeSpAVt0qu/nj3X\nHZEkLXyTTn/98dqWV9V7+nRHkrSQrcvVX08DTm+vDwTOA66ei05JkhamdfmSrn2q6jaAJMcAH6+q\nQ+eqY5KkhWfS27Q8Crhr5PVdwJLuvZEkLWiTjlQ+AlyY5FMM/7P+JcBJc9YrSdKCNOnVX3+e5Ezg\nV1rRa6vq63PXLUnSQjTp9BfAVsCtVfV/gVVJ9pijPkmSFqhJv074aOBI4C2taHPg5LnqlCRpYZp0\npPIS4EXAHQBVdS3epkWSNM2koXJXVRXt9vdJHj53XZIkLVSThsppST4EbJfk9cDZzPKFXUmOT3JD\nkstGynZIsjzJFe3n9q08SY5NsjLJJaM3sExySKt/RZJDRsqfmuTSts6xSbIuOy5J6m/SW9//b+AT\nwCeBJwB/WlXvnWW1E4ADppUdBZxTVUuBc9prgOcDS9vjMOADMIQQcDSwL/B04OipIGp1DhtZb/q2\nJEkb2ayXFCdZBJxVVc8Dlk/acFWdl2TJtOKDgP3a8xOBcxkuADgIOKlNsX0lyXZJFre6y6tqTevL\ncuCAJOcC21TVl1v5ScCLgTMn7Z8kqb9ZRypVdQ/wwyTbdtjeI6vqutbudcDOrXxX7nsfsVWtbG3l\nq8aUj5XksCQrkqxYvXr1Bu+EJGm8Sf9H/X8Cl7aRwh1ThVX1B536Me58SK1H+VhVdRxwHMCyZctm\nrCdJ2jCThso/tceGuj7J4qq6rk1v3dDKVwG7j9TbDbi2le83rfzcVr7bmPqSpHm01lBJ8qiq+l5V\nndhpe6cDhwDvaj8/M1J+RJKPMZyUv6UFz1nAO0dOzu8PvKWq1iS5LckzgAuAVwOzXTggSZpjs51T\n+fTUkySfXJeGk3wU+DLwhCSrkryOIUx+LckVwK+11wBnAFcCKxkuVf59gHaC/u3ARe3xtqmT9sAb\ngQ+3df4DT9JL0rybbfpr9NzFY9al4ao6eIZFzx1Tt4DDZ2jneOD4MeUrgCevS58kSXNrtpFKzfBc\nkqT7mW2k8vNJbmUYsWzZntNeV1VtM6e9kyQtKGsNlapatLE6Ikla+Nbl+1QkSVorQ0WS1I2hIknq\nxlCRJHVjqEiSujFUJEndGCqSpG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHVjqEiSujFUJEndGCqS\npG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHVjqEiSutlsvjsgaYE5Ztv57sGDyzG3zHcPunKkIknq\nxlCRJHVjqEiSujFUJEndGCqSpG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHUzL6GS5Koklya5OMmK\nVrZDkuVJrmg/t2/lSXJskpVJLkmyz0g7h7T6VyQ5ZD72RZL0U/M5Unl2Ve1dVcva66OAc6pqKXBO\new3wfGBpexwGfACGEAKOBvYFng4cPRVEkqT58UCa/joIOLE9PxF48Uj5STX4CrBdksXArwPLq2pN\nVd0MLAcO2NidliT91HyFSgGfT/LVJIe1skdW1XUA7efOrXxX4OqRdVe1spnK7yfJYUlWJFmxevXq\njrshSRo1X7e+f2ZVXZtkZ2B5km+tpW7GlNVayu9fWHUccBzAsmXLxtaRJG24eRmpVNW17ecNwKcY\nzolc36a1aD9vaNVXAbuPrL4bcO1ayiVJ82Sjh0qShyd5xNRzYH/gMuB0YOoKrkOAz7TnpwOvbleB\nPQO4pU2PnQXsn2T7doJ+/1YmSZon8zH99UjgU0mmtn9KVX0uyUXAaUleB3wPeFmrfwbwAmAl8EPg\ntQBVtSbJ24GLWr23VdWajbcbkqTpNnqoVNWVwM+PKb8JeO6Y8gIOn6Gt44Hje/dRkrR+HkiXFEuS\nFjhDRZLUjaEiSerGUJEkdWOoSJK6MVQkSd0YKpKkbgwVSVI3hookqRtDRZLUjaEiSerGUJEkdWOo\nSJK6MVQkSd0YKpKkbgwVSVI3hookqRtDRZLUjaEiSerGUJEkdWOoSJK6MVQkSd0YKpKkbgwVSVI3\nhookqRtDRZLUjaEiSerGUJEkdWOoSJK6MVQkSd0YKpKkbgwVSVI3hookqRtDRZLUjaEiSerGUJEk\ndbPgQyXJAUm+nWRlkqPmuz+StClb0KGSZBHwfuD5wJ7AwUn2nN9eSdKma0GHCvB0YGVVXVlVdwEf\nAw6a5z5J0iZrs/nuwAbaFbh65PUqYN/plZIcBhzWXt6e5NsboW+bgh2BG+e7E7PJX853DzRPFsT7\nkz/LfPdgUo+epNJCD5Vxv426X0HVccBxc9+dTUuSFVW1bL77IY3j+3N+LPTpr1XA7iOvdwOunae+\nSNImb6GHykXA0iR7JHko8Arg9HnukyRtshb09FdV3Z3kCOAsYBFwfFVdPs/d2pQ4pagHMt+f8yBV\n9zsFIUnSelno01+SpAcQQ0WS1I2hovtI8sgkpyS5MslXk3w5yUs6tHtuEi/v1KyS3JPk4iSXJ/lG\nkj9OMutnVZJ3t3XevZ7bvb39XJLklevThhb4iXr1lSTAp4ETq+qVrezRwIvmtWPa1NxZVXsDJNkZ\nOAXYFjh6lvXeAOxUVT/awO0vAV7Ztqt15EhFo54D3FVVH5wqqKrvVtV7k2yR5P8luTTJ15M8G2At\n5Vsm+ViSS5KcCmw5P7ukhayqbmC4G8YRGSxqI5KL2nvrDQBJTgceDlyQ5LeTHJjkgvaePDvJI1u9\nY5L8yVT7SS5LsmTaZt8F/EobLf3XjbGfDyaOVDTqScDXZlh2OEBVPSXJzwGfT/L4tZS/EfhhVe2V\nZK+1tCutVVVd2aa/dma4t98tVfW0JA8DvpTk81X1oiS3j4xwtgeeUVWV5FDgzcCbJtzkUcCfVNVv\nzMHuPOgZKppRkvcDvwzcxXD3gvcCVNW3knwXeHxbPq78V4FjW/klSS7Z+HugB5GpWzLtD+yV5KXt\n9bbAUuA70+rvBpyaZDHw0DHLNUec/tKoy4F9pl5U1eHAc4GdGH+fNdZSDmPuwyatqySPAe4BbmB4\nv/2Xqtq7Pfaoqs+PWe29wPuq6ikM51q2aOV3c9/PvS2mr6gNY6ho1D8DWyR540jZVu3necCrANr0\n1qOAb09Y/mRgr43Qfz3IJNkJ+CBDQBTD3TPemGTztvzxSR4+ZtVtgWva80NGyq+i/eGUZB9gjzHr\n3gY8ossObIIMFd2r/aN9MfCsJN9JciFwInAk8DfAoiSXAqcCr2lX2cxU/gFg6zbt9Wbgwo2/R1qg\ntpy6pBg4G/g88Gdt2YeBbwJfS3IZ8CHGT+MfA3w8yb9w39vffxLYIcnFDOf9/n3MupcAd7fLmT1R\nv468TYskqRtHKpKkbgwVSVI3hookqRtDRZLUjaEiSerGUJE6S/Lf291yL2mXxu6b5I+SbDXBuhPV\nkx6ovKRY6ijJLwLvAfarqh8l2ZHhNiH/CiyrqhtnWf+qSepJD1SOVKS+FgM3Tt1+vYXDS4FdgC8k\n+QJAkg8kWdFGNH/Wyv5gTL3bpxpO8tIkJ7TnL2t32P1GkvM24v5Ja+VIReooydbA+Qy3tzkbOLWq\nvjh9BJJkh6pak2QRcA7wB+3Gm9Pr3V5VW7fnLwV+o6pe0+5gcEBVXZNku6r6wcbeV2kcRypSR1V1\nO/BUhu8AWc1wp9zXjKn68iRfA77O8JUDe67jpr4EnJDk9cCi9e+x1Je3vpc6q6p7gHOBc9uIYvSG\nhiTZA/gT4GlVdXOb0prpbrmjUwn31qmq30uyL/BC4OIke1fVTf32Qlo/jlSkjpI8IcnSkaK9ge9y\n3zvfbgPcAdzSvpHw+SP1p98h9/okT2xfUvWSke08tqouqKo/Zbhh4u7990Zad45UpL62Bt6bZDuG\n7+5YyTAVdjBwZpLrqurZSb7O8P01VzJMZU05brQew7cQ/iNwNXBZax/g3S28wnBO5htzv2vS7DxR\nL0nqxukvSVI3hookqRtDRZLUjaEiSerGUJEkdWOoSJK6MVQkSd38f7J5Bknl8uPZAAAAAElFTkSu\nQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#count_classes = pd.value_counts(df['Class'], sort = True)\n", + "count_classes = pd.value_counts(cc_training.select('default').toPandas()['default'], sort = True)\n", + "count_classes.plot(kind = 'bar', rot=0)\n", + "plt.title(\"Transaction class distribution\")\n", + "plt.xticks(range(2), LABELS)\n", + "plt.xlabel(\"Status\")\n", + "plt.ylabel(\"Frequency\");" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "(trainingData, validData) = cc_training.select('balance','sex','education','marriage','age','label').randomSplit([.7,.3])" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "assembler = VectorAssembler(inputCols=['balance','sex','education','marriage','age'], outputCol=\"assembled\")\n", + "scaler = StandardScaler(inputCol=\"assembled\", outputCol=\"features\")\n", + "pipeline = Pipeline(stages = [assembler, scaler])\n", + "pipelineTraining = pipeline.fit(trainingData)\n", + "cc_data_training = pipelineTraining.transform(trainingData)\n", + "pipelineTest = pipeline.fit(validData)\n", + "cc_data_test = pipelineTest.transform(validData)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[[ 0.07713293, 2.04703291, 1.26444402, 1.91475904, 4.4585972 ]],\n", + "\n", + " [[ 0.07713293, 2.04703291, 1.26444402, 1.91475904, 5.43731365]],\n", + "\n", + " [[ 0.07713293, 2.04703291, 1.26444402, 3.82951807, 2.39241801]],\n", + "\n", + " ..., \n", + " [[ 5.78496992, 4.09406582, 3.79333205, 1.91475904, 4.34985092]],\n", + "\n", + " [[ 5.86210285, 2.04703291, 3.79333205, 1.91475904, 5.87229875]],\n", + "\n", + " [[ 6.01636872, 2.04703291, 1.26444402, 1.91475904, 5.21982111]]])" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "np.array(cc_data_training.select('features').collect())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bigDLModel = Sequential().add(Linear(n_input, n_hidden_1)).add(Linear(n_hidden_1, n_classes)).add(LogSoftMax())\n", + "classnll_criterion = ClassNLLCriterion()\n", + "dlClassifier = DLClassifier(model=bigDLModel, criterion=classnll_criterion, feature_size=[n_input])\n", + "dlClassifier.setLabelCol(\"default\").setMaxEpoch(training_epochs).setBatchSize(batch_size)\n", + "model = dlClassifier.fit(cc_data_training)\n", + "print(\"\\ninitial model training finished.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql import DataFrame, SQLContext\n", + "predictionDF = DataFrame(model.transform(cc_data_test), SQLContext(sc))\n", + "predictionDF" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "predictionDF.cache()\n", + "evaluator = BinaryClassificationEvaluator(rawPredictionCol=\"prediction\")\n", + "auPRC = evaluator.evaluate(predictionDF)\n", + "print(\"\\nArea under precision-recall curve: = \" + str(auPRC))\n", + " \n", + "recall = MulticlassClassificationEvaluator(metricName=\"weightedRecall\").evaluate(predictionDF)\n", + "print(\"\\nrecall = \" + str(recall))\n", + "\n", + "precision = MulticlassClassificationEvaluator(metricName=\"weightedPrecision\").evaluate(predictionDF)\n", + "print(\"\\nPrecision = \" + str(precision))\n", + "predictionDF.unpersist()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "y_pred = np.array(predictionDF.select('prediction').collect())\n", + "y_true = np.array(predictionDF.select('label').collect())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "acc = accuracy_score(y_true, y_pred)\n", + "print(\"The prediction accuracy is %.2f%%\"%(acc*100))\n", + "\n", + "cm = confusion_matrix(y_true, y_pred)\n", + "cm.shape\n", + "df_cm = pd.DataFrame(cm)\n", + "plt.figure(figsize = (10,8))\n", + "sn.heatmap(df_cm, annot=True,fmt='d');" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/feedforward-credit-card-default.ipynb b/notebooks/feedforward-credit-card-default.ipynb new file mode 100644 index 0000000..0bbb29b --- /dev/null +++ b/notebooks/feedforward-credit-card-default.ipynb @@ -0,0 +1,761 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Feedforward Network with Credit Card Default\n", + "\n", + "Let us look at a BigDL example with Credit Card Default. We will train a simple, feedforward neural network with " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import pandas as pd\n", + "import datetime as dt\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from sklearn.metrics import confusion_matrix\n", + "from sklearn.metrics import accuracy_score\n", + "import seaborn as sn\n", + "import pandas as pd\n", + "import random as rd\n", + "import datetime as dt\n", + "\n", + "\n", + "from bigdl.dataset.transformer import *\n", + "from bigdl.dataset.base import *\n", + "from bigdl.nn.layer import *\n", + "from bigdl.nn.criterion import *\n", + "from bigdl.optim.optimizer import *\n", + "from bigdl.util.common import *\n", + "from utils import *\n", + "from bigdl.models.ml_pipeline.dl_classifier import *\n", + "\n", + "\n", + "from pyspark.sql.types import DoubleType\n", + "from pyspark.sql.functions import col, udf\n", + "from pyspark.ml import Pipeline\n", + "from pyspark.ml.feature import VectorAssembler, StandardScaler\n", + "from pyspark.ml.evaluation import BinaryClassificationEvaluator, MulticlassClassificationEvaluator\n", + "\n", + "\n", + "\n", + "\n", + "init_engine()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "learning_rate = 0.01\n", + "training_epochs = 60\n", + "batch_size = 1024\n", + "display_step = 1\n", + "\n", + "# Network Parameters\n", + "n_input = 5\n", + "n_classes = 2\n", + "n_hidden_1 = 3 # 1st layer number of features\n", + "n_hidden_2 = 2 # 1st layer number of features\n", + "\n", + "\n", + "\n", + "filename = \"../data/cc-default/default-simple.csv\"\n", + "\n", + "LABELS = [\"Good\", \"Default\"] " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hidden layer 1 (Guess) : 2.59002006411\n", + "Hidden layer 2 (Guess) : 2.2360679775\n" + ] + } + ], + "source": [ + "# Number of hidden layers\n", + "\n", + "n_hidden_guess = np.sqrt(np.sqrt((n_classes + 2) * n_input) + 2 * np.sqrt(n_input /(n_classes+2.)))\n", + "print(\"Hidden layer 1 (Guess) : \" + str(n_hidden_guess))\n", + "\n", + "n_hidden_guess_2 = n_classes * np.sqrt(n_input / (n_classes + 2.))\n", + "print(\"Hidden layer 2 (Guess) : \" + str(n_hidden_guess_2))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "cc_training = spark.read.csv(filename, header=True, inferSchema=\"true\", mode=\"DROPMALFORMED\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.read_csv(filename)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+---+-------+---+---------+--------+---+-------+\n", + "| id|balance|sex|education|marriage|age|default|\n", + "+---+-------+---+---------+--------+---+-------+\n", + "| 1| 20000| 2| 2| 1| 24| 2|\n", + "| 2| 120000| 2| 2| 2| 26| 2|\n", + "| 3| 90000| 2| 2| 2| 34| 1|\n", + "| 4| 50000| 2| 2| 1| 37| 1|\n", + "| 5| 50000| 1| 2| 1| 57| 1|\n", + "| 6| 50000| 1| 1| 2| 37| 1|\n", + "| 7| 500000| 1| 1| 2| 29| 1|\n", + "| 8| 100000| 2| 2| 2| 23| 1|\n", + "| 9| 140000| 2| 3| 1| 28| 1|\n", + "| 10| 20000| 1| 3| 2| 35| 1|\n", + "| 11| 200000| 2| 3| 2| 34| 1|\n", + "| 12| 260000| 2| 1| 2| 51| 1|\n", + "| 13| 630000| 2| 2| 2| 41| 1|\n", + "| 14| 70000| 1| 2| 2| 30| 2|\n", + "| 15| 250000| 1| 1| 2| 29| 1|\n", + "| 16| 50000| 2| 3| 3| 23| 1|\n", + "| 17| 20000| 1| 1| 2| 24| 2|\n", + "| 18| 320000| 1| 1| 1| 49| 1|\n", + "| 19| 360000| 2| 1| 1| 49| 1|\n", + "| 20| 180000| 2| 1| 2| 29| 1|\n", + "+---+-------+---+---------+--------+---+-------+\n", + "only showing top 20 rows\n", + "\n" + ] + } + ], + "source": [ + "cc_training.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------+------------------+------------------+------------------+------------------+-----------------+------------------+\n", + "|summary| balance| sex| education| marriage| age| default|\n", + "+-------+------------------+------------------+------------------+------------------+-----------------+------------------+\n", + "| count| 30000| 30000| 30000| 30000| 30000| 30000|\n", + "| mean|167484.32266666667|1.6037333333333332|1.8531333333333333|1.5518666666666667| 35.4855| 1.2212|\n", + "| stddev|129747.66156720246|0.4891291960902602|0.7903486597207269|0.5219696006132467|9.217904068090155|0.4150618056909329|\n", + "| min| 10000| 1| 0| 0| 21| 1|\n", + "| max| 1000000| 2| 6| 3| 79| 2|\n", + "+-------+------------------+------------------+------------------+------------------+-----------------+------------------+\n", + "\n" + ] + } + ], + "source": [ + "cc_training.select('balance','sex','education','marriage','age','default').describe().show()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "cc_training = cc_training.select([col(c).cast(\"double\") for c in cc_training.columns])\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----+--------+---+---------+--------+----+-------+\n", + "| id| balance|sex|education|marriage| age|default|\n", + "+----+--------+---+---------+--------+----+-------+\n", + "| 1.0| 20000.0|2.0| 2.0| 1.0|24.0| 2.0|\n", + "| 2.0|120000.0|2.0| 2.0| 2.0|26.0| 2.0|\n", + "| 3.0| 90000.0|2.0| 2.0| 2.0|34.0| 1.0|\n", + "| 4.0| 50000.0|2.0| 2.0| 1.0|37.0| 1.0|\n", + "| 5.0| 50000.0|1.0| 2.0| 1.0|57.0| 1.0|\n", + "| 6.0| 50000.0|1.0| 1.0| 2.0|37.0| 1.0|\n", + "| 7.0|500000.0|1.0| 1.0| 2.0|29.0| 1.0|\n", + "| 8.0|100000.0|2.0| 2.0| 2.0|23.0| 1.0|\n", + "| 9.0|140000.0|2.0| 3.0| 1.0|28.0| 1.0|\n", + "|10.0| 20000.0|1.0| 3.0| 2.0|35.0| 1.0|\n", + "|11.0|200000.0|2.0| 3.0| 2.0|34.0| 1.0|\n", + "|12.0|260000.0|2.0| 1.0| 2.0|51.0| 1.0|\n", + "|13.0|630000.0|2.0| 2.0| 2.0|41.0| 1.0|\n", + "|14.0| 70000.0|1.0| 2.0| 2.0|30.0| 2.0|\n", + "|15.0|250000.0|1.0| 1.0| 2.0|29.0| 1.0|\n", + "|16.0| 50000.0|2.0| 3.0| 3.0|23.0| 1.0|\n", + "|17.0| 20000.0|1.0| 1.0| 2.0|24.0| 2.0|\n", + "|18.0|320000.0|1.0| 1.0| 1.0|49.0| 1.0|\n", + "|19.0|360000.0|2.0| 1.0| 1.0|49.0| 1.0|\n", + "|20.0|180000.0|2.0| 1.0| 2.0|29.0| 1.0|\n", + "+----+--------+---+---------+--------+----+-------+\n", + "only showing top 20 rows\n", + "\n" + ] + } + ], + "source": [ + "cc_training.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEWCAYAAACufwpNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGlFJREFUeJzt3XuYZVV95vHva4MCIrcAprloo7ZG\nVEKwFROTiJcQ1CCaqBFNRB8RY2CSTMwIOjOBqDFmnDgZ1KjEMIAEBTUqJiA2RCQYBVpFLkZDB1Ea\nEBoauUlE8Dd/7FV4KE51ne5e1UXR38/znKfOWXvttdfedfq8tdbevU+qCkmSenjIfHdAkvTgYahI\nkroxVCRJ3RgqkqRuDBVJUjeGiiSpG0NFm6wk+yW5fCNv8+Qkx2zMbU7b/qok+7Xn/zPJBzu1uyjJ\n7Uke1V533c8kH07y1l7tae4YKppI+8CYevwkyZ0jr1813/2bTZLNklSSJVNlVXVuVT1p/no1v6rq\n7VX1e7PVS3J+ktfM0tY9VbV1VX1vQ/uV5NAk505r/9CqeueGtq25t9l8d0ALQ1VtPfU8yVXAoVV1\n9kz1k2xWVXdvjL5pfvm71ihHKuoiyTuSnJrko0luA34nyS8m+UqSHyS5LsmxSTZv9adGDm9IsjLJ\nzUmOHWnv8UnOS3JLkhuTnDKy7H1tGufWJBcl+aWRZZu1aZ3/aMtXJNkFOK9VubyNrn4ryfNaQE6t\n+6QkX2z9vTTJC0eWndz6f2aS25J8Ockeazkev9r2/ZYkVyf53TF1fibJGUlWt/3/bJJdR5a/LslV\nbXtXJnnFbMdmzDZek+S7rd5RY35nJ7TnWyU5JclNbf8vTLJjkr8EfhH4YDtufz3yu/v9JCuBb40b\nCQI7JTmn9f8LSXZv23pckprWl/NbX58CvA/4lba9G0eO/zEj9X+vvW9uSvLpJItb+VrfV9oIqsqH\nj3V6AFcBz5tW9g7gLuBAhj9WtgSeBuzLMCJ+DPDvwBGt/mZAAZ8BtgWWAGum2gU+DhzZ2toCeObI\ntn4X2KG1cSRwDfCwtuwtwDeApW3dvUfqFrBkpJ3nAVe15w8FvgO8Gdi8LbsdeFxbfjJwI7CsLT8V\nOHmG47MHcBvw8rbdHYG9R9o5pj3fCXhJO1bbAP8AfKIt2wa4BVjaXi8G9pzt2Ezrx1PaPjwTeBhw\nLHA3sN/I7+yE9vxw4NOtL4vafm7dlp0PvGak3alj+Tlg+7bOfY5v289bRrb9fuDctuxxQE3r673b\nAA6dqjuyfPS47Q/c0H63WwB/A/zzJO8rH3P/cKSins6vqs9W1U+q6s6quqiqLqiqu6vqSuA44FnT\n1vmLqrqlqq4CzmX4oAD4McMHwuKq+s+q+tLUClX1kapaU8OUy/9i+AB+XFt8KPDWqrqi9ePiqloz\nQd+fyRAs766qH9cwtXcm8IqROp+oqhVV9WPg70f6Ot3vAJ+rqtPavt9YVRdPr1RVq6vqU+1Y3Qq8\nc9rxKeDJSbaoquuq6puzHZtpXgZ8uqq+VFU/At4KZIa6P2YIv8fVcH5kRVXdPkPdKe+sqpur6s4Z\nln922rZ/dWpEsYFeBXy4/W7/EzgKeFaS3UbqzPS+0hwzVNTT1aMvkvxckn9K8v0ktwJvY/jgGvX9\nkec/BKbO3byJYUSwok1FHTLS7puTfCvJLcDNwMNH2t0d+I/16PsuwPeqanRa5rvAriOvZ+rrdBP1\nIcnDM1zV9L12fP6Zth8tZA5mGEF8P8k/Jnl8W3XGYzNmn+79nbSQmClgTwDOBk5Lck2SdyWZ7Zzr\n1ZMur6pbGEYuu8yyziR2YfjdTLV9K8P7YH1+V+rMUFFP0295/SHgMoa/frcB/pSZ/1K+b0PDX+aH\nVtVihg/W45LskeTZwB8DvwVsxzD9cvtIu1cDj52gb9NdC+yeZLR/j2KYWltXM/VhujczTJU9vR2f\n54wurKozq+p5DFNfKxmO54zHZkz71zEEHABJtmaYCryfqrqrqo6pqicCv8wwLTd1Vd9Mx262Yzq6\n7W0ZpqOuBe5oZVuN1P3ZdWj3WuDRI20/guF9sD6/K3VmqGguPYLhr9M7kjwReMOkKyZ5+chJ6x8w\nfNDc09q8m+H8xubAMQwjlSkfBt6R5LEZ7J1kh6q6B7iJ4dzOOP/a2n1Tks2TPAd4AXDapH0ecTJw\nQLsYYLN2wvvnx9R7BMNf0Tcn+RmG0J3a/8VJDmwfvHcxfBDf05bNdGym+zhwUIYLJh7GcA5l7Ad2\nkuckeXKShwC3MkyHTbV5PTMft7U5cNq2z6+q6xhGEd9nuJhjUZLDGAmJtr3d0i7qGOOjwOuS7NXa\n/gvgX6pq1Xr0UZ0ZKppLbwIOYThp/SGGk9uT2he4KMkdDCewD6/h/0CcwTBNcwXDBQO3MvxFPuXd\nDCecz2nLjmM4mQtwNHBKu7rpN0c31ub9DwQOYgisY4FXVtW/r0Ofp9r6TmvrSIbppq8xnDSf7j0M\nf73fxBBqZ44sWwT8t7ZvNwG/BBzRls10bKb34xLgDxmC8Rp++mE+zi6trVuByxmO8Ufbsr8GDm7H\n7T2z7P6okxnC5EZgL4YLLGhTjK9nOM9yI8P5sAtG1lvO8Pu9Psn9+ltVn2OYSv0Uw/F5FD8dVWme\n5b5TyJIkrT9HKpKkbgwVSVI3hookqRtDRZLUzSZ3Q8kdd9yxlixZMt/dkKQF5atf/eqNVbXTbPU2\nuVBZsmQJK1asmO9uSNKCkuS7s9dy+kuS1JGhIknqxlCRJHVjqEiSujFUJEndGCqSpG4MFUlSN4aK\nJKkbQ0WS1M0m9z/qF4olR/3TfHfhQeOqd71wvrsgbTIcqUiSujFUJEndGCqSpG4MFUlSN4aKJKkb\nQ0WS1I2hIknqxlCRJHVjqEiSujFUJEndGCqSpG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHVjqEiS\nujFUJEndGCqSpG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHVjqEiSupmzUEmye5IvJPm3JJcn+cNW\nvkOS5UmuaD+3b+VJcmySlUkuSbLPSFuHtPpXJDlkpPypSS5t6xybJHO1P5Kk2c3lSOVu4E1V9UTg\nGcDhSfYEjgLOqaqlwDntNcDzgaXtcRjwARhCCDga2Bd4OnD0VBC1OoeNrHfAHO6PJGkWcxYqVXVd\nVX2tPb8N+DdgV+Ag4MRW7UTgxe35QcBJNfgKsF2SxcCvA8urak1V3QwsBw5oy7apqi9XVQEnjbQl\nSZoHG+WcSpIlwC8AFwCPrKrrYAgeYOdWbVfg6pHVVrWytZWvGlMuSZoncx4qSbYGPgn8UVXduraq\nY8pqPcrH9eGwJCuSrFi9evVsXZYkrac5DZUkmzMEyt9X1T+04uvb1BXt5w2tfBWw+8jquwHXzlK+\n25jy+6mq46pqWVUt22mnnTZspyRJM5rLq78C/B3wb1X1npFFpwNTV3AdAnxmpPzV7SqwZwC3tOmx\ns4D9k2zfTtDvD5zVlt2W5BltW68eaUuSNA82m8O2nwn8LnBpkotb2VuBdwGnJXkd8D3gZW3ZGcAL\ngJXAD4HXAlTVmiRvBy5q9d5WVWva8zcCJwBbAme2hyRpnsxZqFTV+Yw/7wHw3DH1Czh8hraOB44f\nU74CePIGdFOS1JH/o16S1I2hIknqxlCRJHVjqEiSujFUJEndGCqSpG4MFUlSN4aKJKkbQ0WS1I2h\nIknqxlCRJHVjqEiSujFUJEndGCqSpG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHVjqEiSujFUJEnd\nGCqSpG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHVjqEiSujFUJEndGCqSpG4MFUlSN4aKJKkbQ0WS\n1I2hIknqxlCRJHVjqEiSupmzUElyfJIbklw2UnZMkmuSXNweLxhZ9pYkK5N8O8mvj5Qf0MpWJjlq\npHyPJBckuSLJqUkeOlf7IkmazFyOVE4ADhhT/n+qau/2OAMgyZ7AK4AntXX+JsmiJIuA9wPPB/YE\nDm51Af6ytbUUuBl43RzuiyRpAnMWKlV1HrBmwuoHAR+rqh9V1XeAlcDT22NlVV1ZVXcBHwMOShLg\nOcAn2vonAi/uugOSpHU2UagkeXLHbR6R5JI2PbZ9K9sVuHqkzqpWNlP5zwA/qKq7p5WPleSwJCuS\nrFi9enWv/ZAkTTPpSOWDSS5M8vtJttuA7X0AeCywN3Ad8FetPGPq1nqUj1VVx1XVsqpattNOO61b\njyVJE5soVKrql4FXAbsDK5KckuTX1nVjVXV9Vd1TVT8B/pZheguGkcbuI1V3A65dS/mNwHZJNptW\nLkmaRxOfU6mqK4D/ARwJPAs4Nsm3kvzmpG0kWTzy8iXA1JVhpwOvSPKwJHsAS4ELgYuApe1Kr4cy\nnMw/vaoK+ALw0rb+IcBnJu2HJGlubDZ7FUiyF/Ba4IXAcuDAqvpakl2ALwP/MGadjwL7ATsmWQUc\nDeyXZG+GqaqrgDcAVNXlSU4DvgncDRxeVfe0do4AzgIWAcdX1eVtE0cCH0vyDuDrwN+t895Lkrqa\nKFSA9zFMV721qu6cKqyqa5P8j3ErVNXBY4pn/OCvqj8H/nxM+RnAGWPKr+Sn02eSpAeASUPlBcCd\nI6OHhwBbVNUPq+ojc9Y7SdKCMuk5lbOBLUdeb9XKJEm616ShskVV3T71oj3fam66JElaqCYNlTuS\n7DP1IslTgTvXUl+StAma9JzKHwEfTzL1f0EWA789N12SJC1UE4VKVV2U5OeAJzD8b/ZvVdWP57Rn\nkqQFZ9KRCsDTgCVtnV9IQlWdNCe9kiQtSJP+58ePMNyz62LgnlZcgKEiSbrXpCOVZcCe7fYokiSN\nNenVX5cBPzuXHZEkLXyTjlR2BL6Z5ELgR1OFVfWiOemVJGlBmjRUjpnLTkiSHhwmvaT4i0keDSyt\nqrOTbMVw12BJku416dcJv57h++A/1Ip2BT49V52SJC1Mk56oPxx4JnAr3PuFXTvPVackSQvTpKHy\no6q6a+pF+xpfLy+WJN3HpKHyxSRvBbZs303/ceCzc9ctSdJCNGmoHAWsBi5l+ArgMxi+r16SpHtN\nevXXTxi+Tvhv57Y7kqSFbNJ7f32HMedQquox3XskSVqw1uXeX1O2AF4G7NC/O5KkhWyicypVddPI\n45qq+mvgOXPcN0nSAjPp9Nc+Iy8fwjByecSc9EiStGBNOv31VyPP7wauAl7evTeSpAVt0qu/nj3X\nHZEkLXyTTn/98dqWV9V7+nRHkrSQrcvVX08DTm+vDwTOA66ei05JkhamdfmSrn2q6jaAJMcAH6+q\nQ+eqY5KkhWfS27Q8Crhr5PVdwJLuvZEkLWiTjlQ+AlyY5FMM/7P+JcBJc9YrSdKCNOnVX3+e5Ezg\nV1rRa6vq63PXLUnSQjTp9BfAVsCtVfV/gVVJ9pijPkmSFqhJv074aOBI4C2taHPg5LnqlCRpYZp0\npPIS4EXAHQBVdS3epkWSNM2koXJXVRXt9vdJHj53XZIkLVSThsppST4EbJfk9cDZzPKFXUmOT3JD\nkstGynZIsjzJFe3n9q08SY5NsjLJJaM3sExySKt/RZJDRsqfmuTSts6xSbIuOy5J6m/SW9//b+AT\nwCeBJwB/WlXvnWW1E4ADppUdBZxTVUuBc9prgOcDS9vjMOADMIQQcDSwL/B04OipIGp1DhtZb/q2\nJEkb2ayXFCdZBJxVVc8Dlk/acFWdl2TJtOKDgP3a8xOBcxkuADgIOKlNsX0lyXZJFre6y6tqTevL\ncuCAJOcC21TVl1v5ScCLgTMn7Z8kqb9ZRypVdQ/wwyTbdtjeI6vqutbudcDOrXxX7nsfsVWtbG3l\nq8aUj5XksCQrkqxYvXr1Bu+EJGm8Sf9H/X8Cl7aRwh1ThVX1B536Me58SK1H+VhVdRxwHMCyZctm\nrCdJ2jCThso/tceGuj7J4qq6rk1v3dDKVwG7j9TbDbi2le83rfzcVr7bmPqSpHm01lBJ8qiq+l5V\nndhpe6cDhwDvaj8/M1J+RJKPMZyUv6UFz1nAO0dOzu8PvKWq1iS5LckzgAuAVwOzXTggSZpjs51T\n+fTUkySfXJeGk3wU+DLwhCSrkryOIUx+LckVwK+11wBnAFcCKxkuVf59gHaC/u3ARe3xtqmT9sAb\ngQ+3df4DT9JL0rybbfpr9NzFY9al4ao6eIZFzx1Tt4DDZ2jneOD4MeUrgCevS58kSXNrtpFKzfBc\nkqT7mW2k8vNJbmUYsWzZntNeV1VtM6e9kyQtKGsNlapatLE6Ikla+Nbl+1QkSVorQ0WS1I2hIknq\nxlCRJHVjqEiSujFUJEndGCqSpG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHVjqEiSujFUJEndGCqS\npG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHVjqEiSutlsvjsgaYE5Ztv57sGDyzG3zHcPunKkIknq\nxlCRJHVjqEiSujFUJEndGCqSpG4MFUlSN4aKJKkbQ0WS1I2hIknqxlCRJHUzL6GS5Koklya5OMmK\nVrZDkuVJrmg/t2/lSXJskpVJLkmyz0g7h7T6VyQ5ZD72RZL0U/M5Unl2Ve1dVcva66OAc6pqKXBO\new3wfGBpexwGfACGEAKOBvYFng4cPRVEkqT58UCa/joIOLE9PxF48Uj5STX4CrBdksXArwPLq2pN\nVd0MLAcO2NidliT91HyFSgGfT/LVJIe1skdW1XUA7efOrXxX4OqRdVe1spnK7yfJYUlWJFmxevXq\njrshSRo1X7e+f2ZVXZtkZ2B5km+tpW7GlNVayu9fWHUccBzAsmXLxtaRJG24eRmpVNW17ecNwKcY\nzolc36a1aD9vaNVXAbuPrL4bcO1ayiVJ82Sjh0qShyd5xNRzYH/gMuB0YOoKrkOAz7TnpwOvbleB\nPQO4pU2PnQXsn2T7doJ+/1YmSZon8zH99UjgU0mmtn9KVX0uyUXAaUleB3wPeFmrfwbwAmAl8EPg\ntQBVtSbJ24GLWr23VdWajbcbkqTpNnqoVNWVwM+PKb8JeO6Y8gIOn6Gt44Hje/dRkrR+HkiXFEuS\nFjhDRZLUjaEiSerGUJEkdWOoSJK6MVQkSd0YKpKkbgwVSVI3hookqRtDRZLUjaEiSerGUJEkdWOo\nSJK6MVQkSd0YKpKkbgwVSVI3hookqRtDRZLUjaEiSerGUJEkdWOoSJK6MVQkSd0YKpKkbgwVSVI3\nhookqRtDRZLUjaEiSerGUJEkdWOoSJK6MVQkSd0YKpKkbgwVSVI3hookqRtDRZLUjaEiSerGUJEk\ndbPgQyXJAUm+nWRlkqPmuz+StClb0KGSZBHwfuD5wJ7AwUn2nN9eSdKma0GHCvB0YGVVXVlVdwEf\nAw6a5z5J0iZrs/nuwAbaFbh65PUqYN/plZIcBhzWXt6e5NsboW+bgh2BG+e7E7PJX853DzRPFsT7\nkz/LfPdgUo+epNJCD5Vxv426X0HVccBxc9+dTUuSFVW1bL77IY3j+3N+LPTpr1XA7iOvdwOunae+\nSNImb6GHykXA0iR7JHko8Arg9HnukyRtshb09FdV3Z3kCOAsYBFwfFVdPs/d2pQ4pagHMt+f8yBV\n9zsFIUnSelno01+SpAcQQ0WS1I2hovtI8sgkpyS5MslXk3w5yUs6tHtuEi/v1KyS3JPk4iSXJ/lG\nkj9OMutnVZJ3t3XevZ7bvb39XJLklevThhb4iXr1lSTAp4ETq+qVrezRwIvmtWPa1NxZVXsDJNkZ\nOAXYFjh6lvXeAOxUVT/awO0vAV7Ztqt15EhFo54D3FVVH5wqqKrvVtV7k2yR5P8luTTJ15M8G2At\n5Vsm+ViSS5KcCmw5P7ukhayqbmC4G8YRGSxqI5KL2nvrDQBJTgceDlyQ5LeTHJjkgvaePDvJI1u9\nY5L8yVT7SS5LsmTaZt8F/EobLf3XjbGfDyaOVDTqScDXZlh2OEBVPSXJzwGfT/L4tZS/EfhhVe2V\nZK+1tCutVVVd2aa/dma4t98tVfW0JA8DvpTk81X1oiS3j4xwtgeeUVWV5FDgzcCbJtzkUcCfVNVv\nzMHuPOgZKppRkvcDvwzcxXD3gvcCVNW3knwXeHxbPq78V4FjW/klSS7Z+HugB5GpWzLtD+yV5KXt\n9bbAUuA70+rvBpyaZDHw0DHLNUec/tKoy4F9pl5U1eHAc4GdGH+fNdZSDmPuwyatqySPAe4BbmB4\nv/2Xqtq7Pfaoqs+PWe29wPuq6ikM51q2aOV3c9/PvS2mr6gNY6ho1D8DWyR540jZVu3necCrANr0\n1qOAb09Y/mRgr43Qfz3IJNkJ+CBDQBTD3TPemGTztvzxSR4+ZtVtgWva80NGyq+i/eGUZB9gjzHr\n3gY8ossObIIMFd2r/aN9MfCsJN9JciFwInAk8DfAoiSXAqcCr2lX2cxU/gFg6zbt9Wbgwo2/R1qg\ntpy6pBg4G/g88Gdt2YeBbwJfS3IZ8CHGT+MfA3w8yb9w39vffxLYIcnFDOf9/n3MupcAd7fLmT1R\nv468TYskqRtHKpKkbgwVSVI3hookqRtDRZLUjaEiSerGUJE6S/Lf291yL2mXxu6b5I+SbDXBuhPV\nkx6ovKRY6ijJLwLvAfarqh8l2ZHhNiH/CiyrqhtnWf+qSepJD1SOVKS+FgM3Tt1+vYXDS4FdgC8k\n+QJAkg8kWdFGNH/Wyv5gTL3bpxpO8tIkJ7TnL2t32P1GkvM24v5Ja+VIReooydbA+Qy3tzkbOLWq\nvjh9BJJkh6pak2QRcA7wB+3Gm9Pr3V5VW7fnLwV+o6pe0+5gcEBVXZNku6r6wcbeV2kcRypSR1V1\nO/BUhu8AWc1wp9zXjKn68iRfA77O8JUDe67jpr4EnJDk9cCi9e+x1Je3vpc6q6p7gHOBc9uIYvSG\nhiTZA/gT4GlVdXOb0prpbrmjUwn31qmq30uyL/BC4OIke1fVTf32Qlo/jlSkjpI8IcnSkaK9ge9y\n3zvfbgPcAdzSvpHw+SP1p98h9/okT2xfUvWSke08tqouqKo/Zbhh4u7990Zad45UpL62Bt6bZDuG\n7+5YyTAVdjBwZpLrqurZSb7O8P01VzJMZU05brQew7cQ/iNwNXBZax/g3S28wnBO5htzv2vS7DxR\nL0nqxukvSVI3hookqRtDRZLUjaEiSerGUJEkdWOoSJK6MVQkSd38f7J5Bknl8uPZAAAAAElFTkSu\nQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#count_classes = pd.value_counts(df['Class'], sort = True)\n", + "count_classes = pd.value_counts(cc_training.select('default').toPandas()['default'], sort = True)\n", + "count_classes.plot(kind = 'bar', rot=0)\n", + "plt.title(\"Transaction class distribution\")\n", + "plt.xticks(range(2), LABELS)\n", + "plt.xlabel(\"Status\")\n", + "plt.ylabel(\"Frequency\");" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "(trainingData, validData) = cc_training.select('balance','sex','education','marriage','age','default').randomSplit([.7,.3])" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------+-----+\n", + "| balance|count|\n", + "+--------+-----+\n", + "| 50000.0| 2334|\n", + "| 20000.0| 1386|\n", + "| 30000.0| 1118|\n", + "| 80000.0| 1093|\n", + "|200000.0| 1060|\n", + "|150000.0| 789|\n", + "|100000.0| 734|\n", + "|180000.0| 679|\n", + "|360000.0| 620|\n", + "| 60000.0| 590|\n", + "|230000.0| 543|\n", + "|140000.0| 522|\n", + "|130000.0| 514|\n", + "|500000.0| 508|\n", + "|210000.0| 502|\n", + "|120000.0| 497|\n", + "| 70000.0| 488|\n", + "|160000.0| 471|\n", + "| 90000.0| 446|\n", + "|240000.0| 427|\n", + "|110000.0| 419|\n", + "|170000.0| 384|\n", + "|300000.0| 384|\n", + "|280000.0| 356|\n", + "| 10000.0| 355|\n", + "|260000.0| 351|\n", + "|220000.0| 316|\n", + "|250000.0| 254|\n", + "|290000.0| 245|\n", + "|320000.0| 213|\n", + "|400000.0| 199|\n", + "|310000.0| 192|\n", + "|350000.0| 167|\n", + "|190000.0| 166|\n", + "|270000.0| 161|\n", + "|340000.0| 154|\n", + "| 40000.0| 149|\n", + "|420000.0| 119|\n", + "|330000.0| 117|\n", + "|390000.0| 117|\n", + "+--------+-----+\n", + "only showing top 40 rows\n", + "\n" + ] + } + ], + "source": [ + "trainingData.groupBy('balance').count().sort('count', ascending=False ).show(40)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "assembler = VectorAssembler(inputCols=['balance','sex','education','marriage','age'], outputCol=\"assembled\")\n", + "scaler = StandardScaler(inputCol=\"assembled\", outputCol=\"features\")\n", + "pipeline = Pipeline(stages = [assembler, scaler])\n", + "pipelineTraining = pipeline.fit(trainingData)\n", + "cc_data_training = pipelineTraining.transform(trainingData)\n", + "pipelineTest = pipeline.fit(validData)\n", + "cc_data_test = pipelineTest.transform(validData)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[array([ 0.07714453, 2.04395644, 1.26342173, 1.91339646, 4.12035441]),\n", + " array([ 0.07714453, 2.04395644, 1.26342173, 1.91339646, 4.33721517])]" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cc_data_training.select('features').rdd.map(lambda x: x[0]).map(lambda x: np.array(x)).take(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "9073" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#convert ndarray data into RDD[Sample]\n", + "\n", + "# balance,sex,education,marriage,age,default\n", + "\n", + "def array2rdd(ds):\n", + " #build Sample from ndarrays\n", + " def build_sample(balance,sex,education,marriage,age,default):\n", + " feature = np.array([balance,sex,education,marriage,age]).flatten()\n", + " label = np.array(default)\n", + " return Sample.from_ndarray(feature, label)\n", + " rdd = ds.map(lambda (balance,sex,education,marriage,age,default): build_sample(balance,sex,education,marriage,age,default))\n", + " return rdd\n", + "\n", + "\n", + "def DF2rdd(ds):\n", + " #build Sample from ndarrays\n", + " def build_sample(features, label):\n", + " feature = np.array([balance,sex,education,marriage,age]).flatten()\n", + " label = np.array(default)\n", + " return Sample.from_ndarray(feature, label)\n", + " features = ds.select('features').rdd.map(lambda x: x[0]).map(lambda x: np.array(x)).take(2)\n", + " rdd = ds.map(lambda (balance,sex,education,marriage,age,default): build_sample(balance,sex,education,marriage,age,default))\n", + " return rdd\n", + "\n", + "cc_rdd_train = array2rdd(trainingData.rdd.map(list))\n", + "cc_rdd_train.cache()\n", + "cc_rdd_train.count()\n", + "\n", + "\n", + "\n", + "cc_rdd_test = array2rdd(validData.rdd.map(list))\n", + "cc_rdd_test.cache()\n", + "cc_rdd_test.count()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Row(balance=10000.0, sex=1.0, education=1.0, marriage=1.0, age=41.0, default=1.0),\n", + " Row(balance=10000.0, sex=1.0, education=1.0, marriage=1.0, age=48.0, default=1.0),\n", + " Row(balance=10000.0, sex=1.0, education=1.0, marriage=2.0, age=23.0, default=2.0),\n", + " Row(balance=10000.0, sex=1.0, education=1.0, marriage=2.0, age=23.0, default=2.0),\n", + " Row(balance=10000.0, sex=1.0, education=1.0, marriage=2.0, age=24.0, default=1.0),\n", + " Row(balance=10000.0, sex=1.0, education=1.0, marriage=2.0, age=24.0, default=2.0),\n", + " Row(balance=10000.0, sex=1.0, education=1.0, marriage=2.0, age=24.0, default=2.0),\n", + " Row(balance=10000.0, sex=1.0, education=1.0, marriage=2.0, age=26.0, default=1.0),\n", + " Row(balance=10000.0, sex=1.0, education=1.0, marriage=2.0, age=26.0, default=2.0),\n", + " Row(balance=10000.0, sex=1.0, education=1.0, marriage=2.0, age=33.0, default=1.0),\n", + " Row(balance=10000.0, sex=1.0, education=2.0, marriage=1.0, age=27.0, default=1.0),\n", + " Row(balance=10000.0, sex=1.0, education=2.0, marriage=1.0, age=28.0, default=1.0),\n", + " Row(balance=10000.0, sex=1.0, education=2.0, marriage=1.0, age=29.0, default=1.0),\n", + " Row(balance=10000.0, sex=1.0, education=2.0, marriage=1.0, age=29.0, default=1.0),\n", + " Row(balance=10000.0, sex=1.0, education=2.0, marriage=1.0, age=30.0, default=2.0),\n", + " Row(balance=10000.0, sex=1.0, education=2.0, marriage=1.0, age=30.0, default=2.0),\n", + " Row(balance=10000.0, sex=1.0, education=2.0, marriage=1.0, age=35.0, default=1.0),\n", + " Row(balance=10000.0, sex=1.0, education=2.0, marriage=1.0, age=35.0, default=1.0),\n", + " Row(balance=10000.0, sex=1.0, education=2.0, marriage=1.0, age=36.0, default=1.0),\n", + " Row(balance=10000.0, sex=1.0, education=2.0, marriage=1.0, age=37.0, default=1.0)]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "validData.take(20)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Sample: features: [JTensor: storage: [ 1.00000000e+04 1.00000000e+00 1.00000000e+00 1.00000000e+00\n", + " 3.80000000e+01], shape: [5], float], label: JTensor: storage: [ 2.], shape: [1], float,\n", + " Sample: features: [JTensor: storage: [ 1.00000000e+04 1.00000000e+00 1.00000000e+00 1.00000000e+00\n", + " 4.00000000e+01], shape: [5], float], label: JTensor: storage: [ 2.], shape: [1], float,\n", + " Sample: features: [JTensor: storage: [ 1.00000000e+04 1.00000000e+00 1.00000000e+00 1.00000000e+00\n", + " 4.20000000e+01], shape: [5], float], label: JTensor: storage: [ 1.], shape: [1], float]" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cc_rdd_train.take(3)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "creating: createSequential\n", + "creating: createLinear\n", + "creating: createDropout\n", + "creating: createReLU\n", + "creating: createLinear\n", + "creating: createLogSoftMax\n" + ] + } + ], + "source": [ + "# Create model\n", + "\n", + "def multilayer_perceptron(n_hidden_1, n_hidden_2, n_input, n_classes):\n", + " # Initialize a sequential container\n", + " model = Sequential()\n", + " # Hidden layer with ReLu activation\n", + " model.add(Linear(n_input, n_hidden_1).set_name('mlp_fc1'))\n", + " model.add(Dropout(0.2))\n", + " model.add(ReLU())\n", + " # Hidden layer with ReLu activation\n", + " #model.add(Linear(n_hidden_1, n_hidden_2).set_name('mlp_fc2'))\n", + " #model.add(ReLU())\n", + " # output layer\n", + " #model.add(Linear(n_hidden_2, n_classes).set_name('mlp_fc3'))\n", + " model.add(Linear(n_hidden_1, n_classes).set_name('mlp_fc3'))\n", + " model.add(LogSoftMax())\n", + " return model\n", + "\n", + "model = multilayer_perceptron(n_hidden_1, n_hidden_2, n_input, n_classes)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "creating: createClassNLLCriterion\n", + "creating: createAdagrad\n", + "creating: createMaxEpoch\n", + "creating: createOptimizer\n", + "creating: createEveryEpoch\n", + "creating: createTop1Accuracy\n", + "creating: createTrainSummary\n", + "creating: createSeveralIteration\n", + "creating: createValidationSummary\n", + "('saving logs to ', 'cc-default-20171115-183958')\n" + ] + } + ], + "source": [ + "# Create an Optimizer\n", + "optimizer = Optimizer(\n", + " model=model,\n", + " training_rdd=cc_rdd_train,\n", + " criterion=ClassNLLCriterion(),\n", + " optim_method=Adagrad(learningrate=learning_rate, learningrate_decay=0.0002),\n", + " end_trigger=MaxEpoch(training_epochs),\n", + " batch_size=batch_size)\n", + "\n", + "# Set the validation logic\n", + "optimizer.set_validation(\n", + " batch_size=batch_size,\n", + " val_rdd=cc_rdd_test,\n", + " trigger=EveryEpoch(),\n", + " val_method=[Top1Accuracy()]\n", + ")\n", + "\n", + "app_name='cc-default-'+dt.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n", + "train_summary = TrainSummary(log_dir='/tmp/bigdl_summaries',\n", + " app_name=app_name)\n", + "train_summary.set_summary_trigger(\"Parameters\", SeveralIteration(50))\n", + "val_summary = ValidationSummary(log_dir='/tmp/bigdl_summaries',\n", + " app_name=app_name)\n", + "optimizer.set_train_summary(train_summary)\n", + "optimizer.set_val_summary(val_summary)\n", + "print(\"saving logs to \",app_name)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Optimization Done.\n", + "CPU times: user 50 ms, sys: 10 ms, total: 60 ms\n", + "Wall time: 1min 8s\n" + ] + } + ], + "source": [ + "%%time\n", + "# Boot training process\n", + "trained_model = optimizer.optimize()\n", + "print(\"Optimization Done.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAswAAAK7CAYAAADm9tljAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3XucXXV97//XZ2aSQBIkICMi4RIl\njuClaBVvPXYQL3ip2HN6gWO91T5i+xNbPfbXorVKPcdf7Tmt+rNVz0mPHi+tUA6tNW2xiuj2cooK\nKCIEIwERQpCgXCcEcpnP+WOvnewMkzV7wsys7yKv5+Mx7L3W+q69vns+s4Z3vvPda0VmIkmSJGl6\nQ013QJIkSSqZgVmSJEmqYWCWJEmSahiYJUmSpBoGZkmSJKmGgVmSJEmqYWCWpIJFxI0R8YKm+yFJ\nBzIDsyRJklTDwCxJkiTVMDBLUgtExJKI+GBEbK6+PhgRS6ptR0TEP0fEXRFxR0R8PSKGqm1/GBG3\nRMS9EbEhIk5r9p1IUvuMNN0BSdJA/gh4FnAykMDngHcCfwy8DdgEjFZtnwVkRIwBZwPPyMzNEXE8\nMLyw3Zak9nOEWZLa4VXAezJzS2beDvwJ8Opq2w7gKOC4zNyRmV/PzAR2AUuAkyJiUWbemJnXN9J7\nSWoxA7MktcNjgB/3Lf+4Wgfw34CNwBcj4oaIOAcgMzcCbwHOBbZExPkR8RgkSbNiYJakdtgMHNe3\nfGy1jsy8NzPflpmPBX4J+E+9ucqZ+ZnM/IVq3wT+bGG7LUntZ2CWpHY4D3hnRIxGxBHAu4C/AYiI\nl0fECRERwD10p2LsioixiHh+9eHA+4Ft1TZJ0iwYmCWpHf4LcDlwFfB94DvVOoDVwJeACeBS4COZ\n2aE7f/l9wE+BnwCPAt6xoL2WpIeB6H4uRJIkSdJ0HGGWJEmSahiYJUmSpBoGZkmSJKmGgVmSJEmq\nUdytsVesWJEnnHBC093QDLZu3cqyZcua7oZqWKN2sE7ls0btYJ3KV2KNrrjiip9m5uhM7YoLzEce\neSSXX355093QDDqdDuPj4013QzWsUTtYp/JZo3awTuUrsUYR8eOZWzklQ5IkSaplYJYkSZJqGJgl\nSZKkGgZmSZIkqYaBWZIkSaphYJYkSZJqGJglSZKkGgZmSZIkqYaBWZIkSaphYJYkSZJqGJglSZKk\nGsUF5p9uS971uaub7oYkSZIEFBiYd0wmN9y+teluSJIkSUCBgRlgMrPpLkiSJElAgYE5MDBLkiSp\nHMUFZoBJ87IkSZIKUWRgTkeYJUmSVIgiA7MjzJIkSSpFkYHZEWZJkiSVorjAHBGOMEuSJKkYxQVm\ncIRZkiRJ5SgyMDvCLEmSpFIUGphNzJIkSSpDcYG5e+OSpnshSZIkdRUXmME5zJIkSSpHkYHZKRmS\nJEkqRXGB2SkZkiRJKklxgRkcYZYkSVI5igzM5mVJkiSVorzAHI4wS5IkqRzlBWYcYZYkSVI5igvM\n3Q/9mZglSZJUhuICMzjCLEmSpHLMKjBHxDER8ZWIuDYiromI36vWnxsRt0TEldXXS/v2eXtEbIyI\nDRHx4kGO4wizJEmSSjEyy/Y7gbdl5nci4hDgioi4uNr2gcz88/7GEXEScCbwROAxwJci4vGZuavu\nIAZmSZIklWJWI8yZeWtmfqd6fi9wLXB0zS5nAOdn5gOZ+SNgI3BK3TG8cYkkSZJKErmfo7kRcTzw\nNeBJwH8CXgfcA1xOdxT6zoj4K+Cbmfk31T4fAz6fmRdOea01wBqA5Uce+/Or1nyEDz1/2X71Swtj\nYmKC5cuXN90N1bBG7WCdymeN2sE6la/EGp166qlXZObTZ2o32ykZAETEcuDvgbdk5j0R8VHgPwNZ\nPf4F8Jt0B4ynelBCz8y1wFqAI457fI4sWsz4+Pj+dE0LpNPpWKPCWaN2sE7ls0btYJ3K1+Yazfoq\nGRGxiG5Y/tvM/AeAzLwtM3dl5iTw1+yZdrEJOKZv95XA5pmO4RxmSZIklWK2V8kI4GPAtZn5/r71\nR/U1+2Xg6ur5OuDMiFgSEauA1cC3a48BTDqJWZIkSYWY7ZSM5wKvBr4fEVdW694BnBURJ9OdbnEj\n8EaAzLwmIi4A1tO9wsabZrpCRne/WfZKkiRJmiezCsyZ+Q2mn5d8Uc0+7wXeO6vjzKaxJEmSNI+K\nvNOfc5glSZJUiuICc/c6zAZmSZIklaG4wAzeuESSJEnlKDIw7+/NVCRJkqS5VmRgdoRZkiRJpSgu\nMEc4h1mSJEnlKC4wQ/c6zE7LkCRJUgmKDMzgzUskSZJUhmIDs9MyJEmSVIKCA3PTPZAkSZIKDMy9\n+26nN8iWJElSAYoLzD3OyJAkSVIJig3MzmGWJElSCQoOzE33QJIkSSowMEc1idkRZkmSJJWguMDc\nk5NN90CSJEkqODA7wixJkqQSGJglSZKkGsUF5t51mP3QnyRJkkpQXGDuSUeYJUmSVIBiA7MjzJIk\nSSpBsYHZW2NLkiSpBMUF5j3XYW62H5IkSRIUGJh7Jk3MkiRJKkCxgdnP/EmSJKkExQZmr8MsSZKk\nEhQXmPdch9nALEmSpOYVF5h7nMIsSZKkEhQbmL1xiSRJkkpQbGB2hFmSJEklKC4wO4dZkiRJJSku\nMLP7xiUGZkmSJDWvvMBcMS9LkiSpBAZmSZIkqUaxgdkpGZIkSSpBcYHZD/1JkiSpJMUF5h4vKydJ\nkqQSFBiYu2PM3rhEkiRJJSguMPemZPxs6/ZG+yFJkiRBgYE56Y4sv/HTVzTcE0mSJGmWgTkijomI\nr0TEtRFxTUT8XrX+8Ii4OCKuqx4Pq9ZHRHwoIjZGxFUR8bSZjjEyFDM1kSRJkhbMbEeYdwJvy8wT\ngWcBb4qIk4BzgEsyczVwSbUM8BJgdfW1BvjoTAdYNASnrDqcp6w8dJZdkyRJkuberAJzZt6amd+p\nnt8LXAscDZwBfLJq9kngldXzM4BPZdc3gRURcdRMxzlkyYiXlZMkSVIRRvZ3x4g4Hngq8C3gyMy8\nFbqhOiIeVTU7Gri5b7dN1bpbp7zWGroj0IyOjnLHHT/jnm1Jp9PZ3+5pnk1MTFifwlmjdrBO5bNG\n7WCdytfmGu1XYI6I5cDfA2/JzHsi9jnveLoNDxo6zsy1wFqAsbGxHD3iCLbdcR/j48/bn+5pAXQ6\nHcbHx5vuhmpYo3awTuWzRu1gncrX5hrN+ioZEbGIblj+28z8h2r1bb2pFtXjlmr9JuCYvt1XAptn\nPgY4I0OSJEklmO1VMgL4GHBtZr6/b9M64LXV89cCn+tb/5rqahnPAu7uTd2o7VTE7svLSZIkSU2a\n7ZSM5wKvBr4fEVdW694BvA+4ICLeANwE/Gq17SLgpcBG4D7g9YMcZCjCW2NLkiSpCLMKzJn5Daaf\nlwxw2jTtE3jTbDsVgVfJkCRJUhGKu9MfVFMyzMuSJEkqQJGB2RFmSZIklaLIwOwIsyRJkkpRZGB2\nhFmSJEmlKDIwO8IsSZKkUhQamB1hliRJUhkKDcxhYJYkSVIRigzM3TnMTfdCkiRJKjYwO4dZkiRJ\nZSgyMA8FpIlZkiRJBSg0MDuHWZIkSWUoODA33QtJkiSp0MDsjUskSZJUijIDM37oT5IkSWUoMjD7\noT9JkiSVoszAPOQcZkmSJJWhyMDsHGZJkiSVosjAPOSNSyRJklSIIgNz4AizJEmSylBkYPbGJZIk\nSSpFoYEZjMuSJEkqQZGBOao5zF5aTpIkSU0rMjAPRQD4wT9JkiQ1rtDA3H10HrMkSZKaVmRgjt2B\nudl+SJIkSYUG5mpKhh/9kyRJUsOKDMzOYZYkSVIpCg3M3UfnMEuSJKlphQbmbmJ2DrMkSZKaVmRg\nDkeYJUmSVIhCA3M1h3my4Y5IkiTpgFdkYO7NYfYqGZIkSWpaoYHZOcySJEkqQ6GBufvoHGZJkiQ1\nrcjAHLtHmA3MkiRJalahgbn7aF6WJElS04oMzEOOMEuSJKkQhQbm7qN5WZIkSU0rMjA7h1mSJEml\nKDIw96ZkmJclSZLUtFkH5oj4eERsiYir+9adGxG3RMSV1ddL+7a9PSI2RsSGiHjxQJ3ysnKSJEkq\nxP6MMH8COH2a9R/IzJOrr4sAIuIk4EzgidU+H4mI4ZkOELsD8370TpIkSZpDsw7Mmfk14I4Bm58B\nnJ+ZD2Tmj4CNwCkzdmr3lAwTsyRJkpo1l3OYz46Iq6opG4dV644Gbu5rs6laVyu8NbYkSZIKMTJH\nr/NR4D8DWT3+BfCbQEzT9kExOCLWAGsARkdH+cH69QB869vfZtPyIj+XeMCbmJig0+k03Q3VsEbt\nYJ3KZ43awTqVr801mpPAnJm39Z5HxF8D/1wtbgKO6Wu6Etg8zf5rgbUAY2Nj+aQnPRG+9x2e/vRn\nMPboQ+aii5pjnU6H8fHxpruhGtaoHaxT+axRO1in8rW5RnMyfBsRR/Ut/jLQu4LGOuDMiFgSEauA\n1cC3Z+yUV8mQJElSIWY9whwR5wHjwBERsQl4NzAeESfTnW5xI/BGgMy8JiIuANYDO4E3ZeauAY4C\nGJglSZLUvFkH5sw8a5rVH6tp/17gvbM5hrfGliRJUimK/ESdd/qTJElSKcoMzFWvnJIhSZKkphUZ\nmPdch9nALEmSpGaVGZirR29cIkmSpKYVGZi9NbYkSZJKUXZgbrgfkiRJUqGBufs46ZwMSZIkNazI\nwLznQ38Nd0SSJEkHvCID854bl5iYJUmS1KwiA7MjzJIkSSpFkYF59xxmR5glSZLUsCIDc3iVDEmS\nJBWiyMDsCLMkSZJKUWhg9sYlkiRJKkPRgXlysuGOSJIk6YBXZGAeruZk7NhlYpYkSVKzigzMj1y+\nGICfbt3ecE8kSZJ0oCszMC9bTATcfs/9TXdFkiRJB7giA/PI8BCPXLaELfc+0HRXJEmSdIArMjAD\njB6yhNsNzJIkSWpYsYH5UYc4wixJkqTmFRuYDzlohK0P7Gy6G5IkSTrAFRuYFw0PscMLMUuSJKlh\nxQbmkaFg5y7v9CdJkqRmlRuYh4fYYWCWJElSw4oNzIuGwzv9SZIkqXHFBuaRoSF2GpglSZLUsGID\n86LhYMekUzIkSZLUrIIDsyPMkiRJal6xgXlkOJhMmHSUWZIkSQ0qNjAvGu52zWsxS5IkqUnFBuaR\noQDwWsySJElqVLmBuRphNjBLkiSpScUG5kXD3RFmp2RIkiSpScUG5pEhR5glSZLUvHIDc2+E2UvL\nSZIkqUHFBuZFBmZJkiQVoODAXE3J8DrMkiRJalCxgbk3h9kRZkmSJDWp2MDcm5Lhh/4kSZLUpFkH\n5oj4eERsiYir+9YdHhEXR8R11eNh1fqIiA9FxMaIuCoinjbocXZfh9nLykmSJKlB+zPC/Ang9Cnr\nzgEuyczVwCXVMsBLgNXV1xrgo4MeZNFQ70N/jjBLkiSpObMOzJn5NeCOKavPAD5ZPf8k8Mq+9Z/K\nrm8CKyLiqEGO453+JEmSVIK5msN8ZGbeClA9PqpafzRwc1+7TdW6GY14pz9JkiQVYGSeXz+mWfeg\nIeOIWEN3ygajo6N0Oh1uvHsXAN+98iri1vnupmZrYmKCTqfTdDdUwxq1g3UqnzVqB+tUvjbXaK6S\n6G0RcVRm3lpNudhSrd8EHNPXbiWweerOmbkWWAswNjaW4+Pj/OAn98ClX+cJJz2R8ScPNItDC6jT\n6TA+Pt50N1TDGrWDdSqfNWoH61S+NtdorqZkrANeWz1/LfC5vvWvqa6W8Szg7t7UjZl4HWZJkiSV\nYNYjzBFxHjAOHBERm4B3A+8DLoiINwA3Ab9aNb8IeCmwEbgPeP2gx1k83AvMfuhPkiRJzZl1YM7M\ns/ax6bRp2ibwptkeA+CgRd3AfP+OXfuzuyRJkjQnir3T35JFw4CBWZIkSc0qNjD3Rpgf2OkcZkmS\nJDWn2MC8eHiICEeYJUmS1KxiA3NEcPCiYbZtNzBLkiSpOcUGZoCDFg1z/04DsyRJkppTdmAeGeL+\nHc5hliRJUnPKDsyLhp3DLEmSpEYVHZiXGJglSZLUsKID88GLnJIhSZKkZhUdmJ2SIUmSpKaVH5i9\nSoYkSZIaVHhgHvI6zJIkSWpU4YF52DnMkiRJalTRgXnJyDAP7DQwS5IkqTlFB+bFw8GOXQZmSZIk\nNafswDwyxHZHmCVJktSg8gOzI8ySJElqUNmBeXiYXZPJrslsuiuSJEk6QBUdmBeNBIDzmCVJktSY\nogPz4uFu97xShiRJkppSdGBeMtLtnh/8kyRJUlOKDsyLe4HZKRmSJElqSNGBeVE1JWOHI8ySJElq\nSNGB2RFmSZIkNa3swDzsHGZJkiQ1q+zAPOJVMiRJktSssgNzbw6zUzIkSZLUkLIDs5eVkyRJUsMM\nzJIkSVKNdgRmp2RIkiSpIWUHZucwS5IkqWFFB+bejUu8SoYkSZKaUnRgXlJNybh/x66GeyJJkqQD\nVdGB+fBlixk9ZAlf3XB7012RJEnSAarowDwyPMSv/vxKvrJhC7feva3p7kiSJOkAVHRgBvj1ZxzD\nZMLnv/+TprsiSZKkA1DxgfnYw5eyeHiILfc+0HRXJEmSdAAqPjBHBIcuXcTd27Y33RVJkiQdgIoP\nzAArDl7EXfftaLobkiRJOgC1IzAvNTBLkiSpGa0IzIcevIi7txmYJUmStPDmNDBHxI0R8f2IuDIi\nLq/WHR4RF0fEddXjYbN93UMPXmxgliRJUiPmY4T51Mw8OTOfXi2fA1ySmauBS6rlWelOyfBDf5Ik\nSVp4CzEl4wzgk9XzTwKvnO0LrDh4EVu372LHrsk57ZgkSZI0k8jMuXuxiB8BdwIJ/I/MXBsRd2Xm\nir42d2bmYVP2WwOsARgdHf35Cy64YK/X/dKPd/A3127nQ6cu5RFLYs76q/03MTHB8uXLm+6Galij\ndrBO5bNG7WCdyldijU499dQr+mZF7NPIHB/3uZm5OSIeBVwcET8YZKfMXAusBRgbG8vx8fG9tt99\n5S38zbVXctJTn8EJjyrrG32g6nQ6TK2TymKN2sE6lc8atYN1Kl+bazSnUzIyc3P1uAX4LHAKcFtE\nHAVQPW6Z7euuWLoYwJuXSJIkacHNWWCOiGURcUjvOfAi4GpgHfDaqtlrgc/N9rVXHLwIwGsxS5Ik\nacHN5ZSMI4HPRkTvdT+Tmf8aEZcBF0TEG4CbgF+d7QsfamCWJElSQ+YsMGfmDcDPTbP+Z8BpD+W1\nVyztBmavxSxJkqSF1oo7/R1y0CKGAtZ+7QYmJ+fuqh6SJEnSTFoRmIeHgl9YPcpP7rmfuxxlliRJ\n0gJqRWAGePmTjwLgvu07G+6JJEmSDiStCcwHLx4GYNv2XQ33RJIkSQeS1gTmpVVg3mpgliRJ0gJq\nUWDuXtDDKRmSJElaSC0KzE7JkCRJ0sJrXWC+z8AsSZKkBdSewLzEKRmSJElaeO0JzIscYZYkSdLC\na01gPtgpGZIkSWpAawLzkpEhhofCD/1JkiRpQbUmMEcESxcN86/X/ITrbru36e5IkiTpANGawAzw\nqEcsYeOWCX7jY99quiuSJEk6QLQqMP/yU48GYNdkwx2RJEnSAaNVgfl3xk9g1RHLOOrQg5ruiiRJ\nkg4QrQrMw0PBz608lLu2bW+6K5IkSTpAtCowA6xYupi77tvRdDckSZJ0gGhhYF7EvffvZKcTmSVJ\nkrQA2heYD14EwN3bHGWWJEnS/GtdYD5s2WIA7tjqPGZJkiTNv9YF5lVHLAPg+tsnGu6JJEmSDgSt\nC8yPP/IQhgLW3+rd/iRJkjT/WheYD1o0zONGl3Px+tvYvtMP/kmSJGl+tS4wA5z9/BO49tZ7+Pp1\ntzfdFUmSJD3MtTIwv/iJj2bxyBCXXv+zprsiSZKkh7lWBuaDFg3zzFWHc963b+KWu7Y13R1JkiQ9\njLUyMAO8/SUnsnX7Lr7htAxJkiTNo9YG5ic8+hCWLxlh/eZ7mu6KJEmSHsZaG5iHhoITjzqEq265\nu+muSJIk6WGstYEZ4NmPO4Lv3XwXP5t4oOmuSJIk6WGq1YH5tCc8ismES2/wahmSJEmaH60OzKuP\nXA7Aj392X8M9kSRJ0sNVqwPz0sUjjB6yhM9deYt3/ZMkSdK8aHVgBrj93gf44W0TfOwbP2q6K5Ik\nSXoYan1g/s3nrgLgz/71B94qW5IkSXOu9YH5j19+Ii97ylEAvHvdNWRmwz2SJEnSw0nrA3NE8L5/\n/2Te+LzHcsPtW3nO+77MAzt3Nd0tSZIkPUy0PjADHHLQIn73tNUMBdx69/285INfZ9t2Q7MkSZIe\nuodFYAZYtmSE6/+/l3LWKcdyw0+38pqPf4t/+t7mprslSZKklluQwBwRp0fEhojYGBHnzONx+NN/\n/2Re++zjuPzHd/Lm877L8ef8Cx+65Dp+cvf983VYSZIkPYyNzPcBImIY+DDwQmATcFlErMvM9fN1\nzD8540m842Un8oq//D9suO1e3n/xD3n/xT/kCY8+hJWHLeURB40w9uhDWLF0EcuWjDAUMV9d2ctQ\nsPtYc/XRxP6eT+berxwRTPfO6o4ddPs4mcmuyWQyu/2OCIaiu28mXP2Tndx/9a01vYH+b+vUfu6a\nTJIkCCK62/e0j33uH9XK/tfrvZ/eBz73LE/p3e7j7Pm+ZN9+vdfvvl+YnOw+9mq2fdckk5NZ9Wuw\n99rf5+m2T/3Rm7o8Obn3tu576L6RzGTXJNN+0DUiuHrLTnasv+1B2waxvx+eHYp40Huos/v9ANn/\ns9v3Hif3cYn1Qb53CQxXwwKZ1Vf/Pn377V3D2NNgz0P35zanP97++v6Wney6dv/qNNVk1bfez/Cu\nSRgZCoaGuu9oMrv93zW57/pO977669T/u2F4KBgZ3vPak5PVY3a3B7B4ZGj3+bQrk6C7X+9nZfc5\nR5Ak23dOkglDQ3vWwd7nc+95/7uY+jO75/X3PpeHhrrHG47g/p27yOp99Jv63bnq9p3kD7b0bd+7\nxb5Ol9776/3eyezbc/d7ePDOu99f/3ue5n2ODAfDQ0N9++WD2k/7hvYhyd3nSf972P1epp4QJejr\n69Q6LXxXHvz923t7vaFg97k687Fmd4Dez2L/z2Dv53Io9vxunJyh8/0/r1N/pw4P9f3/Lvf8ruh/\nxatv28kD1/xkprc35f+Tg/0/9EH/T+hvWT099vClPG50+YzHn7ZP831ViYh4NnBuZr64Wn47QGb+\n6XTtx8bGcsOGDXNy7G3bdzE0BJde/zOuvfVevn7d7Wy59wHu2baDLfc+MCfHkCRJUvl+Z/xx/OHp\nT9hrXURckZlPn2nfeR9hBo4Gbu5b3gQ8s79BRKwB1gCMjo7S6XTmvBMnAieu7j7PHOa+nUvZtjN5\nYOfcjfbOJOmO7MzngPbuEeycbtxiML1R5aFqZKQ3CjuZe/4Ft+2+bSxbtnT3PlP/4TXTsYf7X7t/\nvymjxHu95pSVyXSjuVOWp/Snf2Rq6r9S+7dl33vt/Qt6ZKj7PZnNe6379+jUTQ9qmntGxXd/n3LP\n8yH2jCROd8xt27Zx8MEH1/Su3mx/TqeO3s7Yfvd/+upRvdnd77H3V4Hp9q1Z7v/e7R5Mnea1cspj\n/8KDtlWjnkPTHe8huO++bSxduv916un9TPSeZ3Ue78o934PuX5D2fG+me41pX7uvtrtrUp0zO3uj\n2lNeeyiCBHZUBx/q26c7krWnn1TLEdV5Rl/deh1nyjk7TT/7z1mAyepN9dZP5t7HHx7q9nm6Aff+\n19+2bRtLp55L+/hd09P7+e71of9nPKbuM92o/r437f4+TtL9K8Jebfbx166ZTude//p/NvY6B/p/\nPw7wegup15f7pqvTQov9n+s6yTQ/izXf7Gl/NqZZ16tZ5t4/g711u2vft226Q0/389r/O7W//0PV\nxt5537Ptvm0cPIvfd4P+P3S6/wdMt21FbqbTmXmEezoLEZhn/L2cmWuBtdAdYR4fH1+Abumh6HQ6\nWKeyWaN2sE7ls0btYJ3K1+YaLcSH/jYBx/QtrwS8fIUkSZJaYSEC82XA6ohYFRGLgTOBdQtwXEmS\nJOkhm/cpGZm5MyLOBr4ADAMfz8xr5vu4kiRJ0lxYiDnMZOZFwEULcSxJkiRpLj1s7vQnSZIkzQcD\nsyRJklTDwCxJkiTVMDBLkiRJNQzMkiRJUg0DsyRJklTDwCxJkiTViMxsug97iYh7gQ1N90MzOgL4\nadOdUC1r1A7WqXzWqB2sU/lKrNFxmTk6U6MFuXHJLG3IzKc33QnVi4jLrVPZrFE7WKfyWaN2sE7l\na3ONnJIhSZIk1TAwS5IkSTVKDMxrm+6ABmKdymeN2sE6lc8atYN1Kl9ra1Tch/4kSZKkkpQ4wixJ\nkiQVw8AsSZIk1SgqMEfE6RGxISI2RsQ5TffnQBURx0TEVyLi2oi4JiJ+r1p/eERcHBHXVY+HVesj\nIj5U1e2qiHhas+/gwBERwxHx3Yj452p5VUR8q6rR30XE4mr9kmp5Y7X9+Cb7fSCJiBURcWFE/KA6\np57tuVSWiHhr9bvu6og4LyIO8lxqXkR8PCK2RMTVfetmfe5ExGur9tdFxGubeC8PZ/uo03+rfudd\nFRGfjYgVfdveXtVpQ0S8uG990RmwmMAcEcPAh4GXACcBZ0XESc326oC1E3hbZp4IPAt4U1WLc4BL\nMnM1cEm1DN2ara6+1gAfXfguH7B+D7i2b/nPgA9UNboTeEO1/g3AnZl5AvCBqp0Wxv8P/GtmPgH4\nObr18lwqREQcDfwu8PTMfBIwDJyJ51IJPgGcPmXdrM6diDgceDfwTOAU4N29kK058wkeXKeLgSdl\n5lOAHwJvB6iyxJnAE6t9PlIN/BSfAYsJzHR/kDdm5g2ZuR04Hzij4T4dkDLz1sz8TvX8Xrr/gz+a\nbj0+WTX7JPDK6vkZwKey65vAiog4aoG7fcCJiJXAy4D/WS0H8HzgwqrJ1Br1anchcFrVXvMoIh4B\nPA/4GEBmbs/Mu/BcKs0IcHBEjABLgVvxXGpcZn4NuGPK6tmeOy8GLs7MOzLzTrpBbmq400MwXZ0y\n84uZubNa/Cawsnp+BnB+Zj6QmT8CNtLNf8VnwJIC89HAzX3Lm6p1alD158anAt8CjszMW6EbqoFH\nVc2sXTM+CPwBMFktPxK4q++yRfLHAAAgAElEQVSXVH8ddteo2n531V7z67HA7cD/qqbO/M+IWIbn\nUjEy8xbgz4Gb6Ablu4Er8Fwq1WzPHc+p5v0m8PnqeWvrVFJgnu5f6F7zrkERsRz4e+AtmXlPXdNp\n1lm7eRQRLwe2ZOYV/aunaZoDbNP8GQGeBnw0M58KbGXPn5CnY50WWPXn+TOAVcBjgGV0/yw8ledS\n2fZVF+vVoIj4I7rTPP+2t2qaZq2oU0mBeRNwTN/ySmBzQ3054EXEIrph+W8z8x+q1bf1/jxcPW6p\n1lu7hfdc4BURcSPdP109n+6I84rqz8qwdx1216jafigP/lOn5t4mYFNmfqtavpBugPZcKscLgB9l\n5u2ZuQP4B+A5eC6VarbnjudUQ6oPWL4ceFXuuelHa+tUUmC+DFhdfTJ5Md1J4esa7tMBqZqP9zHg\n2sx8f9+mdUDvE8avBT7Xt/411aeUnwXc3fuTmeZHZr49M1dm5vF0z5UvZ+argK8Av1I1m1qjXu1+\npWpf1L/eH44y8yfAzRExVq06DViP51JJbgKeFRFLq999vRp5LpVptufOF4AXRcRh1V8TXlSt0zyK\niNOBPwRekZn39W1aB5xZXW1mFd0PaX6bNmTAzCzmC3gp3U9TXg/8UdP9OVC/gF+g+6eQq4Arq6+X\n0p2ndwlwXfV4eNU+6H669Xrg+3Q/bd74+zhQvoBx4J+r54+l+8tnI/C/gSXV+oOq5Y3V9sc23e8D\n5Qs4Gbi8Op/+ETjMc6msL+BPgB8AVwOfBpZ4LjX/BZxHd175DrojkG/Yn3OH7hzajdXX65t+Xw+3\nr33UaSPdOcm9DPHf+9r/UVWnDcBL+tYXnQG9NbYkSZJUo6QpGZIkSVJxDMySJElSDQOzJEmSVMPA\nLEmSJNUwMEuSJEk1DMySJElSDQOzJEmSVMPALEmSJNUwMEuSJEk1DMySJElSDQOzJEmSVMPALEmS\nJNUwMEuSJEk1DMySJElSDQOzJEmSVMPALEmSJNUwMEuSJEk1DMySJElSDQOzJEmSVMPALEmSJNUw\nMEuSJEk1DMySJElSDQOzJEmSVMPALEmSJNUwMEuSJEk1DMySJElSDQOzJEmSVMPALEmSJNUwMEvS\nDCLixoh4wRy91uKIuLB6zYyI8bl4XUnS/DEwS9LC+wbwG8BPmu7ITCJiuOk+SFLTDMySVCMiPg0c\nC/xTRExExB9U618REddExF0R0YmIE/v2uTEi3h4R6yPizoj4XxFxEEBmbs/MD2bmN4BdAxz/9RFx\nbUTcGxE3RMQbp2w/IyKujIh7IuL6iDi9Wn94ddzNVR/+sVr/uoj4xpTXyIg4oXr+iYj4aERcFBFb\ngVMj4mUR8d3qGDdHxLlT9v+FiPi36ntxc3WMZ0TEbREx0tfuP0TElbP49ktSEQzMklQjM18N3AT8\nUmYuz8z/GhGPB84D3gKMAhfRDdSL+3Z9FfBi4HHA44F37mcXtgAvBx4BvB74QEQ8DSAiTgE+Bfy/\nwArgecCN1X6fBpYCTwQeBXxgFsf8j8B7gUPojoZvBV5THeNlwO9ExCurPhwLfB74S7rfi5OBKzPz\nMuBnwAv7Xvc3qn5JUqsYmCVp9n4d+JfMvDgzdwB/DhwMPKevzV9l5s2ZeQfd8HnW/hwoM/8lM6/P\nrq8CXwT+XbX5DcDHq35MZuYtmfmDiDgKeAnw25l5Z2buqPYd1Ocy8/9Ur3l/ZnYy8/vV8lV0/7Hw\ni1XbVwFfyszzquP8LDN7o8ifpBuSiYjD6f4D4jP7832QpCYZmCVp9h4D/Li3kJmTwM3A0X1tbu57\n/uNqn1mLiJdExDcj4o6IuAt4KXBEtfkY4PppdjsGuCMz79yfY7J334mIZ0bEVyLi9oi4G/jtAfoA\n8DfAL0XEcuDXgK9n5q372SdJaoyBWZJmllOWNwPH9RYiIugGx1v62hzT9/zYap9ZiYglwN/THcE+\nMjNX0J3+EVWTm+lO+ZjqZuDwiFgxzbatdKdq9I7x6GnaTH2/nwHWAcdk5qHAfx+gD2TmLcClwC8D\nr8bpGJJaysAsSTO7DXhs3/IFwMsi4rSIWAS8DXgA+Le+Nm+KiJXVVIR3AH/X2xARS3ofAgQWR8RB\nVeieajGwBLgd2BkRLwFe1Lf9Y8Drq34MRcTREfGEahT388BHIuKwiFgUEc+r9vke8MSIOLnqw7kD\nvP9D6I5Y31/Nm/6Pfdv+FnhBRPxaRIxExCMj4uS+7Z8C/gB4MvDZAY4lScUxMEvSzP4UeGd1FYjf\nz8wNdOfm/iXwU+CX6H4ocHvfPp+hO9/4hurrv/Rt2wBsozuF4wvV8+OYIjPvBX6XbkC/k25QXde3\n/dtUHwQE7ga+2vc6rwZ2AD+g+8HBt1T7/BB4D/Al4Dq6H+qbyf8DvCci7gXeVfWn14eb6E4TeRtw\nB3Al8HN9+3626tNnM3PrAMeSpOJE5tS/vEmSHoqIuBH4rcz8UtN9KUFEXA+80e+HpLZyhFmSNG8i\n4j/QnRP95ab7Ikn7a2TmJpIkzV5EdICTgFdXVxKRpFZySoYkSZJUwykZkiRJUo3ipmSsWLEiTzjh\nhKa7oQFs3bqVZcuWNd0NzcA6tYN1ag9r1Q7WqR2artMVV1zx08wcnaldcYH5yCOP5PLLL2+6GxpA\np9NhfHy86W5oBtapHaxTe1irdrBO7dB0nSLixzO3ckqGJEmSVMvALEmSJNUwMEuSJEk1DMySJElS\nDQOzJEmSVMPALEmSJNUwMEuSJEk1DMySJElSDQOzJEmSVMPALEmSJNUwMEuSJEk1DMySJElSDQOz\nJEmSVMPALEmSJNUwMEuSJEk1DMySJElSDQOzJEmSVMPALEmSJNUwMEuSJEk1DMySJElSDQOzJEmS\nVMPALEmSJNUwMEuSJEk1DMySJElSDQOzJEmSVGOgwBwRp0fEhojYGBHnTLP92Ij4SkR8NyKuioiX\nVusfWa2fiIi/muvOS5IkSfNtxsAcEcPAh4GXACcBZ0XESVOavRO4IDOfCpwJfKRafz/wx8Dvz1mP\nJUmSpAU0yAjzKcDGzLwhM7cD5wNnTGmTwCOq54cCmwEyc2tmfoNucJYkSZJaZ2SANkcDN/ctbwKe\nOaXNucAXI+LNwDLgBXPSO0mSJKlhgwTmmGZdTlk+C/hEZv5FRDwb+HREPCkzJwfpRESsAdYAjI6O\n0ul0BtlNDZuYmLBWLWCd2sE6tYe1agfr1A5tqdMggXkTcEzf8kqqKRd93gCcDpCZl0bEQcARwJZB\nOpGZa4G1AGNjYzk+Pj7IbmpYp9PBWpXPOrWDdWoPa9UO1qkd2lKnQeYwXwasjohVEbGY7of61k1p\ncxNwGkBEnAgcBNw+lx2VJEmSmjDjCHNm7oyIs4EvAMPAxzPzmoh4D3B5Zq4D3gb8dUS8le50jddl\nZgJExI10PxC4OCJeCbwoM9fPz9uRJEmS5tYgUzLIzIuAi6ase1ff8/XAc/ex7/EPoX+SJElSo7zT\nnyRJklTDwCxJkiTVMDBLkiRJNQzMkiRJUg0DsyRJklTDwCxJkiTVMDBLkiRJNQzMkiRJUg0DsyRJ\nklTDwCxJkiTVMDBLkiRJNQzMkiRJUg0DsyRJklTDwCxJkiTVMDBLkiRJNQzMkiRJUg0DsyRJklTD\nwCxJkiTVMDBLkiRJNQzMkiRJUg0DsyRJklTDwCxJkiTVMDBLkiRJNQzMkiRJUg0DsyRJklTDwCxJ\nkiTVGCgwR8TpEbEhIjZGxDnTbD82Ir4SEd+NiKsi4qV9295e7bchIl48l52XJEmS5tvITA0iYhj4\nMPBCYBNwWUSsy8z1fc3eCVyQmR+NiJOAi4Djq+dnAk8EHgN8KSIen5m75vqNSJIkSfNhkBHmU4CN\nmXlDZm4HzgfOmNImgUdUzw8FNlfPzwDOz8wHMvNHwMbq9SRJkqRWmHGEGTgauLlveRPwzCltzgW+\nGBFvBpYBL+jb95tT9j166gEiYg2wBmB0dJROpzNAt9S0iYkJa9UC1qkdrFN7WKt2sE7t0JY6DRKY\nY5p1OWX5LOATmfkXEfFs4NMR8aQB9yUz1wJrAcbGxnJ8fHyAbqlpnU4Ha1U+69QO1qk9rFU7WKd2\naEudBgnMm4Bj+pZXsmfKRc8bgNMBMvPSiDgIOGLAfSVJkqRiDTKH+TJgdUSsiojFdD/Et25Km5uA\n0wAi4kTgIOD2qt2ZEbEkIlYBq4Fvz1XnJUmSpPk24whzZu6MiLOBLwDDwMcz85qIeA9weWauA94G\n/HVEvJXulIvXZWYC10TEBcB6YCfwJq+QIUmSpDYZZEoGmXkR3UvF9a97V9/z9cBz97Hve4H3PoQ+\nSpIkSY3xTn+SJElSDQOzJEmSVMPALEmSJNUwMEuSJEk1DMySJElSDQOzJEmSVMPALEmSJNUwMEuS\nJEk1DMySJElSDQOzJEmSVMPALEmSJNUwMEuSJEk1DMySJElSDQOzJEmSVMPALEmSJNUwMEuSJEk1\nDMySJElSDQOzJEmSVMPALEmSJNUwMEuSJEk1DMySJElSDQOzJEmSVMPALEmSJNUwMEuSJEk1DMyS\nJElSDQOzJEmSVGOgwBwRp0fEhojYGBHnTLP9AxFxZfX1w4i4q2/bn0XE1dXXr89l5yVJkqT5NjJT\ng4gYBj4MvBDYBFwWEesyc32vTWa+ta/9m4GnVs9fBjwNOBlYAnw1Ij6fmffM6buQJEmS5skgI8yn\nABsz84bM3A6cD5xR0/4s4Lzq+UnAVzNzZ2ZuBb4HnP5QOixJkiQtpBlHmIGjgZv7ljcBz5yuYUQc\nB6wCvlyt+h7w7oh4P7AUOBVYP81+a4A1AKOjo3Q6nQG7ryZNTExYqxawTu1gndrDWrWDdWqHttRp\nkMAc06zLfbQ9E7gwM3cBZOYXI+IZwL8BtwOXAjsf9GKZa4G1AGNjYzk+Pj5At9S0TqeDtSqfdWoH\n69Qe1qodrFM7tKVOg0zJ2AQc07e8Eti8j7Znsmc6BgCZ+d7MPDkzX0g3fF+3Px2VJEmSmjBIYL4M\nWB0RqyJiMd1QvG5qo4gYAw6jO4rcWzccEY+snj8FeArwxbnouCRJkrQQZpySkZk7I+Js4AvAMPDx\nzLwmIt4DXJ6ZvfB8FnB+ZvZP11gEfD0iAO4BfiMzHzQlQ5IkSSrVIHOYycyLgIumrHvXlOVzp9nv\nfrpXypAkSZJayTv9SZIkSTUMzJIkSVINA7MkSZJUw8AsSZIk1TAwS5IkSTUMzJIkSVINA7MkSZJU\nw8AsSZIk1TAwS5IkSTUMzJIkSVINA7MkSZJUw8AsSZIk1TAwS5IkSTUMzJIkSVINA7MkSZJUw8As\nSZIk1TAwS5IkSTUMzJIkSVINA7MkSZJUw8AsSZIk1TAwS5IkSTUMzJIkSVINA7MkSZJUw8AsSZIk\n1TAwS5IkSTUMzJIkSVKNgQJzRJweERsiYmNEnDPN9g9ExJXV1w8j4q6+bf81Iq6JiGsj4kMREXP5\nBiRJkqT5NDJTg4gYBj4MvBDYBFwWEesyc32vTWa+ta/9m4GnVs+fAzwXeEq1+RvALwKdOeq/JEmS\nNK8GGWE+BdiYmTdk5nbgfOCMmvZnAedVzxM4CFgMLAEWAbftf3clSZKkhTXjCDNwNHBz3/Im4JnT\nNYyI44BVwJcBMvPSiPgKcCsQwF9l5rXT7LcGWAMwOjpKp9OZxVtQUyYmJqxVC1indrBO7WGt2sE6\ntUNb6jRIYJ5uznHuo+2ZwIWZuQsgIk4ATgRWVtsvjojnZebX9nqxzLXAWoCxsbEcHx8foFtqWqfT\nwVqVzzq1g3VqD2vVDtapHdpSp0GmZGwCjulbXgls3kfbM9kzHQPgl4FvZuZEZk4AnweetT8dlSRJ\nkpowSGC+DFgdEasiYjHdULxuaqOIGAMOAy7tW30T8IsRMRIRi+h+4O9BUzIkSZKkUs0YmDNzJ3A2\n8AW6YfeCzLwmIt4TEa/oa3oWcH5m9k/XuBC4Hvg+8D3ge5n5T3PWe0mSJGmeDTKHmcy8CLhoyrp3\nTVk+d5r9dgFvfAj9kyRJkhrlnf4kSZKkGgZmSZIkqYaBWZIkSaphYJYkSZJqGJglSZKkGgZmSZIk\nqYaBWZIkSaoRe99npHljY2O5YcOGgdr+yT9dw/rN98xzj7Qvd911FytWrGi6G5qBdWoH69Qe1qod\nrFM7NF2nC377OVdk5tNnaucIsyRJklSj1SPMalan02F8fLzpbmgG1qkdrFN7WKt2sE7t0HSdIsIR\nZkmSJOmhMjBLkiRJNQzMkiRJUg0DsyRJklTDwCxJkiTVMDBLkiRJNQzMkiRJUg0DsyRJklTDwCxJ\nkiTVMDBLkiRJNQzMkiRJUg0DsyRJklTDwCxJkiTVMDBLkiRJNQzMkiRJUo2BAnNEnB4RGyJiY0Sc\nM832D0TEldXXDyPirmr9qX3rr4yI+yPilXP9JiRJkqT5MjJTg4gYBj4MvBDYBFwWEesyc32vTWa+\nta/9m4GnVuu/ApxcrT8c2Ah8cS7fgCRJkjSfBhlhPgXYmJk3ZOZ24HzgjJr2ZwHnTbP+V4DPZ+Z9\ns++mJEmS1IxBAvPRwM19y5uqdQ8SEccBq4AvT7P5TKYP0pIkSVKxZpySAcQ063Ifbc8ELszMXXu9\nQMRRwJOBL0x7gIg1wBqA0dFROp3OAN1S0yYmJqxVC1indrBO7WGt2sE6tUNb6jRIYN4EHNO3vBLY\nvI+2ZwJvmmb9rwGfzcwd0+2UmWuBtQBjY2M5Pj4+QLfUtE6ng7Uqn3VqB+vUHtaqHaxTO7SlToNM\nybgMWB0RqyJiMd1QvG5qo4gYAw4DLp3mNfY1r1mSJEkq2oyBOTN3AmfTnU5xLXBBZl4TEe+JiFf0\nNT0LOD8z95quERHH0x2h/upcdVqSJElaKINMySAzLwIumrLuXVOWz93Hvjeyjw8JSpIkSaXzTn+S\nJElSDQOzJEmSVMPALEmSJNUwMEuSJEk1DMySJElSDQOzJEmSVMPALEmSJNUwMEuSJEk1DMySJElS\nDQOzJEmSVMPALEmSJNUwMEuSJEk1DMySJElSDQOzJEmSVMPALEmSJNUwMEuSJEk1DMySJElSDQOz\nJEmSVMPALEmSJNUwMEuSJEk1DMySJElSDQOzJEmSVMPALEmSJNUwMEuSJEk1DMySJElSDQOzJEmS\nVGOgwBwRp0fEhojYGBHnTLP9AxFxZfX1w4i4q2/bsRHxxYi4NiLWR8Txc9d9SZIkaX6NzNQgIoaB\nDwMvBDYBl0XEusxc32uTmW/ta/9m4Kl9L/Ep4L2ZeXFELAcm56rzkiRJ0nwbZIT5FGBjZt6QmduB\n84EzatqfBZwHEBEnASOZeTFAZk5k5n0Psc+SJEnSgonMrG8Q8SvA6Zn5W9Xyq4FnZubZ07Q9Dvgm\nsDIzd0XEK4HfArYDq4AvAedk5q4p+60B1gCMjo7+/AUXXPCQ35jm38TEBMuXL2+6G5qBdWoH69Qe\n1qodrFM7NF2nU0899YrMfPpM7WackgHENOv2lbLPBC7sC8QjwL+jO0XjJuDvgNcBH9vrxTLXAmsB\nxsbGcnx8fIBuqWmdTgdrVT7r1A7WqT2sVTtYp3ZoS50GmZKxCTimb3klsHkfbc+kmo7Rt+93q+kc\nO4F/BJ62Px2VJEmSmjBIYL4MWB0RqyJiMd1QvG5qo4gYAw4DLp2y72ERMVotPx9YP3VfSZIkqVQz\nBuZqZPhs4AvAtcAFmXlNRLwnIl7R1/Qs4PzsmxRdTc34feCSiPg+3ekdfz2Xb0CSJEmaT4PMYSYz\nLwIumrLuXVOWz93HvhcDT9nP/kmSJOn/tnf/sXqedR3H3x9bN4QG10klus7RhaaCqAyWMURNBwIF\nyeof+6N10akz/QOGuBh1yxJA/Afjj6lxog3g0JDVUREbUh3LRvUP2eymy9gPup0Nw7pOh2GbVI2j\n4+sfz3Xg6dnpdU7tj/t+OO9Xcuc813Vf59n15Lvr2WfPfd3n0aD8pj9JkiSpw8AsSZIkdRiYJUmS\npA4DsyRJktRhYJYkSZI6DMySJElSh4FZkiRJ6jAwS5IkSR0GZkmSJKnDwCxJkiR1GJglSZKkDgOz\nJEmS1GFgliRJkjoMzJIkSVKHgVmSJEnqMDBLkiRJHQZmSZIkqcPALEmSJHUYmCVJkqQOA7MkSZLU\nYWCWJEmSOgzMkiRJUoeBWZIkSeowMEuSJEkdBmZJkiSpw8AsSZIkdSwrMCfZkuRAkrkk1yxy/vok\n97TjoSRPT517burcnpM5eUmSJOlUW73UgCSrgBuANwMHgf1J9lTVA/NjqurqqfHvBi6Yeor/qapX\nn7wpS5IkSafPcj5hvgiYq6pHq+pZYBewtTN+O3DTyZicJEmSNLRUVX9Achmwpap+sbV/BnhdVV21\nyNjzgDuA9VX1XOs7AtwDHAE+WFWfWuT3dgA7ANatW/fam2+++YRelE6Pw4cPs2bNmqGnoSVYp9lg\nnWaHtZoN1mk2DF2nSy655O6qunCpcUtuyQCySN+xUvY2YPd8WG6+r6oOJTkfuD3J56vqkaOerGon\nsBNg06ZNtXnz5mVMS0Pbt28f1mr8rNNssE6zw1rNBus0G2alTsvZknEQOHeqvR44dIyx21iwHaOq\nDrWfjwL7OHp/syRJkjRqywnM+4GNSTYkOYNJKH7eX7tIsglYC3xuqm9tkjPb45cAbwAeWPi7kiRJ\n0lgtuSWjqo4kuQq4BVgFfLSq7k/yAeCuqpoPz9uBXXX0puhXAH+a5OtMwvkHp/+6hiRJkjR2y9nD\nTFXtBfYu6Hvvgvb7F/m9fwR+8ATmJ0mSJA3Kb/qTJEmSOgzMkiRJUoeBWZIkSeowMEuSJEkdBmZJ\nkiSpw8AsSZIkdRiYJUmSpA4DsyRJktRhYJYkSZI6DMySJElSh4FZkiRJ6jAwS5IkSR0GZkmSJKnD\nwCxJkiR1GJglSZKkDgOzJEmS1GFgliRJkjoMzJIkSVKHgVmSJEnqMDBLkiRJHQZmSZIkqcPALEmS\nJHUYmCVJkqQOA7MkSZLUYWCWJEmSOpYVmJNsSXIgyVySaxY5f32Se9rxUJKnF5x/cZLHk/zRyZq4\nJEmSdDqsXmpAklXADcCbgYPA/iR7quqB+TFVdfXU+HcDFyx4mt8E/v6kzFiSJEk6jZbzCfNFwFxV\nPVpVzwK7gK2d8duBm+YbSV4LvBT4zIlMVJIkSRrCcgLzOcBjU+2Dre95kpwHbABub+1vA34X+NUT\nm6YkSZI0jCW3ZABZpK+OMXYbsLuqnmvtdwJ7q+qxZLGnaf+AZAewA2DdunXs27dvGdPS0A4fPmyt\nZoB1mg3WaXZYq9lgnWbDrNRpOYH5IHDuVHs9cOgYY7cB75pqvx74sSTvBNYAZyQ5XFVH3ThYVTuB\nnQCbNm2qzZs3L2/2GtS+ffuwVuNnnWaDdZod1mo2WKfZMCt1Wk5g3g9sTLIBeJxJKP7phYOSbALW\nAp+b76uqy6fO/xxw4cKwLEmSJI3ZknuYq+oIcBVwC/AgcHNV3Z/kA0kunRq6HdhVVcfariFJkiTN\nnOV8wkxV7QX2Luh774L2+5d4jhuBG49rdpIkSdLA/KY/SZIkqcPALEmSJHUYmCVJkqQOA7MkSZLU\nYWCWJEmSOgzMkiRJUoeBWZIkSeowMEuSJEkdBmZJkiSpw8AsSZIkdRiYJUmSpA4DsyRJktRhYJYk\nSZI6DMySJElSh4FZkiRJ6jAwS5IkSR0GZkmSJKnDwCxJkiR1pKqGnsNRknwVODD0PLQsLwH+Y+hJ\naEnWaTZYp9lhrWaDdZoNQ9fpvKpat9Sg1adjJsfpQFVdOPQktLQkd1mr8bNOs8E6zQ5rNRus02yY\nlTq5JUOSJEnqMDBLkiRJHWMMzDuHnoCWzVrNBus0G6zT7LBWs8E6zYaZqNPobvqTJEmSxmSMnzBL\nkiRJo2FgliRJkjpGFZiTbElyIMlckmuGns9KluTcJJ9N8mCS+5O8p/WfneTWJA+3n2tbf5L8Yavd\nvUleM+wrWFmSrEryL0k+3dobktzZ6vSXSc5o/We29lw7/7Ih573SJDkrye4kX2hr6/WuqfFJcnV7\n37svyU1JXuCaGl6SjyZ5Msl9U33HvX6SXNHGP5zkiiFey7e6Y9Tqt9t7371J/jrJWVPnrm21OpDk\nrVP9o8mFownMSVYBNwBvA14JbE/yymFntaIdAX6lql4BXAy8q9XjGuC2qtoI3NbaMKnbxnbsAD50\n+qe8or0HeHCq/VvA9a1OTwFXtv4rgaeq6uXA9W2cTp8/AP6uqr4f+GEmNXNNjUiSc4BfAi6sqlcB\nq4BtuKbG4EZgy4K+41o/Sc4G3ge8DrgIeN98yNZJdSPPr9WtwKuq6oeAh4BrAVq22Ab8QPudP24f\nAo0qF44mMDP5F3euqh6tqmeBXcDWgee0YlXVE1X1z+3xV5n8h/0cJjX5WBv2MeCn2uOtwJ/XxB3A\nWUm+5zRPe0VKsh74SeDDrR3gjcDuNmRhnebrtxt4UxuvUyzJi4EfBz4CUFXPVtXTuKbGaDXwHUlW\nAy8EnsA1Nbiq+gfgKwu6j3f9vBW4taq+UlVPMQlxC4OdTtBitaqqz1TVkda8A1jfHm8FdlXV/1bV\nF4E5JplwVLlwTIH5HOCxqfbB1qeBtUuMFwB3Ai+tqidgEqqB727DrN9wfh/4NeDrrf1dwNNTb0zT\ntfhGndr5Z9p4nXrnA18G/qxtn/lwkhfhmhqVqnoc+B3gS0yC8jPA3bimxup414/rahx+Afjb9ngm\najWmwLzY/5H7N+8GlmQN8FfAL1fVf/aGLtJn/U6xJO8Anqyqu6e7FxlayzinU2s18BrgQ1V1AfBf\nfPPy8WKs1QDa5fmtwAbge4EXMbkkvJBratyOVRfrNbAk1zHZ9vnx+a5Fho2uVmMKzAeBc6fa64FD\nA81FQJJvZxKWP15Vn2zd/z5/Wbj9fLL1W79hvAG4NMm/Mrlc9UYmnzif1S4nw9G1+Ead2vnv5PmX\nOHVqHAQOVtWdrb2bSWScT1sAAAGtSURBVIB2TY3LTwBfrKovV9XXgE8CP4JraqyOd/24rgbUbrJ8\nB3B5ffOLQGaiVmMKzPuBje1O5DOYbADfM/CcVqy2B+8jwINV9XtTp/YA83cVXwH8zVT/z7Y7ky8G\nnpm/TKZTp6qurar1VfUyJmvm9qq6HPgscFkbtrBO8/W7rI3305XToKr+DXgsyabW9SbgAVxTY/Ml\n4OIkL2zvg/N1ck2N0/Gun1uAtyRZ264mvKX16RRLsgX4deDSqvrvqVN7gG3tL85sYHKj5j8xtlxY\nVaM5gLczuXPyEeC6oeezkg/gR5lc+rgXuKcdb2eyN+824OH28+w2PkzuZn0E+DyTO8wHfx0r6QA2\nA59uj89n8oYzB3wCOLP1v6C159r584ee90o6gFcDd7V19SlgrWtqfAfwG8AXgPuAvwDOdE0NfwA3\nMdlX/jUmnz5e+f9ZP0z2z8614+eHfl3fiscxajXHZE/yfKb4k6nx17VaHQDeNtU/mlzoV2NLkiRJ\nHWPakiFJkiSNjoFZkiRJ6jAwS5IkSR0GZkmSJKnDwCxJkiR1GJglSZKkDgOzJEmS1PF/4RVHI/5X\nVDcAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "\n", + "loss = np.array(train_summary.read_scalar(\"Loss\"))\n", + "top1 = np.array(val_summary.read_scalar(\"Top1Accuracy\"))\n", + "\n", + "plt.figure(figsize = (12,12))\n", + "plt.subplot(2,1,1)\n", + "plt.plot(loss[:,0],loss[:,1],label='loss')\n", + "plt.xlim(0,loss.shape[0]+10)\n", + "plt.grid(True)\n", + "plt.title(\"loss\")\n", + "plt.subplot(2,1,2)\n", + "plt.plot(top1[:,0],top1[:,1],label='top1')\n", + "plt.xlim(0,loss.shape[0])\n", + "plt.title(\"top1 accuracy\")\n", + "plt.grid(True)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "predictions = trained_model.predict(cc_rdd_test).collect()\n", + "\n", + "def map_predict_label(l):\n", + " return np.array(l).argmax()\n", + "def map_groundtruth_label(l):\n", + " return l.to_ndarray()[0] - 1\n", + "\n", + "y_pred = np.array([ map_predict_label(s) for s in predictions])\n", + "\n", + "y_true = np.array([map_groundtruth_label(s.label) for s in cc_rdd_test.collect()])" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[ 0.40660739 1.09633136]\n", + "0.0\n" + ] + }, + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(str(np.abs(predictions[0])))\n", + "print(str(y_true[0]))\n", + "map_predict_label(predictions[0])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The prediction accuracy is 77.60%\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAHVCAYAAAA5NRumAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGtxJREFUeJzt3Xu0XmV9J/DvL+HihWJAqAKhQpWp\nYlvRQsA6KhXlZhWntRQ7FZbFSadiB+/VXhZTKq22owzOINO0gGiryKK1UBqHUqzFTlGDS6pctISb\nBCKCXKsCIeeZP/KSHsnJBeo5Zz/Zn49rr5z3eff77ucszcrP7+959q7WWgAAerFgvicAAPBYKF4A\ngK4oXgCAriheAICuKF4AgK4oXgCAriheAICuKF4AgK4oXgCArmwz2xdYc+cNbuEL8+CJu794vqcA\no/XwQ7fWXF1rtv6d3XaXH52z3+GxkrwAAF2Z9eQFAJhFU2vnewZzTvICAHRF8gIAPWtT8z2DOSd5\nAQC6InkBgJ5NjS95UbwAQMeathEAwLBJXgCgZyNsG0leAICuSF4AoGcjXPOieAGAnrnDLgDAsEle\nAKBnI2wbSV4AgK5IXgCgZyPcKq14AYCOucMuAMDASV4AoGcjbBtJXgCArkheAKBn1rwAAAyb5AUA\nejbCxwMoXgCgZ9pGAADDJnkBgJ7ZKg0AMGySFwDo2QjXvCheAKBn2kYAAMMmeQGAjrU2vvu8SF4A\ngK5IXgCgZxbsAgBdsWAXAGDYJC8A0LMRto0kLwBAVyQvANCzqfFtlVa8AEDPtI0AAIZN8gIAPbNV\nGgBg2CQvANAza14AAIZN8gIAPRvhmhfFCwD0bITFi7YRANAVyQsAdKy18d1hV/ICAHRF8gIAPRvh\nmhfFCwD0zH1eAACGTfICAD0bYdtI8gIAdEXyAgA9G+GaF8ULAPRM2wgAYNgkLwDQsxG2jSQvAEBX\nJC8A0DNrXgAAtkxVLaqq86vqa1V1bVW9sKp2rqpLquq6yZ87Tc6tqvpQVa2sqq9U1Qumfc9xk/Ov\nq6rjNnddxQsA9GxqanaOLXNakv/bWnt2kucluTbJu5Nc2lrbJ8mlk9dJckSSfSbH0iRnJElV7Zzk\npCQHJlmS5KRHCp6NUbwAQM/a1Owcm1FVOyZ5SZIzk6S19lBr7Z4kRyU5Z3LaOUleM/n5qCQfbet8\nPsmiqtotyWFJLmmt3dVauzvJJUkO39S1FS8AwAaqamlVXTHtWPqoU340yR1Jzq6qL1fVn1bVk5M8\nrbW2Okkmf/7w5Pw9ktwy7fOrJmMbG98oC3YBoGeztGC3tbYsybJNnLJNkhck+fXW2heq6rT8W4to\nJjXTZTYxvlGSFwDg8ViVZFVr7QuT1+dnXTFz+6QdlMmf35p2/p7TPr84yW2bGN8oxQsA9Gye1ry0\n1r6Z5Jaq+rHJ0CFJrklyYZJHdgwdl+SCyc8XJjl2suvooCT3TtpKFyc5tKp2mizUPXQytlHaRgDQ\ns/m9z8uvJ/nzqtouyQ1J3pB1wch5VXV8km8k+YXJucuTHJlkZZLvTs5Na+2uqvq9JCsm553cWrtr\nUxdVvAAAj0tr7cok+8/w1iEznNuSnLCR7zkryVlbel3FCwD0zLONAACGTfICAD0b4bONFC8A0LMR\nFi/aRgBAVyQvANCztsmb0W6VJC8AQFckLwDQM2teAACGTfICAD0bYfKieAGAnrnDLgDAsEleAKBn\nI2wbSV4AgK5IXgCgZyO8SZ3iBQB6pm0EADBskhcA6JnkBQBg2CQvANCzEd6kTvECAB1rU+PbbaRt\nBAB0RfICAD2zYBcAYNgkLwDQsxEu2JW8AABdkbwAQM9GuNtI8QIAPbNgFwBg2CQvANAzyQsAwLBJ\nXgCgZ82CXQCgJ9pGAADDpngZmRtvXpWfP+6E9ceBr/i5fOyTn8q9992fN574mznyF4/PG0/8zdx7\n3/3f97mvXvv1/OSLX5m//fvPrR/71bf9dl542GvzpneeNNe/Bmy1Djv04Fx91WX52jX/mHe984T5\nng49mGqzcwyY4mVk9n7G4vzFOafnL845Peed9aE84QlPyCEv/en86cfOy0H775flnzwzB+2/X878\ns/PWf2bt2rU59cNn50VLXvB93/WGX/r5/MHvvGOufwXYai1YsCAfOu2U/Oyrfjk/8byfyS/+4mvy\nnOfsM9/TgsFRvIzY56+4MnvusVt2f/rT8vefuzxHHfHyJMlRR7w8n7ns8vXnffz8C/OKg1+UnXda\n9H2fP2j/5+dJT3rSnM4ZtmZLDnh+rr/+ptx44zeyZs2anHfeBXn1qw6b72kxdG1qdo4B2+yC3ap6\ndpKjkuyRpCW5LcmFrbVrZ3luzLJPX/oPOfLlL02SfPvue7LrLjsnSXbdZefcdc+9SZLb77gzl172\nTznzQ+/LVdf+y7zNFcZg9z2enltW3bb+9apbV2fJAc+fxxnRhYG3eGbDJpOXqvqNJOcmqSRfTLJi\n8vMnqurdsz89ZsuaNWvy2X/8Qg592Ys3ed77T/vjvPXXfiULFy6co5nBeFXVBmNthNtgYXM2l7wc\nn+S5rbU10wer6oNJrk7yvpk+VFVLkyxNkg9/4L1547Gv+wFMlR+kz33+ijznPzwzu+y8U5LkqTst\nyh133pVdd9k5d9x5V3Ze9JQkydVfuy7vPGndf81333tfPnf5iixcuDCHvOSn523usLW6ddXq7Ll4\n9/WvF++xW1avvn0eZ0QP2gi3Sm+ueJlKsnuSmx81vtvkvRm11pYlWZYka+68wf9tGKDll3w2R77i\n4PWvD/6PB+WCT/9d3vj6o3PBp/8uP/PiFyZJLj7/I+vP+a33fiAvfdEShQvMkhVXXJlnPWvv7LXX\nnrn11m/m6KOPyuuPteMIHm1zxctbklxaVdcluWUy9iNJnpXkzbM5MWbP9x54IJev+HJOetd/Wz/2\nxtcfnbf/zu/nLy+6OLs9bdd88L2/tdnvOfbX3pEbv3FLvvvdB3LIa345J7/nrXnRgT81m1OHrdra\ntWtz4lt+O8v/5uNZuGBBPnLOJ3PNNdaasRkjXPNSm+unVtWCJEuybsFuJVmVZEVrbe2WXEDyAvPj\nibtvej0TMHsefujWDRcwzZLvnHLsrPw7++Tf+uic/Q6P1WZ3G7XWppJ8fg7mAgA8VgPf1jwbPNsI\nAHo2wraRm9QBAF2RvABAz0a4VVryAgB0RfICAD0b4ZoXxQsA9GyEu420jQCArkheAKBnI2wbSV4A\ngK5IXgCgY54qDQD0RdsIAGDYJC8A0DPJCwDAsEleAKBnblIHADBskhcA6NkI17woXgCgY22ExYu2\nEQDQFckLAPRM8gIAMGySFwDomWcbAQBd0TYCABg2yQsA9EzyAgAwbJIXAOhYa+NLXhQvANAzbSMA\ngGGTvABAzyQvAADDJnkBgI55qjQAwMBJXgCgZyNMXhQvANCz8T2XUdsIAOiL5AUAOmbBLgDAwEle\nAKBnI0xeFC8A0DMLdgEAtlxVLayqL1fVRZPXH6mqG6vqysmx32S8qupDVbWyqr5SVS+Y9h3HVdV1\nk+O4zV1T8gIAHRvAgt0Tk1ybZMdpY+9srZ3/qPOOSLLP5DgwyRlJDqyqnZOclGT/JC3Jl6rqwtba\n3Ru7oOQFAHhcqmpxklcm+dMtOP2oJB9t63w+yaKq2i3JYUkuaa3dNSlYLkly+Ka+SPECAD2bmp2j\nqpZW1RXTjqUzXP1/JnlXNlx5c8qkNXRqVW0/GdsjyS3Tzlk1GdvY+EZpGwFAx2arbdRaW5Zk2cbe\nr6qfTfKt1tqXqurgaW+9J8k3k2w3+fxvJDk5Sc10mU2Mb5TkBQB4PF6U5NVVdVOSc5O8rKr+rLW2\netIaejDJ2UmWTM5flWTPaZ9fnOS2TYxvlOIFAHo2S22jzWmtvae1tri1tleSY5J8prX2y5N1LKmq\nSvKaJFdNPnJhkmMnu44OSnJva211kouTHFpVO1XVTkkOnYxtlLYRAPCD9OdVtWvWtYOuTPJfJ+PL\nkxyZZGWS7yZ5Q5K01u6qqt9LsmJy3smttbs2dQHFCwB0rA3gJnWttc8m+ezk55dt5JyW5ISNvHdW\nkrO29HqKFwDo2QCKl7lmzQsA0BXJCwB0bAhto7kmeQEAuiJ5AYCeSV4AAIZN8gIAHRvjmhfFCwB0\nbIzFi7YRANAVyQsAdEzyAgAwcJIXAOhZq/mewZxTvABAx7SNAAAGTvICAB1rU+NrG0leAICuSF4A\noGNjXPOieAGAjrUR7jbSNgIAuiJ5AYCOjbFtJHkBALoieQGAjtkqDQAwcJIXAOhYa/M9g7mneAGA\njmkbAQAMnOQFADomeQEAGDjJCwB0zIJdAKAr2kYAAAMneQGAjnmqNADAwEleAKBjY3yqtOIFADo2\npW0EADBskhcA6JgFuwAAAyd5AYCOuUkdAMDASV4AoGOebQQAdEXbCABg4CQvANAxN6kDABg4yQsA\ndGyMN6lTvABAx8a420jbCADoiuQFADpmwS4AwMBJXgCgYxbsAgBdsWAXAGDgJC8A0LExLtid9eLl\nXfv/5mxfAgAYEckLAHRsjAt2rXkBALoieQGAjlnzAgB0ZYQ7pbWNAIC+SF4AoGNjbBtJXgCArkhe\nAKBjY9wqrXgBgI5NzfcE5oG2EQDQFckLAHSsZXxtI8kLANAVyQsAdGxqhHepU7wAQMemtI0AAIZN\n8gIAHbNgFwBg4CQvANAxN6kDABg4yQsAdGyMa14ULwDQMW0jAICBk7wAQMckLwAAAyd5AYCOWbAL\nAHRlany1i7YRANAXyQsAdMxTpQEAtkBVPaGqvlhV/1xVV1fV707G966qL1TVdVX1yarabjK+/eT1\nysn7e037rvdMxr9eVYdt7tqKFwDoWJulYws8mORlrbXnJdkvyeFVdVCS9yc5tbW2T5K7kxw/Of/4\nJHe31p6V5NTJeamqfZMck+S5SQ5P8uGqWripCyteAKBjU7N0bE5b518nL7edHC3Jy5KcPxk/J8lr\nJj8fNXmdyfuHVFVNxs9trT3YWrsxycokSzZ1bcULAPC4VNXCqroyybeSXJLk+iT3tNYenpyyKske\nk5/3SHJLkkzevzfJU6ePz/CZGVmwCwAdm6rZWbBbVUuTLJ02tKy1tmz6Oa21tUn2q6pFST6V5Dkz\nfNUjXaiZJto2Mb5RihcAYAOTQmXZZk9cd+49VfXZJAclWVRV20zSlcVJbpuctirJnklWVdU2SZ6S\n5K5p44+Y/pkZaRsBQMfma8FuVe06SVxSVU9M8vIk1yb5+ySvnZx2XJILJj9fOHmdyfufaa21yfgx\nk91IeyfZJ8kXN3VtyQsA8HjsluScyc6gBUnOa61dVFXXJDm3qt6b5MtJzpycf2aSj1XVyqxLXI5J\nktba1VV1XpJrkjyc5IRJO2qjFC8A0LH5eqp0a+0rSZ4/w/gNmWG3UGvtgSS/sJHvOiXJKVt6bcUL\nAHTMs40AAAZO8gIAHfNsIwCAgZO8AEDHtvA5RFsVxQsAdMyCXQCAgZO8AEDH5us+L/NJ8gIAdEXy\nAgAds2AXAOiKBbsAAAMneQGAjlmwCwAwcJIXAOiY5AUAYOAkLwDQsTbC3UaKFwDomLYRAMDASV4A\noGOSFwCAgZO8AEDHPNsIAOiKZxsBAAyc5AUAOmbBLgDAwEleAKBjY0xeFC8A0LEx7jbSNgIAuiJ5\nAYCO2SoNADBwkhcA6NgYF+xKXgCArkheAKBjY9xtpHgBgI5NjbB80TYCALoieQGAjlmwCwAwcJIX\nAOjY+Fa8KF4AoGvaRgAAAyd5AYCOebYRAMDASV4AoGNjvEmd4gUAOja+0kXbCADojOQFADpmqzQA\nwMBJXgCgYxbsAgBdGV/pom0EAHRG8gIAHbNgFwBg4CQvANCxMS7YlbwAAF2RvABAx8aXuyheAKBr\nFuwCAAyc5AUAOtZG2DiSvAAAXZG8AEDHxrjmRfECAB1znxcAgIGTvABAx8aXu0heAIDOSF4AoGNj\nXPOieAGAjtltxFZv0W5PzS998E3ZcddFaVNTufwTn8llZ386T3rKk3Ps/z4xOy/eNXetuiPnnHBa\nvnffd/Ljr/ipHPG2o9Nay9TDa/Opkz+aG6/4enbf9xn5hfcenyfs8MRMrZ3KJaf/Va686PL5/vWg\ne4cdenA++MGTs3DBgpx19ifyh390+nxPCQZH8TIyUw+vzYXv/VhWXX1Ttn/yE/K2v/6DfP1zX8mS\n17401/3TVbn0jAtzyK+9Ooe86ahc9L6P51/+31W56pIvJUl2e/aP5LjTT8z7Dnl71nzvofz52z6c\nO2/6Znb84Z3y9ot+P1+77J/zwH3fneffEPq1YMGCfOi0U3L4ka/LqlWr8/nLl+evL/rbXHvtdfM9\nNQbMHXbZ6t13xz1ZdfVNSZIHv/NAbr/+1jzl6Tvnx1+xf1acf1mSZMX5l+UnXrF/kuSh7z64/rPb\nPWn79cva77hxde686ZvrvvNbd+f+b9+XHXbece5+EdgKLTng+bn++pty443fyJo1a3LeeRfk1a86\nbL6nBYPzuJOXqnpDa+3sH+RkmFs7Ld41i/fdKzdfuTI/tOtTct8d9yRZV+DssMu/FSI/cdgBeeW7\njskOT31K/uRX3r/B9/zI856ZbbbdJt+++fY5mztsjXbf4+m5ZdVt61+vunV1lhzw/HmcET0Y45qX\nf0/y8rsbe6OqllbVFVV1xVfvv/7fcQlmy3ZP2j5vOOOt+dTJ5+TBf/3eJs/96sUr8r5D3p6zlv6P\nHPm2o7/vvR13XZT//MET8ol3npHWxhddwg9SVW0w5u8VbGiTyUtVfWVjbyV52sY+11pblmRZkrx1\nr2P8zRuYBdsszBv+z9vypb/6x3z14hVJkvvvuDc77roo991xT3bcdVH+9c77NvjcDV/8Wp76jKfl\nyTv9UL5z9/3Zfocn5r+c/RtZ/oFP5uYvr5zrXwO2OreuWp09F+++/vXiPXbL6tUSTTbNmpcNPS3J\nsUleNcPx7dmdGrPlmPf/am5feWv+4czl68eu+rsv5YDXviRJcsBrX5KrLrkiSbLLM/6tRl383L2y\ncNtt8p2778/CbRfmV/747Vnxl5fln5d/YW5/AdhKrbjiyjzrWXtnr732zLbbbpujjz4qf33R3873\ntBi4qVk6hmxza14uSrJDa+3KR79RVZ+dlRkxq/be/8dywM+/JLdde3Pesfx9SZK/+cNzc+kZF+S4\n09+SA4/+mdx927dzzptOTZL85BEH5oCfe3HWPrw2ax54KB9982lJkv1e+cI8c8mz8+SddsiS1740\nSfLxd5yR2665eX5+MdgKrF27Nie+5bez/G8+noULFuQj53wy11zzL/M9LRicmu1+qrYRzI//ddvn\n5nsKMFoPP3TrhguYZsnrn/Fzs/Lv7Mdu/ss5+x0eK1ulAYCuuEkdAHRsjO0NxQsAdGyMD2bUNgIA\nuiJ5AYCOuc8LAMDASV4AoGNDv6HcbFC8AEDHLNgFANgCVXVWVX2rqq6aNvbfq+rWqrpychw57b33\nVNXKqvp6VR02bfzwydjKqnr3llxb8QIAHWuz9J8t8JEkh88wfmprbb/JsTxJqmrfJMckee7kMx+u\nqoVVtTDJ6UmOSLJvktdNzt0kbSMA4DFrrV1WVXtt4elHJTm3tfZgkhuramWSJZP3VrbWbkiSqjp3\ncu41m/oyyQsAdGy2nipdVUur6oppx9ItnNKbq+ork7bSTpOxPZLcMu2cVZOxjY1vkuIFANhAa21Z\na23/aceyLfjYGUmemWS/JKuTfGAyPtNDHtsmxjdJ2wgAOtbacHYbtdZuf+TnqvqTJBdNXq5Ksue0\nUxcnuW3y88bGN0ryAgAdm0qblePxqKrdpr38T0ke2Yl0YZJjqmr7qto7yT5JvphkRZJ9qmrvqtou\n6xb1Xri560heAIDHrKo+keTgJLtU1aokJyU5uKr2y7rWz01JfjVJWmtXV9V5WbcQ9+EkJ7TW1k6+\n581JLk6yMMlZrbWrN3dtxQsAdGy+7rDbWnvdDMNnbuL8U5KcMsP48iTLH8u1tY0AgK5IXgCgY2N8\nqrTiBQA65tlGAAADJ3kBgI4N6T4vc0XyAgB0RfICAB2br63S80nxAgAdG+NuI20jAKArkhcA6Jit\n0gAAAyd5AYCO2SoNADBwkhcA6NgY17woXgCgY7ZKAwAMnOQFADo2ZcEuAMCwSV4AoGPjy10ULwDQ\ntTHuNtI2AgC6InkBgI5JXgAABk7yAgAdG+OzjRQvANAxbSMAgIGTvABAxzzbCABg4CQvANCxMS7Y\nlbwAAF2RvABAx8a420jxAgAd0zYCABg4yQsAdGyMbSPJCwDQFckLAHRsjDepU7wAQMemLNgFABg2\nyQsAdGyMbSPJCwDQFckLAHRsjGteFC8A0DFtIwCAgZO8AEDHxtg2krwAAF2RvABAx6x5AQAYOMkL\nAHRsjGteFC8A0DFtIwCAgZO8AEDHWpua7ynMOckLANAVyQsAdGxqhGteFC8A0LE2wt1G2kYAQFck\nLwDQsTG2jSQvAEBXJC8A0LExrnlRvABAx8b4eABtIwCgK5IXAOiYZxsBAAyc5AUAOjbGBbuSFwCg\nK5IXAOjYGG9Sp3gBgI5pGwEADJzkBQA65iZ1AAADJ3kBgI6Ncc2L4gUAOjbG3UbaRgBAVyQvANCx\nMbaNJC8AQFckLwDQsTFulVa8AEDHmgW7AADDJnkBgI6NsW0keQEAuiJ5AYCO2SoNADBwkhcA6NgY\ndxspXgCgY9pGAAADp3gBgI611mbl2BJVdXhVfb2qVlbVu2f5V11P8QIAPGZVtTDJ6UmOSLJvktdV\n1b5zcW3FCwB0rM3SsQWWJFnZWruhtfZQknOTHPWD+a02bdYX7J5607k129dg9lTV0tbasvmeB4/d\nqfM9Af5d/N1jSz380K2z8u9sVS1NsnTa0LJH/W9yjyS3THu9KsmBszGXR5O8sDlLN38KMAv83WNe\ntdaWtdb2n3Y8upieqWiak61PihcA4PFYlWTPaa8XJ7ltLi6seAEAHo8VSfapqr2rarskxyS5cC4u\n7CZ1bI6eO8wPf/cYtNbaw1X15iQXJ1mY5KzW2tVzce0a4535AIB+aRsBAF1RvAAAXVG8MKP5uuUz\njF1VnVVV36qqq+Z7LjBUihc2MJ+3fAbykSSHz/ckYMgUL8xk3m75DGPXWrssyV3zPQ8YMsULM5np\nls97zNNcAOD7KF6Yybzd8hkANkfxwkzm7ZbPALA5ihdmMm+3fAaAzVG8sIHW2sNJHrnl87VJzpur\nWz7D2FXVJ5JcnuTHqmpVVR0/33OCofF4AACgK5IXAKArihcAoCuKFwCgK4oXAKArihcAoCuKFwCg\nK4oXAKAr/x94OvEbXj6pvwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "acc = accuracy_score(y_true, y_pred)\n", + "print(\"The prediction accuracy is %.2f%%\"%(acc*100))\n", + "\n", + "cm = confusion_matrix(y_true, y_pred)\n", + "cm.shape\n", + "df_cm = pd.DataFrame(cm)\n", + "plt.figure(figsize = (10,8))\n", + "sn.heatmap(df_cm, annot=True,fmt='d');" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/feedforward-credit-card-fraud-pipeline.ipynb b/notebooks/feedforward-credit-card-fraud-pipeline.ipynb new file mode 100644 index 0000000..ccf223e --- /dev/null +++ b/notebooks/feedforward-credit-card-fraud-pipeline.ipynb @@ -0,0 +1,549 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Feedforward Network with Credit Card Fraud\n", + "\n", + "Credit Card Transactions Fraud Detection Example:\n", + "\n", + "The notebook demonstrates how to develop a fraud detection application with the BigDL deep learning library on Apache Spark. We'll try to introduce some techniques that can be used for training a fraud detection model, but some advanced skills is not applicable since the dataset is highly simplified.\n", + "\n", + "Dataset: Credit Card Fraud Detection https://www.kaggle.com/dalpozz/creditcardfraud\n", + "\n", + "This dataset presents transactions that occurred in two days, where we got 492 frauds out of 284,807 transactions. The dataset is highly unbalanced, the positive class (frauds) account for 0.172% of all transactions.\n", + "\n", + "It contains only numerical input variables which are the result of a PCA transformation. \n", + "\n", + "Unfortunately, due to confidentiality issues, we cannot find the original features and more background information about the data. Features V1, V2, ... V28 are the principal components obtained with PCA, the only features which have not been transformed with PCA are 'Time' and 'Amount'. Feature 'Time' contains the seconds elapsed between each transaction and the first transaction in the dataset. The feature 'Amount' is the transaction Amount, this feature can be used for example-dependant cost-senstive learning. Feature 'Class' is the response variable and it takes value 1 in case of fraud and 0 otherwise.\n", + "\n", + "Contact: yuhao.yang@intel.com\n" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import pandas as pd\n", + "import datetime as dt\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from sklearn.metrics import confusion_matrix\n", + "from sklearn.metrics import accuracy_score\n", + "import seaborn as sn\n", + "import pandas as pd\n", + "import random as rd\n", + "import datetime as dt\n", + "\n", + "\n", + "from bigdl.dataset.transformer import *\n", + "from bigdl.dataset.base import *\n", + "from bigdl.nn.layer import *\n", + "from bigdl.nn.criterion import *\n", + "from bigdl.optim.optimizer import *\n", + "from bigdl.util.common import *\n", + "from bigdl.models.ml_pipeline.dl_classifier import *\n", + "from utils import *\n", + "\n", + "from pyspark.sql.functions import col, udf\n", + "from pyspark.sql.types import DoubleType\n", + "from pyspark.ml import Pipeline\n", + "from pyspark.ml.feature import VectorAssembler, StandardScaler\n", + "from pyspark.ml.evaluation import BinaryClassificationEvaluator, MulticlassClassificationEvaluator\n", + "\n", + "\n", + "init_engine()" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['V1',\n", + " 'V2',\n", + " 'V3',\n", + " 'V4',\n", + " 'V5',\n", + " 'V6',\n", + " 'V7',\n", + " 'V8',\n", + " 'V9',\n", + " 'V10',\n", + " 'V11',\n", + " 'V12',\n", + " 'V13',\n", + " 'V14',\n", + " 'V15',\n", + " 'V16',\n", + " 'V17',\n", + " 'V18',\n", + " 'V19',\n", + " 'V20',\n", + " 'V21',\n", + " 'V22',\n", + " 'V23',\n", + " 'V24',\n", + " 'V25',\n", + " 'V26',\n", + " 'V27',\n", + " 'V28',\n", + " 'Amount']" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "learning_rate = 0.1\n", + "training_epochs = 100\n", + "batch_size = 10000\n", + "display_step = 1\n", + "\n", + "# Network Parameters\n", + "n_input = 29\n", + "n_classes = 2\n", + "n_hidden_1 = 10 # 1st layer number of features\n", + "\n", + "\n", + "LABELS = [\"Normal\", \"Fraud\"]\n", + "\n", + "cols = [\"V\" + str(x) for x in list(range(1,29))] + [\"Amount\"]\n", + "cols\n" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "cc_training = spark.read.csv(\"../data/creditcardfraud/creditcard.csv\", header=True, inferSchema=\"true\", mode=\"DROPMALFORMED\")" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------+-----------------+--------------------+--------------------+-----------------+-------------------+\n", + "|summary| Time| V1| V2| Amount| Class|\n", + "+-------+-----------------+--------------------+--------------------+-----------------+-------------------+\n", + "| count| 284807| 284807| 284807| 284807| 284807|\n", + "| mean| 94813.8596|2.260907382386083...|6.067406817137794...|88.34961925095207| 1.00172748563062|\n", + "| stddev|47488.14595456619| 1.9586958038574867| 1.6513085794769937|250.1201092401879|0.04152718963546483|\n", + "| min| 0| -56.407509631329| -72.7157275629303| 0.0| 1|\n", + "| max| 172792| 2.45492999121121| 22.0577289904909| 25691.16| 2|\n", + "+-------+-----------------+--------------------+--------------------+-----------------+-------------------+\n", + "\n" + ] + } + ], + "source": [ + "cc_training.select('Time', 'V1', 'V2', 'Amount', 'Class').describe().show()" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "cc_training = cc_training.select([col(c).cast(\"double\") for c in cc_training.columns])\n", + "cc_training = cc_training.withColumn(\"label\", cc_training[\"Class\"])\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Dataset Balance\n", + "\n", + "Let us see the dataset balance. We suspect a highly unbalanced dataset -- let us visualize. that." + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAEWCAYAAACwtjr+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAHKxJREFUeJzt3Xm0VOWd7vHvIziPKGgQUIhid9Qk\nqKgYMziL5ho0rd0OrcSFwSS4EhM7LfEmkXZIx9vXobkaE4y0gCNqnCKGRmNi2zHK0bBE1DQnSgQh\nCoKAM+Dv/rHfituizjnFgfcUFs9nrVqn6rff/e531ynOU3tgb0UEZmZmOW3U6AGYmVnzc9iYmVl2\nDhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMatB0iGSZnXxMm+UNKYrl1m1/HmSDknPfyDpp+uo326S\n3pC0S3q9TtdT0s8lXbCu+rM8HDa21tIfksrjfUlvl16f1ujxdURSd0khqX+lFhG/iYi9GjeqxoqI\niyPiax21k/SopK900NeqiNgqIl5a23FJOkvSb6r6PysifrS2fVte3Rs9APvoi4itKs8lzQHOiogH\n22ovqXtErOyKsVlj+XdtFd6ysewkXSLpNkm3SFoO/KOkgyT9XtLrkhZIGitp49S+sqVxtqRWSUsk\njS31t4ekRyQtlbRI0s2laVen3UHLJE2X9JnStO5p99Cf0vQWSTsDj6Qms9LW2N9JOiIFZ2XevST9\nNo13pqQvlqbdmMb/gKTlkh6TNKCd9+Pzad2XSpor6fQabXaQNEXSwrT+90nqU5o+QtKctLwXJJ3c\n0XtTYxlfkfTn1G50jd/ZDen5FpJulvRaWv8nJPWUdBlwEPDT9L5dVfrdfUNSK/B8rS1HoJekh9L4\nH5bULy1rd0lRNZZH01g/CVwNfC4tb1Hp/R9Tav+19Ll5TdLdknqnerufK8vLYWNd5QTgZmBb4DZg\nJfAtoCdwMDAUOLtqnmOB/YB9KALqiFS/FLgf6AH0Ba4pzfM48Clge+AO4HZJm6Zp3wVOTMvaDjgL\neAf4fJq+V9rdc2d5EJI2AX6ZltkL+DZwm6TdS81OBX6QlvsScHGtNyGF0P3AFcAOad1m1mi6EXAd\nsAuwK7AC+PfUxzZp/iMjYmuK9+/pOt6b8jgqf7hPBfoAOwMfq9UWOBPYIvW3A/AN4J2IOB94DPha\net/OLc3zJWB/4JNt9PmPwA8pfv/PApPaaPdXETETOAf4r7S8njXW6yjgIorfcx9gPnBTVbO2PleW\nkcPGusqjEXFfRLwfEW9HxPSIeDwiVkbEC8A44AtV8/xrRCyNiDnAb4BBqb4C6A/0joh3IuK/KzNE\nxKSIWJx23fwfYBugEgpnARdExOw0jhkRsbiOsR8MbAL8W0SsSLsIHwBOLrW5IyJaImIFxR+3QTX6\ngeKP7K8iYnJa90URMaO6UUQsjIi70nu1DPhR1fsTwN6SNouIBRHxbEfvTZWTgLsj4r8j4l3gAkBt\ntF1BEQq7p+MvLRHxRhttK34UEUsi4u02pt9XtezPV7ZA1tJpwM/T7/YdYDTwBUl9S23a+lxZRg4b\n6ypzyy8k/a2k+yX9RdIyim+j1d9U/1J6/hZQOTZ0HrAx0JJ2aQ0v9fvPkp6XtBRYAmxZ6rcf8KdO\njH1n4KX48FVr/0zxzbmjsVarawyStlRxltVL6f35NWk9UvicAowC/iLpl5L2SLO2+d7UWKe//k5S\neLQVvDcADwKTJb0s6ceSOjreO7fe6RGxFFiaxrS2dqb43VT6XkbxOejM78rWIYeNdZXqy4v/DHiG\n4tvyNhS7VNr6Zv3hjopv8mdFRG+KP7jjJA2QdCjwHeDvKHaT9QDeKPU7F9itjrFVmw/0k1Qe3y7A\ny/WMt0pbY6j2z8AA4ID0/hxWnhgRD0TEEUBvoJXi/WzzvanR/wKK4ANA0lYUuwBXExHvRcSYiPgE\n8FmKXaKVswzbeu86ek/Ly96WYvfqfODNVNui1La8e6+e39Wupb63pvgcdOZ3ZeuQw8YaZWuKb7Nv\nSvoEqx+vaZOkvy8dLH+d4g/QqtTnSmARxbf7MRRbNhU/By6RtJsKgyRtHxGrgNeAj7exyN+lfs+T\ntLGkwyj2+0+ud8wlNwJDVZyE0D0daP90jXZbU3zrXiJpB4owrqx/b0nHpT/I71H8gV6VprX13lS7\nHRim4kSNTYFLaOMPuaTDJO0taSNgGcVutUqfr9D2+9ae46qW/WhELKDY6vgLxbGUbpJGUgqPtLy+\nSieT1HALMELSp1Lf/0pxjGdeJ8Zo65DDxhrlPGA4sJziW/ltazDvgcB0SW8CvwBGpf/DMYVid89s\nYA7FH8YFpfn+DbgbeChNGwdslqZdCNyczrb6cnlh6bjCccAwiiAbC5waEf+zBmOu9PVi6ut8it1W\nT1H7IPoVFN/2X6MIuwdK07pRnOywIE3/DMWBc2j7vakex9MUJ2hMpvjWX/kjX8vOqa9lwCyK9/iW\nNO0q4JT0vl3RweqX3UgRMosoTug4PY0rgK9SHMdZRHG87fHSfNMofr+vSFptvBHxK4pdsndRvD+7\n8MFWmDWQfPM0MzPLzVs2ZmaWncPGzMyyc9iYmVl2DhszM8vOF+JMevbsGf3792/0MMzMPlKefPLJ\nRRHRq6N2Dpukf//+tLS0NHoYZmYfKZL+3HEr70YzM7Mu4LAxM7PsHDZmZpadw8bMzLJz2JiZWXYO\nGzMzy85hY2Zm2TlszMwsO4eNmZll5ysIfMT0H31/o4fQVOb8+IuNHoLZBsFbNmZmlp3DxszMsnPY\nmJlZdg4bMzPLzmFjZmbZOWzMzCw7h42ZmWXnsDEzs+wcNmZmlp3DxszMsnPYmJlZdg4bMzPLzmFj\nZmbZOWzMzCw7h42ZmWXnsDEzs+wcNmZmlp3DxszMsnPYmJlZdg4bMzPLzmFjZmbZZQsbSf0kPSzp\nOUmzJH0r1cdIelnSjPQ4tjTP9yS1SvqjpKNL9aGp1ippdKk+QNLjkmZLuk3SJqm+aXrdmqb3z7We\nZmbWsZxbNiuB8yLiE8AQYJSkPdO0KyNiUHpMAUjTTgb2AoYCP5HUTVI34BrgGGBP4JRSP5elvgYC\nS4ARqT4CWBIRuwNXpnZmZtYg2cImIhZExFPp+XLgOaBPO7MMA26NiHcj4kWgFTggPVoj4oWIeA+4\nFRgmScBhwB1p/gnA8aW+JqTndwCHp/ZmZtYAXXLMJu3G2gd4PJXOkfS0pPGSeqRaH2BuabZ5qdZW\nfQfg9YhYWVX/UF9p+tLUvnpcIyW1SGpZuHDhWq2jmZm1LXvYSNoKuBM4NyKWAdcCuwGDgAXA5ZWm\nNWaPTtTb6+vDhYhxETE4Igb36tWr3fUwM7POyxo2kjamCJqbIuIXABHxSkSsioj3gesodpNBsWXS\nrzR7X2B+O/VFwHaSulfVP9RXmr4tsHjdrp2ZmdUr59loAq4HnouIK0r13qVmJwDPpOf3AienM8kG\nAAOBJ4DpwMB05tkmFCcR3BsRATwMnJjmHw7cU+preHp+IvDr1N7MzBqge8dNOu1g4HRgpqQZqXYB\nxdlkgyh2a80BzgaIiFmSJgPPUpzJNioiVgFIOgeYCnQDxkfErNTf+cCtki4B/kARbqSfkyS1UmzR\nnJxxPc3MrAPZwiYiHqX2sZMp7cxzKXBpjfqUWvNFxAt8sBuuXH8HOGlNxmtmZvn4CgJmZpadw8bM\nzLJz2JiZWXYOGzMzy85hY2Zm2TlszMwsO4eNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXYOGzMz\ny85hY2Zm2TlszMwsO4eNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXYOGzMzy85hY2Zm2TlszMws\nO4eNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXbZwkZSP0kPS3pO0ixJ30r17SVNkzQ7/eyR6pI0\nVlKrpKcl7Vvqa3hqP1vS8FJ9P0kz0zxjJam9ZZiZWWPk3LJZCZwXEZ8AhgCjJO0JjAYeioiBwEPp\nNcAxwMD0GAlcC0VwABcCBwIHABeWwuPa1LYy39BUb2sZZmbWANnCJiIWRMRT6fly4DmgDzAMmJCa\nTQCOT8+HAROj8HtgO0m9gaOBaRGxOCKWANOAoWnaNhHxWEQEMLGqr1rLMDOzBuiSYzaS+gP7AI8D\nO0XEAigCCdgxNesDzC3NNi/V2qvPq1GnnWVUj2ukpBZJLQsXLuzs6pmZWQeyh42krYA7gXMjYll7\nTWvUohP1ukXEuIgYHBGDe/XqtSazmpnZGsgaNpI2pgiamyLiF6n8StoFRvr5aqrPA/qVZu8LzO+g\n3rdGvb1lmJlZA+Q8G03A9cBzEXFFadK9QOWMsuHAPaX6GemstCHA0rQLbCpwlKQe6cSAo4Cpadpy\nSUPSss6o6qvWMszMrAG6Z+z7YOB0YKakGal2AfBjYLKkEcBLwElp2hTgWKAVeAs4EyAiFku6GJie\n2l0UEYvT868DNwCbAw+kB+0sw8zMGiBb2ETEo9Q+rgJweI32AYxqo6/xwPga9RZg7xr112otw8zM\nGsNXEDAzs+wcNmZmlp3DxszMsnPYmJlZdg4bMzPLzmFjZmbZOWzMzCy7usJG0mr/l8XMzKxe9W7Z\n/FTSE5K+IWm7rCMyM7OmU1fYRMRngdMoLojZIulmSUdmHZmZmTWNuo/ZRMRs4PvA+cAXgLGSnpf0\n5VyDMzOz5lDvMZtPSbqS4m6bhwHHpds9HwZcmXF8ZmbWBOq9EOfVwHXABRHxdqUYEfMlfT/LyMzM\nrGnUGzbHAm9HxCoASRsBm0XEWxExKdvozMysKdR7zOZBinvGVGyRamZmZh2qN2w2i4g3Ki/S8y3y\nDMnMzJpNvWHzpqR9Ky8k7Qe83U57MzOzv6r3mM25wO2S5qfXvYF/yDMkMzNrNnWFTURMl/S3wN9Q\n3Or5+YhYkXVkZmbWNOrdsgHYH+if5tlHEhExMcuozMysqdQVNpImAbsBM4BVqRyAw8bMzDpU75bN\nYGDPiIicgzEzs+ZU79lozwAfyzkQMzNrXvVu2fQEnpX0BPBupRgRX8oyKjMzayr1hs2YnIMwM7Pm\nVu+pz7+VtCswMCIelLQF0C3v0MzMrFnUe4uBrwJ3AD9LpT7A3bkGZWZmzaXeEwRGAQcDy+CvN1Lb\nsb0ZJI2X9KqkZ0q1MZJeljQjPY4tTfuepFZJf5R0dKk+NNVaJY0u1QdIelzSbEm3Sdok1TdNr1vT\n9P51rqOZmWVSb9i8GxHvVV5I6k7x/2zacwMwtEb9yogYlB5TUn97AicDe6V5fiKpm6RuwDXAMcCe\nwCmpLcBlqa+BwBJgRKqPAJZExO4UN3a7rM51NDOzTOoNm99KugDYXNKRwO3Afe3NEBGPAIvr7H8Y\ncGtEvBsRLwKtwAHp0RoRL6SwuxUYJkkUdwm9I80/ATi+1NeE9PwO4PDU3szMGqTesBkNLARmAmcD\nU4DO3qHzHElPp91sPVKtDzC31GZeqrVV3wF4PSJWVtU/1FeavjS1NzOzBqkrbCLi/Yi4LiJOiogT\n0/POXE3gWorL3gwCFgCXp3qtLY/oRL29vlYjaaSkFkktCxcubG/cZma2Fuq9NtqL1PiDHREfX5OF\nRcQrpT6vA36ZXs4D+pWa9gUqtzOoVV8EbCepe9p6Kbev9DUvHVvaljZ250XEOGAcwODBg30pHjOz\nTNbk2mgVmwEnAduv6cIk9Y6IBenlCRSXwQG4F7hZ0hXAzsBA4AmKrZSBkgYAL1OcRHBqRISkh4ET\nKY7jDAfuKfU1HHgsTf+1r+lmZtZY9f6nzteqSldJehT4YVvzSLoFOAToKWkecCFwiKRBFFtJcyiO\n/xARsyRNBp4FVgKjImJV6uccYCrFfyIdHxGz0iLOB26VdAnwB+D6VL8emCSplWKL5uR61tHMzPKp\ndzfavqWXG1Fs6Wzd3jwRcUqN8vU1apX2lwKX1qhPoTghobr+AsXZatX1dyi2vMzMbD1R7260y0vP\nV1Jslfz9Oh+NmZk1pXp3ox2aeyBmZta86t2N9p32pkfEFetmOGZm1ozW5Gy0/SnO9AI4DniED/+H\nSzMzs5rW5OZp+0bEciguqAncHhFn5RqYmZk1j3ovV7ML8F7p9XtA/3U+GjMza0r1btlMAp6QdBfF\n/5E5AZiYbVRmZtZU6j0b7VJJDwCfS6UzI+IP+YZlZmbNpN7daABbAMsi4t8prjs2INOYzMysydR7\nW+gLKS4P871U2hi4MdegzMysudS7ZXMC8CXgTYCImE8Hl6sxMzOrqDds3ktXTg4ASVvmG5KZmTWb\nesNmsqSfUdxD5qvAg8B1+YZlZmbNpN6z0f6vpCOBZcDfAD+MiGlZR2ZmZk2jw7CR1A2YGhFHAA4Y\nMzNbYx3uRks3MXtL0rZdMB4zM2tC9V5B4B1gpqRppDPSACLim1lGZWZmTaXesLk/PczMzNZYu2Ej\naZeIeCkiJnTVgMzMrPl0dMzm7soTSXdmHouZmTWpjsJGpecfzzkQMzNrXh2FTbTx3MzMrG4dnSDw\naUnLKLZwNk/PSa8jIrbJOjozM2sK7YZNRHTrqoGYmVnzWpP72ZiZmXWKw8bMzLJz2JiZWXYOGzMz\nyy5b2EgaL+lVSc+UattLmiZpdvrZI9UlaaykVklPS9q3NM/w1H62pOGl+n6SZqZ5xkpSe8swM7PG\nybllcwMwtKo2GngoIgYCD6XXAMcAA9NjJHAtFMEBXAgcCBwAXFgKj2tT28p8QztYhpmZNUi2sImI\nR4DFVeVhQOU6axOA40v1iVH4PcUdQXsDRwPTImJxRCyhuJ/O0DRtm4h4LN2uemJVX7WWYWZmDdLV\nx2x2iogFAOnnjqneB5hbajcv1dqrz6tRb28Zq5E0UlKLpJaFCxd2eqXMzKx968sJAqpRi07U10hE\njIuIwRExuFevXms6u5mZ1amrw+aVtAuM9PPVVJ8H9Cu16wvM76Det0a9vWWYmVmDdHXY3AtUzigb\nDtxTqp+RzkobAixNu8CmAkdJ6pFODDgKmJqmLZc0JJ2FdkZVX7WWYWZmDVLvnTrXmKRbgEOAnpLm\nUZxV9mNgsqQRwEvASan5FOBYoBV4CzgTICIWS7oYmJ7aXRQRlZMOvk5xxtvmwAPpQTvLMDOzBskW\nNhFxShuTDq/RNoBRbfQzHhhfo94C7F2j/lqtZZiZWeOsLycImJlZE3PYmJlZdg4bMzPLzmFjZmbZ\nOWzMzCw7h42ZmWXnsDEzs+wcNmZmlp3DxszMsnPYmJlZdg4bMzPLzmFjZmbZOWzMzCw7h42ZmWXn\nsDEzs+wcNmZmlp3DxszMsnPYmJlZdg4bMzPLzmFjZmbZOWzMzCw7h42ZmWXnsDEzs+wcNmZmlp3D\nxszMsnPYmJlZdg4bMzPLriFhI2mOpJmSZkhqSbXtJU2TNDv97JHqkjRWUqukpyXtW+pneGo/W9Lw\nUn2/1H9rmlddv5ZmZlbRyC2bQyNiUEQMTq9HAw9FxEDgofQa4BhgYHqMBK6FIpyAC4EDgQOACysB\nldqMLM03NP/qmJlZW9an3WjDgAnp+QTg+FJ9YhR+D2wnqTdwNDAtIhZHxBJgGjA0TdsmIh6LiAAm\nlvoyM7MGaFTYBPCfkp6UNDLVdoqIBQDp546p3geYW5p3Xqq1V59Xo74aSSMltUhqWbhw4VqukpmZ\ntaV7g5Z7cETMl7QjME3S8+20rXW8JTpRX70YMQ4YBzB48OCabczMbO01ZMsmIuann68Cd1Ecc3kl\n7QIj/Xw1NZ8H9CvN3heY30G9b426mZk1SJeHjaQtJW1deQ4cBTwD3AtUzigbDtyTnt8LnJHOShsC\nLE272aYCR0nqkU4MOAqYmqYtlzQknYV2RqkvMzNrgEbsRtsJuCudjdwduDkifiVpOjBZ0gjgJeCk\n1H4KcCzQCrwFnAkQEYslXQxMT+0uiojF6fnXgRuAzYEH0sPMzBqky8MmIl4APl2j/hpweI16AKPa\n6Gs8ML5GvQXYe60Ha2Zm68T6dOqzmZk1KYeNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXYOGzMz\ny85hY2Zm2TlszMwsO4eNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXYOGzMzy85hY2Zm2TlszMws\nO4eNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXYOGzMzy85hY2Zm2TlszMwsO4eNmZll57AxM7Ps\nHDZmZpZd04aNpKGS/iipVdLoRo/HzGxD1pRhI6kbcA1wDLAncIqkPRs7KjOzDVf3Rg8gkwOA1oh4\nAUDSrcAw4NmGjsqsmY3ZttEjaC5jljZ6BOtUs4ZNH2Bu6fU84MDqRpJGAiPTyzck/bELxrah6Aks\navQgOqLLGj0Ca4CPxGeTf1GjR1CvXetp1KxhU+u3FKsVIsYB4/IPZ8MjqSUiBjd6HGbV/NlsjKY8\nZkOxJdOv9LovML9BYzEz2+A1a9hMBwZKGiBpE+Bk4N4Gj8nMbIPVlLvRImKlpHOAqUA3YHxEzGrw\nsDY03j1p6yt/NhtAEasdyjAzM1unmnU3mpmZrUccNmZmlp3DxlYjKSRdXnr9T5LGdPEYbpB0Ylcu\n0z56JK2SNKP06J9hGf0lPbOu+93QOGyslneBL0vq2ZmZJTXliSe2Xno7IgaVHnPKE/1ZXH/4F2G1\nrKQ4Y+fbwP8uT5C0KzAe6AUsBM6MiJck3QAsBvYBnpK0HBgA9Ab2AL4DDKG4Xt3LwHERsULSD4Hj\ngM2B3wFnh89asbUg6SvAF4HNgC0lfQm4B+gBbAx8PyLuSVtBv4yIvdN8/wRsFRFjJO1H8Tl/C3i0\ny1eiCXnLxtpyDXCapOoLXl0NTIyITwE3AWNL0/YAjoiI89Lr3Sj+0Q8DbgQejohPAm+nOsDVEbF/\n+ge/OfC/sqyNNavNS7vQ7irVDwKGR8RhwDvACRGxL3AocLmkjq4F8x/ANyPioDzD3vA4bKymiFgG\nTAS+WTXpIODm9HwS8NnStNsjYlXp9QMRsQKYSfH/nX6V6jOB/un5oZIelzQTOAzYa52thG0IyrvR\nTijVp0XE4vRcwI8kPQ08SHHtxJ3a6jB9wdouIn6bSpNyDHxD491o1p6rgKcovuW1pbzL682qae8C\nRMT7klaUdo+9D3SXtBnwE2BwRMxNJyFstk5Gbhu68mfxNIrdvvulXbdzKD5nK/nwF+7KZ0/UuJai\nrR1v2Vib0jfDycCIUvl3FJf/geIf8drsz678414kaSvAZ59ZDtsCr6agOZQPrlL8CrCjpB0kbUra\nhRsRrwNLJVW22k/r8hE3IW/ZWEcuB84pvf4mMF7Sd0knCHS244h4XdJ1FLvV5lBc085sXbsJuE9S\nCzADeB4ghc9FwOPAi5V6cibF5/wtiste2Vry5WrMzCw770YzM7PsHDZmZpadw8bMzLJz2JiZWXYO\nGzMzy85hY9YAkj4m6VZJf5L0rKQpkvbw1YWtWfn/2Zh1sXRdrruACRFxcqoNop1LqJh91HnLxqzr\nHQqsiIifVgoRMQOYW3md7qHyX5KeSo/PpHpvSY+kC08+I+lzkrql+/88I2mmpG93/SqZtc9bNmZd\nb2/gyQ7avAocGRHvSBoI3AIMBk4FpkbEpZK6AVsAg4A+pUvlb5dv6Gad47AxWz9tDFyddq+torh9\nAxSX9BkvaWPg7oiYIekF4OOS/h9wP/CfDRmxWTu8G82s680C9uugzbcpLhT5aYotmk0AIuIR4PMU\nN6CbJOmMiFiS2v0GGAX8PM+wzTrPYWPW9X4NbCrpq5WCpP354GrEUFypeEFEvA+cTnE/oMqdUl+N\niOuA64F90+27N4qIO4EfAPt2zWqY1c+70cy6WESEpBOAqySNpriT5Bzg3FKznwB3SjoJeJgP7s9y\nCPBdSSuAN4AzKG4G9h+SKl8ev5d9JczWkK/6bGZm2Xk3mpmZZeewMTOz7Bw2ZmaWncPGzMyyc9iY\nmVl2DhszM8vOYWNmZtn9fyQaEZIK2LqrAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#count_classes = pd.value_counts(df['Class'], sort = True)\n", + "count_classes = pd.value_counts(cc_training.select('Class').toPandas()['Class'], sort = True)\n", + "count_classes.plot(kind = 'bar', rot=0)\n", + "plt.title(\"Transaction class distribution\")\n", + "plt.xticks(range(2), LABELS)\n", + "plt.xlabel(\"Class\")\n", + "plt.ylabel(\"Frequency\");" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [], + "source": [ + "# get the time to split the data.\n", + "splitTime = cc_training.stat.approxQuantile(\"Time\", [0.7], 0.001)[0]\n", + "\n", + "trainingData =cc_training.filter(\"Time < \" + str(splitTime))\n", + "validData = cc_training.filter(\"Time >= \" + str(splitTime))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Feature analysis:\n", + "Normally it would improve the model if we could derive more features from the raw transaction records. E.g.\n", + "days to last transaction,\n", + "distance with last transaction,\n", + "amount percentage over the last 1 month / 3months\n", + "...\n", + "\n", + "Yet with the public dataset, we can hardly derive any extention features from the PCA result. So here we only introduce several general practices:\n", + "\n", + "Usually there's a lot of categorical data in the raw dataset, E.g. post code, card type, merchandise id, seller id, etc. \n", + "\n", + "1). For categorical feature with limited candidate values, like card type, channel id, just use OneHotEncoder. \n", + "\n", + "2). For categorical feature with many candidate values, like merchandise id, post code or even phone number, suggest to use Weight of Evidence. \n", + "\n", + "3). You can also use FeatureHasher from Spark MLlib which will be release with Spark 2.3.\n", + "For this dataset, essentially it's a classification problem with highly unbalanced data set.\n", + "\n", + "## Approach\n", + "\n", + "We will build a feature transform pipeline with Apache Spark and some of our transformers.\n", + "We will run some inital statistical analysis and split the dataset for training and validation.\n", + "We will build the model with BigDL.\n", + "We will compare different strategy to handle the unbalance.\n", + "Details of each step is as follows:\n", + "\n", + "### step 1. Build an inital pipeline for feature transform.\n", + "For each training records, we intend to aggregate all the features into one Spark Vector, which will then be sent to BigDL model for the training. First we'd like to introduce one handy transformer that we developed to help user build custom Transformers for Spark ML Pipeline.\n", + "```\n", + "class FuncTransformer (\n", + " override val uid: String,\n", + " val func: UserDefinedFunction\n", + ") extends Transformer with HasInputCol with HasOutputCol with DefaultParamsWritable {\n", + "```\n", + "FuncTransformer takes an udf as the constructor parameter and use the udf to perform the actual transform. The transformer can be saved/loaded as other transformer and can be integrated into a pipeline normally. It can be used widely in many use cases like conditional conversion(if...else...), , type conversion, to/from Array, to/from Vector and many string ops. Some examples:\n", + "```\n", + "val labelConverter = new FuncTransformer(udf { i: Double => if (i >= 1) 1 else 0 })\n", + "```\n", + "\n", + "```\n", + "val shifter = new FuncTransformer(udf { i: Double => i + 1 })\n", + "```\n", + "\n", + "```\n", + "val toVector = new FuncTransformer(udf { i: Double => Vectors.dense(i) })\n", + "```\n", + "\n", + "We will use VectorAssembler to compose the all the Vx columns and append the Amount column. Then use StandardScaler to normlize the training records. Since in BigDL, the criterion generally only accepts 1, 2, 3... as the Label, so we will replace all the 0 with 2 in the training data." + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [], + "source": [ + "assembler = VectorAssembler(inputCols=cols, outputCol=\"assembled\")\n", + "scaler = StandardScaler(inputCol=\"assembled\", outputCol=\"features\")\n", + "pipeline = Pipeline(stages = [assembler, scaler])\n", + "pipelineTraining = pipeline.fit(trainingData)\n", + "data_training = pipelineTraining.transform(trainingData)\n", + "pipelineTest = pipeline.fit(validData)\n", + "data_test = pipelineTest.transform(validData)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### step 2. split the dataset into training and validation dataset.\n", + "\n", + "Unlike some other training dataset, where the data does not have a time of occurance. For this case, we can know the sequence of the transactions from the Time column. Thus randomly splitting the data into training and validation does not make much sense, since in real world applications, we can only use the history transactions for training and use the latest transactions for validation. Thus we'll split the dataset according the time of occurance." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### step 3. Build the model with BigDL\n", + "From the research community and industry feedback, a simple neural network turns out be the perfect candidate for the fraud detection training. We will quickly build a multiple layer Perceptron with linear layers.\n", + "```\n", + " val bigDLModel = Sequential()\n", + " .add(Linear(29, 10))\n", + " .add(Linear(10, 2))\n", + " .add(LogSoftMax())\n", + " val criterion = ClassNLLCriterion()\n", + "```\n", + "\n", + "BigDL provides DLEstimator and DLClassifier for users with Apache Spark MLlib experience, which provides high level API for training a BigDL Model with the Apache Spark Estimator/Transfomer pattern, thus users can conveniently fit BigDL into a ML pipeline. The fitted model DLModel and DLClassiferModel contains the trained BigDL model and extends the Spark ML Model class. Alternatively users may also construct a DLModel with a pre-trained BigDL model to use it in Spark ML Pipeline for prediction.\n", + "\n", + "DLClassifier is a specialized DLEstimator that simplifies the data format for classification tasks. It only supports label column of DoubleType, and the fitted DLClassifierModel will have the prediction column of DoubleType.\n", + "\n", + "For this case we'll just use DLClassifier for the training. Note that users can set differet optimization mothod, batch size and epoch number.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "creating: createSequential\n", + "creating: createLinear\n", + "creating: createLinear\n", + "creating: createLogSoftMax\n", + "creating: createClassNLLCriterion\n", + "creating: createDLClassifier\n", + "\n", + "initial model training finished.\n" + ] + } + ], + "source": [ + "\n", + "bigDLModel = Sequential().add(Linear(n_input, n_hidden_1)).add(Linear(n_hidden_1, n_classes)).add(LogSoftMax())\n", + "classnll_criterion = ClassNLLCriterion()\n", + "dlClassifier = DLClassifier(model=bigDLModel, criterion=classnll_criterion, feature_size=[n_input])\n", + "dlClassifier.setLabelCol(\"label\").setMaxEpoch(training_epochs).setBatchSize(batch_size)\n", + "model = dlClassifier.fit(data_training)\n", + "print(\"\\ninitial model training finished.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DataFrame[Time: double, V1: double, V2: double, V3: double, V4: double, V5: double, V6: double, V7: double, V8: double, V9: double, V10: double, V11: double, V12: double, V13: double, V14: double, V15: double, V16: double, V17: double, V18: double, V19: double, V20: double, V21: double, V22: double, V23: double, V24: double, V25: double, V26: double, V27: double, V28: double, Amount: double, Class: double, label: double, assembled: vector, features: vector, prediction: double]" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from pyspark.sql import DataFrame, SQLContext\n", + "predictionDF = DataFrame(model.transform(data_test), SQLContext(sc))\n", + "predictionDF" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "Now we have finished the training of our first model (which is certainly not the best, keep reading!).\n", + "\n", + "We'll need to think about how do evaluate the trained model:\n", + "\n", + "Given the class imbalance ratio, we recommend measuring the accuracy using the Area Under the Precision-Recall Curve (AUPRC). Confusion matrix accuracy is not meaningful for unbalanced classification. Since even if the model predicts all the records as normal transactions, it will still get an accuracy above 99%." + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Area under precision-recall curve: = 1.0\n", + "\n", + "recall = 0.998458934096\n", + "\n", + "Precision = 0.998246759394\n" + ] + }, + { + "data": { + "text/plain": [ + "DataFrame[Time: double, V1: double, V2: double, V3: double, V4: double, V5: double, V6: double, V7: double, V8: double, V9: double, V10: double, V11: double, V12: double, V13: double, V14: double, V15: double, V16: double, V17: double, V18: double, V19: double, V20: double, V21: double, V22: double, V23: double, V24: double, V25: double, V26: double, V27: double, V28: double, Amount: double, Class: double, label: double, assembled: vector, features: vector, prediction: double]" + ] + }, + "execution_count": 46, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "predictionDF.cache() \n", + "evaluator = BinaryClassificationEvaluator(rawPredictionCol=\"prediction\")\n", + "auPRC = evaluator.evaluate(predictionDF)\n", + "print(\"\\nArea under precision-recall curve: = \" + str(auPRC))\n", + " \n", + "recall = MulticlassClassificationEvaluator(metricName=\"weightedRecall\").evaluate(predictionDF)\n", + "print(\"\\nrecall = \" + str(recall))\n", + "\n", + "precision = MulticlassClassificationEvaluator(metricName=\"weightedPrecision\").evaluate(predictionDF)\n", + "print(\"\\nPrecision = \" + str(precision)) \n", + "predictionDF.unpersist()" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": {}, + "outputs": [], + "source": [ + "y_pred = np.array(predictionDF.select('prediction').collect())\n", + "y_true = np.array(predictionDF.select('label').collect())" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The prediction accuracy is 99.85%\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAHVCAYAAAAJuQqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAH45JREFUeJzt3Xu4XVV5L+DfZyIYqAoqx0LiUWpT\nL7SVekGUqggqoK3QejlwWkFLT5RKL0eLYm3rpdRLW7V6jlpjxbtExQtoQeq16lFulRQFpESoJYAo\n5VKxQkj2OH9kZq1d3EmAstkZk/fNM5+sNdZca4398OTZH79vjDmrtRYAgN7dZaEnAABwe1DUAACj\noKgBAEZBUQMAjIKiBgAYBUUNADAKihoAYBQUNQDAKChqAIBRWDzfX3DTVRe7ZDEsgCW7PW6hpwB3\nWuvXXVZ31HfN1+/Zu97nZ+6wn+H2IqkBAEZh3pMaAGAezWxY6BlsMyQ1AMAoSGoAoGdtZqFnsM2Q\n1AAAoyCpAYCezUhqNlHUAEDHmvbThPYTADAKkhoA6Jn204SkBgAYBUkNAPTMmpoJRQ0A9MwVhSe0\nnwCAUZDUAEDPtJ8mJDUAwChIagCgZ7Z0TyhqAKBjrig8pf0EAIyCpAYAeqb9NCGpAQBGQVIDAD2z\npmZCUgMAjIKkBgB65jYJE4oaAOiZ9tOE9hMAMAqSGgDomS3dE5IaAGAUJDUA0DNraiYUNQDQM+2n\nCe0nAGAUJDUA0LHWXKdmE0kNADAKkhoA6JmFwhOKGgDomYXCE9pPAMAoSGoAoGfaTxOSGgBgFCQ1\nANCzGVu6N1HUAEDPtJ8mtJ8AgFGQ1ABAz2zpnpDUAACjIKkBgJ5ZUzMhqQEARkFSAwA9s6ZmQlED\nAD1T1ExoPwEAoyCpAYCOteaKwptIagCAUVDUAEDPZmbm59iKqnpQVa2edfx7Vf1BVb2yqi6bNf7U\nWe95WVWtqaoLq+qAWeMHDmNrqurYWeO7V9UZVXVRVX24qrbb0pwUNQDQszYzP8fWvra1C1tre7bW\n9kzyiCT/keQTw8tv2vRaa+2UJKmqhyY5NMkeSQ5M8raqWlRVi5K8NclBSR6a5LDh3CR5/fBZy5Nc\nk+TILc1JUQMA/Fftn+Q7rbXvbuGcg5Osaq3d2Fq7JMmaJHsNx5rW2sWttXVJViU5uKoqyX5JThze\n/94kh2xpEooaAOjZPLWfqmpFVZ0961ixhVkcmuSEWc+Prqpzq+r4qtp5GFua5NJZ56wdxjY3fu8k\n17bW1t9sfLMUNQDAT2itrWytPXLWsXKu84Z1Lk9P8tFh6O1JHphkzyRXJHnDplPn+prbML5ZtnQD\nQM8W/t5PByX5RmvtyiTZ9HeSVNU7k3x6eLo2yf1mvW9ZksuHx3ONX5Vkp6paPKQ1s8+fk6QGAHq2\nQLufZjkss1pPVbXrrNd+Lcm3hscnJzm0qravqt2TLE9yZpKzkiwfdjptl42trJNbay3JF5M8c3j/\nEUlO2tJEJDUAwG1SVTskeXKS588a/ouq2jMbW0X/sum11tp5VfWRJOcnWZ/khW24cmBVHZ3ktCSL\nkhzfWjtv+KyXJllVVcclOSfJu7Y4n42F0Py56aqL5/cLgDkt2e1xCz0FuNNav+6yudaDzIsfn/Z/\n5+X37JIDjr7Dfobbi/YTADAK2k8A0DN36Z6Q1AAAoyCpAYCeSWomFDUA0LOFv07NNkP7CQAYBUkN\nAPRM+2lCUgMAjIKkBgB6Zk3NhKIGAHqm/TSh/QQAjIKkBgB6pv00IakBAEZBUgMAPbOmZkJRAwA9\nU9RMaD8BAKMgqQGAnrW20DPYZkhqAIBRkNQAQM+sqZmQ1AAAoyCpAYCeSWomFDUA0DNXFJ7QfgIA\nRkFSAwA9036akNQAAKMgqQGAnrn43oSiBgB6pv00of0EAIyCpAYAeiapmZDUAACjIKkBgJ65+N6E\nogYAOtZm7H7aRPsJABgFSQ0A9MxC4QlJDQAwCpIaAOiZhcITkhoAYBQkNQDQM7ufJhQ1ANAzC4Un\ntJ8AgFGQ1ABAzyQ1E5IaAGAUJDUA0LNmofAmihoA6Jn204T2EwAwCpKaO4n3rfpEPvapz6SqsvyB\nD8hxf/SivPov/0/OXv3N/NSOOyZJ/vzlL8qDf+6Bk/d884IL8xsrXpS/evWxecoTH5ckeePb3pUv\nf+2sJMnzn3tYDnrSE5IkLz/uDVv8LGDz1vzz6fnh9ddnw4aZrF+/Pns/5ql5/Wv/OE/7lSdn3bp1\nufji7+bI335Rrrvu3xd6qmyLXKdmQlFzJ3DlD67KB088KSd98B252/bb58V/8pqc+rl/SJK8+IVH\nTgqW2TZs2JA3ve3d2Wevh0/G/uFrZ+b8C7+TE9/z1qy76aY894UvyeMe88hJIbO5zwK27klPflb+\n7d+umTz/3Oe/nD/649dmw4YNee1r/ijHvvTovOyPXrOAM4Rtn/bTncT6DRty443rsn79hvz4hhuz\ny33utcXzP3TiyXnyvvvkXjvvNBn7ziX/mkf90i9k8eJF2WHJ3fKg5bvnq6f/43xPHe6UPvu5L2fD\nhg1JktPP+EaWLt11gWfENqvNzM/Roa0WNVX14Kp6aVW9parePDx+yB0xOW4f993lPnnuYc/Ik379\n8Dzx4P+Zu++4Q/Z59COSJG95x3vza4cflde/+R1Zt25dko3Jzue//LU8+5Cn/qfPedDP7p6vnH52\nfnzDDbnm2uty1jfOzfe+/4PJ63N9FrB1rbWcesoJOeP0U/PbR/7GT7z+vOcems+c9sUFmBldmGnz\nc3Roi+2nqnppksOSrEpy5jC8LMkJVbWqtfa6eZ4ft4Pr/v2H+eJXTs9pH3137n73n8qL//g1+dRp\nX8gfvOB5uc+9d85NN92UV77+LXnXBz6ao37rN/L6N78j//uo38qiRYv+0+fs8+hH5Fvf/uf85vNf\nnJ13umcetseDJ+ds7rOArXv8vofkiiuuzC673DufOXVVLrxwTb7y1TOSJC879veyfv36fOhDH1/g\nWcK2b2trao5Mskdr7abZg1X1xiTnJZmzqKmqFUlWJMnb3nBcfvvww26HqXJbnX726izd7b6TVtL+\nT3hsVn/z/PzqAfslSbbbbrsc8rSn5D0nfCxJct63L8oxr9j4n/aa6/49X/n6WVm0aFH2f/xj8/wj\nDsvzj9j43/Mlr3x97r9stySZtLNu/lnA1l1xxZVJkh/84N9y0kmn5lGP2jNf+eoZec5znpWnPfVJ\nefIBz17gGbIta7Z0T2ytqJlJsluS795sfNfhtTm11lYmWZkkN111cZ8Z1ojset9dcu63vp0f33BD\n7rb99jnj7NXZ48HL84Orrs4u97lXWmv5wpe/luU/c/8kyWknvmfy3pcf94Y8YZ+9sv/jH5sNGzbk\nh9f/KDvd8x65cM0l+ec1l+Sxf/yHSbLZzwK2bIcdluQud7lLrr/+R9lhhyV58pOekOP+/E054Cn7\n5pg//J3st/8z8uMf37DQ04QubK2o+YMkn6+qi5JcOoz99yQ/m+To+ZwYt59f3OPBefITfznPft7v\nZtGiRXnwzz0wzzr4oLzgxX+aa669Lq21PGj5z+QVx/zuFj9n/foNOfx3NhYxP7XDDnndnx6TxYs3\ntp9e+qq/uFWfBWx03/vukhM/+q4kyeLFi7Jq1Sdz2t9/Kd8+/6vZfvvt85lTVyVJzjjjG3nh0ccu\n5FTZVnW6/mU+VNvK5ZWr6i5J9kqyNEklWZvkrNbahlvyBZIaWBhLdrO9HhbK+nWX1R31XT/688Pn\n5ffsji9/3x32M9xetnqdmtbaTJLT74C5AAC3Vqfbr+eDi+8BQM+0nyZcfA8AGAVJDQD0zJbuCUkN\nADAKkhoA6Jk1NROKGgDomd1PE9pPAMAoSGoAoGfaTxOSGgBgFCQ1ANAxd+meUtQAQM+0nya0nwCA\nUVDUAEDPZtr8HLdAVe1UVSdW1ber6oKqekxV3auqPltVFw1/7zycW1X1lqpaU1XnVtXDZ33OEcP5\nF1XVEbPGH1FV3xze85aq2uKdwxU1AMBt9eYkn2mtPTjJw5JckOTYJJ9vrS1P8vnheZIclGT5cKxI\n8vYkqap7JXlFkkcn2SvJKzYVQsM5K2a978AtTUZRAwA9azPzc2xFVd0jyeOTvCtJWmvrWmvXJjk4\nyXuH096b5JDh8cFJ3tc2Oj3JTlW1a5IDkny2tXZ1a+2aJJ9NcuDw2j1aa19vrbUk75v1WXNS1AAA\nt8XPJPlBkndX1TlV9bdVtWOS+7bWrkiS4e//Npy/NMmls96/dhjb0vjaOcY3S1EDAD2bpzU1VbWi\nqs6eday42TcvTvLwJG9vrf1Skh9l2mqay1zrYdptGN8sW7oBoGNtnrZ0t9ZWJlm5hVPWJlnbWjtj\neH5iNhY1V1bVrq21K4YW0vdnnX+/We9fluTyYXzfm41/aRhfNsf5myWpAQButdba95JcWlUPGob2\nT3J+kpOTbNrBdESSk4bHJyc5fNgFtXeS64b21GlJnlJVOw8LhJ+S5LThtR9W1d7DrqfDZ33WnCQ1\nANCzhb343u8m+WBVbZfk4iTPy8bA5CNVdWSSf03yrOHcU5I8NcmaJP8xnJvW2tVV9WdJzhrOe3Vr\n7erh8VFJ3pNkSZJTh2OzFDUAwG3SWlud5JFzvLT/HOe2JC/czOccn+T4OcbPTvLzt3Q+ihoA6Jl7\nP00oagCgZ+79NGGhMAAwCpIaAOiZpGZCUgMAjIKkBgA6tnFTEYmiBgD6pv00of0EAIyCpAYAeiap\nmZDUAACjIKkBgI7N1126eySpAQBGQVIDAD2T1EwoagCgZ+5nOaH9BACMgqQGADpmofCUpAYAGAVJ\nDQD0TFIzoagBgJ5ZKDyh/QQAjIKkBgA6ZqHwlKQGABgFSQ0A9MyamglFDQB0TPtpSvsJABgFSQ0A\n9Ez7aUJSAwCMgqQGADrWJDUTihoA6JmiZkL7CQAYBUkNAHRM+2lKUgMAjIKkBgB6JqmZkNQAAKMg\nqQGAjllTM6WoAYCOKWqmtJ8AgFGQ1ABAxyQ1U5IaAGAUJDUA0LNWCz2DbYaiBgA6pv00pf0EAIyC\npAYAOtZmtJ82kdQAAKMgqQGAjllTM6WoAYCONbufJrSfAIBRkNQAQMe0n6YkNQDAKEhqAKBjtnRP\nSWoAgFGQ1ABAx1pb6BlsOxQ1ANAx7acp7ScAYBQkNQDQMUnNlKQGABgFSQ0AdMxC4SlFDQB0TPtp\nSvsJABgFSQ0AdMxduqckNQDAKEhqAKBj7tI9pagBgI7NaD9NaD8BAKMgqQGAjlkoPCWpAQBGQVID\nAB1z8b0pSQ0AMAqKGgDoWGvzc9xSVbWoqs6pqk8Pz99TVZdU1erh2HMYr6p6S1Wtqapzq+rhsz7j\niKq6aDiOmDX+iKr65vCet1TVFmMp7ScA6Ng20H76/SQXJLnHrLFjWmsn3uy8g5IsH45HJ3l7kkdX\n1b2SvCLJI5O0JP9YVSe31q4ZzlmR5PQkpyQ5MMmpm5uIpAYAuE2qalmSpyX521tw+sFJ3tc2Oj3J\nTlW1a5IDkny2tXb1UMh8NsmBw2v3aK19vbXWkrwvySFb+gJFDQB0bKbVvBxVtaKqzp51rJjj6/86\nyUuS3Py6xn8+tJjeVFXbD2NLk1w665y1w9iWxtfOMb5ZihoA4Ce01la21h4561g5+/Wq+pUk32+t\n/ePN3vqyJA9O8qgk90ry0k1vmetrbsP4ZilqAKBjrdW8HLfAPkmeXlX/kmRVkv2q6gOttSuGFtON\nSd6dZK/h/LVJ7jfr/cuSXL6V8WVzjG+WogYAOrZQu59aay9rrS1rrT0gyaFJvtBa+81hLUyGnUqH\nJPnW8JaTkxw+7ILaO8l1rbUrkpyW5ClVtXNV7ZzkKUlOG177YVXtPXzW4UlO2tKc7H4CAG5PH6yq\nXbKxfbQ6yQuG8VOSPDXJmiT/keR5SdJau7qq/izJWcN5r26tXT08PirJe5IsycZdT5vd+ZQk1W7N\nZvTb4KarLp7fLwDmtGS3xy30FOBOa/26y+6wfdar7//0efk9u+d3T17wveK3lvYTADAK2k8A0DF3\n6Z5S1ABAx+Z5FUlXtJ8AgFGQ1ABAx2a0nybmvaixAwMAuCNIagCgYxYKT1lTAwCMgqQGADpmTc2U\nogYAOmZH95T2EwAwCpIaAOiY9tOUpAYAGAVJDQB0zJbuKUUNAHRsZqEnsA3RfgIARkFSAwAda9F+\n2kRSAwCMgqQGADo24+p7E4oaAOjYjPbThPYTADAKkhoA6JiFwlOSGgBgFCQ1ANAxF9+bktQAAKMg\nqQGAjllTM6WoAYCOaT9NaT8BAKMgqQGAjklqpiQ1AMAoSGoAoGMWCk8pagCgYzNqmgntJwBgFCQ1\nANAxd+mektQAAKMgqQGAjrWFnsA2RFEDAB1znZop7ScAYBQkNQDQsZmyUHgTSQ0AMAqSGgDomIXC\nU5IaAGAUJDUA0DG7n6YUNQDQMfd+mtJ+AgBGQVIDAB1z76cpSQ0AMAqSGgDomC3dU4oaAOiYhcJT\n2k8AwChIagCgY65TMyWpAQBGQVIDAB2zUHhKUQMAHbNQeEr7CQAYBUkNAHTMQuEpSQ0AMAqSGgDo\nmKRmSlIDAIyCpAYAOtbsfppQ1ABAx7SfprSfAIBRkNQAQMckNVOSGgBgFCQ1ANAx936aUtQAQMfc\n+2lK+wkAGAVFDQB0bGaejq2pqrtV1ZlV9U9VdV5VvWoY372qzqiqi6rqw1W13TC+/fB8zfD6A2Z9\n1suG8Qur6oBZ4wcOY2uq6titzUlRAwDcFjcm2a+19rAkeyY5sKr2TvL6JG9qrS1Pck2SI4fzj0xy\nTWvtZ5O8aTgvVfXQJIcm2SPJgUneVlWLqmpRkrcmOSjJQ5McNpy7WYoaAOjYQiU1baPrh6d3HY6W\nZL8kJw7j701yyPD44OF5htf3r6oaxle11m5srV2SZE2SvYZjTWvt4tbauiSrhnM3S1EDAB1r83RU\n1YqqOnvWseLm3z0kKquTfD/JZ5N8J8m1rbX1wylrkywdHi9NcmmSDK9fl+Tes8dv9p7NjW+W3U8A\nwE9ora1MsnIr52xIsmdV7ZTkE0keMtdpw99z7dNqWxifK3jZ4g52RQ0AdGxb2NLdWru2qr6UZO8k\nO1XV4iGNWZbk8uG0tUnul2RtVS1Ocs8kV88a32T2ezY3PiftJwDgVquqXYaEJlW1JMmTklyQ5ItJ\nnjmcdkSSk4bHJw/PM7z+hdZaG8YPHXZH7Z5keZIzk5yVZPmwm2q7bFxMfPKW5iSpAYCOLeC9n3ZN\n8t5hl9Jdknyktfbpqjo/yaqqOi7JOUneNZz/riTvr6o12ZjQHJokrbXzquojSc5Psj7JC4e2Vqrq\n6CSnJVmU5PjW2nlbmlBtLJLmz+LtlrqCMwB3KuvXXXaHNYVed//fnJffs8d+9wPbQGPr1pHUAEDH\nJAdTihoA6NiMsmbCQmEAYBQkNQDQsQVcKLzNkdQAAKMgqQGAjllRM6WoAYCOaT9NaT8BAKMgqQGA\njm0L937aVkhqAIBRkNQAQMdcfG9KUQMAHVPSTGk/AQCjIKkBgI7Z0j0lqQEARkFSAwAds1B4SlED\nAB1T0kxpPwEAoyCpAYCOWSg8JakBAEZBUgMAHbNQeEpSAwCMgqQGADomp5lS1ABAxywUntJ+AgBG\nQVIDAB1rGlATkhoAYBQkNQDQMWtqphQ1ANAx16mZ0n4CAEZBUgMAHZPTTElqAIBRkNQAQMesqZlS\n1ABAx+x+mtJ+YuL3f+9/5Z9WfyGrz/l8PvD+t2b77bfPAx5wv3ztq5/KBed9NR/64Ntz17vedaGn\nCaOybNlu+dzffzTfPPdL+afVX8jvHn1kkuRhD9sj/+8rn8rZZ/19Tv/6KXnUI/dc4JnCtk9RQ5Jk\nt91+Oke/8Lfy6L2fmj1/af8sWrQo/+PZB+e1r3l5/vot78xD9vjlXHPNdfmt5x220FOFUVm/fn2O\necmr8gu/uG/2+eVfzVFHPTcPecjyvO41L8+fHffGPPJRT8mrXvVXed1rX77QU2Ub1ebpT48UNUws\nXrw4S5bcLYsWLcoOS5bke9+7Mk/cd5987GN/lyR5//s/moOffsACzxLG5Xvf+37OWf2tJMn11/8o\n3/72RVm620+ntZa73+PuSZJ73PPuufyKKxdymtCF27ympqqe11p79+05GRbO5Zd/L29809/kku+c\nmR//+IZ89nP/kH/8xrm59trrsmHDhiTJ2suuyG5Lf3qBZwrjdf/7L8ueD/v5nHHmOXnRH74ip3z6\nQ/mL1/1J7nKXyuOecPBCT49tlDU1U/+VpOZVm3uhqlZU1dlVdfbMzI/+C1/BHWWnne6Zp//qAfnZ\nn9s797v/w7PjjjvkwAP3+4nzWuszkoRt3Y477pCPfPidedEfviI//OH1ef6Kw/PiY16Z3R/4qLz4\nmFflne94w0JPEbZ5W0xqqurczb2U5L6be19rbWWSlUmyeLulfgt2YP/9H5dL/uVfc9VVVydJPvHJ\nU/OYvR+ZnXa6ZxYtWpQNGzZk2dJdc8XlInC4vS1evDgf/fA7c8IJn8gnP3lqkuTw5zwr//tFf5ok\nOfHET2Xl3/zlQk6RbViv61/mw9aSmvsmOTzJr85x/Nv8To070qX/elke/eiHZ8mSuyVJ9nviL+eC\nC/45X/qHr+UZz3hakuQ5z3lWTv7U3y/kNGGU3rnyDbng22vy129eORm7/Ior84THPybJxn+PF625\nZKGmxzZuZp6OHm1tTc2nk/xUa231zV+oqi/Ny4xYEGeedU4+/vG/y1lnnpb169dn9erz8s6//WBO\nOfXz+dAH3pZXv/IlWf1P5+X4d5+w0FOFUdnnsY/Kc37zmTn3m+fn7LM2/k/Dn/zJ6/KCFxyTN77x\n1Vm8eHFuvOGGHHXUSxZ4prDtq/leI6H9BMCdzfp1l9Ud9V3Puf+vz8vv2fd/9+N32M9we7GlGwAY\nBbdJAICOaYdMKWoAoGNuaDml/QQAjIKkBgA65jo1U5IaAGAUJDUA0LFeL5Q3HxQ1ANAxC4WntJ8A\ngFGQ1ABAxywUnpLUAACjIKkBgI5ZKDwlqQEARkFSAwAda82amk0UNQDQMVu6p7SfAIBRkNQAQMcs\nFJ6S1AAAoyCpAYCOufjelKIGADpmofCU9hMAMAqSGgDomOvUTElqAIBRkNQAQMds6Z6S1ABAx9o8\n/dmaqjq+qr5fVd+aNfbKqrqsqlYPx1NnvfayqlpTVRdW1QGzxg8cxtZU1bGzxnevqjOq6qKq+nBV\nbbe1OSlqAIDb4j1JDpxj/E2ttT2H45QkqaqHJjk0yR7De95WVYuqalGStyY5KMlDkxw2nJskrx8+\na3mSa5IcubUJKWoAoGMzafNybE1r7ctJrr6F0zw4yarW2o2ttUuSrEmy13Csaa1d3Fpbl2RVkoOr\nqpLsl+TE4f3vTXLI1r5EUQMA3J6Orqpzh/bUzsPY0iSXzjpn7TC2ufF7J7m2tbb+ZuNbpKgBgI61\n1ublqKoVVXX2rGPFLZjO25M8MMmeSa5I8oZhvOaa+m0Y3yK7nwCAn9BaW5lk5a18z5WbHlfVO5N8\neni6Nsn9Zp26LMnlw+O5xq9KslNVLR7Smtnnb5akBgA6tlBrauZSVbvOevprSTbtjDo5yaFVtX1V\n7Z5keZIzk5yVZPmw02m7bFxMfHLbeEXBLyZ55vD+I5KctLXvl9QAQMcW6oaWVXVCkn2T3Keq1iZ5\nRZJ9q2rPbGwV/UuS5ydJa+28qvpIkvOTrE/ywtbahuFzjk5yWpJFSY5vrZ03fMVLk6yqquOSnJPk\nXVud03xfXnnxdktdvxmAO5X16y6ba03IvNh32ZPm5ffsl9Z+7g77GW4vkhoA6NiMez9NWFMDAIyC\npAYAOianmVLUAEDHbutOpTHSfgIARkFSAwAdk9RMSWoAgFGQ1ABAx+b7enM9UdQAQMe0n6a0nwCA\nUZDUAEDHFureT9siSQ0AMAqSGgDomIXCU5IaAGAUJDUA0DG7n6YUNQDQMe2nKe0nAGAUJDUA0DHt\npylJDQAwCpIaAOiYi+9NKWoAoGMzFgpPaD8BAKMgqQGAjmk/TUlqAIBRkNQAQMesqZlS1ABAx7Sf\nprSfAIBRkNQAQMe0n6YkNQDAKEhqAKBj1tRMSWoAgFGQ1ABAx6ypmVLUAEDHtJ+mtJ8AgFGQ1ABA\nx1qbWegpbDMkNQDAKEhqAKBjM9bUTChqAKBjze6nCe0nAGAUJDUA0DHtpylJDQAwCpIaAOiYNTVT\nihoA6JjbJExpPwEAoyCpAYCOuffTlKQGABgFSQ0AdMxC4SlJDQAwCpIaAOiYi+9NKWoAoGPaT1Pa\nTwDAKEhqAKBjLr43JakBAEZBUgMAHbOmZkpRAwAds/tpSvsJABgFSQ0AdEz7aUpSAwCMgqQGADpm\nS/eUogYAOtYsFJ7QfgIARkFSAwAd036aktQAAKMgqQGAjtnSPSWpAQBGQVIDAB2z+2lKUQMAHdN+\nmtJ+AgBGQVIDAB2T1ExJagCAUZDUAEDH5DRTJbZiS6pqRWtt5ULPA+5s/NuDW0/7ia1ZsdATgDsp\n//bgVlLUAACjoKgBAEZBUcPW6OnDwvBvD24lC4UBgFGQ1AAAo6CoAQBGQVHDnKrqwKq6sKrWVNWx\nCz0fuLOoquOr6vtV9a2Fngv0RlHDT6iqRUnemuSgJA9NclhVPXRhZwV3Gu9JcuBCTwJ6pKhhLnsl\nWdNau7i1ti7JqiQHL/Cc4E6htfblJFcv9DygR4oa5rI0yaWznq8dxgBgm6WoYS41x5i9/wBs0xQ1\nzGVtkvvNer4syeULNBcAuEUUNczlrCTLq2r3qtouyaFJTl7gOQHAFilq+AmttfVJjk5yWpILknyk\ntXbews4K7hyq6oQkX0/yoKpaW1VHLvScoBdukwAAjIKkBgAYBUUNADAKihoAYBQUNQDAKChqAIBR\nUNQAAKOgqAEARuH/A/5MjiNvjWetAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "acc = accuracy_score(y_true, y_pred)\n", + "print(\"The prediction accuracy is %.2f%%\"%(acc*100))\n", + "\n", + "cm = confusion_matrix(y_true, y_pred)\n", + "cm.shape\n", + "df_cm = pd.DataFrame(cm)\n", + "plt.figure(figsize = (10,8))\n", + "sn.heatmap(df_cm, annot=True,fmt='d');" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To this point, we have finished the training and evaluation with a simple BigDL model. We can see that even though the recall and precision are high, the area under precision-recall curve is not optimistic. That's because we haven't really apply any technique to handle the imbalanced training data.\n", + "\n", + "Next we'll try to optimize the training process.\n", + "\n", + "### step 4. handle the data imbalance\n", + "There are several ways to approach this classification problem taking into consideration this unbalance.\n", + "\n", + "Collect more data? Nice strategy but not applicable in this case.\n", + "\n", + "Resampling the dataset Essentially this is a method that will process the data to have an approximate 50-50 ratio. One way to achieve this is by OVER-sampling, which is adding copies of the under-represented class (better when there're little data) Another is UNDER-sampling, which deletes instances from the over-represented class (better when there are lots of data)\n", + "Apart from under and over sampling, there is a very popular approach called SMOTE (Synthetic Minority Over-Sampling Technique), which is a combination of oversampling and undersampling, but the oversampling approach is not by replicating minority class but constructing new minority class data instance via an algorithm.\n", + "\n", + "We'll start with Resampling.\n", + "\n", + "Since there're 492 frauds out of 284,807 transactions, to build a reasonable training dataset, we'll use UNDER-sampling for normal transactions and use OVER-sampling for fraud transactions. By using the sampling rate as fraud -> 10, normal -> 0.05, we can get a training dataset of (5K fraud + 14K normal) transactions. We can use the training data to fit a model.\n", + "\n", + "Yet we'll soon find that since there're only 5% of all the normal transactions are included in the training data, the model can only cover 5% of all the normal transactions, which is obviousely not optimistic. So how can we get a better converage for the normal transactions without breaking the ideal ratio in the training dataset?\n", + "\n", + "An immediate improvement would be to train multiple models. For each model, we will run the resampling from the original dataset and get a new training data set. After training, we can select best voting strategy for all the models to make the prediction.\n", + "\n", + "We'll use Ensembling of neural networks. That's where a Bagging classifier becomes handy. Bagging is an Estimator we developed for ensembling of multiple other Estimator.\n", + "\n", + "```\n", + "package org.apache.spark.ml.ensemble\n", + "\n", + "class Bagging[M <: Model[M]](override val uid: String)\n", + " extends Estimator[BaggingModel[M]]\n", + " with BaggingParams[M] {\n", + "For usage, user need to set the specific Estimator to use and the number of models to be trained:\n", + " val estimator = new Bagging()\n", + " .setPredictor(dlClassifier)\n", + " .setLabelCol(\"Class\")\n", + " .setIsClassifier(true)\n", + " .setNumModels(10)\n", + "```\n", + "\n", + "Internally, Bagging will train $(numModels) models. Each model is trained with the resampled data from the original dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/feedforward-credit-card-fraud.ipynb b/notebooks/feedforward-credit-card-fraud.ipynb new file mode 100644 index 0000000..712d290 --- /dev/null +++ b/notebooks/feedforward-credit-card-fraud.ipynb @@ -0,0 +1,618 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Feedforward Network with Credit Card Fraud\n", + "\n", + "Let us look at a BigDL example with Credit Card Fraud. We will train a simple, feedforward neural network with the credit card dataset.\n", + "\n", + "Let's look briefly at the credit card dataset\n", + "\n", + "| Time\" | \"V1\" | \"V2\" | \"V3\" | \"V4\" | \"V5\" | \"V6\" | \"V7\" | \"V8\" | \"V9\" | \"V10\" | \"V11\" | \"V12\" | \"V13\" | \"V14\" | \"V15\" | \"V16\" | \"V17\" | \"V18\" | \"V19\" | \"V20\" | \"V21\" | \"V22\" | \"V23\" | \"V24\" | \"V25\" | \"V26\" | \"V27\" | \"V28\" | \"Amount\" | \"Class\" | \n", + "|-------|-------------------|---------------------|------------------|-------------------|--------------------|---------------------|---------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|-------------------|--------------------|--------------------|--------------------|--------------------|---------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|----------------------|---------------------|----------|---------| \n", + "| 0 | -1.3598071336738 | -0.0727811733098497 | 2.53634673796914 | 1.37815522427443 | -0.338320769942518 | 0.462387777762292 | 0.239598554061257 | 0.0986979012610507 | 0.363786969611213 | 0.0907941719789316 | -0.551599533260813 | -0.617800855762348 | -0.991389847235408 | -0.311169353699879 | 1.46817697209427 | -0.470400525259478 | 0.207971241929242 | 0.0257905801985591 | 0.403992960255733 | 0.251412098239705 | -0.018306777944153 | 0.277837575558899 | -0.110473910188767 | 0.0669280749146731 | 0.128539358273528 | -0.189114843888824 | 0.133558376740387 | -0.0210530534538215 | 149.62 | 1 | \n", + "| 0 | 1.19185711131486 | 0.26615071205963 | 0.16648011335321 | 0.448154078460911 | 0.0600176492822243 | -0.0823608088155687 | -0.0788029833323113 | 0.0851016549148104 | -0.255425128109186 | -0.166974414004614 | 1.61272666105479 | 1.06523531137287 | 0.48909501589608 | -0.143772296441519 | 0.635558093258208 | 0.463917041022171 | -0.114804663102346 | -0.183361270123994 | -0.145783041325259 | -0.0690831352230203 | -0.225775248033138 | -0.638671952771851 | 0.101288021253234 | -0.339846475529127 | 0.167170404418143 | 0.125894532368176 | -0.00898309914322813 | 0.0147241691924927 | 2.69 | 1 | \n", + "| 1 | -1.35835406159823 | -1.34016307473609 | 1.77320934263119 | 0.379779593034328 | -0.503198133318193 | 1.80049938079263 | 0.791460956450422 | 0.247675786588991 | -1.51465432260583 | 0.207642865216696 | 0.624501459424895 | 0.066083685268831 | 0.717292731410831 | -0.165945922763554 | 2.34586494901581 | -2.89008319444231 | 1.10996937869599 | -0.121359313195888 | -2.26185709530414 | 0.524979725224404 | 0.247998153469754 | 0.771679401917229 | 0.909412262347719 | -0.689280956490685 | -0.327641833735251 | -0.139096571514147 | -0.0553527940384261 | -0.0597518405929204 | 378.66 | 1 | \n", + "\n", + "\n", + "Notice the PCA dimensions here (Principal Component Analysis). PCA here is done for several reasons: as a dimensionality reduction technique, but also as a way to anonymize very sensitive financial data.\n", + "\n", + "Of course, it would be better if we could see the actual dimensions, but on the ohter hand, the PCA dimensions should contain most of the \"signal\" of the original data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import pandas as pd\n", + "import datetime as dt\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from sklearn.metrics import confusion_matrix\n", + "from sklearn.metrics import accuracy_score\n", + "import seaborn as sn\n", + "import pandas as pd\n", + "import random as rd\n", + "import datetime as dt\n", + "\n", + "\n", + "from bigdl.dataset.transformer import *\n", + "from bigdl.dataset.base import *\n", + "from bigdl.nn.layer import *\n", + "from bigdl.nn.criterion import *\n", + "from bigdl.optim.optimizer import *\n", + "from bigdl.util.common import *\n", + "from utils import *\n", + "\n", + "from pyspark.sql.functions import col\n", + "\n", + "\n", + "init_engine()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## Parameteters\n", + "\n", + "We are using a very large batch size here because the dataset is very lopsided." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "learning_rate = 0.1\n", + "training_epochs = 100\n", + "batch_size = 10000\n", + "display_step = 1\n", + "\n", + "# Network Parameters\n", + "n_input = 29\n", + "n_classes = 2\n", + "n_hidden_1 = 10 # 1st layer number of features\n", + "\n", + "LABELS = [\"Normal\", \"Fraud\"]\n", + "\n", + "cols = [\"V\" + str(x) for x in list(range(1,28))] + [\"Amount\"]\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Sizing the Hidden Layer(s)\n", + "\n", + "Sizing hidden layers can be a challenge. The best way to figure this out is to do it\n", + "empirically. However, we may need a \"rule of thumb\" to start. Here is a good rule of thumb:\n", + "\n", + "First Hidden Layer:\n", + "```\n", + "n_hidden_1 = np.sqrt(np.sqrt((n_classes + 2) * n_input) + 2 * np.sqrt(n_input /(n_classes+2.)))\n", + "```\n", + "\n", + "Second Hidden Layer: (if needed)\n", + "```\n", + "n_hidden_2 = n_classes * np.sqrt(n_input / (n_classes + 2.))\n", + "```\n", + "\n", + "This dataset may not need two hidden layers. We'll start with one and see how that goes." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hidden layer 1 (Guess) : 4.0193898071\n", + "Hidden layer 2 (Guess) : 5.38516480713\n" + ] + } + ], + "source": [ + "# Number of hidden layers\n", + "\n", + "n_hidden_guess = np.sqrt(np.sqrt((n_classes + 2) * n_input) + 2 * np.sqrt(n_input /(n_classes+2.)))\n", + "print(\"Hidden layer 1 (Guess) : \" + str(n_hidden_guess))\n", + "\n", + "n_hidden_guess_2 = n_classes * np.sqrt(n_input / (n_classes + 2.))\n", + "print(\"Hidden layer 2 (Guess) : \" + str(n_hidden_guess_2))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "cc_training = spark.read.csv(\"../data/creditcardfraud/creditcard.csv\", header=True, inferSchema=\"true\", mode=\"DROPMALFORMED\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------+-----------------+--------------------+--------------------+-----------------+-------------------+\n", + "|summary| Time| V1| V2| Amount| Class|\n", + "+-------+-----------------+--------------------+--------------------+-----------------+-------------------+\n", + "| count| 284807| 284807| 284807| 284807| 284807|\n", + "| mean| 94813.8596|2.260907382386083...|6.067406817137794...|88.34961925095207| 1.00172748563062|\n", + "| stddev|47488.14595456619| 1.9586958038574867| 1.6513085794769937|250.1201092401879|0.04152718963546483|\n", + "| min| 0| -56.407509631329| -72.7157275629303| 0.0| 1|\n", + "| max| 172792| 2.45492999121121| 22.0577289904909| 25691.16| 2|\n", + "+-------+-----------------+--------------------+--------------------+-----------------+-------------------+\n", + "\n" + ] + } + ], + "source": [ + "cc_training.select('Time', 'V1', 'V2', 'Amount', 'Class').describe().show()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "cc_training = cc_training.select([col(c).cast(\"double\") for c in cc_training.columns])\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----+------------------+-------------------+------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------------------+-------------------+------------------+------------------+-------------------+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------+-----+\n", + "|Time| V1| V2| V3| V4| V5| V6| V7| V8| V9| V10| V11| V12| V13| V14| V15| V16| V17| V18| V19| V20| V21| V22| V23| V24| V25| V26| V27| V28|Amount|Class|\n", + "+----+------------------+-------------------+------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------------------+-------------------+------------------+------------------+-------------------+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------+-----+\n", + "| 0.0| -1.3598071336738|-0.0727811733098497| 2.53634673796914| 1.37815522427443| -0.338320769942518| 0.462387777762292| 0.239598554061257| 0.0986979012610507| 0.363786969611213| 0.0907941719789316|-0.551599533260813|-0.617800855762348| -0.991389847235408| -0.311169353699879| 1.46817697209427| -0.470400525259478| 0.207971241929242| 0.0257905801985591| 0.403992960255733| 0.251412098239705| -0.018306777944153| 0.277837575558899| -0.110473910188767| 0.0669280749146731| 0.128539358273528| -0.189114843888824| 0.133558376740387|-0.0210530534538215|149.62| 1.0|\n", + "| 0.0| 1.19185711131486| 0.26615071205963| 0.16648011335321| 0.448154078460911| 0.0600176492822243|-0.0823608088155687| -0.0788029833323113| 0.0851016549148104|-0.255425128109186| -0.166974414004614| 1.61272666105479| 1.06523531137287| 0.48909501589608| -0.143772296441519| 0.635558093258208| 0.463917041022171| -0.114804663102346| -0.183361270123994| -0.145783041325259|-0.0690831352230203| -0.225775248033138| -0.638671952771851| 0.101288021253234| -0.339846475529127| 0.167170404418143| 0.125894532368176|-0.00898309914322813| 0.0147241691924927| 2.69| 1.0|\n", + "| 1.0| -1.35835406159823| -1.34016307473609| 1.77320934263119| 0.379779593034328| -0.503198133318193| 1.80049938079263| 0.791460956450422| 0.247675786588991| -1.51465432260583| 0.207642865216696| 0.624501459424895| 0.066083685268831| 0.717292731410831| -0.165945922763554| 2.34586494901581| -2.89008319444231| 1.10996937869599| -0.121359313195888| -2.26185709530414| 0.524979725224404| 0.247998153469754| 0.771679401917229| 0.909412262347719| -0.689280956490685| -0.327641833735251| -0.139096571514147| -0.0553527940384261|-0.0597518405929204|378.66| 1.0|\n", + "| 1.0|-0.966271711572087| -0.185226008082898| 1.79299333957872| -0.863291275036453|-0.0103088796030823| 1.24720316752486| 0.23760893977178| 0.377435874652262| -1.38702406270197|-0.0549519224713749|-0.226487263835401| 0.178228225877303| 0.507756869957169| -0.28792374549456| -0.631418117709045| -1.0596472454325| -0.684092786345479| 1.96577500349538| -1.2326219700892| -0.208037781160366| -0.108300452035545|0.00527359678253453| -0.190320518742841| -1.17557533186321| 0.647376034602038| -0.221928844458407| 0.0627228487293033| 0.0614576285006353| 123.5| 1.0|\n", + "| 2.0| -1.15823309349523| 0.877736754848451| 1.548717846511| 0.403033933955121| -0.407193377311653| 0.0959214624684256| 0.592940745385545| -0.270532677192282| 0.817739308235294| 0.753074431976354|-0.822842877946363| 0.53819555014995| 1.3458515932154| -1.11966983471731| 0.175121130008994| -0.451449182813529| -0.237033239362776|-0.0381947870352842| 0.803486924960175| 0.408542360392758|-0.00943069713232919| 0.79827849458971| -0.137458079619063| 0.141266983824769| -0.206009587619756| 0.502292224181569| 0.219422229513348| 0.215153147499206| 69.99| 1.0|\n", + "| 2.0|-0.425965884412454| 0.960523044882985| 1.14110934232219| -0.168252079760302| 0.42098688077219|-0.0297275516639742| 0.476200948720027| 0.260314333074874| -0.56867137571251| -0.371407196834471| 1.34126198001957| 0.359893837038039| -0.358090652573631| -0.137133700217612| 0.517616806555742| 0.401725895589603| -0.0581328233640131| 0.0686531494425432|-0.0331937877876282| 0.0849676720682049| -0.208253514656728| -0.559824796253248|-0.0263976679795373| -0.371426583174346| -0.232793816737034| 0.105914779097957| 0.253844224739337| 0.0810802569229443| 3.67| 1.0|\n", + "| 4.0| 1.22965763450793| 0.141003507049326|0.0453707735899449| 1.20261273673594| 0.191880988597645| 0.272708122899098|-0.00515900288250983| 0.0812129398830894| 0.464959994783886|-0.0992543211289237| -1.41690724314928|-0.153825826253651| -0.75106271556262| 0.16737196252175| 0.0501435942254188| -0.443586797916727| 0.00282051247234708| -0.61198733994012|-0.0455750446637976| -0.21963255278686| -0.167716265815783| -0.270709726172363| -0.154103786809305| -0.780055415004671| 0.75013693580659| -0.257236845917139| 0.0345074297438413|0.00516776890624916| 4.99| 1.0|\n", + "| 7.0|-0.644269442348146| 1.41796354547385| 1.0743803763556| -0.492199018495015| 0.948934094764157| 0.428118462833089| 1.12063135838353| -3.80786423873589| 0.615374730667027| 1.24937617815176|-0.619467796121913| 0.291474353088705| 1.75796421396042| -1.32386521970526| 0.686132504394383|-0.0761269994382006| -1.2221273453247| -0.358221569869078| 0.324504731321494| -0.156741852488285| 1.94346533978412| -1.01545470979971| 0.057503529867291| -0.649709005559993| -0.415266566234811|-0.0516342969262494| -1.20692108094258| -1.08533918832377| 40.8| 1.0|\n", + "| 7.0| -0.89428608220282| 0.286157196276544|-0.113192212729871| -0.271526130088604| 2.6695986595986| 3.72181806112751| 0.370145127676916| 0.851084443200905|-0.392047586798604| -0.410430432848439|-0.705116586646536|-0.110452261733098| -0.286253632470583| 0.0743553603016731| -0.328783050303565| -0.210077268148783| -0.499767968800267| 0.118764861004217| 0.57032816746536| 0.0527356691149697| -0.0734251001059225| -0.268091632235551| -0.204232669947878| 1.0115918018785| 0.373204680146282| -0.384157307702294| 0.0117473564581996| 0.14240432992147| 93.2| 1.0|\n", + "| 9.0| -0.33826175242575| 1.11959337641566| 1.04436655157316| -0.222187276738296| 0.49936080649727| -0.24676110061991| 0.651583206489972| 0.0695385865186387|-0.736727316364109| -0.366845639206541| 1.01761446783262| 0.836389570307029| 1.00684351373408| -0.443522816876142| 0.150219101422635| 0.739452777052119| -0.540979921943059| 0.47667726004282| 0.451772964394125| 0.203711454727929| -0.246913936910008| -0.633752642406113| -0.12079408408185| -0.385049925313426|-0.0697330460416923| 0.0941988339514961| 0.246219304619926| 0.0830756493473326| 3.68| 1.0|\n", + "|10.0| 1.44904378114715| -1.17633882535966| 0.913859832832795| -1.37566665499943| -1.97138316545323| -0.62915213889734| -1.4232356010359| 0.0484558879088564| -1.72040839292037| 1.62665905834133| 1.1996439495421|-0.671439778462005| -0.513947152539479|-0.0950450453999549| 0.230930409124119| 0.0319674667862076| 0.253414715863197| 0.854343814324194| -0.221365413645481| -0.387226474431156|-0.00930189652490052| 0.313894410791098| 0.0277401580170247| 0.500512287104917| 0.25136735874921| -0.129477953726618| 0.0428498709381461| 0.0162532619375515| 7.8| 1.0|\n", + "|10.0| 0.38497821518095| 0.616109459176472|-0.874299702595052|-0.0940186259679115| 2.92458437838817| 3.31702716826156| 0.470454671805879| 0.53824722837695|-0.558894612428441| 0.30975539423728|-0.259115563735702|-0.326143233995877|-0.0900467227020648| 0.362832368569793| 0.928903660629178| -0.129486811402759| -0.809978925963589| 0.359985390219981| 0.70766382644648| 0.12599157561542| 0.049923685888971| 0.238421512225103|0.00912986861262866| 0.996710209581086| -0.767314827174801| -0.492208295340017| 0.042472441919027|-0.0543373883732122| 9.99| 1.0|\n", + "|10.0| 1.249998742053| -1.22163680921816| 0.383930151282291| -1.23489868766892| -1.48541947377961| -0.753230164566149| -0.689404975426345| -0.227487227519552| -2.09401057344842| 1.32372927445937| 0.227666231237246|-0.242681998944186| 1.20541680770748| -0.317630527025074| 0.725674990179153| -0.815612186027305| 0.873936447614439| -0.847788598847099| -0.683192626267037| -0.102755941505071| -0.231809239223849| -0.483285330117712| 0.0846676908596583| 0.392830885335013| 0.161134553588505| -0.354990039673962| 0.0264155490776107| 0.0424220887282304| 121.5| 1.0|\n", + "|11.0| 1.0693735878819| 0.287722129331455| 0.828612726634281| 2.71252042961718| -0.178398016248009| 0.337543730282968| -0.0967168617395962| 0.115981735546597|-0.221082566236194| 0.460230444301678|-0.773656930526689| 0.32338724546722|-0.0110758870883779| -0.178485175177916| -0.65556427824926| -0.19992517131173| 0.1240054151819| -0.980496201537345| -0.982916082135047| -0.153197231044512| -0.0368755317335273| 0.0744124028162195|-0.0714074332998586| 0.104743752596029| 0.548264725394119| 0.104094153162781| 0.0214910583643189| 0.021293311477486| 27.5| 1.0|\n", + "|12.0| -2.7918547659339| -0.327770756658658| 1.64175016056605| 1.76747274389883| -0.136588446465306| 0.80759646826532| -0.422911389711497| -1.90710747624096| 0.755712908314791| 1.1510869876677| 0.844555470974377| 0.7929439518176| 0.370448092803246| -0.734975105820311| 0.406795710431001| -0.303057623825763| -0.155868714793874| 0.778265457041536| 2.22186801373788| -1.58212204356551| 1.15166304848789| 0.222181966098225| 1.02058620426601| 0.0283166513238872| -0.232746324289105| -0.23555721754117| -0.16477751177654|-0.0301536365592253| 58.8| 1.0|\n", + "|12.0|-0.752417042956605| 0.345485415344747| 2.05732291276727| -1.46864329840046| -1.1583936804082|-0.0778498291166733| -0.608581418236123|0.00360348436201849|-0.436166983515744| 0.747730827192802|-0.793980602837221|-0.770406728847129| 1.04762699748088| -1.06660368148653| 1.10695345662141| 1.66011355713381| -0.279265373246772| -0.419994141181313| 0.432535348618175| 0.263450864446125| 0.499624954671111| 1.35365048557231| -0.256573280448308|-0.0650837078816517|-0.0391243535426488|-0.0870864732146962| -0.180997500092721| 0.129394059390202| 15.99| 1.0|\n", + "|12.0| 1.10321543528383|-0.0402962145973447| 1.2673320885949| 1.28909146962552| -0.735997163604068| 0.288069162976262| -0.586056786337461| 0.189379713679593| 0.782332891785191| -0.267975066537173|-0.450311279515466| 0.936707714991982| 0.708380406186981| -0.468647287707221| 0.354574063407955| -0.246634655717582|-0.00921237772707382| -0.595912405700819| -0.57568162226261| -0.113910176982092| -0.0246120063374677| 0.196001952806192| 0.0138016541409422| 0.103758331023198| 0.364297540595235| -0.382260574113217| 0.092809187460487| 0.0370505169810008| 12.99| 1.0|\n", + "|13.0|-0.436905071360625| 0.918966212909322| 0.92459077438817| -0.727219053596792| 0.915678718106307| -0.127867352079254| 0.707641607333935| 0.0879623554672504| -0.66527135413364| -0.737979823596458| 0.32409781346169| 0.277192107214981| 0.252624256310781| -0.291896460370468| -0.184520169327133| 1.14317370716197| -0.92870926272403| 0.680469592634687| 0.0254364616880793|-0.0470212823165035| -0.194795823794671| -0.672637997017793| -0.156857514491897| -0.888386320943716| -0.342413218776576| -0.049026728633951| 0.0796923991551505| 0.131023789452311| 0.89| 1.0|\n", + "|14.0| -5.40125766315825| -5.45014783420644| 1.18630463143652| 1.73623880012095| 3.04910587764025| -1.76340557365201| -1.55973769907953| 0.160841747266769| 1.23308974041888| 0.345172827050629| 0.917229867699146| 0.970116716069048| -0.266567764915222| -0.479129929276704| -0.526608502569153| 0.47200411177674| -0.725480944982201| 0.075081351540202| -0.406866573198217| -2.19684802485647| -0.503600328973703| 0.984459785590244| 2.45858857639219| 0.0421188969891572| -0.481630823956716| -0.621272013713977| 0.392053289557744| 0.949594245504846| 46.8| 1.0|\n", + "|15.0| 1.4929359769862| -1.02934573189487| 0.45479473374366| -1.43802587991702| -1.55543410136344| -0.720961147043557| -1.08066413038614|-0.0531271179483221| -1.9786815953872| 1.63807603690446| 1.07754241162743| -0.63204651464934| -0.41695716661602| 0.0520105153724404|-0.0429789228232019| -0.166432496451972| 0.304241418614353| 0.554432499062278| 0.0542295152184719| -0.387910172646258| -0.177649846438814| -0.175073809074822| 0.0400022190621329| 0.295813862676508| 0.33293059939425| -0.220384850672322| 0.0222984359135846|0.00760225559997897| 5.0| 1.0|\n", + "+----+------------------+-------------------+------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------------------+-------------------+------------------+------------------+-------------------+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------+-----+\n", + "only showing top 20 rows\n", + "\n" + ] + } + ], + "source": [ + "cc_training.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Lopsided Data distribution\n", + "\n", + "Let us look at the very lopsided data distribution." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAEWCAYAAACwtjr+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAHKxJREFUeJzt3Xm0VOWd7vHvIziPKGgQUIhid9Qk\nqKgYMziL5ho0rd0OrcSFwSS4EhM7LfEmkXZIx9vXobkaE4y0gCNqnCKGRmNi2zHK0bBE1DQnSgQh\nCoKAM+Dv/rHfituizjnFgfcUFs9nrVqn6rff/e531ynOU3tgb0UEZmZmOW3U6AGYmVnzc9iYmVl2\nDhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMatB0iGSZnXxMm+UNKYrl1m1/HmSDknPfyDpp+uo326S\n3pC0S3q9TtdT0s8lXbCu+rM8HDa21tIfksrjfUlvl16f1ujxdURSd0khqX+lFhG/iYi9GjeqxoqI\niyPiax21k/SopK900NeqiNgqIl5a23FJOkvSb6r6PysifrS2fVte3Rs9APvoi4itKs8lzQHOiogH\n22ovqXtErOyKsVlj+XdtFd6ysewkXSLpNkm3SFoO/KOkgyT9XtLrkhZIGitp49S+sqVxtqRWSUsk\njS31t4ekRyQtlbRI0s2laVen3UHLJE2X9JnStO5p99Cf0vQWSTsDj6Qms9LW2N9JOiIFZ2XevST9\nNo13pqQvlqbdmMb/gKTlkh6TNKCd9+Pzad2XSpor6fQabXaQNEXSwrT+90nqU5o+QtKctLwXJJ3c\n0XtTYxlfkfTn1G50jd/ZDen5FpJulvRaWv8nJPWUdBlwEPDT9L5dVfrdfUNSK/B8rS1HoJekh9L4\nH5bULy1rd0lRNZZH01g/CVwNfC4tb1Hp/R9Tav+19Ll5TdLdknqnerufK8vLYWNd5QTgZmBb4DZg\nJfAtoCdwMDAUOLtqnmOB/YB9KALqiFS/FLgf6AH0Ba4pzfM48Clge+AO4HZJm6Zp3wVOTMvaDjgL\neAf4fJq+V9rdc2d5EJI2AX6ZltkL+DZwm6TdS81OBX6QlvsScHGtNyGF0P3AFcAOad1m1mi6EXAd\nsAuwK7AC+PfUxzZp/iMjYmuK9+/pOt6b8jgqf7hPBfoAOwMfq9UWOBPYIvW3A/AN4J2IOB94DPha\net/OLc3zJWB/4JNt9PmPwA8pfv/PApPaaPdXETETOAf4r7S8njXW6yjgIorfcx9gPnBTVbO2PleW\nkcPGusqjEXFfRLwfEW9HxPSIeDwiVkbEC8A44AtV8/xrRCyNiDnAb4BBqb4C6A/0joh3IuK/KzNE\nxKSIWJx23fwfYBugEgpnARdExOw0jhkRsbiOsR8MbAL8W0SsSLsIHwBOLrW5IyJaImIFxR+3QTX6\ngeKP7K8iYnJa90URMaO6UUQsjIi70nu1DPhR1fsTwN6SNouIBRHxbEfvTZWTgLsj4r8j4l3gAkBt\ntF1BEQq7p+MvLRHxRhttK34UEUsi4u02pt9XtezPV7ZA1tJpwM/T7/YdYDTwBUl9S23a+lxZRg4b\n6ypzyy8k/a2k+yX9RdIyim+j1d9U/1J6/hZQOTZ0HrAx0JJ2aQ0v9fvPkp6XtBRYAmxZ6rcf8KdO\njH1n4KX48FVr/0zxzbmjsVarawyStlRxltVL6f35NWk9UvicAowC/iLpl5L2SLO2+d7UWKe//k5S\neLQVvDcADwKTJb0s6ceSOjreO7fe6RGxFFiaxrS2dqb43VT6XkbxOejM78rWIYeNdZXqy4v/DHiG\n4tvyNhS7VNr6Zv3hjopv8mdFRG+KP7jjJA2QdCjwHeDvKHaT9QDeKPU7F9itjrFVmw/0k1Qe3y7A\ny/WMt0pbY6j2z8AA4ID0/hxWnhgRD0TEEUBvoJXi/WzzvanR/wKK4ANA0lYUuwBXExHvRcSYiPgE\n8FmKXaKVswzbeu86ek/Ly96WYvfqfODNVNui1La8e6+e39Wupb63pvgcdOZ3ZeuQw8YaZWuKb7Nv\nSvoEqx+vaZOkvy8dLH+d4g/QqtTnSmARxbf7MRRbNhU/By6RtJsKgyRtHxGrgNeAj7exyN+lfs+T\ntLGkwyj2+0+ud8wlNwJDVZyE0D0daP90jXZbU3zrXiJpB4owrqx/b0nHpT/I71H8gV6VprX13lS7\nHRim4kSNTYFLaOMPuaTDJO0taSNgGcVutUqfr9D2+9ae46qW/WhELKDY6vgLxbGUbpJGUgqPtLy+\nSieT1HALMELSp1Lf/0pxjGdeJ8Zo65DDxhrlPGA4sJziW/ltazDvgcB0SW8CvwBGpf/DMYVid89s\nYA7FH8YFpfn+DbgbeChNGwdslqZdCNyczrb6cnlh6bjCccAwiiAbC5waEf+zBmOu9PVi6ut8it1W\nT1H7IPoVFN/2X6MIuwdK07pRnOywIE3/DMWBc2j7vakex9MUJ2hMpvjWX/kjX8vOqa9lwCyK9/iW\nNO0q4JT0vl3RweqX3UgRMosoTug4PY0rgK9SHMdZRHG87fHSfNMofr+vSFptvBHxK4pdsndRvD+7\n8MFWmDWQfPM0MzPLzVs2ZmaWncPGzMyyc9iYmVl2DhszM8vOF+JMevbsGf3792/0MMzMPlKefPLJ\nRRHRq6N2Dpukf//+tLS0NHoYZmYfKZL+3HEr70YzM7Mu4LAxM7PsHDZmZpadw8bMzLJz2JiZWXYO\nGzMzy85hY2Zm2TlszMwsO4eNmZll5ysIfMT0H31/o4fQVOb8+IuNHoLZBsFbNmZmlp3DxszMsnPY\nmJlZdg4bMzPLzmFjZmbZOWzMzCw7h42ZmWXnsDEzs+wcNmZmlp3DxszMsnPYmJlZdg4bMzPLzmFj\nZmbZOWzMzCw7h42ZmWXnsDEzs+wcNmZmlp3DxszMsnPYmJlZdg4bMzPLzmFjZmbZZQsbSf0kPSzp\nOUmzJH0r1cdIelnSjPQ4tjTP9yS1SvqjpKNL9aGp1ippdKk+QNLjkmZLuk3SJqm+aXrdmqb3z7We\nZmbWsZxbNiuB8yLiE8AQYJSkPdO0KyNiUHpMAUjTTgb2AoYCP5HUTVI34BrgGGBP4JRSP5elvgYC\nS4ARqT4CWBIRuwNXpnZmZtYg2cImIhZExFPp+XLgOaBPO7MMA26NiHcj4kWgFTggPVoj4oWIeA+4\nFRgmScBhwB1p/gnA8aW+JqTndwCHp/ZmZtYAXXLMJu3G2gd4PJXOkfS0pPGSeqRaH2BuabZ5qdZW\nfQfg9YhYWVX/UF9p+tLUvnpcIyW1SGpZuHDhWq2jmZm1LXvYSNoKuBM4NyKWAdcCuwGDgAXA5ZWm\nNWaPTtTb6+vDhYhxETE4Igb36tWr3fUwM7POyxo2kjamCJqbIuIXABHxSkSsioj3gesodpNBsWXS\nrzR7X2B+O/VFwHaSulfVP9RXmr4tsHjdrp2ZmdUr59loAq4HnouIK0r13qVmJwDPpOf3AienM8kG\nAAOBJ4DpwMB05tkmFCcR3BsRATwMnJjmHw7cU+preHp+IvDr1N7MzBqge8dNOu1g4HRgpqQZqXYB\nxdlkgyh2a80BzgaIiFmSJgPPUpzJNioiVgFIOgeYCnQDxkfErNTf+cCtki4B/kARbqSfkyS1UmzR\nnJxxPc3MrAPZwiYiHqX2sZMp7cxzKXBpjfqUWvNFxAt8sBuuXH8HOGlNxmtmZvn4CgJmZpadw8bM\nzLJz2JiZWXYOGzMzy85hY2Zm2TlszMwsO4eNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXYOGzMz\ny85hY2Zm2TlszMwsO4eNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXYOGzMzy85hY2Zm2TlszMws\nO4eNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXbZwkZSP0kPS3pO0ixJ30r17SVNkzQ7/eyR6pI0\nVlKrpKcl7Vvqa3hqP1vS8FJ9P0kz0zxjJam9ZZiZWWPk3LJZCZwXEZ8AhgCjJO0JjAYeioiBwEPp\nNcAxwMD0GAlcC0VwABcCBwIHABeWwuPa1LYy39BUb2sZZmbWANnCJiIWRMRT6fly4DmgDzAMmJCa\nTQCOT8+HAROj8HtgO0m9gaOBaRGxOCKWANOAoWnaNhHxWEQEMLGqr1rLMDOzBuiSYzaS+gP7AI8D\nO0XEAigCCdgxNesDzC3NNi/V2qvPq1GnnWVUj2ukpBZJLQsXLuzs6pmZWQeyh42krYA7gXMjYll7\nTWvUohP1ukXEuIgYHBGDe/XqtSazmpnZGsgaNpI2pgiamyLiF6n8StoFRvr5aqrPA/qVZu8LzO+g\n3rdGvb1lmJlZA+Q8G03A9cBzEXFFadK9QOWMsuHAPaX6GemstCHA0rQLbCpwlKQe6cSAo4Cpadpy\nSUPSss6o6qvWMszMrAG6Z+z7YOB0YKakGal2AfBjYLKkEcBLwElp2hTgWKAVeAs4EyAiFku6GJie\n2l0UEYvT868DNwCbAw+kB+0sw8zMGiBb2ETEo9Q+rgJweI32AYxqo6/xwPga9RZg7xr112otw8zM\nGsNXEDAzs+wcNmZmlp3DxszMsnPYmJlZdg4bMzPLzmFjZmbZOWzMzCy7usJG0mr/l8XMzKxe9W7Z\n/FTSE5K+IWm7rCMyM7OmU1fYRMRngdMoLojZIulmSUdmHZmZmTWNuo/ZRMRs4PvA+cAXgLGSnpf0\n5VyDMzOz5lDvMZtPSbqS4m6bhwHHpds9HwZcmXF8ZmbWBOq9EOfVwHXABRHxdqUYEfMlfT/LyMzM\nrGnUGzbHAm9HxCoASRsBm0XEWxExKdvozMysKdR7zOZBinvGVGyRamZmZh2qN2w2i4g3Ki/S8y3y\nDMnMzJpNvWHzpqR9Ky8k7Qe83U57MzOzv6r3mM25wO2S5qfXvYF/yDMkMzNrNnWFTURMl/S3wN9Q\n3Or5+YhYkXVkZmbWNOrdsgHYH+if5tlHEhExMcuozMysqdQVNpImAbsBM4BVqRyAw8bMzDpU75bN\nYGDPiIicgzEzs+ZU79lozwAfyzkQMzNrXvVu2fQEnpX0BPBupRgRX8oyKjMzayr1hs2YnIMwM7Pm\nVu+pz7+VtCswMCIelLQF0C3v0MzMrFnUe4uBrwJ3AD9LpT7A3bkGZWZmzaXeEwRGAQcDy+CvN1Lb\nsb0ZJI2X9KqkZ0q1MZJeljQjPY4tTfuepFZJf5R0dKk+NNVaJY0u1QdIelzSbEm3Sdok1TdNr1vT\n9P51rqOZmWVSb9i8GxHvVV5I6k7x/2zacwMwtEb9yogYlB5TUn97AicDe6V5fiKpm6RuwDXAMcCe\nwCmpLcBlqa+BwBJgRKqPAJZExO4UN3a7rM51NDOzTOoNm99KugDYXNKRwO3Afe3NEBGPAIvr7H8Y\ncGtEvBsRLwKtwAHp0RoRL6SwuxUYJkkUdwm9I80/ATi+1NeE9PwO4PDU3szMGqTesBkNLARmAmcD\nU4DO3qHzHElPp91sPVKtDzC31GZeqrVV3wF4PSJWVtU/1FeavjS1NzOzBqkrbCLi/Yi4LiJOiogT\n0/POXE3gWorL3gwCFgCXp3qtLY/oRL29vlYjaaSkFkktCxcubG/cZma2Fuq9NtqL1PiDHREfX5OF\nRcQrpT6vA36ZXs4D+pWa9gUqtzOoVV8EbCepe9p6Kbev9DUvHVvaljZ250XEOGAcwODBg30pHjOz\nTNbk2mgVmwEnAduv6cIk9Y6IBenlCRSXwQG4F7hZ0hXAzsBA4AmKrZSBkgYAL1OcRHBqRISkh4ET\nKY7jDAfuKfU1HHgsTf+1r+lmZtZY9f6nzteqSldJehT4YVvzSLoFOAToKWkecCFwiKRBFFtJcyiO\n/xARsyRNBp4FVgKjImJV6uccYCrFfyIdHxGz0iLOB26VdAnwB+D6VL8emCSplWKL5uR61tHMzPKp\ndzfavqWXG1Fs6Wzd3jwRcUqN8vU1apX2lwKX1qhPoTghobr+AsXZatX1dyi2vMzMbD1R7260y0vP\nV1Jslfz9Oh+NmZk1pXp3ox2aeyBmZta86t2N9p32pkfEFetmOGZm1ozW5Gy0/SnO9AI4DniED/+H\nSzMzs5rW5OZp+0bEciguqAncHhFn5RqYmZk1j3ovV7ML8F7p9XtA/3U+GjMza0r1btlMAp6QdBfF\n/5E5AZiYbVRmZtZU6j0b7VJJDwCfS6UzI+IP+YZlZmbNpN7daABbAMsi4t8prjs2INOYzMysydR7\nW+gLKS4P871U2hi4MdegzMysudS7ZXMC8CXgTYCImE8Hl6sxMzOrqDds3ktXTg4ASVvmG5KZmTWb\nesNmsqSfUdxD5qvAg8B1+YZlZmbNpN6z0f6vpCOBZcDfAD+MiGlZR2ZmZk2jw7CR1A2YGhFHAA4Y\nMzNbYx3uRks3MXtL0rZdMB4zM2tC9V5B4B1gpqRppDPSACLim1lGZWZmTaXesLk/PczMzNZYu2Ej\naZeIeCkiJnTVgMzMrPl0dMzm7soTSXdmHouZmTWpjsJGpecfzzkQMzNrXh2FTbTx3MzMrG4dnSDw\naUnLKLZwNk/PSa8jIrbJOjozM2sK7YZNRHTrqoGYmVnzWpP72ZiZmXWKw8bMzLJz2JiZWXYOGzMz\nyy5b2EgaL+lVSc+UattLmiZpdvrZI9UlaaykVklPS9q3NM/w1H62pOGl+n6SZqZ5xkpSe8swM7PG\nybllcwMwtKo2GngoIgYCD6XXAMcAA9NjJHAtFMEBXAgcCBwAXFgKj2tT28p8QztYhpmZNUi2sImI\nR4DFVeVhQOU6axOA40v1iVH4PcUdQXsDRwPTImJxRCyhuJ/O0DRtm4h4LN2uemJVX7WWYWZmDdLV\nx2x2iogFAOnnjqneB5hbajcv1dqrz6tRb28Zq5E0UlKLpJaFCxd2eqXMzKx968sJAqpRi07U10hE\njIuIwRExuFevXms6u5mZ1amrw+aVtAuM9PPVVJ8H9Cu16wvM76Det0a9vWWYmVmDdHXY3AtUzigb\nDtxTqp+RzkobAixNu8CmAkdJ6pFODDgKmJqmLZc0JJ2FdkZVX7WWYWZmDVLvnTrXmKRbgEOAnpLm\nUZxV9mNgsqQRwEvASan5FOBYoBV4CzgTICIWS7oYmJ7aXRQRlZMOvk5xxtvmwAPpQTvLMDOzBskW\nNhFxShuTDq/RNoBRbfQzHhhfo94C7F2j/lqtZZiZWeOsLycImJlZE3PYmJlZdg4bMzPLzmFjZmbZ\nOWzMzCw7h42ZmWXnsDEzs+wcNmZmlp3DxszMsnPYmJlZdg4bMzPLzmFjZmbZOWzMzCw7h42ZmWXn\nsDEzs+wcNmZmlp3DxszMsnPYmJlZdg4bMzPLzmFjZmbZOWzMzCw7h42ZmWXnsDEzs+wcNmZmlp3D\nxszMsnPYmJlZdg4bMzPLriFhI2mOpJmSZkhqSbXtJU2TNDv97JHqkjRWUqukpyXtW+pneGo/W9Lw\nUn2/1H9rmlddv5ZmZlbRyC2bQyNiUEQMTq9HAw9FxEDgofQa4BhgYHqMBK6FIpyAC4EDgQOACysB\nldqMLM03NP/qmJlZW9an3WjDgAnp+QTg+FJ9YhR+D2wnqTdwNDAtIhZHxBJgGjA0TdsmIh6LiAAm\nlvoyM7MGaFTYBPCfkp6UNDLVdoqIBQDp546p3geYW5p3Xqq1V59Xo74aSSMltUhqWbhw4VqukpmZ\ntaV7g5Z7cETMl7QjME3S8+20rXW8JTpRX70YMQ4YBzB48OCabczMbO01ZMsmIuann68Cd1Ecc3kl\n7QIj/Xw1NZ8H9CvN3heY30G9b426mZk1SJeHjaQtJW1deQ4cBTwD3AtUzigbDtyTnt8LnJHOShsC\nLE272aYCR0nqkU4MOAqYmqYtlzQknYV2RqkvMzNrgEbsRtsJuCudjdwduDkifiVpOjBZ0gjgJeCk\n1H4KcCzQCrwFnAkQEYslXQxMT+0uiojF6fnXgRuAzYEH0sPMzBqky8MmIl4APl2j/hpweI16AKPa\n6Gs8ML5GvQXYe60Ha2Zm68T6dOqzmZk1KYeNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXYOGzMz\ny85hY2Zm2TlszMwsO4eNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXYOGzMzy85hY2Zm2TlszMws\nO4eNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXYOGzMzy85hY2Zm2TlszMwsO4eNmZll57AxM7Ps\nHDZmZpZd04aNpKGS/iipVdLoRo/HzGxD1pRhI6kbcA1wDLAncIqkPRs7KjOzDVf3Rg8gkwOA1oh4\nAUDSrcAw4NmGjsqsmY3ZttEjaC5jljZ6BOtUs4ZNH2Bu6fU84MDqRpJGAiPTyzck/bELxrah6Aks\navQgOqLLGj0Ca4CPxGeTf1GjR1CvXetp1KxhU+u3FKsVIsYB4/IPZ8MjqSUiBjd6HGbV/NlsjKY8\nZkOxJdOv9LovML9BYzEz2+A1a9hMBwZKGiBpE+Bk4N4Gj8nMbIPVlLvRImKlpHOAqUA3YHxEzGrw\nsDY03j1p6yt/NhtAEasdyjAzM1unmnU3mpmZrUccNmZmlp3DxlYjKSRdXnr9T5LGdPEYbpB0Ylcu\n0z56JK2SNKP06J9hGf0lPbOu+93QOGyslneBL0vq2ZmZJTXliSe2Xno7IgaVHnPKE/1ZXH/4F2G1\nrKQ4Y+fbwP8uT5C0KzAe6AUsBM6MiJck3QAsBvYBnpK0HBgA9Ab2AL4DDKG4Xt3LwHERsULSD4Hj\ngM2B3wFnh89asbUg6SvAF4HNgC0lfQm4B+gBbAx8PyLuSVtBv4yIvdN8/wRsFRFjJO1H8Tl/C3i0\ny1eiCXnLxtpyDXCapOoLXl0NTIyITwE3AWNL0/YAjoiI89Lr3Sj+0Q8DbgQejohPAm+nOsDVEbF/\n+ge/OfC/sqyNNavNS7vQ7irVDwKGR8RhwDvACRGxL3AocLmkjq4F8x/ANyPioDzD3vA4bKymiFgG\nTAS+WTXpIODm9HwS8NnStNsjYlXp9QMRsQKYSfH/nX6V6jOB/un5oZIelzQTOAzYa52thG0IyrvR\nTijVp0XE4vRcwI8kPQ08SHHtxJ3a6jB9wdouIn6bSpNyDHxD491o1p6rgKcovuW1pbzL682qae8C\nRMT7klaUdo+9D3SXtBnwE2BwRMxNJyFstk5Gbhu68mfxNIrdvvulXbdzKD5nK/nwF+7KZ0/UuJai\nrR1v2Vib0jfDycCIUvl3FJf/geIf8drsz678414kaSvAZ59ZDtsCr6agOZQPrlL8CrCjpB0kbUra\nhRsRrwNLJVW22k/r8hE3IW/ZWEcuB84pvf4mMF7Sd0knCHS244h4XdJ1FLvV5lBc085sXbsJuE9S\nCzADeB4ghc9FwOPAi5V6cibF5/wtiste2Vry5WrMzCw770YzM7PsHDZmZpadw8bMzLJz2JiZWXYO\nGzMzy85hY9YAkj4m6VZJf5L0rKQpkvbw1YWtWfn/2Zh1sXRdrruACRFxcqoNop1LqJh91HnLxqzr\nHQqsiIifVgoRMQOYW3md7qHyX5KeSo/PpHpvSY+kC08+I+lzkrql+/88I2mmpG93/SqZtc9bNmZd\nb2/gyQ7avAocGRHvSBoI3AIMBk4FpkbEpZK6AVsAg4A+pUvlb5dv6Gad47AxWz9tDFyddq+torh9\nAxSX9BkvaWPg7oiYIekF4OOS/h9wP/CfDRmxWTu8G82s680C9uugzbcpLhT5aYotmk0AIuIR4PMU\nN6CbJOmMiFiS2v0GGAX8PM+wzTrPYWPW9X4NbCrpq5WCpP354GrEUFypeEFEvA+cTnE/oMqdUl+N\niOuA64F90+27N4qIO4EfAPt2zWqY1c+70cy6WESEpBOAqySNpriT5Bzg3FKznwB3SjoJeJgP7s9y\nCPBdSSuAN4AzKG4G9h+SKl8ev5d9JczWkK/6bGZm2Xk3mpmZZeewMTOz7Bw2ZmaWncPGzMyyc9iY\nmVl2DhszM8vOYWNmZtn9fyQaEZIK2LqrAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#count_classes = pd.value_counts(df['Class'], sort = True)\n", + "count_classes = pd.value_counts(cc_training.select('Class').toPandas()['Class'], sort = True)\n", + "count_classes.plot(kind = 'bar', rot=0)\n", + "plt.title(\"Transaction class distribution\")\n", + "plt.xticks(range(2), LABELS)\n", + "plt.xlabel(\"Class\")\n", + "plt.ylabel(\"Frequency\");" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Splitting the data\n", + "\n", + "Instead of using a random split, we are going to segment the dataset based on time. This is becaause a model will be trained on historical data and deployed on new data for prediction, so it makes sense for validation data to work much the same way." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# get the time to split the data.\n", + "splitTime = cc_training.stat.approxQuantile(\"Time\", [0.7], 0.001)[0]\n", + "\n", + "trainingData =cc_training.filter(\"Time < \" + str(splitTime))\n", + "validData = cc_training.filter(\"Time >= \" + str(splitTime))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Converting to RDD of Sample\n", + "\n", + "BigDL requires all input data to be in its native Sample format (for the RDD API). The Pipeline api, shown in feedforward-iris-pipeline -- allows data to be in Spark Dataframes.\n", + "\n", + "To convert the data into type Sample, we will use a helper method here to convert. The sample function requires two numpy arrays:\n", + "\n", + "1. Feature array (of all the features in double type)\n", + "2. Label array: the label value (usually just a single number in the array).\n", + "\n", + "Then we convert these to RDDs of type Sample." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training Count:199152\n", + "Test Count:85655\n" + ] + } + ], + "source": [ + "#convert ndarray data into RDD[Sample]\n", + "\n", + "# time,V1,V2,V3,V4,V5,V6,V7,V8,V9,V10,V11,V12,V13,V14,V15,V16,V17,V18,V19,V20,V21,V22,V23,V24,V25,V26,V27,V28,amount,prediction\n", + "def array2rdd(ds):\n", + " #build Sample from ndarrays\n", + " def build_sample(time,V1,V2,V3,V4,V5,V6,V7,V8,V9,V10,V11,V12,V13,V14,V15,V16,V17,V18,V19,V20,V21,V22,V23,V24,V25,V26,V27,V28,amount,prediction):\n", + " feature = np.array([V1,V2,V3,V4,V5,V6,V7,V8,V9,V10,V11,V12,V13,V14,V15,V16,V17,V18,V19,V20,V21,V22,V23,V24,V25,V26,V27,V28,amount]).flatten()\n", + " label = np.array(prediction)\n", + " return Sample.from_ndarray(feature, label)\n", + " rdd = ds.map(lambda (time,V1,V2,V3,V4,V5,V6,V7,V8,V9,V10,V11,V12,V13,V14,V15,V16,V17,V18,V19,V20,V21,V22,V23,V24,V25,V26,V27,V28,amount,prediction): build_sample(time,V1,V2,V3,V4,V5,V6,V7,V8,V9,V10,V11,V12,V13,V14,V15,V16,V17,V18,V19,V20,V21,V22,V23,V24,V25,V26,V27,V28,amount,prediction))\n", + " return rdd\n", + "\n", + "cc_rdd_train = array2rdd(trainingData.rdd.map(list))\n", + "cc_rdd_train.cache()\n", + "print(\"Training Count:\" + str(cc_rdd_train.count()))\n", + "\n", + "\n", + "\n", + "cc_rdd_test = array2rdd(validData.rdd.map(list))\n", + "cc_rdd_test.cache()\n", + "print(\"Test Count:\" + str(cc_rdd_test.count()))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setting up our network\n", + "\n", + "Here we are going to actually set up our network. It will be single hidden layer network, with input layer (based on our 4-inputs), and output layer (softmax with 3 classes). \n", + "\n", + "The hidden layer has been set up to 10, and we will use ReLU activation for that hidden layer. ReLU is currently preferred as a activation layer for these types of networks as it is linear in the postive direction, avoiding the vanishing gradient problem.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "creating: createSequential\n", + "creating: createLinear\n", + "creating: createReLU\n", + "creating: createLinear\n", + "creating: createLogSoftMax\n" + ] + } + ], + "source": [ + "# Create model\n", + "\n", + "def multilayer_perceptron(n_hidden_1, n_input, n_classes):\n", + " # Initialize a sequential container\n", + " model = Sequential()\n", + " # Hidden layer with ReLu activation\n", + " model.add(Linear(n_input, n_hidden_1).set_name('mlp_fc1'))\n", + " model.add(ReLU())\n", + " # output layer\n", + " model.add(Linear(n_hidden_1, n_classes).set_name('mlp_fc3'))\n", + " model.add(LogSoftMax())\n", + " return model\n", + "\n", + "model = multilayer_perceptron(n_hidden_1, n_input, n_classes)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Training the Model\n", + "\n", + "Now we need to set up our training. We are going to use the following:\n", + "\n", + "* Loss Function: ClassNLLCriterion\n", + "* Optimization: AdaGrad \n", + "\n", + "For validation, we will validating against top1Accuracy for every epoch." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "creating: createClassNLLCriterion\n", + "creating: createDefault\n", + "creating: createSGD\n", + "creating: createMaxEpoch\n", + "creating: createOptimizer\n", + "creating: createEveryEpoch\n", + "creating: createTop1Accuracy\n", + "creating: createTrainSummary\n", + "creating: createSeveralIteration\n", + "creating: createValidationSummary\n", + "('saving logs to ', 'creditcardfraud-20171115-132426')\n" + ] + } + ], + "source": [ + "# Create an Optimizer\n", + "optimizer = Optimizer(\n", + " model=model,\n", + " training_rdd=cc_rdd_train,\n", + " criterion=ClassNLLCriterion(),\n", + " optim_method=SGD(learningrate=learning_rate),\n", + " end_trigger=MaxEpoch(training_epochs),\n", + " batch_size=batch_size)\n", + "\n", + "# Set the validation logic\n", + "optimizer.set_validation(\n", + " batch_size=batch_size,\n", + " val_rdd=cc_rdd_test,\n", + " trigger=EveryEpoch(),\n", + " val_method=[Top1Accuracy()]\n", + ")\n", + "\n", + "app_name='creditcardfraud-'+dt.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n", + "train_summary = TrainSummary(log_dir='/tmp/bigdl_summaries',\n", + " app_name=app_name)\n", + "train_summary.set_summary_trigger(\"Parameters\", SeveralIteration(50))\n", + "val_summary = ValidationSummary(log_dir='/tmp/bigdl_summaries',\n", + " app_name=app_name)\n", + "optimizer.set_train_summary(train_summary)\n", + "optimizer.set_val_summary(val_summary)\n", + "print(\"saving logs to \",app_name)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Optimization Done.\n", + "CPU times: user 60 ms, sys: 10 ms, total: 70 ms\n", + "Wall time: 3min 1s\n" + ] + } + ], + "source": [ + "%%time\n", + "# Boot training process\n", + "trained_model = optimizer.optimize()\n", + "print(\"Optimization Done.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAugAAAK7CAYAAACgQHrJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3XucXXV97//XZ2ZygwTCZYCE+zUK\niBEQUaskXgEVvNWCrYK9UM/R369aa9XiT63Wo22t2ovVQ48c8VSRVKtNFRW8jOgRq6AJ90uIQWBC\nAoTcrzPz+f2x1yR7JjPJJOyZvdbK6/l47Mfs/V1rr/3dn+yZvOc73/VdkZlIkiRJKoeOdndAkiRJ\n0g4GdEmSJKlEDOiSJElSiRjQJUmSpBIxoEuSJEklYkCXJEmSSsSALkn7iIhYFhEvaXc/JEm7ZkCX\nJEmSSsSALkmSJJWIAV2S9jERMSUiPh0RvcXt0xExpdh2aER8MyJWR8SqiPhxRHQU294TEY9ExLqI\nuDciXtzedyJJ9dTV7g5IkibclcC5wFwggf8A3g/8f8C7gIeB7mLfc4GMiDnA24FnZ2ZvRBwHdE5s\ntyVp3+AIuiTte34X+HBmrszMx4C/BN5UbNsGzAKOzcxtmfnjzEygH5gCnBoRkzJzWWY+0JbeS1LN\nGdAlad8zG3iw6fGDRRvA3wJLgBsiYmlEvBcgM5cA7wA+BKyMiK9ExGwkSS1nQJekfU8vcGzT42OK\nNjJzXWa+KzNPAF4F/OngXPPM/HJm/lbx3AT+emK7LUn7BgO6JO17rgXeHxHdEXEo8AHgXwEi4pUR\ncVJEBLCWxtSW/oiYExEvKk4m3QxsKrZJklrMgC5J+56/Am4BbgNuB35ZtAGcDHwPWA/cDPxzZvbQ\nmH/+ceBx4FHgMOAvJrTXkrSPiMa5P5IkSZLKwBF0SZIkqUQM6JIkSVKJGNAlSZKkEjGgS5IkSSXS\n1e4OjKeZM2fmSSed1O5u1MaGDRvYf//9292NWrCWrWMtW8t6to61bB1r2TrWsrWa63nrrbc+npnd\nrThurQP64Ycfzi233NLubtRGT08P8+bNa3c3asFato61bC3r2TrWsnWsZetYy9ZqrmdEPLjrvcfO\nKS6SJElSiRjQJUmSpBIxoEuSJEklYkCXJEmSSsSALkmSJJWIAV2SJEkqEQO6JEmSVCIGdEmSJKlE\nDOiSJElSiRjQJUmSpBIxoEuSJEklYkBX5Vz789/wuR890O5uSJIkjYuudndA2lPv+/fbAXjreSe2\nuSeSJEmt5wi6JEmSVCIGdEmSJKlEDOiSJElSiRjQJUmSpBIxoEuSJEklYkCXJEmSSsSALkmSJJWI\nAV2SJEkqEQO6JEmSVCIGdEmSJKlEDOiSJElSiRjQJUmSpBJpW0CPiKsjYmVE3NHUdl1ELCpuyyJi\nUdF+XERsatr2uXb1W5IkSRpPXW187S8A/wR8cbAhM39n8H5E/B2wpmn/BzJz7oT1TpIkSWqDtgX0\nzLwpIo4baVtEBPAG4EUT2SdJkiSp3SIz2/fijYD+zcw8fVj7C4FPZubZTfvdCdwHrAXen5k/HuWY\nVwBXAHR3d5+1YMGCcer9vmf9+vVMnz693d3g8u9sAOAL5+/f5p7svbLUsg6sZWtZz9axlq1jLVvH\nWrZWcz3nz59/62B2faraOcVlVy4Frm16vBw4JjOfiIizgG9ExGmZuXb4EzPzKuAqgDlz5uS8efMm\nor/7hJ6eHkpRz+98C6AcfdlLpallDVjL1rKerWMtW8dato61bK3xqmfpVnGJiC7gtcB1g22ZuSUz\nnyju3wo8AJzSnh5KkiRJ46d0AR14CXBPZj482BAR3RHRWdw/ATgZWNqm/kmSJEnjpp3LLF4L3AzM\niYiHI+IPik2XMHR6C8ALgdsiYjHwVeCtmblq4norSZIkTYx2ruJy6Sjtl4/Q9jXga+PdJ0mSJKnd\nyjjFRZIkSdpnGdAlSZKkEjGgS5IkSSViQJckSZJKxIAuSZIklYgBXZIkSSoRA7okSZJUIgZ0SZIk\nqUQM6JIkSVKJGNAlSZKkEjGgS5IkSSViQJckSZJKxIAuSZIklYgBXZIkSSoRA7okSZJUIgZ0SZIk\nqUQM6JIkSVKJGNAlSZKkEjGgS5IkSSViQJckSZJKxIAuSZIklYgBXZIkSSoRA7okSZJUIgZ0SZIk\nqUQM6JIkSVKJGNAlSZKkEjGgS5IkSSXS1oAeEVdHxMqIuKOp7UMR8UhELCpuFzZte19ELImIeyPi\n5e3ptSRJkjR+2j2C/gXg/BHaP5WZc4vb9QARcSpwCXBa8Zx/jojOCeupJEmSNAHaGtAz8yZg1Rh3\nvxj4SmZuycxfA0uAc8atc5IkSVIbtHsEfTRvj4jbiikwBxVtRwIPNe3zcNEmSZIk1UZkZns7EHEc\n8M3MPL14fDjwOJDAR4BZmfn7EfEZ4ObM/Ndiv88D12fm14Yd7wrgCoDu7u6zFixYMFFvpfbWr1/P\n9OnT290NLv/OBgC+cP7+be7J3itLLevAWraW9Wwda9k61rJ1rGVrNddz/vz5t2bm2a04blcrDtJK\nmbli8H5E/AvwzeLhw8DRTbseBfSO8PyrgKsA5syZk/PmzRu3vu5renp6KEU9v/MtgHL0ZS+VppY1\nYC1by3q2jrVsHWvZOtaytcarnqWb4hIRs5oevgYYXOFlIXBJREyJiOOBk4GfT3T/JEmSpPHU1hH0\niLgWmAccGhEPAx8E5kXEXBpTXJYBfwyQmXdGxALgLqAPeFtm9rej35IkSdJ4aWtAz8xLR2j+/C72\n/yjw0fHrkaokM4mIdndDkiSppUo3xUUaqzaf3yxJkjQuDOiqLPO5JEmqIwO6KqvdS4RKkiSNBwO6\nJEmSVCIGdFWW4+eSJKmODOiqLGe4SJKkOjKgq7LSMXRJklRDBnRVliPokiSpjgzokiRJUokY0FVZ\njqBLkqQ6MqCrspyDLkmS6siArspyBF2SJNWRAV2VZT6XJEl1ZEBXZaVD6JIkqYYM6Kos47kkSaoj\nA7oqywF0SZJURwZ0SZIkqUQM6KouR9AlSVINGdBVWa6DLkmS6siArspyDrokSaojA7oqy3wuSZLq\nyICuynIddEmSVEcGdFWW8VySJNWRAV2V5QC6JEmqIwO6KstVXCRJUh0Z0FVd5nNJklRDBnRVlvlc\nkiTVUdsCekRcHRErI+KOpra/jYh7IuK2iPh6RMws2o+LiE0Rsai4fa5d/VZ5OAddkiTVUTtH0L8A\nnD+s7Ubg9Mw8A7gPeF/Ttgcyc25xe+sE9VGSJEmaUG0L6Jl5E7BqWNsNmdlXPPwZcNSEd0yV4Umi\nkiSpjso8B/33gW83PT4+In4VET+KiBe0q1MqD6e4SJKkOop2Xo0xIo4DvpmZpw9rvxI4G3htZmZE\nTAGmZ+YTEXEW8A3gtMxcO8IxrwCuAOju7j5rwYIF4/wu9h3r169n+vTp7e4Gl39nAwB/d940DplW\n5t8xR1eWWtaBtWwt69k61rJ1rGXrWMvWaq7n/Pnzb83Ms1tx3K5WHKSVIuIy4JXAi7P47SEztwBb\nivu3RsQDwCnALcOfn5lXAVcBzJkzJ+fNmzdBPa+/np4eSlHP73wLgOecey5HHbRfmzuzd0pTyxqw\nlq1lPVvHWraOtWwda9la41XPUg0/RsT5wHuAizJzY1N7d0R0FvdPAE4GlranlyoLp7hIkqQ6atsI\nekRcC8wDDo2Ih4EP0li1ZQpwY0QA/KxYseWFwIcjog/oB96amatGPLAkSZJUYW0L6Jl56QjNnx9l\n368BXxvfHkmSJEntV6opLtKecIqLJEmqIwO6JEmSVCIGdFWWFyqSJEl1ZECXJEmSSsSArspyDrok\nSaojA7okSZJUIgZ0VZYD6JIkqY4M6JIkSVKJGNBVWekkdEmSVEMGdEmSJKlEDOiqLMfPJUlSHRnQ\nJUmSpBIxoEuSJEklYkBXZXmOqCRJqiMDuiRJklQiBnRVmEPokiSpfgzokiRJUokY0FVZzkGXJEl1\nZECXJEmSSsSArspyAF2SJNWRAV2SJEkqEQO6Kss56JIkqY4M6JIkSVKJGNBVWeksdEmSVEMGdEmS\nJKlEDOiqLOegS5KkOjKgS5IkSSViQFdlOYIuSZLqqK0BPSKujoiVEXFHU9vBEXFjRNxffD2oaI+I\n+IeIWBIRt0XEme3ruSRJkjQ+2j2C/gXg/GFt7wW+n5knA98vHgNcAJxc3K4APjtBfVRJuYqLJEmq\no7YG9My8CVg1rPli4Jri/jXAq5vav5gNPwNmRsSsiempJEmSNDEi2zyRNyKOA76ZmacXj1dn5sym\n7U9m5kER8U3g45n5k6L9+8B7MvOWYce7gsYIO93d3WctWLBgYt7IPmD9+vVMnz693d3g8u9sAOAv\nnzeVYw/obHNv9k5ZalkH1rK1rGfrWMvWsZatYy1bq7me8+fPvzUzz27FcbtacZAJEiO07fTbRWZe\nBVwFMGfOnJw3b944d2vf0dPTQynq+Z1vAXDWWWdz+pEHtrkze6c0tawBa9la1rN1rGXrWMvWsZat\nNV71bPcc9JGsGJy6UnxdWbQ/DBzdtN9RQO8E902SJEkaV2UM6AuBy4r7lwH/0dT+5mI1l3OBNZm5\nvB0dlCRJksZLW6e4RMS1wDzg0Ih4GPgg8HFgQUT8AfAb4LeL3a8HLgSWABuBt0x4h1UqroMuSZLq\nqK0BPTMvHWXTi0fYN4G3jW+PJEmSpPYq4xQXaUxcB12SJNWRAV2SJEkqEQO6Kss56JIkqY4M6JIk\nSVKJGNAlSZKkEjGgq7Kc4SJJkurIgC5JkiSViAFdlZWeJSpJkmrIgC5JkiSViAFdleX4uSRJqiMD\nuiRJklQiBnRVllPQJUlSHRnQJUmSpBIxoKvCHEKXJEn1Y0CXJEmSSsSArspyDrokSaojA7okSZJU\nIgZ0VZYD6JIkqY4M6JIkSVKJGNBVWc5BlyRJdWRAV6WkqVySJNWcAV2VZViXJEl1ZEBXpZjJJUlS\n3RnQVVlmdUmSVEcGdFWKoVySJNWdAV2V5XQXSZJURwZ0VYonhkqSpLrrancHRhIRc4DrmppOAD4A\nzAT+CHisaP+LzLx+grunkkgnvEiSpBoqZUDPzHuBuQAR0Qk8AnwdeAvwqcz8RBu7pzYykkuSpLqr\nwhSXFwMPZOaD7e6ISsa0LkmSaijKPqc3Iq4GfpmZ/xQRHwIuB9YCtwDvyswnh+1/BXAFQHd391kL\nFiyY2A7X2Pr165k+fXpb+9A3kPzhDRsB+PNnT+XUQzrb2p+9VYZa1oW1bC3r2TrWsnWsZetYy9Zq\nruf8+fNvzcyzW3HcUgf0iJgM9AKnZeaKiDgceJzG2OlHgFmZ+fujPX/OnDl57733Tkxn9wE9PT3M\nmzevrX3Y2jfAKe//NgBf+sPn8PyTDm1rf/ZWGWpZF9aytaxn61jL1rGWrWMtW6u5nhHRsoBe9iku\nF9AYPV8BkJkrMrM/MweAfwHOaWvvNOE8MVSSJNVd2QP6pcC1gw8iYlbTttcAd0x4jyRJkqRxVMpV\nXAAiYj/gpcAfNzX/TUTMpTHFZdmwbdoHNM/IKvHsLEmSpL1W2oCemRuBQ4a1valN3ZEkSZImRNmn\nuEijcj66JEmqIwO6JEmSVCIGdFWKc9AlSVLdGdAlSZKkEjGgq1Ka5507gC5JkurIgK5K6RswlkuS\npHozoKtS+vqbRtCLSei3LFvFus3b2tUlSZKkljKgq1K29Q8Mebxm4zZe/7mb+X+u/VWbeiRJktRa\nBnRVSnNAT2BtMXJ+/4r1beqRJElSaxnQVSnb+ofOQd+0rR+AaZM729EdSZKkljOgq1L6mqe4JGza\n2gjo+xnQJUlSTRjQVSmjjaBP7TKgS5KkejCgq1KGzkHPHQHdEXRJklQTBnRVSt/A0FVctgzOQZ/k\nR1mSJNWDqUaVsrWveR102Lh1MKA7gi5JkurBgK5KGT6CPjjlpavTj7IkSaoHU40qZcgc9GzcADoj\n2tQjSZKk1jKgq1KGr+LSXyT0Dj/JkiSpJow1qpThVxIdGGgE9HAEXZIk1YQBXZXSN3wEvQjoHeZz\nSZJUEwZ0VcrQOejJYF7vcARdkiTVhAFdlZLDHg9sH0E3oEuSpHowoKtacujdwZNEzeeSJKkuDOiq\nlGS0OegmdEmSVA8GdFVK5tD7A54kKkmSasaArkoZPgd9xxQXE7okSaoHA7oqJYcl9IHhiV2SJKni\nutrdgdFExDJgHdAP9GXm2RFxMHAdcBywDHhDZj7Zrj5q4g2dg57bp7jk8OQuSZJUUWUfQZ+fmXMz\n8+zi8XuB72fmycD3i8fahwzP4YNTXMznkiSpLsoe0Ie7GLimuH8N8Oo29kVt0DxS3nySqPlckiTV\nRZR1akBE/Bp4kkb2+p+ZeVVErM7MmU37PJmZBw173hXAFQDd3d1nLViwYCK7XWvr169n+vTpbe3D\n9x7cxr/evRWAt82dwpLV/Xx3WR8vO7aLNz59Slv7tifKUMu6sJatZT1bx1q2jrVsHWvZWs31nD9/\n/q1Nsz6ektLOQQeen5m9EXEYcGNE3DOWJ2XmVcBVAHPmzMl58+aNYxf3LT09PbS7ng/+dBncfScA\np512GhuXrYJlyzjyqKOYN++0tvZtT5ShlnVhLVvLeraOtWwda9k61rK1xquepZ3ikpm9xdeVwNeB\nc4AVETELoPi6sn09VDsM/4vPjpNE29EbSZKk1itlQI+I/SNixuB94GXAHcBC4LJit8uA/2hPD9Uu\nQ9ZwyR0niUqSJNVFWae4HA58vbj4TBfw5cz8TkT8AlgQEX8A/Ab47Tb2UW2w0youA4PtBnVJklQP\npQzombkUeOYI7U8AL574Hqkshq6Cnq7iIkmSaqeUU1yk0QwfKXcddEmSVDcGdFXW0HXQTeiSJKke\nDOiqlOEj5QOOoEuSpJoxoKtSmkfKE+gvHg4Y0CVJUk0Y0FUpO42gb0/mJnRJklQPBnRVytB10JN+\nL1QkSZJqxoCuStlpHXTnoEuSpJoxoKtShq/W4ioukiSpbgzoqhRH0CVJUt0Z0FVZmbBl20Djfpv7\nIkmS1CoGdFXK8CuJbunrL9rb0RtJkqTWM6CrUpqDeJJs3j6CbkKXJEn1YEBXpQyP4YMj6OZzSZJU\nFwZ0VcqQEfSELX3OQZckSfViQFelDJ/Ksn2Ki5PQJUlSTRjQVSk7j6AXJ4m2qT+SJEmtZkBXpey8\nisvgCHo7eiNJktR6BnRVSnMOH8hkaxHQB0zokiSpJgzoqpTmHD44eg5OcZEkSfVhQFelNJ8k2j8w\nZFF0SZKkWjCgq1KaR9Cbp7V4oSJJklQXBnRVytA56E3t5nNJklQTBnRVytBlFnPEdkmSpCozoKtS\nmqeyOMVFkiTVkQFd1dKUw/sHmprN55IkqSYM6KqU4eugj9QuSZJUZQZ0VcrQeefOQZckSfVTuoAe\nEUdHxA8j4u6IuDMi/qRo/1BEPBIRi4rbhe3uqybe0GUWh2yZ6K5IkiSNi652d2AEfcC7MvOXETED\nuDUibiy2fSozP9HGvqnNmmN484WKHEGXJEl1UbqAnpnLgeXF/XURcTdwZHt7pbIYdZnFNvRFkiRp\nPESWeOgxIo4DbgJOB/4UuBxYC9xCY5T9yRGecwVwBUB3d/dZCxYsmKDe1t/69euZPn16W/vwpbu3\ncOODfQC88oRJfHPpNgDOOLSTPz17aju7tkfKUMu6sJatZT1bx1q2jrVsHWvZWs31nD9//q2ZeXYr\njlu6EfRBETEd+BrwjsxcGxGfBT5CY7D0I8DfAb8//HmZeRVwFcCcOXNy3rx5E9bnuuvp6aHd9exZ\neyc8uAyAo485BpY+AMBBBx/MvHnntLFne6YMtawLa9la1rN1rGXrWMvWsZatNV71LN1JogARMYlG\nOP9SZv47QGauyMz+zBwA/gWoThrTuHCZRUmSVEelC+gREcDngbsz85NN7bOadnsNcMdE903tN9rS\nimWeqiVJkrQnyjjF5fnAm4DbI2JR0fYXwKURMZfGYOky4I/b0z2102iruEiSJNVF6QJ6Zv4EiBE2\nXT/RfVH5DF0H3WUWJUlS/ZRuiou0K8koU1ychS5JkmrCgK5KcQRdkiTVnQFdldI87dwriUqSpDoy\noKtikijOUBhwioskSaohA7oqJRM6ioQ+2pKLkiRJVWZAV6Vk7ljixwsVSZKkOjKgq1KS3D6C3j8w\nZIMkSVItGNBVKZlsH0JvnuIy4BwXSZJUEwZ0VUoCHUVA73eKiyRJqiEDuiqlMQe9kdCHrOLiCLok\nSaoJA7oqpTEHvXF/cFpLhCPokiSpPgzoqpaEGLbMYmeEyyxKkqTaMKCrUprOEd1+JdGOjnAEXZIk\n1YYBXZWSufOVRDsjvFKRJEmqDQO6KiVpnuLSaOvsiCEnjEqSJFWZAV2VkslOJ4lO7uqgz4QuSZJq\nwoCuSmkeQd8e0Ds72DbksqKSJEnVZUBXpWTuWGZx8CTRyV0d9BnQJUlSTRjQVSnFyueN+8Wslsld\nHWzrd4qLJEmqBwO6KqOvfwBGmoPuFBdJklQjXe3ugDQW2/oHOPnKbwNw+AFTAE8SlSRJ9eQIuiph\nw5a+7fc7Bk8SLQbNp3Q5gi5JkurDgK5K2LStf/v9jVsb95tH0A3okiSpLgzoqoTN23YE8DWbtgE7\nAvqUrg76PElUkiTVhAFdlbBpa/9ObQNNq7j0DSSZhnRJklR9BnRVwua+kQL6jlVcAJdalCRJtWBA\nVyVsHnEEfcccdIC+AeehS5Kk6qtcQI+I8yPi3ohYEhHvbXd/NDGaTxIddMcja4EdAd0RdEmSVAeV\nCugR0Ql8BrgAOBW4NCJObW+vNBGaTxIdbnJnJ4AruUiSpFqo2oWKzgGWZOZSgIj4CnAxcNdIO6/a\nnHz0W3ex/5Qupk/pIiIIIILia2y/z5BtjfaO4j7N+w/u07Qfw7cRjec2Djzk9QYN3ovt+xWvVWwM\nGpeyT7LpOTueP+QgIz8c8nojbYcd00Qmde74XW34uZaDfbjj8X46739shO1Dbdraz5a+fg7ab/KQ\nOsVInRhh0Ls/k//141/zo/se47XPOpJnHXsQ9z26boTeNwyOoH/79uVMm9zF1r4BZkzt2qmuOx4z\n5PFgS1dH87/n0H+Poc+L7Y9He421m7Zx34r1POOoA7av297stsf64N6Vo76n0V6nrJq/FybaXU/0\nM2nJ49sfb+sfYMOWfg7ab1J7OjSSkv/7Nbv7iX6mPPDEqNtzpG/anXcakz35m9fwn7lVcM+qfqYu\nbdTSc9j3XPO/dXMttbM9+Xzt7ntce+aJTeMzOBhVWvkiIl4PnJ+Zf1g8fhPwnMx8+0j77z/75Ox+\n86cnsosaR087Ygaf+O1nsmlbP9OndPHJG+9jycr1vOMlJ/MnX1nU7u5JkqR9zOtPmcQnfv9lAETE\nrZl5diuOW7UR9JHGTob8hhERVwBXAHR3d3P1y/ejbwAGZ0gM/j6Sg0/Mwfu5/f727bnja/MLZY5y\nf8jxIDOH7DdSx7P5wbC2ISOnuxm13unwu9l/UBTbBobtMHyUKoCNmzax37Rpox5nUGdH4/HWkWqe\nIx97uMP3C6Z2Bau3JJ0BG7YlB0/r5/H7fwXABuB3jwGOgXzyPv7mhdPoK16vI6B/yD/KsH+jJoO/\noDbXIHf1vGHtQ/YZdvBJHTDa79WbNm5i2n4j13K012m3ZPRvwHb+nr9p0yamDftcbhto1L8MdlWa\nkb4f2m3jxp3rOdxY+tzqt9Xuz9neGOmzqb1jLXcY7WfxWFnL1to/N9HT09Py41YtoD8MHN30+Cig\nt3mHzLwKuApgzpw5+aL58yeudzXX09PDvHnz2t2NWrCWrWMtW8t6to61bB1r2TrWsrXGq54lGWMa\ns18AJ0fE8RExGbgEWNjmPkmSJEktU6kR9Mzsi4i3A98FOoGrM/PONndLkiRJaplKBXSAzLweuL7d\n/ZAkSZLGQ9WmuEiSJEm1ZkCXJEmSSsSALkmSJJWIAV2SJEkqEQO6JEmSVCIGdEmSJKlEDOiSJElS\niRjQJUmSpBKJzGx3H8ZNRKwD7m13P2rkUODxdneiJqxl61jL1rKerWMtW8dato61bK3meh6bmd2t\nOGjlriS6h+7NzLPb3Ym6iIhbrGdrWMvWsZatZT1bx1q2jrVsHWvZWuNVT6e4SJIkSSViQJckSZJK\npO4B/ap2d6BmrGfrWMvWsZatZT1bx1q2jrVsHWvZWuNSz1qfJCpJkiRVTd1H0CVJkqRKMaBLkiRJ\nJVLbgB4R50fEvRGxJCLe2+7+lF1EHB0RP4yIuyPizoj4k6L9QxHxSEQsKm4XNj3nfUV9742Il7ev\n9+UTEcsi4vaiZrcUbQdHxI0RcX/x9aCiPSLiH4pa3hYRZ7a39+USEXOaPn+LImJtRLzDz+bYRMTV\nEbEyIu5oatvjz2JEXFbsf39EXNaO99Juo9TybyPinqJeX4+ImUX7cRGxqenz+bmm55xV/HxYUtQ7\n2vF+2m2Ueu7x97X/349ay+ua6rgsIhYV7X42d2EXeWhif25mZu1uQCfwAHACMBlYDJza7n6V+QbM\nAs4s7s8A7gNOBT4E/NkI+59a1HUKcHxR7852v4+y3IBlwKHD2v4GeG9x/73AXxf3LwS+DQRwLvBf\n7e5/WW/F9/ajwLF+NsdcsxcCZwJ3NLXt0WcROBhYWnw9qLh/ULvfW0lq+TKgq7j/1021PK55v2HH\n+Tnw3KLO3wYuaPd7K1E99+j72v/vR6/lsO1/B3yguO9nc9e1HC0PTejPzbqOoJ8DLMnMpZm5FfgK\ncHGb+1Rqmbk8M39Z3F8H3A0cuYunXAx8JTO3ZOavgSU06q7RXQxcU9y/Bnh1U/sXs+FnwMyImNWO\nDlbAi4EHMvPBXezjZ7NJZt4ErBrWvKefxZcDN2bmqsx8ErgROH/8e18uI9UyM2/IzL7i4c+Ao3Z1\njKKeB2Tmzdn4X/yL7Kj/PmWUz+ZoRvu+9v97dl3LYhT8DcC1uzqGn82GXeShCf25WdeAfiTwUNPj\nh9l12FSTiDgOeBbwX0XT24s/21w9+CcdrPHuJHBDRNwaEVcUbYdn5nJo/AAADivareXYXcLQ/2T8\nbO6dPf0sWtOx+X0aI2mDjo/yM1LxAAAgAElEQVSIX0XEjyLiBUXbkTTqN8ha7mxPvq/9bO7eC4AV\nmXl/U5ufzTEYlocm9OdmXQP6SHOmXE9yDCJiOvA14B2ZuRb4LHAiMBdYTuPPZGCNd+f5mXkmcAHw\ntoh44S72tZZjEBGTgYuAfyua/Gy23mi1s6a7ERFXAn3Al4qm5cAxmfks4E+BL0fEAVjL3dnT72vr\nuXuXMnRgw8/mGIyQh0bddYS2p/zZrGtAfxg4uunxUUBvm/pSGRExicaH8UuZ+e8AmbkiM/szcwD4\nF3ZMFbDGu5CZvcXXlcDXadRtxeDUleLrymJ3azk2FwC/zMwV4GfzKdrTz6I13YXi5K9XAr9bTA2g\nmIrxRHH/VhrzpE+hUcvmaTDWsslefF/72dyFiOgCXgtcN9jmZ3P3RspDTPDPzboG9F8AJ0fE8cWo\n2yXAwjb3qdSKOWqfB+7OzE82tTfPhX4NMHiG+ELgkoiYEhHHAyfTOLlknxcR+0fEjMH7NE4iu4NG\nzQbP4r4M+I/i/kLgzcWZ4OcCawb/jKYhhowC+dl8Svb0s/hd4GURcVAx5eBlRds+LyLOB94DXJSZ\nG5vauyOis7h/Ao3P4dKinusi4tzi5+6b2VH/fd5efF/7//2uvQS4JzO3T13xs7lro+UhJvrn5nie\nCdvOG42zau+j8Zvhle3uT9lvwG/R+NPLbcCi4nYh8H+A24v2hcCspudcWdT3XvbBM713UcsTaKwk\nsBi4c/DzBxwCfB+4v/h6cNEewGeKWt4OnN3u91C2G7Af8ARwYFObn82x1e5aGn/S3kZjROcP9uaz\nSGN+9ZLi9pZ2v68S1XIJjXmmgz83P1fs+7ri+38x8EvgVU3HOZtG8HwA+CeKq3rva7dR6rnH39f+\nfz9yLYv2LwBvHbavn81d13K0PDShPzejOIAkSZKkEqjrFBdJkiSpkgzokiRJUokY0CVJkqQSMaBL\nkiRJJWJAlyRJkkrEgC5JkiSViAFdkiRJKhEDuiRJklQiBnRJkiSpRAzokiRJUokY0CVJkqQSMaBL\nkiRJJWJAlyRJkkrEgC5JkiSViAFdkiRJKhEDuiRJklQiBnRJkiSpRAzokiRJUokY0CVJkqQSMaBL\nkiRJJWJAlyRJkkrEgC5JkiSViAFdkiRJKhEDuiRJklQiBnRJkiSpRAzokiRJUokY0CVJkqQSMaBL\nkiRJJWJAl6QJFBHLIuIlLTrW5Ij4anHMjIh5rTiuJKm9DOiSVG0/AX4PeLTdHdmdiOhsdx8kqQoM\n6JI0QSLi/wDHAP8ZEesj4s+L9osi4s6IWB0RPRHx9KbnLIuI90XEXRHxZET874iYCpCZWzPz05n5\nE6B/DK//loi4OyLWRcTSiPjjYdsvjohFEbE2Ih6IiPOL9oOL1+0t+vCNov3yiPjJsGNkRJxU3P9C\nRHw2Iq6PiA3A/Ih4RUT8qniNhyLiQ8Oe/1sR8dOiFg8Vr/HsiFgREV1N+70uIhbtQfklqTIM6JI0\nQTLzTcBvgFdl5vTM/JuIOAW4FngH0A1cTyPAT2566u8CLwdOBE4B3r+XXVgJvBI4AHgL8KmIOBMg\nIs4Bvgi8G5gJvBBYVjzv/wD7AacBhwGf2oPXfCPwUWAGjdH+DcCbi9d4BfDfIuLVRR+OAb4N/CON\nWswFFmXmL4AngJc2Hff3in5JUu0Y0CWpvX4H+FZm3piZ24BPANOA5zXt80+Z+VBmrqIRdi/dmxfK\nzG9l5gPZ8CPgBuAFxeY/AK4u+jGQmY9k5j0RMQu4AHhrZj6ZmduK547Vf2Tm/y2OuTkzezLz9uLx\nbTR+OTmv2Pd3ge9l5rXF6zyRmYOj5NfQCOVExME0fmH58t7UQZLKzoAuSe01G3hw8EFmDgAPAUc2\n7fNQ0/0Hi+fssYi4ICJ+FhGrImI1cCFwaLH5aOCBEZ52NLAqM5/cm9dkaN+JiOdExA8j4rGIWAO8\ndQx9APhX4FURMR14A/DjzFy+l32SpFIzoEvSxMphj3uBYwcfRETQCKqPNO1zdNP9Y4rn7JGImAJ8\njcYI/eGZOZPGdJoodnmIxhSa4R4CDo6ImSNs20Bj6svgaxwxwj7D3++XgYXA0Zl5IPC5MfSBzHwE\nuBl4DfAmnN4iqcYM6JI0sVYAJzQ9XgC8IiJeHBGTgHcBW4CfNu3ztog4qpja8RfAdYMbImLK4Emj\nwOSImFqE/OEmA1OAx4C+iLgAeFnT9s8Dbyn60RERR0bE04pR6m8D/xwRB0XEpIh4YfGcxcBpETG3\n6MOHxvD+Z9AYkd9czHt/Y9O2LwEviYg3RERXRBwSEXObtn8R+HPgGcDXx/BaklRJBnRJmlgfA95f\nrFLyZ5l5L4251f8IPA68isZJpFubnvNlGvPFlxa3v2radi+wicaUmO8W949lmMxcB/y/NH4heJJG\nMF7YtP3nFCeOAmuAHzUd503ANuAeGieavqN4zn3Ah4HvAffTOAl0d/478OGIWAd8oOjPYB9+Q2Pa\nzbuAVcAi4JlNz/160aevZ+aGMbyWJFVSZA7/66MkqSwiYhnwh5n5vXb3pQwi4gHgj62HpDpzBF2S\nVAkR8Toac9p/0O6+SNJ46tr9LpIktVdE9ACnAm8qVrqRpNpyioskSZJUIk5xkSRJkkqk1lNcZs6c\nmSeddFK7u1EbGzZsYP/99293N2rDeraW9Wwda9la1rO1rGfrWMvWuvXWWx/PzO5WHKvWAf3www/n\nlltuaXc3aqOnp4d58+a1uxu1YT1by3q2jrVsLevZWtazdaxla0XEg7vfa2yc4iJJkiSViAFdkiRJ\nKhEDuiRJklQiBnRJkiSpRAzokiRJUokY0CVJkqQSMaBLkiRJJWJAlyRJkkrEgC5JkiSViAFdkiRJ\nKhEDuiRJklQiBvSa2NY/wCv/8cfceNeKdndFkiRJT4EBvSYeXbOZOx5Zy8+WPtHurkiSJOkpMKDX\nxCOrNwHQW3yVJElSNRnQa2L5miKgr9nc5p5IkiTpqTCg10Tv6s3FV0fQJUmSqsyAXhODU1weW7eF\nLX39be6NJEmS9pYBvSaWN42cr1izpY09kSRJ0lNhQK+J3tWbmTGlC9gxmi5JkqTqMaDXRO/qTTzr\n2IO235ckSVI1jSmgR8T5EXFvRCyJiPeOsP3YiPh+RNwWET0RcVTTtr+OiDuK2+80tb8oIn5ZtF8T\nEV1Fe0TEPxSvdVtEnNn0nP6IWFTcFj61t14fazdvY92WPs46phHQB1d0kSRJUvXsNqBHRCfwGeAC\n4FTg0og4ddhunwC+mJlnAB8GPlY89xXAmcBc4DnAuyPigIjoAK4BLsnM04EHgcuKY10AnFzcrgA+\n2/Q6mzJzbnG7aG/ecB0tL1ZwOaF7fw7ZfzKPrHapRUmSpKoaywj6OcCSzFyamVuBrwAXD9vnVOD7\nxf0fNm0/FfhRZvZl5gZgMXA+cAiwJTPvK/a7EXhdcf9iGmE/M/NnwMyImLUX722fMTilZfbMacya\nOdUpLpIkSRXWNYZ9jgQeanr8MI3R8GaLaQTsvwdeA8yIiEOK9g9GxCeB/YD5wF3A48CkiDg7M28B\nXg8cvYvXOxJYDkyNiFuAPuDjmfmN4Z2NiCtojLzT3d1NT0/PGN5itfX8ZhsAD979KyZv28qS3nXj\n8r7Xr1+/T9RzoljP1rKerWMtW8t6tpb1bB1rWV5jCegxQlsOe/xnwD9FxOXATcAjQF9m3hARzwZ+\nCjwG3Fy0Z0RcAnwqIqYAN9AI3bt7vWMyszciTgB+EBG3Z+YDQ3bMvAq4CmDOnDk5b968MbzFavvF\nd++h856lXPyy+dy29S7+7ZaHOO+884gYqZR7r6enh32hnhPFeraW9Wwda9la1rO1rGfrWMvyGssU\nl4fZMboNcBTQ27xDZvZm5msz81nAlUXbmuLrR4s54y+lEb7vL9pvzswXZOY5NEL9/bt7vcwc/LoU\n6AGeNfa3Wl+9qzdzxAFT6ewIZs+cyoat/azd3Lf7J0qSJKl0xhLQfwGcHBHHR8Rk4BJgyAoqEXFo\nceInwPuAq4v2zmKqCxFxBnAGjdFyIuKw4usU4D3A54rnLwTeXKzmci6wJjOXR8RBxb5ExKHA82lM\nl9nn9a7exOyZU4HGPHRwJRdJkqSq2u0Ul8zsi4i3A98FOoGrM/POiPgwcEtmLgTmAR+LiKQxGv62\n4umTgB8XUy3WAr+XmYNDu++OiFfS+CXhs5n5g6L9euBCYAmwEXhL0f504H9GxEDxnI9npgEd6F2z\niTOLJRYHA3rv6k087YgD2tktSZIk7YWxzEEnM6+nEZyb2z7QdP+rwFdHeN5mGiu5jHTMdwPvHqE9\n2RHwm9t/CjxjLP3dl/QPJI+u2bw9mM8+sPHVpRYlSZKqySuJVtzj67ewrT+ZfWBjikv3jCl0dQTL\nXWpRkiSpkgzoFde8BjpAZ0dwxIGuhS5JklRVBvSK6y2msgwGdGhMc+l1ioskSVIlGdArbvsI+oFN\nAX3mVHpdxUWSJKmSDOgV17tmE/tP7uSAaTvO9509cxqPrtlM/8Dw60lJkiSp7AzoFddYA33akKuG\nzpo5jb6B5LF1W9rYM0mSJO0NA3rF9a7ezKym+ecARxYXLXKaiyRJUvUY0Ctu+ZpN2wP5oOaLFUmS\nJKlaDOgVtnlbP4+v3zrkBFGAWQca0CVJkqrKgF5hy9fsvMQiwAFTu5g+pculFiVJkirIgF5hg1cL\nnTVsiktENJZaHMcR9M3b+sft2JIkSfsyA3qFPVIE8COHjaBDY5rLeJ0kesOdj/KMD32XRQ+tHpfj\nS5Ik7csM6BU2OIXliAOn7rRt9sxpLB+HKS4r123mvf9+O9v6k6/e+lDLjy9JkrSvM6BX2PI1mzh0\n+hSmdHXutO3ImVN5YsPWlk5FyUze89Xb2LCljzOPmcm3blvOtv6Blh1fkiRJBvRKe2T1zkssDhqP\nlVx6Hurjh/c+xvsueBpvPe9Enty4jZ8sebxlx5ckSZIBvdIGryI6ksH2wZVenqqlj63n2nu38oKT\nD+XNzz2O8+Z0c8DULv5zUW9Lji9JkqQGA3pFZSbL12zePlI+3OCJo4+0YAS9r3+Ady5YzKQO+NvX\nP5OOjmBKVycXnD6L7975qCu6SJIktZABvaLWbNrGxq39zB5lisvhB04BWjPF5TM/fIDFD63mslOn\nDDkh9aK5s9mwtZ8f3LPyKb+GJEmSGgzoFbWrJRYBpnR10j1jylNeyWXRQ6v5hx/cz6vnzuacWV1D\ntp17wiF0z5jCQqe5SJIktYwBvaIGg/esUQI6NOahP5W10Ddu7eOd1y3i8BlT+MuLT99pe2dH8Ipn\nzOIH965k7eZte/06kiRJ2qFr97uojAaD92hTXABmHziVe1es2+Vx7l6+lhvvWjHitsUPrebXj2/g\ny3/0HA6cNmnEfS6aO5sv/HQZ373jUX777KPH2HtJkqT6+M4dj7b0eAb0inpk9SYmd3Zw6P5TRt1n\n9sxp9Nz7GJlJRIy4z5Vfv51f/mbkK4JGwJ+8+GSed+Kho77Gs46eydEHT2Ph4l4DuiRJ2uc8umYz\n7/nabS09pgG9onpXb+aIA6fS0TFy8IZGQN+0rZ/VG7dx0P6Td9r+0KqN/PI3q3n3y+fw1vNOHPEY\nnbs4PkBEcNEzZ/O5Hy3l8fVbOHT66L8wSJIk1cnAQPLury5ma19rL9zoHPSKWr560y6nt0BjiguM\nvtTif97WOLnzomfOprMjRryNxUXPPJL+geT625fvwTuQJEmqti/evIwf3/8473/l01t6XAN6Re3q\nIkWDdnexooWLennWMTM5+uD9nlJf5hwxgzmHz3A1F0mStM9YsnIdH/v2PbzoaYfxxnOOaemxDegV\n1Nc/wKNrNzN7lIsUDRoM6COthX7/inXc8+g6Lnrm7Jb06aK5s7nlwSd5+MmNLTmeJElSWW3tG+Ad\n1y1i/yldfPx1zxj1XL+9ZUCvoJXrtjCQ7HYE/ZD9JzO5s2PEgL5wcS8dAa84Y1ZL+vSqMxpB/z8X\nO81FkiTV2z98/37ueGQt/+M1z+CwGbuecrw3DOgVNBi4dzcHvaMjmDVzKr3DprhkJgsX9/LcEw9p\n2YfqmEP2Y+7RM1m42GkukiSpvm59cBX/3LOE3z7rKM4//YhxeQ0DegXt7iqizWYfOG2nEfTbHl7D\ng09sbNn0lkEXPXM2dy9fy5KVu157XZIkqYrWb+njndct5siDpvHBi04bt9cxoFfQ4Emfu7qK6KBZ\nM6eyfFhAX7i4l0mdwfmntWZ6y6BXnjGLjsCTRSVJUi391Tfv4qEnN/LJN8xl+pTxW63cddBL6Af3\nrODqnyzjHy59FgePsH557+pNHDC1a0wfjCNnTuPRtZvp6x+gq7OD/oHkm7f1ct4ph3HgfiNfHXRv\nHXbAVJ574iF8fdEjvO1FJzGlq7Olx5ckSbt2410r+B/X3z2mdbk3b97M1J/9YAJ6VR+PrN7Ef5t3\nIs8+7uBxfR0Desk8umYz77xuMWs2beP937idz7zxzJ3ODB7LEouDZs+cxkDCinVbOHLmNH7+61Ws\nWLuFK1/R2uktg6544YlcdvXP+eQN9/G+C1u7JqgkSRrd8jWbeNeCRXTPmMK5Jxyy2/0fffRRjjhi\n9/tph1kHTuX/ffHJ4/46YwroEXE+8PdAJ/C/MvPjw7YfC1wNdAOrgN/LzIeLbX8NvKLY9SOZeV3R\n/iLgE8Bk4FbgDzKzLxpp9O+BC4GNwOWZ+cviOZcB7y+O9VeZec1eveuSykz+/Gu3saWvn999zjF8\n6b9+wzcWPcJrnnXUkP16V28ec0CfVVysaPnqTRw5cxoLF/cybVInL3n6YS3vP8B5p3Tzxuccw1U/\nXsqLnnYYzxnDDwhJkvTUDAwkf/Zvi+kbSD5/2bM57tD9d/ucnp4nmTfvmRPQO+2p3c5Bj4hO4DPA\nBcCpwKURceqw3T4BfDEzzwA+DHyseO4rgDOBucBzgHdHxAER0QFcA1ySmacDDwKXFce6ADi5uF0B\nfLY41sHAB4vjnAN8MCIO2sv3XUr/+rMHuem+x7jywqfz4YtP5+xjD+ID37hzpyuB9q7Z/VVEBw2e\nSPrI6k1s7Rvg23cs56WnHs5+k8fvjydXXvh0jjl4P/50wWLWbd42bq8jSZIarrl5Gf93yRO8/xWn\njimcq9zGcpLoOcCSzFyamVuBrwAXD9vnVOD7xf0fNm0/FfhRZvZl5gZgMXA+cAiwJTPvK/a7EXhd\ncf9iGmE/M/NnwMyImAW8HLgxM1dl5pPFc87fw/dbWg88tp6PXn83Lzylm98791g6O4JPvmEuA5n8\n2YLFDAwkABu39rF647axj6Bvv1jRZn6y5DFWb9zW8tVbhtt/SheffMNclq/ZxIf/865xfS1JkvZ1\nS1au4+PFFS0vPefodndHLTCWYdQjgYeaHj9MYxS72WIaAfvvgdcAMyLikKL9gxHxSWA/YD5wF/A4\nMCkizs7MW4DXA4OfqJFe78hdtA8REVfQGHmnu7ubnp6eMbzF9uobSD76X5vpZIDXzF7Pj370o+3b\n3nBKJ//7jie48ovf4+XHTaJ3feOkj9W9y+jpeXhMx9+vC265awk3LU72nwQ8ehc9K+/e436uX79+\nj+r5iuMn8W+3PswRA49x1uGe7jDcntZTu2Y9W8datpb1bC3rOVTfQPJXP9vMpBjgolnrhmSI3bGW\n5TWW1DTStUtz2OM/A/4pIi4HbgIeAfoy84aIeDbwU+Ax4OaiPSPiEuBTETEFuAHo283rjaUfZOZV\nwFUAc+bMyXnz5u363ZXAp793H79ecz+feeOZO13Z87xMHvriLXzt/sd5ywXn0rFmM/zk57zkuWdy\nzvFjO4P4mEU3sXnyZG57dDWvmns0L3nRGXvVz56eHvakns/7rQF+/dn/y5fu28ybLjx3XK60VWV7\nWk/tmvVsHWvZWtaztaznUJ+84V6WrV3C537vTM4/fc+WT7aW5TWWKS4Ps2N0G+AoYMhC15nZm5mv\nzcxnAVcWbWuKrx/NzLmZ+VIaIfv+ov3mzHxBZp5DI9Tfv5vX220/qmjxQ6v5xx8s4dVzZ+8UzgEi\ngo+99gxmTOnindct4sFVG4HdX0W02ZEzp/HTB55gw9b+cZ/e0mxyVwefesNc1m/p431fu53MnX6f\nkiRJe+mXv3mSz/Q8wOvOPGqPw7nKbSwB/RfAyRFxfERMBi4BFjbvEBGHFid+AryPxoouRERnMdWF\niDgDOIPGaDkRcVjxdQrwHuBzxfMXAm+OhnOBNZm5HPgu8LKIOKg4OfRlRVtlbdrazzuvW8RhM6bw\nlxefPup+3TOm8LHXPoM7e9fy99+7jwg4/ICxB/RZM6eSCYfNmDLhq6qcfPgM3nv+0/j+PSv5yi8e\n2v0TJEnSbm3c2sefXreIIw6YygcvGr52h6put1NciqUP304jDHcCV2fmnRHxYeCWzFwIzAM+FhFJ\nYzT8bcXTJwE/LtbxXktj+cXBqSzvjohX0vgl4bOZObhS/vU0llhcQmOZxbcU/VgVER+h8QsDwIcz\nc9Xev/X2+/i372bp4xv40h8+hwOn7fqiQS877QjecPZRLLjlYY44YCqTOsd+EdjBE0pfccYsOjtG\nmik0vi5/3nF8/54VfOSbd/G8Ew/h2EM8u1ySpN1Zs3EbSx9fP+K2L//Xb3hw1Uau/aNzOWBqay88\nqPYb05l7mXk9jeDc3PaBpvtfBb46wvM201jJZaRjvht49wjtyY6AP3zb1RSj81W3ZOV6rrn5QS5/\n3nE8/6RDx/ScD7zqNG5e+sT2pRPH6oRDpwPw6rk7nVM7ITo6gr99/TN5+adv4p3XLWLBHz+Xrj34\nBUOSpH3Nqg1bOf/TN7Fy3ZZR9/mjFxw/pgsSqXpcWqNNFi56hI6A/z7/xDE/Z/qULv79vz1/j+dy\nv+zUw7nxnS/k5MNn7Gk3W2b2zGl85OLTecd1i/ifNy3lbfNPaltfJEkqs8zkff9+G6s3buPvL5nL\nASP8lX3apE7OGefLzat9DOhtkJksXNzLc088ZI9XNumeMWWPX6+jI9oazgddPHc237t7BZ+68T5e\neHI3zzjqwHZ3SZKk0vnqrQ/z3TtX8L4LnsbFbfrrt9rLeQZtcNvDa1j2xMYJXVGlDCKCv3r16Rw6\nfQrvuO5XbN7W3+4uSZJUKg+t2shf/uddnHP8wfzhC05od3fUJgb0Nli4uJdJncH5p+17SyLN3G8y\nf/vbZ/DAYxv4+LfvaXd3JEkqjf6B5F0LFgPwyTc8sy0LO6gcDOgTrH8g+eZtvZx3ymEcuN++edb1\nC07u5vLnHccXfrqMm+57rN3dkSSpFP7lx0v5+bJV/OVFp3HUQfu1uztqIwP6BPv5r1exYu0WLp67\nb01vGe69FzyNkw6bzru/upjVG7e2uzuSJLXVnb1r+Lsb7uXCZxzBa8903vm+zoA+wRYu7mW/yZ28\n5OmHt7srbTV1Uief/p25PLF+K1d+4w6vMipJ2mdt3ta4cOFB+03mo69+BsX1Y7QPM6BPoK19A3z7\njuW89NTDmTa5s93dabvTjzyQd770FL5123L+Y1Fvu7sjSVJbfOK793LfivX8zevP4KD9J7e7OyoB\nl1mcQD++/zH+//buPDyu+srz//uoSi553yQ27wRjMMEYbCCEkIjQCUsWGkgHmKydEKanQz/dJPD7\nhcnODCFJEzKZSTrd9C90N/mFAIGQdroJkBAEYTOWjQUGY7wgGVvGthYvkqylVGf+qCtTFmWrJN3S\nreXzep56XLpbnTrPler41Pd+756uvrKbveVI/ut7j+ePr+7i6/++jnPeMZOjpwxv2kkREZGwtXb0\n8KPHNtKbTOX9tXr7U/x6zXY+fc48ahcdlffXk+KgAn0MrWhoZur4Ss5bWBN1KAUjHqvgGx9ezKU/\neZrntrRqvlcREYnc3Su3ctezTRw9Zfj3HhmJ8xZWc9PFJ4/Ja0lxUIE+Rg709vP7V3Zy6dLjGBfX\nyKJMMyelv87rGYNOhYiIyJEM3EzwrPkzuO+vzok6HClTqhTHyB/W76Srt5+PaHjL2yTi6fH4Pbpx\nkYiIROzVN/ezcVcHHynz2dYkWirQx8iKhmaOnpLg7AUzow6l4FRVpk9DddBFRCRqKxqaiVUYl7zz\nmKhDkTKmAn0M7O3q44kNu/nwkuN0V7AsBjro3eqgi4hIhNyd3zY0854Tqpk5aWzGn4tkowJ9DDz8\n8g56+1OaveUwKmNGhamDLiIi0VqzdQ/b2g/o81oipwJ9DKxoaGbezAksmT016lAKkpmRiMfUQRcR\nkUj9tqGZRLyCD55S3jcTlOipQM+zXfu7eXZzKx897TjdGewIqior1EEXEZHIJPtT/MeLO3j/SUcx\nuaoy6nCkzKlAz7P/fHEHKUdflw1BHXQREYnSc1vaaOno0ee1FAQV6Hn2u5fe5KRjJrPw6MlRh1LQ\nEuqgi4hIhFY0bGdSIs75J+lunhI9Feh59uqb+1g+f3rUYRS8KnXQRUQkIj3Jfn637k0+uPhoqipj\nUYcjogI9n/Z09bKvO8n8mROjDqXgqYMuIiJReWLDbvZ3J3VzIikYKtDzqLG1C4B5KtCHpA66iIhE\nZUVDM9MnVPKeE6qjDkUEUIGeV02tnQDMmzkh4kgKnzroIiIShc6eJH9Yv5NLTj2WypjKIikMOhPz\nqCnooM+doQJ9KOlZXFSgi4jI2PrD+p109+lmglJYVKDnUWNrJ8dMqdIFJzlId9A1xEVERMbWbxua\nOXZqFWfOnxF1KCIHqUDPo62tXRrekqOqeIweddBFRGQM7enq5YnXdvPhJcdSUaGbCUrhUIGeR42t\nXZrBJUfqoIuIyFh7eN2b9PU7Hz1tVtShiBxCBXqedPQkaenoYa466Dmp0hh0EREZYysamllQPZF3\nzpoSdSgih1CBnidbgwtE1UHPjTroIiIylvYe6OO5La18eMmxmGl4ixQWFeh5oikWh6cqHqOv3+lP\nedShiIhIGViztZ2Uw216fA0AACAASURBVDnHz4w6FJG3UYGeJ01twRSLKtBzkqhMn4rqoouIyFio\nb2wjVmEsnTst6lBE3kYFep40tXYyc+I4plRVRh1KUaiKBwW6xqGLiMgYqG9s55TjpjBhXDzqUETe\nJqcC3cwuMrMNZrbJzL6SZf08M3vMzF40szozm52x7ntmti54XJmx/AIzW2Nma83sKTM7IYdj9Qfb\nrzWzFaN76/nV2NKl7vkwJIK54rvVQRcRkTzrTaZo2LaHZfOmRx2KSFZDFuhmFgN+AlwMLAauNrPF\ngza7DbjL3ZcANwO3Bvt+CDgDWAqcDdxoZgOXSv8U+IS7LwXuBr52pGMFDrj70uDx0WG/2zG0tU1T\nLA5HVaU66CIiMjZebt5Ld19KNyeSgpVLB/0sYJO7b3H3XuAe4NJB2ywGHgueP56xfjHwhLsn3b0T\naAAuCtY5MFCsTwWahzhW0eju66d57wFdIDoMibg66CIiMjZWN7UDsFwddClQuQy8mgW8kfHzNtLd\n8EwNwBXAj4DLgMlmNjNY/k0zux2YAJwPvBLscw3wkJkdAPYB7zrSsdy9Fagys3ogCXzX3X8zOFgz\nuxa4FqCmpoa6uroc3mK4mjtSuEPXrq3U1TUPvUOR6OjoyFs+X9uVBOCZlat4c2osL69RaPKZz3Kk\nfIZHuQyX8hmuMPL5uxe6qRlvvLLmuYNFSTnSuVm4cinQs00OOnguvBuAH5vZZ4Enge1A0t0fNbMz\ngWeA3cCzpItrgOuBS9x9pZndCNxOumjPeqxgn7nu3mxmxwN/NLOX3H3zIYG53wHcAbBo0SKvra3N\n4S2G67H1O+Gpei56zzLOmFs6/zuvq6sjX/ms3NQCa1ZyyqlLObtMprzKZz7LkfIZHuUyXMpnuEab\nT3fnhqf+wHtPOoba2qXhBVaEdG4WrlwK9G3AnIyfZ/PWcBQA3L0ZuBzAzCYBV7j73mDdLcAtwbq7\ngY1mVgOc5u4rg0PcCzycw7Gag3+3mFkdcDpwSIFeCBqDmxTNm6EhLrk6OAY9qTHoIiKSP02tXbR0\n9LJsfuk00KT05DIGfRWw0MwWmNk44CrgkBlUzKzazAaOdRNwZ7A8Fgx1wcyWAEuAR4F2YKqZnRjs\n8wFg/RDHmm5miYFtgHOhML+Z2trayeREnBkTx0UdStE4OAa9T2PQRUQkf1Y1tgGwfJ4uEJXCNWQH\n3d2TZnYd8AgQA+5095fN7Gag3t1XALXArWbmpIelfDHYvRL4U3AL3X3AJ909CWBmXwAeMLMU6YL9\nc8E+hzvWycA/BdtXkB6DXpAFemNrF/OqJ+jWwcOgDrqIiIyF1U3tTKmKs/CoSVGHInJYOc3O7+4P\nAQ8NWvaNjOf3A/dn2a+b9Kws2Y75IPBgluWHO9YzwKm5xBu1ptZOTjluatRhFBV10EVEZCzUN7Wz\nbN50KirURJPCpTuJhizZn2Jbu6ZYHK5EXB10ERHJr/bOXjbt6mC55j+XAqcCPWTNe7pJplwF+jAd\nvJOoOugiIpInmv9cioUK9JA1tXUCME93ER0WddBFRCTf6pvaqYwZp82ZFnUoIkekAj1kA1MszleB\nPiwHC3R10EVEJE/qG9s45bipVFWWxw3xpHipQA9ZU0sniXgFR01ORB1KUTEzEvEKddBFRCQvepL9\nvLh9L2dq/nMpAirQQ9bU1sW8mRN0dfgIVFXGNAZdRETyYt32vfQmUyzT/OdSBFSgh6yptZO5MzS8\nZSTUQRcRkXxZ1Zi+QHSZLhCVIqACPUSplLO1rYv5msFlRNRBFxGRfKlvbGdB9URqNARVioAK9BDt\n2t9Dd1+KedXqoI+EOugiIpIP7s7qpjZ1z6VoqEAPUWNrMMXiDHXQR0IddBERyYfNuztp7+rTBaJS\nNFSgh2irplgcFXXQRUQkH1Y3tQHoAlEpGirQQ9TY2km8wjhuWlXUoRQlddBFRCQfVjW2M31CJe+o\nUQNNioMK9BA1tXUxe/p44jGldSTUQRcRkXxY3dTOsnkzMNMUyFIcVEmGqKm1k3ka3jJiVZUxFegi\nIhKqlo4eXm/pZLnGn0sRUYEeEnenqSV9kyIZmUS8QkNcREQkVPXB/OfLNYOLFJF41AGUivauPvb3\nJNVBH4WEOugiIjICew/08eCabfT2v/0z5E8bWxgXr+DU2VMjiExkZFSgh0RTLI6eOugiIjJc7s7f\n3vMCdRt2H3abPzv5KBLx2BhGJTI6KtBDcnCKxWoV6COlMegiIjJcv1i5lboNu/n6hxdz1Zlzsm4z\nvlLFuRQXFeghaWztxAxmT1eBPlKJeAW9yRSplFNRoSvtRUTkyLbs7uCW/1zPeQur+ct3z9dnh5QM\nXSQakqbWLo6dUkWV/pc+YgO5yzaGUEREJFNff4rr713LuHgFt/3FaSrOpaSoQA+JplgcvUQ8fTpq\nHLqIiAzlJ49vomHbXr5z2akcPUU3CJTSogI9JE2tmmJxtBKV6dNR49BFRORIXtjazv/54yYuO30W\nH1pybNThiIROBXoI9nf30drZqw76KFUFV9irgy4iIofTk3S+dF8Dx0yp4tuXnhJ1OCJ5oYtEQ9A0\nMIOLOuijog66iIgM5Z4NvTS2Jrn7mncxpaoy6nBE8kId9BAMFOhzVaCPijroIiJyJI+/uovH30hy\nzXsWcM47ZkYdjkjeqIMegqa24CZFGuIyKuqgi4iUN3fnlv9cz+MbdmVdv2NvN7MnGTdcuGiMIxMZ\nWyrQQ7BzbzdTquJMSiidozEwzaI66CIi5en+1dv4/556nXe/YybTJ4572/rT5kzjrIltuiuolDxV\nlCFo6eylelIi6jCK3sA0iz196qCLiJSbN9q6+PZvX+HsBTP4+efPJnaYec3r6urGNjCRCGgMegha\nO3qYOent/9OX4TnYQU+qgy4iUk76U86X72vAgB98/LTDFuci5UIFeghaO3qZOVEd9NFSB11EpDzd\n8eQWnm9s49uXnsLs6ZpwQUQFeghaOnqonqwO+mipgy4iUn5ebt7L7b/fwCWnHsNlp8+KOhyRgqAC\nfZSS/Snau/rUQQ+BOugiIuWlu6+f6+9dy/QJ47jlz0/FTENbRCDHAt3MLjKzDWa2ycy+kmX9PDN7\nzMxeNLM6M5udse57ZrYueFyZsfwCM1tjZmvN7CkzOyGHY33GzDYGj8+M7q2Ho62rF4BqjUEfNXXQ\nRUTKy98/soHXdnbw/Y8tyTpri0i5GrJAN7MY8BPgYmAxcLWZLR602W3AXe6+BLgZuDXY90PAGcBS\n4GzgRjObEuzzU+AT7r4UuBv42hDHmgF8MzjOWcA3zWz6SN50mFo70gX6TM3iMmrjYuqgi4iUi6c3\ntfCzp17n0+fMo3bRUVGHI1JQcplm8Sxgk7tvATCze4BLgVcytlkMXB88fxz4TcbyJ9w9CSTNrAG4\nCLgPcGCgWJ8KNA9xrAuB37t7WxDH74Nj/TKnd5onLR09AJpmMQQVFca4eIU66CIiJWLz7g7aO3vf\ntjyZcm74VQPHV0/kpotPjiAykcKWS4E+C3gj4+dtpLvYmRqAK4AfAZcBk81sZrD8m2Z2OzABOJ+3\nCvtrgIfM7ACwD3jXEMfKFkfkV5O81UHXV3NhSMQr1EEXESkB//FiM9fd/cJh18crjAf+27sZP043\nHRIZLJcCPdsVGz7o5xuAH5vZZ4Enge1A0t0fNbMzgWeA3cCzQDLY53rgEndfaWY3AreTLtqzHivH\nODCza4FrAWpqavJ+Q4OVjX0AvLp2FW9UlvbFLR0dHXnPZ4X307h1G3V1u/P6OoVgLPJZTpTP8CiX\n4SrHfLZ3p/ja0wc4fmoFly/M3sCqGW+0b15L3ebhHbsc85kvymXhyqVA3wbMyfh5Nm8NRwHA3ZuB\nywHMbBJwhbvvDdbdAtwSrLsb2GhmNcBp7r4yOMS9wMNHOpaZbQNqB8VRNzhYd78DuANg0aJFXltb\nO3iTUK18+FUqN27hkj+rLfmrz+vq6sh3Piev/CMzamZQW7s0r69TCMYin+VE+QyPchmucsunu/Pp\nO58nRS8/+8J5LKieGOrxyy2f+aRcFq5cZnFZBSw0swVmNg64CliRuYGZVZvZwLFuAu4MlseC4SmY\n2RJgCfAo0A5MNbMTg30+AKw/0rGAR4APmtn04OLQDwbLItWyv4eZExMlX5yPlarKGD1JDXERESlW\nP3+uiT9tbOGrHzo59OJcpFwM2UF396SZXUe6GI4Bd7r7y2Z2M1Dv7itId7ZvNTMnPSzli8HulcCf\nguJ1H/DJ4IJRzOwLwANmliJdsH8u2Cfrsdy9zcz+B+n/MADcPHDBaJRaO3s1/jxEiXgF3X26SFRE\npBht3t3Bdx5aT+2iGj5x9tyowxEpWrkMccHdHwIeGrTsGxnP7wfuz7JfN+lZWbId80HgwSzLsx4r\nWHcnb3XUC0JrR4+mWAyROugiIsWprz/Fl+5dS1VljO9fsUTfLIuMgu4kOkotHb1U6+YKoVEHXUSk\nOP34j5to2LaX71x2KkdNqYo6HJGipgJ9FNydlo4eqiergx6WRLxCHXQRkSKz9o09/PjxTVx++iwu\nOfXYqMMRKXoq0Eehs7efnmSKmeqgh6aqMqYOuohIETnQ28+X7l3L0ZMTfOvSU6IOR6Qk5DQGXbJr\nDe4iqjHo4VEHXURk9Oob2/iPF3eMyWtt3LWfLS2d3H3N2UypqhyT1xQpdSrQR6EluItotWZxCY06\n6CIio7OtvYu//JdV9PanqKrM/106zeDLHziRd59QnffXEikXKtBHoSXooFergx4addBFREYulXK+\nfF8DKXf+8KX3MWfGhKhDEpER0Bj0UWgNOuiaBz086qCLiIzcz556nZWvt/HNj56i4lykiKlAH4WB\nMegzdJFoaAY66O4edSgiIkVl/Y59/P0jG7jwlKP5i2Wzow5HREZBBfootHb2MqUqTiKe/zF+5SIR\njJfUMBcRkdz1JPu5/t61TBlfyXcuO1U3CRIpcirQR2F3R4/Gn4csEU+fkirQRURy94NHX+PVN/fz\n/Y+dqpnFREqACvRRaO3o0fjzkA3MONCjcegiIjl5dnMr//ynLfyXs+fy/pOOjjocEQmBCvRRaO3o\nZeZEdSrCpA66iEju9nX3ccOvGpg3YwJf+9DJUYcjIiFRgT4KrZ29VE9WBz1MAx10zeQiIjK0b/37\ny7y5r5sfXrmUCeM0c7JIqVCBPkLJ/hTtXeqgh00ddBGR3Dy9qYVfv7CdL55/AqfPnR51OCISIhXo\nI9TW1Yu77iIaNnXQRURy8/iruxgXr+Cva98RdSgiEjIV6CP01k2K1EEPkzroIiK5qW9q57TZUw82\nNkSkdKhAH6GBAl3TLIZLHXQRkaEd6O1n3fa9LJ8/I+pQRCQPVKCPUEtwF1FNsxiuRKU66CIiQ2nY\ntodkylk+T2PPRUqRCvQRGijQq3WRaKiq4gN3ElUHXUTkcFY3tQOwTAW6SElSgT5CrZ29VMaMKeM1\nrVWYBjro3X3qoIuIHM6qxjYWHjWJaRP0La5IKVKBPkKtHT3MnJjAzKIOpaQc7KBrDLqISFaplLOm\nqZ3l89U9FylVKtBHqLWjV+PP8+BgB11j0EVEstq4q4N93UmWzdMFoiKlSgX6CLV09GiKxTxIHOyg\nq0AXEclmVWMbAGeqgy5SslSgj1BLRy/VE9VBD1uswqiMGd26SFREJKvVTe1UT0owd8aEqEMRkTxR\ngT4C7k5rZw/Vk9VBz4dEPKYOuojIYaxqbGP5vOm6BkqkhKlAH4Gu3n66+1LMVAc9L6oqK9RBFxHJ\nYue+bra1H9AFoiIlTgX6CLx1kyJ10PNBHXQRkezqG9Pzn+sOoiKlTQX6CLR09AK6i2i+JNRBFxHJ\nalVjG1WVFZxy3JSoQxGRPFKBPgKtQQe9Rh30vFAHXUQku9VN7SydM43KmD6+RUqZfsNHoLVTHfR8\nqqqsoEcddBGRQ3T2JHllxz6Wa/5zkZKnAn0EWvanO+gzdJFoXiTiFeqgi4gMsvaNPfSnXBeIipQB\nFegj0NrZy+Sq+MGb6ki4qipjGoMuIjJIfWM7ZnDGPBXoIqUupwLdzC4ysw1mtsnMvpJl/Twze8zM\nXjSzOjObnbHue2a2LnhcmbH8AjNbY2ZrzewpMzshWD7XzB43sxeC410SLJ9vZgeC7dea2T+O/u2P\nTEtHj8af55E66CIib1ff1Maioyczpaoy6lBEJM+GLNDNLAb8BLgYWAxcbWaLB212G3CXuy8BbgZu\nDfb9EHAGsBQ4G7jRzAYuPf8p8Al3XwrcDXwtWP414D53Px24CviHjNfZ7O5Lg8dfDfvdhqS1o1fj\nz/NIHXQRkUP1p5wXtu7R8BaRMpFLB/0sYJO7b3H3XuAe4NJB2ywGHgueP56xfjHwhLsn3b0TaAAu\nCtY5MFCsTwWah1heMFo6epg5UR30fFEHXUTkUK++uY+OnqQuEBUpE7kU6LOANzJ+3hYsy9QAXBE8\nvwyYbGYzg+UXm9kEM6sGzgfmBNtdAzxkZtuATwHfDZZ/C/hksPwh4G8yXmdBMPTlCTM7L5c3mA+t\nneqg55M66CIih1rdlL5B0TKNPxcpC/EctrEsy3zQzzcAPzazzwJPAtuBpLs/amZnAs8Au4FngWSw\nz/XAJe6+0sxuBG4nXbRfDfyru//AzM4Bfm5m7wR2AHPdvdXMlgG/MbNT3H3fIcGaXQtcC1BTU0Nd\nXV0ObzF3/SmnvbOXjpYd1NW1hnrsQtfR0RF6PrPZtaOHrp7kmLxWlMYqn+VC+QyPchmuMPL5n2u7\nmZ4wNjWsZLNl+1guHzo/w6NcFq5cCvRtvNX1BpjNoGEn7t4MXA5gZpOAK9x9b7DuFuCWYN3dwEYz\nqwFOc/eVwSHuBR4Onn+eYBiMuz9rZlVAtbvvAnqC5avNbDNwIlA/KJY7gDsAFi1a5LW1tTm8xdzt\n3t+DP/oHlr/zRGrPmR/qsQtdXV0dYeczm9W9G3ikaRPve9/7sBL+IBqrfJYL5TM8ymW4wsjnf3/2\nMd69aDrnn39GOEEVMZ2f4VEuC1cuQ1xWAQvNbIGZjSN94eaKzA3MrNrMBo51E3BnsDwWDHXBzJYA\nS4BHgXZgqpmdGOzzAWB98HwrcEGwz8lAFbDbzGqCC1Yxs+OBhcCW4b/l0WkJ7iI6U7O45E0iXoE7\n9PUP/qJGRKT8bN9zgOa93SzX8BaRsjFkB93dk2Z2HfAIEAPudPeXzexmoN7dVwC1wK1m5qSHuHwx\n2L0S+FPQBd0HfNLdkwBm9gXgATNLkS7YPxfs82Xgn83setJDaT7r7m5m7wVuNrMk0A/8lbu3jT4F\nw9PaEdxFVDcpypuqyvT88t3JfsbFNVW/iJS3+sb0R92Z83WBqEi5yGWIC+7+EOkLNjOXfSPj+f3A\n/Vn26yY9k0u2Yz4IPJhl+SvAuVmWPwA8kEu8+dTame6gV09WBz1fEkFR3tOXSn9/IiJSxlY3tTNh\nXIyTjpkcdSgiMkZyKtDlLS1BB71a0yzmTSLooPdoJhcRKRP9KWffgb6s655/vY3T504jHtM3iiLl\nQgX6MLV09BCvMKaMV+ryZaCD3q250EWkTPz1L1bzyMs7D7v+by9YOIbRiEjUVGUOU2tHDzMnjSvp\n2UWiVqUOuoiUkb7+FE++1sJ5C6u54KSj3rY+FqvgI0uOjSAyEYmKCvRhau3opVozuOSVOugiUk7W\n79jHgb5+Pr58Dh857biowxGRAqABbcPU0tmrKRbzLBFXB11EyseqxvRdQpfP1zSKIpKmAn2YWvb3\nUK0pFvOqqjJjFhcRkRK3uqmNWdPGc+zU8VGHIiIFQgX6MLg7rZ3pMeiSP+qgi0i5cHfqG9vVPReR\nQ6hAH4au3n66+1Iag55nAx10jUEXkVL3RtsBdu3v0V1CReQQKtCH4eBdRFWg55XmQReRclHflL5L\n6HLdJVREMqhAH4bdHem7iGqIS35VaRYXESkT9U3tTE7EOfFo3SVURN6iAn0YWoMCXXcRzS910EWk\nXKxubOf0edOJVejeGiLyFhXow9DamR7iUj1ZHfR8UgddRMrB3q4+Nuzcz5kafy4ig6hAH4aBDvoM\nTbOYV/FYBbEKUwddREramq3p+c+XaQYXERlEBfowtHT0MrkqfnAaQMmfqniFOugiUtLqm9qIVxhL\n50yLOhQRKTAq0IehpaNHUyyOkURlTB10ESlpqxrbOeW4KUwYF486FBEpMCrQh6G1o5dqzeAyJtRB\nF5FS1ptM0fDGHpbN0/SKIvJ2KtCHobWzh5mawWVMpDvoKtBFpDS93LyXnmSKMzX+XESyUIE+DC0d\nvZoDfYwk4hV092mIi4iUpvpGXSAqIoenAj1Hyf4U7V29uovoGFEHXURKWX1TG3NnTOCoyVVRhyIi\nBUgFeo7au/pwhxp10MdElTroIlKi3J3VTe0sV/dcRA5DBXqOWoI50NVBHxvqoItIqWps7aKlo5fl\nukBURA5DBXqO1u/YB8CC6okRR1IequIV9KiDLiIlqL6xDUAddBE5LBXoOapvamdyIs6JR0+OOpSy\noA66iJSq1U3tTB1fyQk1k6IORUQKlAr0HK1ubOeMedOJVVjUoZQFddBFpFStamxj2bzpVOjzREQO\nQwV6DvZ29bFh536Wz9PXkWMlUVlBtzroIlJi2jt72by7U8NbROSIVKDnYM1WzVc71hLxmDroIlJy\nVjelP090gaiIHIkK9BysamwjXmEsnTMt6lDKRpU66CJSglY1tVEZM5bMnhp1KCJSwFSg56C+qZ1T\njpvChHHxqEMpG4l4jP6Uk+xXkS4ipWN1YzunzppKVWUs6lBEpICpQB9CbzJFwxt7WKavI8dUVWX6\n1FQXXURKRXdfPy9u28vy+fo8EZEjU4E+hHXNe+lJpjhT48/HVCKe7i5pHLqIlIp12/fS259imSYc\nEJEhqEAfwupGXSAaBXXQRaTUPPjCdszQjGAiMiQV6EOob2pj7owJHDW5KupQyoo66CJSSp54bTe/\nWLmVz527gJmTElGHIyIFTgX6Ebg79Y3tmq82Agc76H3qoItIcevodW78VQMLj5rEjRcuijocESkC\nORXoZnaRmW0ws01m9pUs6+eZ2WNm9qKZ1ZnZ7Ix13zOzdcHjyozlF5jZGjNba2ZPmdkJwfK5Zva4\nmb0QHO+SjH1uCmLYYGYXju6tD62xtYvWzl7NVxuBgx30pDroIlK83J27XumhrbOXH165VLO3iEhO\nhizQzSwG/AS4GFgMXG1miwdtdhtwl7svAW4Gbg32/RBwBrAUOBu40cymBPv8FPiEuy8F7ga+Fiz/\nGnCfu58OXAX8Q3CsxcHPpwAXAf8QxJY39Y1tAOqgRyChDrqIlIAVDc08/2Y/13/gRN45S3Ofi0hu\ncumgnwVscvct7t4L3ANcOmibxcBjwfPHM9YvBp5w96S7dwINpItrAAcGivWpQPMQyy8F7nH3Hnd/\nHdgUxJY39Y3tTB1fyQk1k/L5MpKFOugiUuya9xzg679ZxwnTKviv7z0+6nBEpIjkcuedWcAbGT9v\nI90Nz9QAXAH8CLgMmGxmM4Pl3zSz24EJwPnAK8E+1wAPmdkBYB/wrmD5t4BHzexvgInAn2XE8dyg\nOGYNDtbMrgWuBaipqaGuri6Ht5jdk+u7mD+pgieffGLExyglHR0do8rncDTtSxfmq9e+CDtK8wZR\nY5nPcqB8hke5HL2UO7fVd9PTl+ITpzhP/enJqEMqGTo/w6NcFq5cKh/LsswH/XwD8GMz+yzwJLAd\nSLr7o2Z2JvAMsBt4FkgG+1wPXOLuK83sRuB20kX71cC/uvsPzOwc4Odm9s4c48Dd7wDuAFi0aJHX\n1tbm8Bbfrq2zlx0P/55PnXcCtbUnjOgYpaauro6R5nO4Nu3qgGee4IRFJ1O79G3/DysJY5nPcqB8\nhke5HL1/efp1Xml9he9cdirHHdiifIZI52d4lMvClcsQl23AnIyfZ/PWsBMA3L3Z3S8Pxo1/NVi2\nN/j3Fndf6u4fIF1kbzSzGuA0d18ZHOJe4N3B888D9wX7PgtUAdW5xBGm1U3p+c91gWg0BmZx6dEY\ndBEpMpt27ee7v3uV9590FFefNWfoHUREBsmlQF8FLDSzBWY2jvSFmisyNzCzajMbONZNwJ3B8lgw\n1AUzWwIsAR4F2oGpZnZisM8HgPXB863ABcE+J5Mu0HcHr3mVmSXMbAGwEHh++G85N/VNbVTGjCWz\ndVFPFDQGXUSKUbI/xfX3NjBhXIzvXnEqZtm+/BURObIhh7i4e9LMrgMeAWLAne7+spndDNS7+wqg\nFrjVzJz0EJcvBrtXAn8K/kDtAz7p7kkAM/sC8ICZpUgX7J8L9vky8M9mdj3pISyfdXcHXjaz+0iP\nYU8CX3T3vFVv9Y3tnDprqqbEiojmQReRYrRm6x5e2r6X2/7iNN3gTkRGLKer79z9IeChQcu+kfH8\nfuD+LPt1k57JJdsxHwQezLL8FeDcw+xzC3BLLjGPRndfPy9t28tnz52f75eSw1AHXUSKUX1Tenre\n9590VMSRiEgx051Es1i3fS+9/SmWzdP851GpjBkVpg66iBSX1Y3tHF8zkRkTx0UdiogUMRXoWaxq\nHLhAVAV6VMyMRDymDrqIFI1UyqlvaudMTS4gIqOkAj2L1U1tHF89kZmTElGHUtaqKivUQReRorF5\ndwd7D/SxTHefFpFRUoE+SCrlrG5q1/CWAqAOuogUk/omffsqIuFQgT7IlpYO2rv6OHO+vqKMWlVl\nBT1JddBFpDisamxj5sRxLKieGHUoIlLkVKAPUh+MP9dXlNFLxGN096mDLiLFYeDbV819LiKjpQJ9\nkPqmdmZMHMfx6oBELqEOuogUiV37u2lq7WK5mjsiEoKc5kEvNTf/9hUefGFb1nX7u5PULjpKHZAC\nUKUOuogUidUDs39peKSIhKAsC/TT5kwlmcremTXgY8vmjG1AklWisoKOnmTUYYiIDKm+qZ1EvIJ3\nHjc16lBEpASUwCaIQQAADjpJREFUZYF+6dJZXLp0VtRhyBAS8RgtHb1RhyEiMqT6xjZOmz2NcXGN\nHBWR0dNfEilY6THoGuIiIoXtQG8/Lzfv0/hzEQmNCnQpWFXxGD26UZGIFLi1b+whmXIV6CISGhXo\nUrDUQReRYlDf2AbAsrm6QFREwqECXQpWehYXddBFpLDVN7Vz4tGTmDqhMupQRKREqECXgqUOuogU\nuv6Us2ZrO8vmqXsuIuFRgS4Fqyoeo6/f6U951KGIiGT12s797O9OcqbGn4tIiFSgS8FKVKZPT3XR\nRaRQ1TcFNyhSB11EQqQCXQpWVTCfsMahi0ihWt3YRs3kBHNmjI86FBEpISrQpWAlKmOAOugiUrhW\nNbZz5vzpmFnUoYhICVGBLgWrqlIddBEpXDv2HmD7ngO6QFREQqcCXQpWIq4OuogUrvrGgfHnukBU\nRMKlAl0KljroIlLIVje1M74yxuLjpkQdioiUGBXoUrAOdtD71EEXkcJT39TG0jnTqIzpo1REwqW/\nKlKwDnbQk+qgi0hh6ehJ8krzPpZr/nMRyQMV6FKw1EEXkUK1duseUg7L5+sCUREJnwp0KVjqoItI\noapvasMMTp87LepQRKQEqUCXgqUOuogUqudfb+OkY6Ywpaoy6lBEpASpQJeClQjuJNqjDrqIFJA/\nvrqTZza3cuEpR0cdioiUKBXoUrAG7iTarQ66iBSI1o4e/p/7X+KkYybz32rfEXU4IlKi4lEHIHI4\n6qCLSCFxd2769UvsO9DH/3/NWQeH4YmIhE0ddClYBwt0ddBFpAD8avU2Hn1lJzdeuIiTjtHNiUQk\nf1SgS8EyMxLxCnXQRSRyb7R18e0VL3PO8TP5/HsWRB2OiJS4nAp0M7vIzDaY2SYz+0qW9fPM7DEz\ne9HM6sxsdsa675nZuuBxZcbyC8xsjZmtNbOnzOyEYPkPg2Vrzew1M9uTsU9/xroVo3vrUgyqKmMa\ngy4ikepPOdffu5YKM277+GlUVFjUIYlIiRtyDLqZxYCfAB8AtgGrzGyFu7+SsdltwF3u/m9m9n7g\nVuBTZvYh4AxgKZAAnjCz37n7PuCnwKXuvt7M/hr4GvBZd78+47X/Bjg943UOuPvS0bxhKS7qoItI\n1P7pyc3UN7XzwytPY9a08VGHIyJlIJcO+lnAJnff4u69wD3ApYO2WQw8Fjx/PGP9YuAJd0+6eyfQ\nAFwUrHNgYBDfVKA5y2tfDfwylzcipUkddBGJ0rrte/nh71/jQ0uO5c+Xzoo6HBEpE+buR97A7GPA\nRe5+TfDzp4Cz3f26jG3uBla6+4/M7HLgAaAaWAZ8k3T3fQLwPPATd/+BmZ0H/AY4AOwD3hV01geO\nOQ94Dpjt7v3BsiSwFkgC33X332SJ91rgWoCamppl99133/CzIll1dHQwadKkMX3N//5UF8dNrOC6\n06vG9HXHQhT5LGXKZ3iKOZc9/Uf+TBuOZApuWXmArj74n+eOZ9K4kQ1tKeZ8FiLlMzzKZbjOP//8\n1e6+PIxj5TLNYra/SIP/At4A/NjMPgs8CWwHku7+qJmdCTwD7AaeJV1cA1wPXOLuK83sRuB24JqM\nY14F3D9QnAfmunuzmR0P/NHMXnL3zYcE5n4HcAfAokWLvLa2Noe3KLmoq6tjrPM586WnmDxpHLW1\nZ43p646FKPJZypTP8BRrLm/69Yv88vk3Qj/uv33uLN53Ys2I9y/WfBYq5TM8ymXhyqVA3wbMyfh5\nNoOGo7h7M3A5gJlNAq5w973BuluAW4J1dwMbzawGOM3dVwaHuBd4eNDrXgV8Mcvr4O5bzKyO9Pj0\nzUjJ0hh0EcnFv6/dzi+ff4PLTp/FomMmh3bcRUdPHlVxLiIyErkU6KuAhWa2gHRn/Crgv2RuYGbV\nQJu7p4CbgDuD5TFgmru3mtkSYAnwaLDbVDM70d1fIz0EZn3G8RYB00l33AeWTQe63L0neL1zge+P\n4D1LEamqjNHVmxx6QxEpWzv2HuDrv1nHGXOn8fcfW0I8phmERaS4DVmgu3vSzK4DHgFiwJ3u/rKZ\n3QzUu/sKoBa41cyc9BCXgc53JfAnM4P0OPNPunsSwMy+ADxgZimgHfhcxsteDdzjhw6QPxn4p2D7\nCtJj0DNnkpESlIhX0N6lDrqIZJdKOTf8qoFkyrn940tVnItIScilg467PwQ8NGjZNzKe3w/cn2W/\nbtIzuWQ75oPAg4dZ960sy54BTs0lXikdmsVFRI7kX59p5OlNrdx6+anMr54YdTgiIqFQq0EKmsag\ni8jhbNy5n+8+/CoXnHQUV505Z+gdRESKhAp0KWiJyhjdfSrQReRQvckUf3fvWiYn4nz3iiUEQylF\nREpCTkNcRKKS7qBriIuIHOp//eE1Xm7exx2fWkbN5ETU4YiIhEoddCloVZUxetRBF5EMqxrb+Mcn\nNnPl8jl88JRjog5HRCR06qBLQUvEK+jtT/EvT7+e9Y5ZxWxjUx+NT78edRglQ/kMT6Hn8mdPv87s\n6RP4+keyzkEgIlL0VKBLQZtfPQGAb/+2RGfUXF+i7ysqymd4CjiXE8bFuOtzZzEpoY8wESlN+usm\nBe2y02fz/pOOJpXyoTcuMk8//TTnnntu1GGUDOUzPIWey6rKGOPHxaIOQ0Qkb1SgS8GbOr4y6hDy\nYtI4Y/rEcVGHUTKUz/AolyIi0dJFoiIiIiIiBUQFuoiIiIhIAVGBLiIiIiJSQFSgi4iIiIgUEBXo\nIiIiIiIFRAW6iIiIiEgBUYEuIiIiIlJAVKCLiIiIiBQQFegiIiIiIgVEBbqIiIiISAFRgS4iIiIi\nUkDM3aOOIW/MbD+wIeo4Skg10BJ1ECVE+QyX8hke5TJcyme4lM/wKJfhWuTuk8M4UDyMgxSwDe6+\nPOogSoWZ1Suf4VE+w6V8hke5DJfyGS7lMzzKZbjMrD6sY2mIi4iIiIhIAVGBLiIiIiJSQEq9QL8j\n6gBKjPIZLuUzXMpneJTLcCmf4VI+w6Nchiu0fJb0RaIiIiIiIsWm1DvoIiIiIiJFRQW6iIiIiEgB\nKdkC3cwuMrMNZrbJzL4SdTyFzszmmNnjZrbezF42s78Nln/LzLab2drgcUnGPjcF+d1gZhdGF31h\nMrNGM3spyFt9sGyGmf3ezDYG/04PlpuZ/e8gny+a2RnRRl9YzGxRxjm41sz2mdnf6fzMnZndaWa7\nzGxdxrJhn49m9plg+41m9pko3kvUDpPLvzezV4N8PWhm04Ll883sQMY5+o8Z+ywL/kZsCvJtUbyf\nqB0mn8P+3dbnftph8nlvRi4bzWxtsFzn5xEcoTbK/99Ody+5BxADNgPHA+OABmBx1HEV8gM4Fjgj\neD4ZeA1YDHwLuCHL9ouDvCaABUG+Y1G/j0J6AI1A9aBl3we+Ejz/CvC94PklwO8AA94FrIw6/kJ9\nBL/fbwLzdH4OK2/vBc4A1mUsG9b5CMwAtgT/Tg+eT4/6vRVILj8IxIPn38vI5fzM7QYd53ngnCDP\nvwMujvq9FVA+h/W7rc/9I+dz0PofAN8Inuv8PHIuD1cb5f1vZ6l20M8CNrn7FnfvBe4BLo04poLm\n7jvcfU3wfD+wHph1hF0uBe5x9x53fx3YRDrvcmSXAv8WPP834M8zlt/lac8B08zs2CgCLAIXAJvd\nvekI2+j8HMTdnwTaBi0e7vl4IfB7d29z93bg98BF+Y++sGTLpbs/6u7J4MfngNlHOkaQzynu/qyn\nP8Hv4q38l5XDnJuHc7jfbX3uB46Uz6AL/nHgl0c6hs7PtCPURnn/21mqBfos4I2Mn7dx5GJTMpjZ\nfOB0YGWw6Lrgq5o7B77GQTnOhQOPmtlqM7s2WHa0u++A9C8+cFSwXPnM3VUc+uGi83Pkhns+Kq+5\n+RzpLtqABWb2gpk9YWbnBctmkc7fAOXy7Ybzu61zMzfnATvdfWPGMp2fORhUG+X9b2epFujZxklp\nPskcmNkk4AHg79x9H/BT4B3AUmAH6a/GQDnOxbnufgZwMfBFM3vvEbZVPnNgZuOAjwK/Chbp/MyP\nw+VPeR2CmX0VSAK/CBbtAOa6++nAl4C7zWwKyuVQhvu7rXzm5moObXDo/MxBltrosJtmWTai87NU\nC/RtwJyMn2cDzRHFUjTMrJL0CfgLd/81gLvvdPd+d08B/8xbwwSU4yG4e3Pw7y7gQdK52zkwdCX4\nd1ewufKZm4uBNe6+E3R+hmC456PyegTBhV8fBj4RDAsgGIrRGjxfTXqc9Imkc5k5DEa5zDCC322d\nm0MwszhwOXDvwDKdn0PLVhsxBn87S7VAXwUsNLMFQcftKmBFxDEVtGBc2s+A9e5+e8byzHHQlwED\nV4WvAK4ys4SZLQAWkr6gRAAzm2hmkweek76AbB3pvA1cvf0Z4N+D5yuATwdXgL8L2Dvw9Zkc4pDu\nj87PURvu+fgI8EEzmx4MOfhgsKzsmdlFwP8LfNTduzKW15hZLHh+POlzcUuQz/1m9q7g7++neSv/\nZW8Ev9v63B/anwGvuvvBoSs6P4/scLURY/G3c6yuhB3rB+kraV8j/b/Br0YdT6E/gPeQ/rrlRWBt\n8LgE+DnwUrB8BXBsxj5fDfK7gTK8unuIfB5PehaBBuDlgXMQmAk8BmwM/p0RLDfgJ0E+XwKWR/0e\nCu0BTABagakZy3R+5p6/X5L+OruPdDfn8yM5H0mPr94UPP4y6vdVQLncRHqM6cDfz38Mtr0i+BvQ\nAKwBPpJxnOWkC8/NwI8J7u5dbo/D5HPYv9v63D98PoPl/wr81aBtdX4eOZeHq43y/rfTgp1ERERE\nRKQAlOoQFxERERGRoqQCXURERESkgKhAFxEREREpICrQRUREREQKiAp0EREREZECogJdRERERKSA\nqEAXERERESkg/xeCBzSYfQK7xQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "\n", + "loss = np.array(train_summary.read_scalar(\"Loss\"))\n", + "top1 = np.array(val_summary.read_scalar(\"Top1Accuracy\"))\n", + "\n", + "plt.figure(figsize = (12,12))\n", + "plt.subplot(2,1,1)\n", + "plt.plot(loss[:,0],loss[:,1],label='loss')\n", + "plt.xlim(0,loss.shape[0]+10)\n", + "plt.grid(True)\n", + "plt.title(\"loss\")\n", + "plt.subplot(2,1,2)\n", + "plt.plot(top1[:,0],top1[:,1],label='top1')\n", + "plt.xlim(0,loss.shape[0])\n", + "plt.title(\"top1 accuracy\")\n", + "plt.grid(True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## See visualizations on tensorboard as well\n", + "\n", + "We can run tensorboard at the command prompt and see visualizations too -- much prettier ones in fact than is available here.\n", + "\n", + "To do, run at the command line:\n", + "\n", + "$ tensorboard --logdir=\"/tmp/bigdl_summaries\"\n", + "\n", + "And then, you can go to your browser:\n", + "\n", + "http://YOURHOSTNAME:6006" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Show Confusion Matrix and Accuracy score on Validation Data\n", + "\n", + "Here, we are going to run predictions on our validation data to determine our accuracy on validation data and our confusion matrix.\n", + "\n", + "Nobody really cares what our error (loss) is on training data. It's validation data that really matters. So, calculating accuracy.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "predictions = trained_model.predict(cc_rdd_test).collect()\n", + "\n", + "def map_predict_label(l):\n", + " return np.array(l).argmax()\n", + "def map_groundtruth_label(l):\n", + " return l.to_ndarray()[0] - 1\n", + "\n", + "y_pred = np.array([ map_predict_label(s) for s in predictions])\n", + "\n", + "y_true = np.array([map_groundtruth_label(s.label) for s in cc_rdd_test.collect()])" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The prediction accuracy is 99.90%\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAHVCAYAAAAJuQqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAH0tJREFUeJzt3XvYpWVdL/Dvb2YaGOwyPKUwQx6n\nDC0RFchDqRgHU7EyN+7LJGPvMQM7mYm13ZppO0szzzUkImogmgqShqiZHZRDMSInZQSVGQ6GHDwh\nMLz3/mOeWesV3jk48c479zOfD9dzzVr3etZa98vl+P74/u77eaq1FgCA3i1a6AkAANwVFDUAwCgo\nagCAUVDUAACjoKgBAEZBUQMAjIKiBgAYBUUNADAKihoAYBSWzPcX3Hbd5S5ZDAtg2d5PWOgpwC5r\nw63ra0d913z9nv2hez9oh/0MdxVJDQAwCvOe1AAA82jm9oWewU5DUgMAjIKkBgB61mYWegY7DUkN\nADAKkhoA6NmMpGYTRQ0AdKxpP01oPwEAoyCpAYCeaT9NSGoAgFGQ1ABAz6ypmVDUAEDPXFF4QvsJ\nABgFSQ0A9Ez7aUJSAwCMgqQGAHpmS/eEogYAOuaKwlPaTwDAKEhqAKBn2k8TkhoAYBQkNQDQM2tq\nJiQ1AMAoSGoAoGdukzChqAGAnmk/TWg/AQCjIKkBgJ7Z0j0hqQEARkFSAwA9s6ZmQlEDAD3TfprQ\nfgIARkFSAwAda811ajaR1AAAoyCpAYCeWSg8oagBgJ5ZKDyh/QQAjIKkBgB6pv00IakBAEZBUgMA\nPZuxpXsTRQ0A9Ez7aUL7CQAYBUkNAPTMlu4JSQ0AMAqSGgDomTU1E5IaAGAUJDUA0DNraiYUNQDQ\nM0XNhPYTADAKkhoA6Fhrrii8iaQGAPiBVdVPVNWaWcc3q+p3quqVVbV+1vhTZ73nZVW1tqq+WFWH\nzho/bBhbW1XHzRp/YFWdXVWXVdX7qmrpluakqAGAns3MzM+xFa21L7bW9mut7ZfkUUm+m+RDw8tv\n2PRaa+2jSVJV+yY5MsnDkhyW5G1VtbiqFid5a5LDk+yb5DnDuUny2uGzVia5IcnRW5qTogYAetZm\n5uf4wRyc5Mutta9u4ZwjkpzSWrultXZFkrVJDhiOta21y1trtyY5JckRVVVJnpzkA8P735XkmVua\nhKIGAPjvOjLJybOeH1tVF1TVCVV1j2FseZIrZ52zbhjb3Pi9ktzYWttwh/HNUtQAQM/mqf1UVauq\n6rxZx6q5vn5Y5/KMJO8fht6e5MFJ9ktydZLXbzp1jre37RjfLLufAIA7aa2tTrJ6G049PMl/ttau\nHd537aYXqur4JGcMT9cl2WfW+1YkuWp4PNf4dUn2rKolQ1oz+/w5SWoAoGcLv6bmOZnVeqqqvWa9\n9otJLhwen57kyKraraoemGRlknOSnJtk5bDTaWk2trJOb621JP+U5FnD+49KctqWJiKpAYCeLeAV\nhatqjyQ/n+QFs4b/vKr2y8ZW0Vc2vdZau6iqTk1ycZINSY5pw0V2qurYJGcmWZzkhNbaRcNnvTTJ\nKVX16iTnJ3nHFuezsRCaP7ddd/n8fgEwp2V7P2GhpwC7rA23rp9rPci8uPnjb5uX37PLDvnNHfYz\n3FUkNQDQsx98+/VoWVMDAIyCpAYAeuYu3ROSGgBgFCQ1ANAzSc2EogYAemah8IT2EwAwCpIaAOiZ\n9tOEpAYAGAVJDQD0zJqaCUUNAPRM+2lC+wkAGAVJDQD0TPtpQlIDAIyCpAYAemZNzYSiBgB6pqiZ\n0H4CAEZBUgMAPWttoWew05DUAACjIKkBgJ5ZUzMhqQEARkFSAwA9k9RMKGoAoGeuKDyh/QQAjIKk\nBgB6pv00IakBAEZBUgMAPXPxvQlFDQD0TPtpQvsJABgFSQ0A9ExSMyGpAQBGQVIDAD1z8b0JRQ0A\ndKzN2P20ifYTADAKkhoA6JmFwhOSGgBgFCQ1ANAzC4UnJDUAwChIagCgZ3Y/TShqAKBnFgpPaD8B\nAKMgqQGAnklqJiQ1AMAoSGoAoGfNQuFNFDUA0DPtpwntJwBgFCQ1u4iTTvlQ/v4j/5iqysoHPyCv\n/sPfy6v+4s05b80X8sN3u1uS5DV/9Ht56I8/OJ/6l8/mzceflEW1KIsXL85xv70q+z/i4ZPP+vZ3\nvpNn/M8X5OCffWz+6MW/+X3fc+wfvDLrrromH37PX+/Qnw/G4tBDnpi//MtXZfGiRTnhnSfnz//i\nrQs9JXZ2rlMzoajZBVz7X9flvR84Lae992+y+2675cUv/9N87BP/nCR58TFH55AnPeH7zj/oUfvl\nSY8/KFWVL669Ir//8j/NR04+fvL6m49/dx79yJ+60/ec9el/yx57LJvfHwZGbNGiRXnTG1+Tw576\nnKxbd3U+99mP5iNnfDyXXHLZQk8NuqD9tIvYcPvtueWWW7Nhw+25+Xu35D73vudmz91jj2WpqiTJ\nzd/7XjI8TpKLLr0s37j+hjz2Mft/33u++92bc9L7PpgXHHXk/PwAsAs44DGPzJe//JVcccXXcttt\nt+XUU0/LM55+6EJPi51dm5mfo0NbTWqq6qFJjkiyPElLclWS01trl8zz3LiL3Pc+986vPeeX85Rf\nel52321pHvuY/fO4Ax+Vj5716bzpb96Vt7/z73LQo/bL777w+Vm6dGmS5BP//G9541+fmG/ccGPe\n9rpXJUlmZmbyF285Pv/v5S/J2eet+b7vePPxJ+WoI38pu++++w7/+WAs9l5+v1y57qrJ83Xrr84B\nj3nkAs6ILmg/TWwxqamqlyY5JUklOSfJucPjk6vquPmfHneFm775rfzTv3wuZ77/nfnUae/Nzd+7\nJR8581P5nd94fj5y8vF539++MTd981t5x3veP3nPU37ucfnIycfnTX/2f/OW409KkpzywTPysz/z\nmOx13/t83+df+qUv52vrr8pTfu5xO/TngrGpWanoJs12XdhmW0tqjk7ysNbabbMHq+ovk1yU5M/m\nelNVrUqyKkne9vpX53897zl3wVTZXp87b02W733f3PMeeyZJDv65x2bNFy7O0w99cpJk6dKleeYv\nHJITT/77O7330fv9VK5cf3VuuPGmfP7CS/IfF1yUUz54Rr578/dy2223ZY89ds9e9/vRXHzp2hzy\ny0fl9ttvzzduuCm/duwf5MS3/PkO/Tmhd+vXXZ19Vuw9eb5i+V65+uprF3BG9KDZ0j2xtaJmJsne\nSb56h/G9htfm1FpbnWR1ktx23eX+M2OB7XXf++SCCy/Nzd/7Xnbfbbecfd6aPOyhK/Nf112f+9z7\nnmmt5VOf+fesfND9kyRfW3dV9lm+V6oqF39xbW67bUP2/JG757WvfOnkMz/8D2floksvy+++8NeT\nJEf+4tOSJOuvvjbHvOQVChrYDueetyYPecgD84AH7JP166/Js599RH71eccs9LSgG1sran4nySer\n6rIkVw5jP5bkIUmOnc+Jcdf56Yc9ND//pMfn2c9/URYvXpyH/viD8ytHHJ7fePH/zQ033pTWWn5i\n5YPyipe8KEly1qf/Nad/7JNZsmRJdt9taV73quPmjMWBu9btt9+e3/6d/5OP/sPfZfGiRTnxXe/L\nxRd/aaGnxc7OmpqJ2lq/tqoWJTkgGxcKV5J1Sc5trd2+LV8gqYGFsWzvJ2z9JGBebLh1/Q77L8Hv\nvOZ58/J79m5/dFJ3/zW71d1PrbWZJJ/bAXMBAH5QnW6/ng8uvgcAPdN+mnDxPQBgFCQ1ANAzW7on\nJDUAwChIagCgZ9bUTChqAKBndj9NaD8BAKMgqQGAnmk/TUhqAIBRkNQAQMfcpXtKUQMAPdN+mtB+\nAgBGQVEDAD2bafNzbIOq2rOqPlBVl1bVJVX1M1V1z6o6q6ouG/68x3BuVdWbqmptVV1QVfvP+pyj\nhvMvq6qjZo0/qqq+MLznTVW1xTuHK2oAgO31xiT/2Fp7aJJHJLkkyXFJPtlaW5nkk8PzJDk8ycrh\nWJXk7UlSVfdM8ookByY5IMkrNhVCwzmrZr3vsC1NRlEDAD1rM/NzbEVV3T3JzyZ5R5K01m5trd2Y\n5Igk7xpOe1eSZw6Pj0hyUtvoc0n2rKq9khya5KzW2vWttRuSnJXksOG1u7fWPttaa0lOmvVZc1LU\nAAB3UlWrquq8WceqO5zyoCT/leSdVXV+Vf1tVd0tyX1ba1cnyfDnjw7nL09y5az3rxvGtjS+bo7x\nzbL7CQB6Nk+7n1prq5Os3sIpS5Lsn+RFrbWzq+qNmbaa5jLXepi2HeObJakBgI61mTYvxzZYl2Rd\na+3s4fkHsrHIuXZoHWX48+uzzt9n1vtXJLlqK+Mr5hjfLEUNAPADa61dk+TKqvqJYejgJBcnOT3J\nph1MRyU5bXh8epLnDbugDkpy09CeOjPJIVV1j2GB8CFJzhxe+1ZVHTTsenrerM+ak/YTAPRsYS++\n96Ik762qpUkuT/L8bAxMTq2qo5N8LcmvDOd+NMlTk6xN8t3h3LTWrq+qP0ly7nDeq1pr1w+PX5jk\nxCTLknxsODZLUQMAbJfW2pokj57jpYPnOLclOWYzn3NCkhPmGD8vycO3dT6KGgDomXs/TShqAKBn\n7v00YaEwADAKkhoA6JmkZkJSAwCMgqQGADq2cVMRiaIGAPqm/TSh/QQAjIKkBgB6JqmZkNQAAKMg\nqQGAjm3jHbV3CZIaAGAUJDUA0DNJzYSiBgB65n6WE9pPAMAoSGoAoGMWCk9JagCAUZDUAEDPJDUT\nihoA6JmFwhPaTwDAKEhqAKBjFgpPSWoAgFGQ1ABAz6ypmVDUAEDHtJ+mtJ8AgFGQ1ABAz7SfJiQ1\nAMAoSGoAoGNNUjOhqAGAnilqJrSfAIBRkNQAQMe0n6YkNQDAKEhqAKBnkpoJSQ0AMAqSGgDomDU1\nU4oaAOiYomZK+wkAGAVJDQB0TFIzJakBAEZBUgMAPWu10DPYaShqAKBj2k9T2k8AwChIagCgY21G\n+2kTSQ0AMAqSGgDomDU1U4oaAOhYs/tpQvsJABgFSQ0AdEz7aUpSAwCMgqQGADpmS/eUpAYAGAVJ\nDQB0rLWFnsHOQ1EDAB3TfprSfgIARkFSAwAdk9RMSWoAgFGQ1ABAxywUnlLUAEDHtJ+mtJ8AgFGQ\n1ABAx9yle0pSAwCMgqQGADrmLt1TihoA6NiM9tOE9hMAMAqSGgDomIXCU5IaAGAUJDUA0DEX35uS\n1AAA262qFlfV+VV1xvD8xKq6oqrWDMd+w3hV1Zuqam1VXVBV+8/6jKOq6rLhOGrW+KOq6gvDe95U\nVVus4CQ1ANCxneDeT7+d5JIkd5819pLW2gfucN7hSVYOx4FJ3p7kwKq6Z5JXJHl0kpbkP6rq9Nba\nDcM5q5J8LslHkxyW5GObm4ikBgA61mZqXo5tUVUrkvxCkr/dhtOPSHJS2+hzSfasqr2SHJrkrNba\n9UMhc1aSw4bX7t5a+2xrrSU5Kckzt/QFihoA4E6qalVVnTfrWDXHaX+V5A+S3PESgK8ZWkxvqKrd\nhrHlSa6cdc66YWxL4+vmGN8s7ScA6Nh8XXyvtbY6yerNvV5VT0vy9dbaf1TVE2e99LIk1yRZOrz/\npUlelWSuibbtGN8sSQ0AsD0el+QZVfWVJKckeXJVvae1dvXQYrolyTuTHDCcvy7JPrPevyLJVVsZ\nXzHH+GYpagCgY63VvBxb/972stbaitbaA5IcmeRTrbXnDmthMuxUemaSC4e3nJ7kecMuqIOS3NRa\nuzrJmUkOqap7VNU9khyS5MzhtW9V1UHDZz0vyWlbmpP2EwB0bCfY/XRH762q+2Rj+2hNkt8Yxj+a\n5KlJ1ib5bpLnJ0lr7fqq+pMk5w7nvaq1dv3w+IVJTkyyLBt3PW1251OSVJvnfxu3XXf5zvevG3YB\ny/Z+wkJPAXZZG25dv8OuiHfBA54+L79nf/orH+nuqn6SGgDomLt0T1lTAwCMgqQGADrmLt1TihoA\n6NhOuFB4wWg/AQCjIKkBgI5ZKDw170WNbaUAwI4gqQGAjlkoPGVNDQAwCpIaAOiYNTVTihoA6Jgd\n3VPaTwDAKEhqAKBj2k9TkhoAYBQkNQDQMVu6pxQ1ANCxmYWewE5E+wkAGAVJDQB0rEX7aRNJDQAw\nCpIaAOjYjKvvTShqAKBjM9pPE9pPAMAoSGoAoGMWCk9JagCAUZDUAEDHXHxvSlIDAIyCpAYAOmZN\nzZSiBgA6pv00pf0EAIyCpAYAOiapmZLUAACjIKkBgI5ZKDylqAGAjs2oaSa0nwCAUZDUAEDH3KV7\nSlIDAIyCpAYAOtYWegI7EUUNAHTMdWqmtJ8AgFGQ1ABAx2bKQuFNJDUAwChIagCgYxYKT0lqAIBR\nkNQAQMfsfppS1ABAx9z7aUr7CQAYBUkNAHTMvZ+mJDUAwChIagCgY7Z0TylqAKBjFgpPaT8BAKMg\nqQGAjrlOzZSkBgAYBUkNAHTMQuEpRQ0AdMxC4SntJwBgFCQ1ANAxC4WnJDUAwChIagCgY5KaKUkN\nADAKkhoA6Fiz+2lCUQMAHdN+mtJ+AgBGQVIDAB2T1ExJagCAUZDUAEDH3PtpSlEDAB1z76cp7ScA\nYBQUNQDQsZl5OramqnavqnOq6vNVdVFV/fEw/sCqOruqLquq91XV0mF8t+H52uH1B8z6rJcN41+s\nqkNnjR82jK2tquO2NidFDQCwPW5J8uTW2iOS7JfksKo6KMlrk7yhtbYyyQ1Jjh7OPzrJDa21hyR5\nw3BeqmrfJEcmeViSw5K8raoWV9XiJG9NcniSfZM8Zzh3sxQ1ANCxhUpq2kbfHp7+0HC0JE9O8oFh\n/F1Jnjk8PmJ4nuH1g6uqhvFTWmu3tNauSLI2yQHDsba1dnlr7dYkpwznbpaiBgA61ubpqKpVVXXe\nrGPVHb97SFTWJPl6krOSfDnJja21DcMp65IsHx4vT3Jlkgyv35TkXrPH7/CezY1vlt1PAMCdtNZW\nJ1m9lXNuT7JfVe2Z5ENJfnKu04Y/59qn1bYwPlfwssUd7IoaAOjYzrClu7V2Y1V9OslBSfasqiVD\nGrMiyVXDaeuS7JNkXVUtSfIjSa6fNb7J7PdsbnxO2k8AwA+squ4zJDSpqmVJnpLkkiT/lORZw2lH\nJTlteHz68DzD659qrbVh/Mhhd9QDk6xMck6Sc5OsHHZTLc3GxcSnb2lOkhoA6NgC3vtpryTvGnYp\nLUpyamvtjKq6OMkpVfXqJOcnecdw/juSvLuq1mZjQnNkkrTWLqqqU5NcnGRDkmOGtlaq6tgkZyZZ\nnOSE1tpFW5pQbSyS5s+SpctdwRmAXcqGW9fvsKbQn93/ufPye/a4r75nJ2hs/WAkNQDQMcnBlKIG\nADo2o6yZsFAYABgFSQ0AdGwBFwrvdCQ1AMAoSGoAoGNW1EwpagCgY9pPU9pPAMAoSGoAoGM7w72f\ndhaSGgBgFCQ1ANAxF9+bUtQAQMeUNFPaTwDAKEhqAKBjtnRPSWoAgFGQ1ABAxywUnlLUAEDHlDRT\n2k8AwChIagCgYxYKT0lqAIBRkNQAQMcsFJ6S1AAAoyCpAYCOyWmmFDUA0DELhae0nwCAUZDUAEDH\nmgbUhKQGABgFSQ0AdMyamilFDQB0zHVqprSfAIBRkNQAQMfkNFOSGgBgFCQ1ANAxa2qmFDUA0DG7\nn6a0n5j47d/63/n8mk9lzfmfzHve/dbstttuk9f+6g1/khuv/9ICzg7GacWKvfOJj78/X7jg0/n8\nmk/lRcce/X2v/97vviAbbl2fe93rHgs0Q+iHooYkyd573y/HHvPrOfCgp2a/Rx6cxYsX5388+4gk\nyaP2/+nsueePLPAMYZw2bNiQl/zBH+enfvqJedzjn54XvvDX8pM/uTLJxoLnKQf/bL761XULPEt2\nZm2e/umRooaJJUuWZNmy3bN48eLssWxZrr76mixatCiv/bOX57iXvXqhpwejdM01X8/5ay5Mknz7\n29/JpZdeluV73y9J8vrXvTLH/eFr0lqfv2BgR9vuoqaqnn9XToSFddVV1+Qv3/DXueLL52Td187P\nTd/8Zs76xGdyzG8+Px854+O55pqvL/QUYfTuf/8V2e8RD8/Z55yfpz3t57N+/dW54IKLF3pa7ORm\n5uno0X8nqfnjzb1QVauq6ryqOm9m5jv/ja9gR9lzzx/JM55+aB7y4wdln/vvn7vdbY8897nPyrN+\n+Wl5y1tPWOjpwejd7W575NT3HZ/f+/1XZMOGDfnD434rr/zj1y30tKArW9z9VFUXbO6lJPfd3Pta\na6uTrE6SJUuXy007cPDBT8gVX/larrvu+iTJhz78sbzi5S/OsmW754uX/FuSZI89luXSi/81D933\n8Qs5VRidJUuW5P3vOz4nn/yhfPjDH8vDH/7QPOABP5b/PO+sJMmKFXvl3LPPzM887hdy7bX/tcCz\nZWfT6/qX+bC1Ld33TXJokhvuMF5J/n1eZsSCuPJr63Pggftn2bLdc/PN38uTn/T4/NUbV+etb3vn\n5Jwbr/+SggbmwfGrX59LLl2bv3rj6iTJhRdemr1XPGLy+tovfS4H/szh+cY37vh/xdBvq2g+bK39\ndEaSH26tffUOx1eSfHreZ8cOc8655+eDH/yHnHvOmVlz/iezaNGiHP+3713oacHoPe6xj8mvPvdZ\nedKTHpvzzv14zjv34zn8sCcv9LSgSzXfq+q1nwDY1Wy4dX3tqO/61fv/0rz8nn33Vz+4w36Gu4ot\n3QDAKLhNAgB0TDtkSlEDAB1zQ8sp7ScAYBQkNQDQMdepmZLUAACjIKkBgI65+N6UogYAOmah8JT2\nEwAwCpIaAOiYhcJTkhoAYBQkNQDQMQuFpyQ1AMAoSGoAoGOtWVOziaIGADpmS/eU9hMAMAqSGgDo\nmIXCU5IaAGAUJDUA0DEX35tS1ABAxywUntJ+AgBGQVIDAB1znZopSQ0AMAqSGgDomC3dU5IaAOhY\nm6d/tqaqTqiqr1fVhbPGXllV66tqzXA8ddZrL6uqtVX1xao6dNb4YcPY2qo6btb4A6vq7Kq6rKre\nV1VLtzYnRQ0AsD1OTHLYHONvaK3tNxwfTZKq2jfJkUkeNrznbVW1uKoWJ3lrksOT7JvkOcO5SfLa\n4bNWJrkhydFbm5CiBgA6NpM2L8fWtNY+k+T6bZzmEUlOaa3d0lq7IsnaJAcMx9rW2uWttVuTnJLk\niKqqJE9O8oHh/e9K8sytfYmiBgC4k6paVVXnzTpWbeNbj62qC4b21D2GseVJrpx1zrphbHPj90py\nY2ttwx3Gt0hRAwAda63N17G6tfboWcfqbZjO25M8OMl+Sa5O8vphvOaa+naMb5HdTwDAXaK1du2m\nx1V1fJIzhqfrkuwz69QVSa4aHs81fl2SPatqyZDWzD5/syQ1ANCxhVpTM5eq2mvW019Msmln1OlJ\njqyq3arqgUlWJjknyblJVg47nZZm42Li09vGKwr+U5JnDe8/KslpW/t+SQ0AdGyhbmhZVScneWKS\ne1fVuiSvSPLEqtovG1tFX0nygiRprV1UVacmuTjJhiTHtNZuHz7n2CRnJlmc5ITW2kXDV7w0ySlV\n9eok5yd5x1bnNN+XV16ydLnrNwOwS9lw6/q51oTMiyeueMq8/J799LpP7LCf4a4iqQGAjs2499OE\nNTUAwChIagCgY3KaKUUNAHRse3cqjZH2EwAwCpIaAOiYpGZKUgMAjIKkBgA6Nt/Xm+uJogYAOqb9\nNKX9BACMgqQGADq2UPd+2hlJagCAUZDUAEDHLBSektQAAKMgqQGAjtn9NKWoAYCOaT9NaT8BAKMg\nqQGAjmk/TUlqAIBRkNQAQMdcfG9KUQMAHZuxUHhC+wkAGAVJDQB0TPtpSlIDAIyCpAYAOmZNzZSi\nBgA6pv00pf0EAIyCpAYAOqb9NCWpAQBGQVIDAB2zpmZKUgMAjIKkBgA6Zk3NlKIGADqm/TSl/QQA\njIKkBgA61trMQk9hpyGpAQBGQVIDAB2bsaZmQlEDAB1rdj9NaD8BAKMgqQGAjmk/TUlqAIBRkNQA\nQMesqZlS1ABAx9wmYUr7CQAYBUkNAHTMvZ+mJDUAwChIagCgYxYKT0lqAIBRkNQAQMdcfG9KUQMA\nHdN+mtJ+AgBGQVIDAB1z8b0pSQ0AMAqSGgDomDU1U4oaAOiY3U9T2k8AwChIagCgY9pPU5IaAGAU\nJDUA0DFbuqcUNQDQsWah8IT2EwAwCpIaAOiY9tOUpAYAGAVJDQB0zJbuKUkNADAKkhoA6JjdT1OK\nGgDomPbTlPYTADAKihoA6FhrbV6ObVFVh1XVF6tqbVUdN88/6lYpagCAH1hVLU7y1iSHJ9k3yXOq\nat+FnJOiBgA61ubp2AYHJFnbWru8tXZrklOSHHHX/FTbZ94XCm+4dX3N93cwf6pqVWtt9ULPA3Y1\n/u6xrebr92xVrUqyatbQ6jv8b3J5kitnPV+X5MD5mMu2ktSwNau2fgowD/zdY0G11la31h4967hj\nkT1XMbWgW7EUNQDA9liXZJ9Zz1ckuWqB5pJEUQMAbJ9zk6ysqgdW1dIkRyY5fSEn5OJ7bI2ePiwM\nf/fYqbXWNlTVsUnOTLI4yQmttYsWck7lSoQAwBhoPwEAo6CoAQBGQVHDnHa2S1/DrqKqTqiqr1fV\nhQs9F+iNooY72RkvfQ27kBOTHLbQk4AeKWqYy0536WvYVbTWPpPk+oWeB/RIUcNc5rr09fIFmgsA\nbBNFDXPZ6S59DQBbo6hhLjvdpa8BYGsUNcxlp7v0NQBsjaKGO2mtbUiy6dLXlyQ5daEvfQ27iqo6\nOclnk/xEVa2rqqMXek7QC7dJAABGQVIDAIyCogYAGAVFDQAwCooaAGAUFDUAwCgoagCAUVDUAACj\n8P8BT8lVBMLf+IcAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "acc = accuracy_score(y_true, y_pred)\n", + "print(\"The prediction accuracy is %.2f%%\"%(acc*100))\n", + "\n", + "cm = confusion_matrix(y_true, y_pred)\n", + "cm.shape\n", + "df_cm = pd.DataFrame(cm)\n", + "plt.figure(figsize = (10,8))\n", + "#sn.heatmap(df_cm, annot=True,fmt='d');" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/feedforward-iris-pipeline.ipynb b/notebooks/feedforward-iris-pipeline.ipynb new file mode 100644 index 0000000..b481de8 --- /dev/null +++ b/notebooks/feedforward-iris-pipeline.ipynb @@ -0,0 +1,525 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Feedforward Neural Network on the Iris Dataset\n", + "# Using Pipeline API\n", + "\n", + "Fisher's Iris dataset \n", + "\n", + "We are going to try the same Iris dataset as on feedforward-iris.ipynb, but this time using\n", + "Spark MLLib's Pipeline-based API.\n", + "\n", + "This dataset contains 150 samples, with 4 dimensions, as follows:\n", + "\n", + "1. Petal Length (c1)\n", + "2. Petal Width (c2)\n", + "3. Sepal Length (c3)\n", + "4. Sepal Width (c4)\n", + "\n", + "There are 3 output classes: Setosa, Versicolor, and Virginica.\n", + "In our output datset, we have simplified this data by making classes simply 1, 2, 3.\n", + "\n", + "Here's an example of what the dataset looks like\n", + "\n", + "| c1 | c2 | c3 | c4 | label | \n", + "|-----|-----|-----|-----|-------| \n", + "| 6.4 | 2.8 | 5.6 | 2.2 | 3 | \n", + "| 5.0 | 2.3 | 3.3 | 1.0 | 2 | \n", + "| 4.9 | 2.5 | 4.5 | 1.7 | 3 | \n", + "| 4.9 | 3.1 | 1.5 | 0.1 | 1 | \n", + "| 5.7 | 3.8 | 1.7 | 0.3 | 1 | \n", + "| 4.4 | 3.2 | 1.3 | 0.2 | 1 | \n", + "| 5.4 | 3.4 | 1.5 | 0.4 | 1 | \n", + "| 6.9 | 3.1 | 5.1 | 2.3 | 3 | \n", + "| 6.7 | 3.1 | 4.4 | 1.4 | 2 | \n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import pandas as pd\n", + "import datetime as dt\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from sklearn.metrics import confusion_matrix\n", + "from sklearn.metrics import accuracy_score\n", + "import seaborn as sn\n", + "import pandas as pd\n", + "import random as rd\n", + "import datetime as dt \n", + "\n", + "\n", + "\n", + "from bigdl.dataset.transformer import *\n", + "from bigdl.dataset.base import *\n", + "from bigdl.nn.layer import *\n", + "from bigdl.nn.criterion import *\n", + "from bigdl.optim.optimizer import *\n", + "from bigdl.util.common import *\n", + "from utils import *\n", + "from bigdl.models.ml_pipeline.dl_classifier import *\n", + "\n", + "from pyspark.sql.types import DoubleType\n", + "from pyspark.sql.functions import col, udf\n", + "from pyspark.ml import Pipeline\n", + "from pyspark.ml.feature import VectorAssembler, StandardScaler\n", + "from pyspark.ml.evaluation import BinaryClassificationEvaluator, MulticlassClassificationEvaluator\n", + "\n", + "\n", + "init_engine()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Parameters\n", + "\n", + "These are parameters that can be changed:\n", + "\n", + "Explanation:\n", + "\n", + "1. Learning Rate: This indicates the \"step size\" of the gradient descent-type optimization functions. The larger the step size, the less of a chance of getting caught in a local minimum but too large may lead to very poor results.\n", + "\n", + "2. Training Epochs: the number of cycles of training that we use.\n", + "\n", + "3. Batch size: The size of the batch (with replacement) that we take per training iteration.\n", + "\n", + "4. N_input: Number of input dimensions\n", + "\n", + "5. N_Classes: Number of output classes for classification problem.\n", + "\n", + "6. N_Hidden_1: Number of hidden layer neurons." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "learning_rate = 0.1\n", + "training_epochs = 100\n", + "batch_size = 16\n", + "display_step = 1\n", + "\n", + "# Network Parameters\n", + "n_input = 4\n", + "n_classes = 3\n", + "n_hidden = 3 # 1st layer number of features\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Sizing the Hidden Layer(s)\n", + "\n", + "Sizing hidden layers can be a challenge. The best way to figure this out is to do it\n", + "empirically. However, we may need a \"rule of thumb\" to start. Here is a good rule of thumb:\n", + "\n", + "First Hidden Layer:\n", + "```\n", + "n_hidden_1 = np.sqrt(np.sqrt((n_classes + 2) * n_input) + 2 * np.sqrt(n_input /(n_classes+2.)))\n", + "```\n", + "\n", + "Second Hidden Layer: (if needed)\n", + "```\n", + "n_hidden_2 = n_classes * np.sqrt(n_input / (n_classes + 2.))\n", + "```\n", + "\n", + "In this case, we have a **VERY** simple dataset. We may not need two hidden layers. Let's see what we have.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hidden layer 1 (Guess) : 2.50219710195\n", + "Hidden layer 2 (Guess) : 2.683281573\n" + ] + } + ], + "source": [ + "# Number of hidden layers\n", + "\n", + "n_hidden_guess = np.sqrt(np.sqrt((n_classes + 2) * n_input) + 2 * np.sqrt(n_input /(n_classes+2.)))\n", + "print(\"Hidden layer 1 (Guess) : \" + str(n_hidden_guess))\n", + "\n", + "n_hidden_guess_2 = n_classes * np.sqrt(n_input / (n_classes + 2.))\n", + "print(\"Hidden layer 2 (Guess) : \" + str(n_hidden_guess_2))\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Result\n", + "\n", + "Each hidden layer should around 2-3 input neurons." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "np.sqrt(np.sqrt((n_classes+2) * n_input) + 2 * np.sqrt(n_input /(n_classes+2)))\n", + "#and in the second hidden layer, the optimal number of hidden nodes is: m*sqrt[N/(m+2)],\n", + "n_classes * np.sqrt(n_input / (n_classes + 2.))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load the Data\n", + "\n", + "Let's load the data into Spark Dataframes, and we'll cast everything to be double." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "iris_training = spark.read.csv(\"../data/iris/iris_training.csv\", header=True, inferSchema=\"true\", mode=\"DROPMALFORMED\")\n", + "iris_test = spark.read.csv(\"../data/iris/iris_test.csv\", header=True, inferSchema=\"true\", mode=\"DROPMALFORMED\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "iris_training = iris_training.select([col(c).cast(\"double\") for c in iris_training.columns])\n", + "iris_test = iris_test.select([col(c).cast(\"double\") for c in iris_test.columns])\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setting up a Spark MLlib pipeline\n", + "\n", + "We will use a Spark MLLib pipeline to do two things:\n", + "\n", + "1. Extract a feature vector into a new column called \"assembled\" (using VectorAssembler).\n", + "2. Use StandardScaler to scale the vector values by mean and unit variance." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "assembler = VectorAssembler(inputCols=['c1','c2','c3','c4'], outputCol=\"assembled\")\n", + "scaler = StandardScaler(inputCol=\"assembled\", outputCol=\"features\")\n", + "pipeline = Pipeline(stages = [assembler, scaler])\n", + "pipelineTraining = pipeline.fit(iris_training)\n", + "iris_data_training = pipelineTraining.transform(iris_training)\n", + "pipelineTest = pipeline.fit(iris_test)\n", + "iris_data_test = pipelineTraining.transform(iris_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------+-----+\n", + "|features |label|\n", + "+----------------------------------------------------------------------------+-----+\n", + "|[7.3683612551017434,6.554983394668502,3.07337626655598,2.813157930275381] |3.0 |\n", + "|[5.7565322305482365,5.384450645620555,1.8110967285062027,1.2787081501251731]|2.0 |\n", + "|[5.641401585937273,5.8526637452397345,2.469677357053913,2.1738038552127943] |3.0 |\n", + "|[5.641401585937273,7.257303044097271,0.8232257856846377,0.12787081501251732]|1.0 |\n", + "|[6.56244674282499,8.896048892764396,0.9329892237759226,0.38361244503755193] |1.0 |\n", + "|[5.065748362882449,7.491409593906861,0.7134623475933526,0.25574163002503464]|1.0 |\n", + "|[6.2170548089920965,7.959622693526039,0.8232257856846377,0.5114832600500693]|1.0 |\n", + "|[7.944014478156568,7.257303044097271,2.798967671327768,2.941028745287898] |3.0 |\n", + "|[7.713753188934637,7.257303044097271,2.4147956380082705,1.7901914101752423] |2.0 |\n", + "|[5.871662875159201,8.661942342954807,0.8232257856846377,0.5114832600500693] |1.0 |\n", + "+----------------------------------------------------------------------------+-----+\n", + "only showing top 10 rows\n", + "\n" + ] + } + ], + "source": [ + "iris_data_training.select('features', 'label').show(10, False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Use the Pipeline API to set up neural network classifier\n", + "\n", + "We will now use the Pipeline API to set up a neural network classifier. The parameters and results are very similar to the previous example, but this time around we are using the pipeline API.\n", + "\n", + "This is totally integrated with our existing Spark ML workflows! " + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "creating: createSequential\n", + "creating: createLinear\n", + "creating: createLinear\n", + "creating: createLogSoftMax\n", + "creating: createClassNLLCriterion\n", + "creating: createDLClassifier\n", + "\n", + "initial model training finished.\n" + ] + } + ], + "source": [ + "\n", + "bigDLModel = Sequential().add(Linear(n_input, n_hidden)).add(Linear(n_hidden, n_classes)).add(LogSoftMax())\n", + "classnll_criterion = ClassNLLCriterion()\n", + "dlClassifier = DLClassifier(model=bigDLModel, criterion=classnll_criterion, feature_size=[n_input])\n", + "dlClassifier.setLabelCol(\"label\").setMaxEpoch(training_epochs).setBatchSize(16).setLearningRate(learning_rate)\n", + "model = dlClassifier.fit(iris_data_training)\n", + "print(\"\\ninitial model training finished.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DataFrame[c1: double, c2: double, c3: double, c4: double, label: double, assembled: vector, features: vector, prediction: double]" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Coerce the outpu to be a Dataframe object -- return object is a JavaObject by defaul.\n", + "\n", + "from pyspark.sql import DataFrame, SQLContext\n", + "predictionDF = DataFrame(model.transform(iris_data_test), SQLContext(sc))\n", + "predictionDF" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+---+---+---+---+-----+-----------------+--------------------+----------+\n", + "| c1| c2| c3| c4|label| assembled| features|prediction|\n", + "+---+---+---+---+-----+-----------------+--------------------+----------+\n", + "|5.9|3.0|4.2|1.5| 2.0|[5.9,3.0,4.2,1.5]|[6.79270803204692...| 2.0|\n", + "|6.9|3.1|5.4|2.1| 3.0|[6.9,3.1,5.4,2.1]|[7.94401447815656...| 3.0|\n", + "|5.1|3.3|1.7|0.5| 1.0|[5.1,3.3,1.7,0.5]|[5.87166287515920...| 1.0|\n", + "|6.0|3.4|4.5|1.6| 2.0|[6.0,3.4,4.5,1.6]|[6.90783867665788...| 2.0|\n", + "|5.5|2.5|4.0|1.3| 2.0|[5.5,2.5,4.0,1.3]|[6.33218545360306...| 2.0|\n", + "|6.2|2.9|4.3|1.3| 2.0|[6.2,2.9,4.3,1.3]|[7.13809996587981...| 2.0|\n", + "|5.5|4.2|1.4|0.2| 1.0|[5.5,4.2,1.4,0.2]|[6.33218545360306...| 1.0|\n", + "|6.3|2.8|5.1|1.5| 3.0|[6.3,2.8,5.1,1.5]|[7.25323061049077...| 2.0|\n", + "|5.6|3.0|4.1|1.3| 2.0|[5.6,3.0,4.1,1.3]|[6.44731609821402...| 2.0|\n", + "|6.7|2.5|5.8|1.8| 3.0|[6.7,2.5,5.8,1.8]|[7.71375318893463...| 3.0|\n", + "|7.1|3.0|5.9|2.1| 3.0|[7.1,3.0,5.9,2.1]|[8.17427576737849...| 3.0|\n", + "|4.3|3.0|1.1|0.1| 1.0|[4.3,3.0,1.1,0.1]|[4.95061771827148...| 1.0|\n", + "|5.6|2.8|4.9|2.0| 3.0|[5.6,2.8,4.9,2.0]|[6.44731609821402...| 3.0|\n", + "|5.5|2.3|4.0|1.3| 2.0|[5.5,2.3,4.0,1.3]|[6.33218545360306...| 2.0|\n", + "|6.0|2.2|4.0|1.0| 2.0|[6.0,2.2,4.0,1.0]|[6.90783867665788...| 2.0|\n", + "|5.1|3.5|1.4|0.2| 1.0|[5.1,3.5,1.4,0.2]|[5.87166287515920...| 1.0|\n", + "|5.7|2.6|3.5|1.0| 2.0|[5.7,2.6,3.5,1.0]|[6.56244674282499...| 2.0|\n", + "|4.8|3.4|1.9|0.2| 1.0|[4.8,3.4,1.9,0.2]|[5.52627094132630...| 1.0|\n", + "|5.1|3.4|1.5|0.2| 1.0|[5.1,3.4,1.5,0.2]|[5.87166287515920...| 1.0|\n", + "|5.7|2.5|5.0|2.0| 3.0|[5.7,2.5,5.0,2.0]|[6.56244674282499...| 3.0|\n", + "+---+---+---+---+-----+-----------------+--------------------+----------+\n", + "only showing top 20 rows\n", + "\n" + ] + } + ], + "source": [ + "predictionDF.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Evaluate the Model\n", + "\n", + "Calculate Precision, Recall and the Area Under the Precision / Recall curve.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Area under precision-recall curve: = 1.0\n", + "\n", + "recall = 0.933333333333\n", + "\n", + "Precision = 0.933333333333\n" + ] + }, + { + "data": { + "text/plain": [ + "DataFrame[c1: double, c2: double, c3: double, c4: double, label: double, assembled: vector, features: vector, prediction: double]" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "predictionDF.cache()\n", + "evaluator = BinaryClassificationEvaluator(rawPredictionCol=\"prediction\")\n", + "auPRC = evaluator.evaluate(predictionDF)\n", + "print(\"\\nArea under precision-recall curve: = \" + str(auPRC))\n", + " \n", + "recall = MulticlassClassificationEvaluator(metricName=\"weightedRecall\").evaluate(predictionDF)\n", + "print(\"\\nrecall = \" + str(recall))\n", + "\n", + "precision = MulticlassClassificationEvaluator(metricName=\"weightedPrecision\").evaluate(predictionDF)\n", + "print(\"\\nPrecision = \" + str(precision))\n", + "predictionDF.unpersist()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## More evaluations: Accuracy and Confusion Matrix\n", + "\n", + "Use predict on the validation dataset, and then calcualte the accuracy and the\n", + "confusion matrix." + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "y_pred = np.array(predictionDF.select('prediction').collect())\n", + "y_true = np.array(predictionDF.select('label').collect())" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The prediction accuracy is 93.33%\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAHVCAYAAADSAqClAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGuRJREFUeJzt3XuwZWV5J+Df29BgAiReKIVuesQE\nNJg4ygxiHGcUQwlG0cZcBDMKcZh04pgMVE2ZmIylFWeS0krFKYNWSGckaBKMRCVgRANhNKCDSssQ\nw8ULCIG+CEEENCED3eebP3rLHJvTt0OfXuvr9TxVq87ea69z1ntwy3n5fZddrbUAAIzZsqELAADY\nGQ0LADB6GhYAYPQ0LADA6GlYAIDR07AAAKOnYQEARk/DAgCMnoYFABi9/Zf6Bvec/CJb6bJHHfap\nW4YuAWCHNj+0ofbWvR6+5+tL8nd2+aE/tNd+h10hYQEARm/JExYAYAnNbRm6gr1CwgIAjJ6EBQB6\n1uaGrmCvkLAAAKMnYQGAns1NI2HRsABAx5ohIQCAcZCwAEDPJjIkJGEBAEZPwgIAPZvIHBYNCwD0\nzE63AAALq6rzq+ruqrph3rnfqaovV9WXquriqnr8dr739qr6u6q6vqrW7cr9NCwA0LM2tzTHzl2Q\n5KXbnLsiyY+11v5lkq8m+fUdfP+LW2vPaa0dtys307AAALuttXZVknu3OXd5a23z7Onnkhyxp+5n\nDgsA9Gy8y5r/Q5IPbee1luTyqmpJ/qC1tnZnP0zDAgAdW6qdbqtqTZI1806t3ZXGYva9/zXJ5iR/\nup1LXtBa21hVT05yRVV9eZbYbJeGBQB4lFlzsksNynxVdWaSU5Kc2Fpr2/nZG2df766qi5Mcn0TD\nAgD7rBENCVXVS5P8WpIXtdb+aTvXHJRkWWvt27PHJyV5+85+tkm3AMBuq6oPJrkmyTOqan1VnZXk\nPUkOydZhnuur6rzZtSuq6rLZtz4lyWeq6m+TfCHJx1trn9zZ/SQsANCzgXa6ba29ZoHT79vOtRuT\nvGz2+OtJnr2795OwAACjJ2EBgJ5NZGt+DQsA9GwiH35oSAgAGD0JCwD0bETLmpeShAUAGD0JCwD0\nbCJzWDQsANAzQ0IAAOMgYQGAjrU2jX1YJCwAwOhJWACgZybdAgCjZ9ItAMA4SFgAoGcTGRKSsAAA\noydhAYCezU1jWbOGBQB6ZkgIAGAcJCwA0DPLmgEAxkHCAgA9M4cFAGAcJCwA0LOJzGHRsABAzybS\nsBgSAgBGT8ICAB1rbRo73UpYAIDRk7AAQM8mModFwwIAPbMPCwDAOEhYAKBnExkSkrAAAKMnYQGA\nnk1kDouGBQB6ZkgIAGAcJCwA0LOJDAlJWACA0ZOwAEDPzGEBABgHCQsA9GwiCYuGBQB6ZtItAMA4\nSFgAoGcTGRKSsAAAoydhAYCemcPC3vK4V/1sHr/2gjz+D/4oh7z5rcnyA4Yuic6dfNIJufGGq/Ll\nmz6TX33TG4cuh32A99SIzc0tzTEyGpaBLXvSofm+U3869/3ymtz3i69P9luWA0/4iaHLomPLli3L\n7737t3LKK16bZz37xTnttFNzzDFHD10WHfOeYgx2OiRUVT+SZHWSlUlako1JLm2t3bzEtU3Hfvul\nDjwwbfOW1IEHZu6b9wxdER07/rnH5tZbb89tt92RJLnookvyylecnJtv/trAldEr76mRMySUVNWv\nJfmzJJXkC0munT3+YFW9eenL2/fNffOePPjhP8sT//iiPPGDH83cP/5jHr5u3dBl0bEVKw/Lnes3\nPvJ8/YZNWbHisAEronfeU4zBzoaEzkry3NbaO1prfzI73pHk+NlrC6qqNVW1rqrWfWD9pj1Z7z6n\nDj44Bzz/3+beM0/PvT/3U6nHPS4H/sRLhi6LjlXVo8611gaohH2F99TImcOSJJlLsmKB84fPXltQ\na21ta+241tpxZxxx+GOpb5+3/NjjMveNTWn3359s2ZKHPnt19n/mjw1dFh3bsH5TVh3x//9ve8TK\nw7Np010DVkTvvKdGTsOSJDknyZVV9YmqWjs7PpnkyiRnL315+765u+/K/sc8MznwwCTJ8uf8q2y5\n4+8HroqeXbvu+hx11NNy5JGrsnz58rz61avzsb+8fOiy6Jj3FGOww0m3rbVPVtXTs3UIaGW2zl9Z\nn+Ta1tqWvVDfPm/zV27OQ1f/TR7/3j9MtmzJ5ltuyT9/4mNDl0XHtmzZkrPPeUsu+/iF2W/Zslzw\n/g/lppu+OnRZdMx7auQmMjxXSz0Oec/JL5rGP0n2msM+dcvQJQDs0OaHNjx64s8SefBDv7kkf2e/\n77S37bXfYVfY6RYAejbC+SZLwcZxAMDoSVgAoGcTSVg0LADQMzvdAgCMg4QFAHo2kSEhCQsAMHoS\nFgDo2UQ2jtOwAEDPDAkBAIyDhAUAeiZhAQAYBwkLAPRsIhvHaVgAoGNtbhqrhAwJAQCjJ2EBgJ6Z\ndAsAMA4SFgDo2UQm3UpYAIDRk7AAQM+sEgIARm9ubmmOnaiq86vq7qq6Yd65J1bVFVX1tdnXJ2zn\ne8+cXfO1qjpzV35NDQsAsBgXJHnpNufenOTK1trRSa6cPf8eVfXEJG9L8rwkxyd52/Yam/k0LADQ\ns4ESltbaVUnu3eb06iTvnz1+f5JTF/jWk5Nc0Vq7t7X2rSRX5NGNz6NoWACAR6mqNVW1bt6xZhe+\n7SmttU1JMvv65AWuWZnkznnP18/O7ZBJtwDQs7Y0k25ba2uTrF2CH10L3W5n3yRhAYCeDTQktB13\nVdXhSTL7evcC16xPsmre8yOSbNzZD9awAAB7yqVJvrvq58wklyxwzV8lOamqnjCbbHvS7NwOaVgA\noGdzbWmOnaiqDya5Jskzqmp9VZ2V5B1JXlJVX0vyktnzVNVxVfU/k6S1dm+S/5bk2tnx9tm5HTKH\nBQDYba2112znpRMXuHZdkv847/n5Sc7fnftpWACgZxP5LCENCwD0zNb8AADjIGEBgI61xS9B7oqE\nBQAYPQkLAPTMHBYAgHGQsABAzyxrBgBGz5AQAMA4SFgAoGeWNQMAjIOEBQB6NpE5LBoWAOjZRFYJ\nGRICAEZPwgIAPZvIkJCEBQAYPQkLAHRsKp/WrGEBgJ4ZEgIAGAcJCwD0TMICADAOEhYA6JmN4wAA\nxkHCAgA9m8gcFg0LAHSsTaRhMSQEAIyehAUAeiZhAQAYBwkLAPTMZwkBAKNnSAgAYBwkLADQMwkL\nAMA4SFgAoGOtTSNh0bAAQM8MCQEAjIOEBQB6NpGEZckblsM+dctS34KJeXDj1UOXwD7kh5++eugS\ngF0gYQGAjvm0ZgCAkZCwAEDPJpKwaFgAoGfT+OxDQ0IAwPhJWACgYybdAgCMhIQFAHo2kYRFwwIA\nPTPpFgBgHCQsANAxk24BAEZCwgIAPZvIHBYNCwB0zJAQAMBISFgAoGcTGRKSsAAAoydhAYCOtYkk\nLBoWAOjZRBoWQ0IAwOhJWACgY1MZEpKwAACjJ2EBgJ5JWAAAxkHCAgAdm8ocFg0LAHRsKg2LISEA\nYPQkLADQMQkLAMBISFgAoGethq5gr9CwAEDHDAkBAIyEhAUAOtbmpjEkJGEBAEZPwgIAHZvKHBYN\nCwB0rE1klZAhIQBgt1XVM6rq+nnHA1V1zjbXnFBV98+75q2LvZ+EBQA6NtSQUGvtK0mekyRVtV+S\nDUkuXuDSq1trpzzW+0lYAIDH6sQkt7bW/n6pbqBhAYCOtblakqOq1lTVunnHmh2UcXqSD27ntedX\n1d9W1Seq6kcX+3saEgIAHqW1tjbJ2p1dV1UHJHllkl9f4OXrkjy1tfadqnpZkr9IcvRi6pGwAEDH\nWluaYzf8ZJLrWmt3Pbq29kBr7Tuzx5clWV5Vhy7m95SwAEDHRrDT7WuyneGgqjosyV2ttVZVx2dr\nUPLNxdxEwwIALEpVfX+SlyT5xXnnfilJWmvnJfmZJG+oqs1JHkxyemu7md/MaFgAoGNDJiyttX9K\n8qRtzp037/F7krxnT9zLHBYAYPQkLADQscUNsPRHwwIAHRvBpNu9wpAQADB6EhYA6JhPawYAGAkJ\nCwB0bKhPa97bNCwA0LE5Q0IAAOMgYQGAjpl0CwAwEhIWAOiYjeMAAEZCwgIAHfNZQgDA6BkSAgAY\nCQkLAHTMxnEAACMhYQGAjk1l4zgNCwB0bCqrhAwJAQCjJ2EBgI6ZdAsAMBIalhE4+aQTcuMNV+XL\nN30mv/qmNw5dDh16y2+/Ky98+ek59bW/9Mi5c9d+IK864w356TPfmF845zdy9z98c8AK6dnvnPv2\nXPeVT+eKz3506FJYQGu1JMfYaFgGtmzZsvzeu38rp7zitXnWs1+c0047Ncccc/TQZdGZU1/2kpz3\nrv/+Pede/+9/Ohd/4Pfzkfe/Ny96wfPy+3904UDV0bs/v/CSnPGzbxi6DLajtaU5xkbDMrDjn3ts\nbr319tx22x15+OGHc9FFl+SVrzh56LLozHHPeVZ+8AcO+Z5zBx900COPH3zwn1Pj+w8mOvGFa76Y\n+751/9BlMHEm3Q5sxcrDcuf6jY88X79hU45/7rEDVsS+5N1/cEEu/eSVOeSgg3L+ue8YuhxgCZh0\nuxNV9fodvLamqtZV1bq5uX9c7C0moRb4z942xiyOLp39iz+fKy/+47z8pBfnwo98bOhyABbtsQwJ\n/eb2XmitrW2tHddaO27ZsoO2dxlJNqzflFVHrHjk+RErD8+mTXcNWBH7opefdEL++tOfHboMYAlM\nZdLtDoeEqupL23spyVP2fDnTc+2663PUUU/LkUeuyoYN38irX706rzvDSiEeu7+/c0OeumplkuRT\nV38uT3vqEQNXBLB4O5vD8pQkJyf51jbnK8n/XpKKJmbLli05+5y35LKPX5j9li3LBe//UG666atD\nl0Vn3vS2d+Ta//Ol3HffAznx1NfmP531ulx9zbW5/Y71qWWVFYc9OW99068MXSadOvcP35nnv+C5\necKTHp/P3/DXedc73psP/cnFQ5fFzFTmsNSO5ktU1fuS/FFr7TMLvHZha+3ndnaD/Q9YaUIGe9SD\nG68eugT2IT/89NVDl8A+6I57/26vdRGfW/FTS/J39sc3fnRUndAOE5bW2lk7eG2nzQoAwJ5gWTMA\ndGwqQ0I2jgMARk/CAgAdG+MS5KWgYQGAjs0NXcBeYkgIABg9CQsAdKxlGkNCEhYAYPQkLADQsbmJ\nbM+qYQGAjs0ZEgIAGAcJCwB0zKRbAICRkLAAQMdsHAcAMBISFgDo2FTmsGhYAKBjhoQAAEZCwgIA\nHZOwAACMhIQFADpm0i0AMHpz0+hXDAkBAOMnYQGAjvm0ZgCAkZCwAEDH2tAF7CUaFgDomH1YAABG\nQsICAB2bK5NuAQBGQcICAB2byqRbCQsAMHoSFgDo2FRWCWlYAKBjPksIAGAkJCwA0DGfJQQAMBIS\nFgDo2FSWNWtYAKBjJt0CAIyEhAUAOjaVfVgkLADA6ElYAKBjQ066rarbk3w7yZYkm1trx23zeiV5\nd5KXJfmnJD/fWrtuMffSsABAx0Yw6fbFrbV7tvPaTyY5enY8L8nvz77uNkNCAMBSWZ3kA22rzyV5\nfFUdvpgfpGEBgI7NLdGxi1qSy6vqi1W1ZoHXVya5c97z9bNzu82QEADwKLMGZH4Tsra1tnaby17Q\nWttYVU9OckVVfbm1dtX8H7PAj17UtBsNCwB0bKmWNc+ak20blG2v2Tj7endVXZzk+CTzG5b1SVbN\ne35Eko2LqceQEACw26rqoKo65LuPk5yU5IZtLrs0yRm11Y8nub+1tmkx95OwAEDH2nCrhJ6S5OKt\nK5ezf5ILW2ufrKpfSpLW2nlJLsvWJc23ZOuy5tcv9mYaFgDo2FA73bbWvp7k2QucP2/e45bkjXvi\nfoaEAIDRk7AAQMd8lhAAwEhIWACgY0N+ltDepGEBgI6N4LOE9gpDQgDA6ElYAKBjJt0CAIyEhAUA\nOjaVhEXDAgAdm8oqIUNCAMDoSVgAoGOWNQMAjISEBQA6NpVJtxIWAGD0JCwA0LGprBLSsNCdH376\n6qFLYB9y+aGrhi4BHpO5ibQshoQAgNGTsABAx0y6BQAYCQkLAHRsGjNYNCwA0DVDQgAAIyFhAYCO\n+SwhAICRkLAAQMemsnGchgUAOjaNdsWQEADQAQkLAHTMsmYAgJGQsABAx0y6BQBGbxrtiiEhAKAD\nEhYA6JhJtwAAIyFhAYCOTWXSrYQFABg9CQsAdGwa+YqGBQC6ZtItAMBISFgAoGNtIoNCEhYAYPQk\nLADQsanMYdGwAEDH7MMCADASEhYA6Ng08hUJCwDQAQkLAHRsKnNYNCwA0LGprBIyJAQAjJ6EBQA6\nZqdbAICRkLAAQMfMYQEAGAkJCwB0bCpzWDQsANAxQ0IAACMhYQGAjs21aQwJSVgAgNGTsABAx6aR\nr2hYAKBrU/nwQ0NCAMDoSVgAoGNT2YdFwgIAjJ6EBQA6NpWN4zQsANAxk24BAEZCwgIAHTPpFgBg\nJCQsANCxqUy6lbAAAKMnYQGAjrWJfFqzhgUAOmZZMwDASEhYAKBjJt0CAGxHVa2qqk9V1c1VdWNV\nnb3ANSdU1f1Vdf3seOti7ydhAYCODbhx3OYk/6W1dl1VHZLki1V1RWvtpm2uu7q1dspjvZmGBQA6\nNtSk29bapiSbZo+/XVU3J1mZZNuGZY8wJAQAPEpVramqdfOONTu49sgkxyb5/AIvP7+q/raqPlFV\nP7rYeiQsANCxpdqHpbW2NsnanV1XVQcn+UiSc1prD2zz8nVJntpa+05VvSzJXyQ5ejH1SFgAgEWp\nquXZ2qz8aWvto9u+3lp7oLX2ndnjy5Isr6pDF3MvCQsAdGyoZc1VVUnel+Tm1tq7tnPNYUnuaq21\nqjo+W4OSby7mfhoWAOjYgKuEXpDkdUn+rqqun537jST/Iklaa+cl+Zkkb6iqzUkeTHJ6W+QYloYF\nANhtrbXPJKmdXPOeJO/ZE/fTsABAx3yWEHvNySedkBtvuCpfvukz+dU3vXHocujc75z79lz3lU/n\nis8+av4bLMoBT1uZIy8595Hj6Os+nCecuXrospgYDcvAli1blt9792/llFe8Ns969otz2mmn5phj\nFrXiC5Ikf37hJTnjZ98wdBnsQx66bUNuX/0rW49XnZ324D/n21dcM3RZzLTWluQYm502LFX1I1V1\n4myd9fzzL126sqbj+Ocem1tvvT233XZHHn744Vx00SV55StOHrosOvaFa76Y+751/9BlsI/6/uc/\nOw/d8Y1s3nj30KUwMTtsWKrqPye5JMmvJLmhquZngL+9lIVNxYqVh+XO9Rsfeb5+w6asWHHYgBUB\nbN8PvPxFeeDjnx66DOaZS1uSY2x2Nun2F5L869kOdUcm+XBVHdlae3d2MDN4tn3vmiSp/X4wy5Yd\ntIfK3fdsXcb+vcYYxQFk+f45+MTn5R9+94KhK2GeAZc171U7a1j2m7dD3e1VdUK2Ni1PzQ4alvnb\n+e5/wMpp/JNcpA3rN2XVESseeX7EysOzadNdA1YEsLCDX3hc/u+Nt2bLN+8buhQmaGdzWL5RVc/5\n7pNZ83JKkkOTPGspC5uKa9ddn6OOelqOPHJVli9fnle/enU+9peXD10WwKP8wCkvygN/+TdDl8E2\n5lpbkmNsdtawnJHkG/NPtNY2t9bOSPLCJatqQrZs2ZKzz3lLLvv4hbnhS5/Ohz/8sdx001eHLouO\nnfuH78xf/NWf5IeOOjKfv+Gvc9prXzV0SewD6nEH5qB/c2y+fflnhy6Fiaqlni9hSIg9bcXBTxy6\nBPYhlx+6augS2Af9yFcv2+EOsHvSv1t54pL8nb16w5V77XfYFXa6BYCOjXFFz1KwcRwAMHoSFgDo\nmIQFAGAkJCwA0LGpbDaqYQGAjhkSAgAYCQkLAHRsKp8lJGEBAEZPwgIAHZvKpFsJCwAwehIWAOjY\nVFYJaVgAoGOGhAAARkLCAgAdm8qQkIQFABg9CQsAdGwqG8dpWACgY3Mm3QIAjIOEBQA6NpUhIQkL\nADB6EhYA6NhU5rBoWACgY4aEAABGQsICAB2bypCQhAUAGD0JCwB0zBwWAICRkLAAQMemModFwwIA\nHTMkBAAwEhIWAOhYa3NDl7BXSFgAgNGTsABAx+YmModFwwIAHWsTWSVkSAgAGD0JCwB0bCpDQhIW\nAGD0JCwA0LGpzGHRsABAx6ayNb8hIQBg9CQsANAxnyUEADASEhYA6NhUJt1KWACA0ZOwAEDHprJx\nnIYFADpmSAgAYCQkLADQMRvHAQCMhIQFADo2lTksGhYA6NhUVgkZEgIARk/CAgAdm8qQkIQFABg9\nCQsAdGwqy5o1LADQsWbSLQDAOEhYAKBjUxkSkrAAAKMnYQGAjlnWDAAwEhIWAOjYVFYJaVgAoGOG\nhAAARkLDAgAda60tybErquqlVfWVqrqlqt68wOsHVtWHZq9/vqqOXOzvqWEBAHZbVe2X5L1JfjLJ\nM5O8pqqeuc1lZyX5VmvtqCT/I8k7F3s/DQsAdKwt0bELjk9yS2vt6621h5L8WZLV21yzOsn7Z48/\nnOTEqqrd/y33wqTbzQ9tWFRhU1RVa1pra4eug32D9xN7mvfUOC3V39mqWpNkzbxTa7f5339lkjvn\nPV+f5Hnb/JhHrmmtba6q+5M8Kck9u1uPhGVc1uz8Ethl3k/sad5TE9JaW9taO27esW2zulCjtG04\nsyvX7BINCwCwGOuTrJr3/IgkG7d3TVXtn+QHk9y7mJtpWACAxbg2ydFV9bSqOiDJ6Uku3eaaS5Oc\nOXv8M0n+V1vkxjE2jhsXY8PsSd5P7GneUzxiNifll5P8VZL9kpzfWruxqt6eZF1r7dIk70vyx1V1\nS7YmK6cv9n41lR3yAIB+GRICAEZPwwIAjJ6GZQR2trUx7I6qOr+q7q6qG4auhX1DVa2qqk9V1c1V\ndWNVnT10TUyPOSwDm21t/NUkL8nW5V/XJnlNa+2mQQujW1X1wiTfSfKB1tqPDV0P/auqw5Mc3lq7\nrqoOSfLFJKf69xR7k4RleLuytTHsstbaVVnkPgewkNbaptbadbPH305yc7buYAp7jYZleAttbexf\nBMAozT5t99gknx+2EqZGwzK8PbZtMcBSqqqDk3wkyTmttQeGrodp0bAMb1e2NgYYVFUtz9Zm5U9b\nax8duh6mR8MyvF3Z2hhgMFVV2bpj6c2ttXcNXQ/TpGEZWGttc5Lvbm18c5KLWms3DlsVPauqDya5\nJskzqmp9VZ01dE107wVJXpfkJ6rq+tnxsqGLYlosawYARk/CAgCMnoYFABg9DQsAMHoaFgBg9DQs\nAMDoaVgAgNHTsAAAo/f/AM8FR00mEaaaAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "acc = accuracy_score(y_true, y_pred)\n", + "print(\"The prediction accuracy is %.2f%%\"%(acc*100))\n", + "\n", + "cm = confusion_matrix(y_true, y_pred)\n", + "cm.shape\n", + "df_cm = pd.DataFrame(cm)\n", + "plt.figure(figsize = (10,8))\n", + "sn.heatmap(df_cm, annot=True,fmt='d');" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/feedforward-iris.ipynb b/notebooks/feedforward-iris.ipynb new file mode 100644 index 0000000..7872678 --- /dev/null +++ b/notebooks/feedforward-iris.ipynb @@ -0,0 +1,702 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Feedforward Neural Network on the Iris Dataset\n", + "\n", + "Fisher's Iris dataset \n", + "\n", + "This dataset contains 150 samples, with 4 dimensions, as follows:\n", + "\n", + "1. Petal Length (c1)\n", + "2. Petal Width (c2)\n", + "3. Sepal Length (c3)\n", + "4. Sepal Width (c4)\n", + "\n", + "There are 3 output classes: Setosa, Versicolor, and Virginica.\n", + "In our output datset, we have simplified this data by making classes simply 1, 2, 3.\n", + "\n", + "Here's an example of what the dataset looks like\n", + "\n", + "| c1 | c2 | c3 | c4 | label | \n", + "|-----|-----|-----|-----|-------| \n", + "| 6.4 | 2.8 | 5.6 | 2.2 | 3 | \n", + "| 5.0 | 2.3 | 3.3 | 1.0 | 2 | \n", + "| 4.9 | 2.5 | 4.5 | 1.7 | 3 | \n", + "| 4.9 | 3.1 | 1.5 | 0.1 | 1 | \n", + "| 5.7 | 3.8 | 1.7 | 0.3 | 1 | \n", + "| 4.4 | 3.2 | 1.3 | 0.2 | 1 | \n", + "| 5.4 | 3.4 | 1.5 | 0.4 | 1 | \n", + "| 6.9 | 3.1 | 5.1 | 2.3 | 3 | \n", + "| 6.7 | 3.1 | 4.4 | 1.4 | 2 | \n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import pandas as pd\n", + "import datetime as dt\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from sklearn.metrics import confusion_matrix\n", + "from sklearn.metrics import accuracy_score\n", + "from sklearn.metrics import roc_curve, auc\n", + "\n", + "\n", + "import seaborn as sn\n", + "import pandas as pd\n", + "import random as rd\n", + "import datetime as dt\n", + "\n", + "from bigdl.dataset.transformer import *\n", + "from bigdl.dataset.base import *\n", + "from bigdl.nn.layer import *\n", + "from bigdl.nn.criterion import *\n", + "from bigdl.optim.optimizer import *\n", + "from bigdl.util.common import *\n", + "from utils import *\n", + "\n", + "from pyspark.sql.functions import col\n", + "\n", + "LABELS = [\"Setosa\", \"Virginica\", \"Versicolor\"]\n", + "\n", + "\n", + "init_engine()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Parameters\n", + "\n", + "These are parameters that can be changed:\n", + "\n", + "Explanation:\n", + "\n", + "1. Learning Rate: This indicates the \"step size\" of the gradient descent-type optimization functions. The larger the step size, the less of a chance of getting caught in a local minimum but too large may lead to very poor results.\n", + "\n", + "2. Training Epochs: the number of cycles of training that we use.\n", + "\n", + "3. Batch size: The size of the batch (with replacement) that we take per training iteration.\n", + "\n", + "4. N_input: Number of input dimensions\n", + "\n", + "5. N_Classes: Number of output classes for classification problem.\n", + "\n", + "6. N_Hidden_1: Number of hidden layer neurons." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "learning_rate = 0.1\n", + "training_epochs = 20\n", + "batch_size = 16\n", + "display_step = 1\n", + "\n", + "# Network Parameters\n", + "n_input = 4 # Number of input dimensions\n", + "n_classes = 3 # Number of output classes (Setosa, Virginica, Versicolor)\n", + "n_hidden_1 = 3 # Hidden layer number of features\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## A Simple Neural Network\n", + "\n", + "Here's a picture of a simple neural network, like what we have in this example:\n", + "\n", + "\n", + "\n", + "As you can see, we have a total of 3 layers:\n", + "\n", + "1. Input layer (sized as number of features -- in this case 4)\n", + "2. Hidden Layer (size we have to specify as part of the model).\n", + "3. Output Layer (Number of output classes we are trying to classify -- in this case 3)\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Sizing the Hidden Layer(s)\n", + "\n", + "Sizing hidden layers can be a challenge. The best way to figure this out is to do it\n", + "empirically. However, we may need a \"rule of thumb\" to start. Here is a good rule of thumb:\n", + "\n", + "First Hidden Layer:\n", + "```\n", + "n_hidden_1 = np.sqrt(np.sqrt((n_classes + 2) * n_input) + 2 * np.sqrt(n_input /(n_classes+2.)))\n", + "```\n", + "\n", + "Second Hidden Layer: (if needed)\n", + "```\n", + "n_hidden_2 = n_classes * np.sqrt(n_input / (n_classes + 2.))\n", + "```\n", + "\n", + "In this case, we have a **VERY** simple dataset. We may not need two hidden layers. Let's see what we have." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hidden layer 1 (Guess) : 2.50219710195\n", + "Hidden layer 2 (Guess) : 2.683281573\n" + ] + } + ], + "source": [ + "# Number of hidden layers\n", + "\n", + "n_hidden_guess = np.sqrt(np.sqrt((n_classes + 2) * n_input) + 2 * np.sqrt(n_input /(n_classes+2.)))\n", + "print(\"Hidden layer 1 (Guess) : \" + str(n_hidden_guess))\n", + "\n", + "n_hidden_guess_2 = n_classes * np.sqrt(n_input / (n_classes + 2.))\n", + "print(\"Hidden layer 2 (Guess) : \" + str(n_hidden_guess_2))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading the Data\n", + "\n", + "Let's use Spark Dataframes to load the data. We can take advantage of the header and infer the schema." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "iris_training = spark.read.csv(\"../data/iris/iris_training.csv\", header=True, inferSchema=\"true\", mode=\"DROPMALFORMED\")\n", + "iris_test = spark.read.csv(\"../data/iris/iris_test.csv\", header=True, inferSchema=\"true\", mode=\"DROPMALFORMED\")\n", + "\n", + "#Force everything to be of type double\n", + "iris_training = iris_training.select([col(c).cast(\"double\") for c in iris_training.columns])\n", + "iris_test = iris_test.select([col(c).cast(\"double\") for c in iris_test.columns])\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "iris_k_train = iris_training.rdd.map(list)\n", + "iris_k_test = iris_test.rdd.map(list)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Examine Training and Validation Class Distribution\n", + "\n", + "Let's look at the training class distribution. Are the different input classes relatively balanced?" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAG3JJREFUeJzt3XmcVOWd7/HPV0DBgCKhVQQdXHCL\nC2LLmJdOYtR4NWrUiTPGyXgxUUnm6muicYzGqwazmsxEkzg3URIX4opi3IjGHRNmItoqIgajGVyC\nEGjjAihRgd/94zytRdlVfbrtU0Vzvu/Xq159luec8zt1uupXz/OcRRGBmZmV13rNDsDMzJrLicDM\nrOScCMzMSs6JwMys5JwIzMxKzonAzKzknAis10nqJ2m5pK16s2xvk7SdpKadPy3pREkz0nCvvg+S\nzpV0SRru1f2UtI2k5b21Pms+JwIjfQF1vFZLWlEx/rnuri8iVkXE4Ih4sTfL9oSkHSVNk/QXSa9J\nmi3pVElr1f9+3vdB0oGSns+xvm9GxJd6IzZJCyTtV7Hu+RExuDfWbWuHterDYM2RvoAGpw/3i8Dh\nFdOuqS4vqX/jo+w+SWOAh4D5wC4RMRQ4FvgosGEzYytSXzk+tvZwIrAuSfqWpKmSrpO0DPhnSR+V\n9FD6lb1I0o8lDUjl+0sKSaPT+NVp/p2Slkn6naStu1s2zT9E0jOSXpd0saT/knR8jdC/CTwYEV+N\niEUAETEvIo6JiPc1baSmmnlpu/8j6cSKeZtKuiPt7yuSflMx72xJCyUtlfR05a/nqvW3SJqeyj0E\nVO5X9ftwWEUsCySdJmlj4HZgq4oa26Y1js+3JF1Ztf2TUpwLJZ1WMf1qSZMqxt+tdUi6DtgCuDNt\n7yvVTU2SRqX9ekXSs5K+UDHvWymuq9O+zJU0rsbxsiZxIrC8jgKuBTYGpgIrgS8Dw4F9gIOBL9ZZ\n/p+Ac4FhZLWOb3a3rKRNgRuAM9J2nwPG11nPgcC0+ru1hsXAocBGwEnAxZJ2S/POIKtZtACbp/iQ\n9BGy/R4XERsBh6SYO/NTYFlafiLwhRrlAK4AToiIIcBuZAntdeBw4MWKGtuSVL76+HTmY8B2KcZz\naiWsShFxLLAQOCRt78JOik0lOxZbAMcA35f08Yr5RwJXAUOBO4Efd7VdaywnAstrZkTcHhGrI2JF\nRDwSEbMiYmVEzAcmAx+vs/y0iGiLiHeAa4CxPSh7GDA7Im5N8y4CXq6znmHAorw7mPZvfmTuB+4D\n/i7Nfofsi26riHg7Ih5M01cCA4GPSOofEc+l92MNqbZ0JHBuRLwZEXPIvhxreQfYWdKQiHglIh7r\nIvw1jk+NMuenbT8BTCFrJvtAUm1tPHBWRPw1xXkFcFxFsQcj4q6IWEW2z/WOvTWBE4Hl9afKkdQJ\n+ytJf5a0FPgG2a/0Wv5cMfwmUK+zsVbZLSrjiOyOiQvqrOcVYESd+WtIzTGzUhPHa8BBvLdPFwAv\nAPelZqMzUgx/AE4n2/8lqRlk805WvxnQjzXfxxfqhHMU8GngRUkzJP1tF+H/qYv51WVeIHs/P6gt\ngJcj4o2qdY+sGK8+nh/qhe1aL3IisLyqTz+8FJgLbJeaRM4DVHAMi4BRHSOSxJpfONXuBT6TZ8WS\nBpE1I30X2Cx1LN9N2qeIWBoRp0XEaLJf9md2NH9ExNURsQ9Zm3+/tI5qi4HVwJYV02qeKppqW58G\nNgWmA9d3zKq1SI7drN72wjT8Bmt2nlcnsnrrXggMl1T55b4V8FKOeGwt4URgPTUEeB14Q9JO1O8f\n6C3TgXGSDk9nxnyZrM2+lvOA/SR9t+NXuqTtJV0rqbpGsgGwPtAOrJJ0GHBAx8y0zW1T8nkdWJXK\n7STpE5I2AFak16rqQFJT1i3A+ZIGSdqFNZtP3pXm/5OkjdJyyyrWuZjsi3dInf2u5dy07l2BCbzX\nlzAbOFTSJpJGAP9atdxiYJvOVhgRzwFtwHckbSBpLPB5siY96yOcCKynTif7MllGVjuo1UHZayJi\nMVln5IXAX4BtgceBt2qUf4bsVNHtgd+n5p4byE4pfbOq7GvAacDNZE1KR5Mlng47APcDy4H/An4U\nETPJEsj3yfoq/gxsApxTYxf+Jc1fDFxG1pZeywTghdTsdgIpaUTEXOAm4Pl0BtOmddZRbSZZh/fd\nwHdTPwjAlcA8siadX/Ne7aPDd8gS2GuSTu1kvccAY8j2fxpwdkQ80I24rMnkB9NYXyWpH1nTxNER\n8dtmx2PWV7lGYH2KpIMlbZyaYs4lO2vn4SaHZdanORFYX7MvWfPGy2TXLhwZEZ02DZlZPm4aMjMr\nOdcIzMxKrk/cnGr48OExevToZodhZtanPProoy9HRL1TrIE+kghGjx5NW1tbs8MwM+tTJNW7ev1d\nbhoyMys5JwIzs5JzIjAzKzknAjOzknMiMDMrOScCM7OScyIwMys5JwIzs5JzIjAzK7k+cWVxo40+\n61fNDqFQz19waLNDKM6kjZsdQbEmvd7sCGwd5BqBmVnJORGYmZWcE4GZWck5EZiZlZwTgZlZyRWe\nCCT1k/S4pOlpfGtJsyQ9K2mqpPWLjsHMzGprRI3gy8C8ivHvARdFxBjgVeCEBsRgZmY1FJoIJI0C\nDgV+nsYF7A9MS0WmAEcWGYOZmdVXdI3gh8BXgdVp/MPAaxGxMo0vAEYWHIOZmdVRWCKQdBiwJCIe\nrZzcSdGosfxESW2S2trb2wuJ0czMiq0R7AN8WtLzwPVkTUI/BIZK6ri1xShgYWcLR8TkiGiNiNaW\nlpYCwzQzK7fCEkFEfC0iRkXEaOCzwP0R8TngAeDoVGwCcGtRMZiZWdeacR3BmcBXJP2RrM/gsibE\nYGZmSUPuPhoRM4AZaXg+ML4R2zUzs675ymIzs5JzIjAzKzknAjOzknMiMDMrOScCM7OScyIwMys5\nJwIzs5JzIjAzKzknAjOzknMiMDMrOScCM7OScyIwMys5JwIzs5JzIjAzK7mG3IbazCyPXafs2uwQ\nCvXkhCebHUKnXCMwMyu5Ih9eP1DSw5KekPSUpPPT9CslPSdpdnqNLSoGMzPrWpFNQ28B+0fEckkD\ngJmS7kzzzoiIaQVu28zMciosEUREAMvT6ID0iqK2Z2ZmPVNoH4GkfpJmA0uAeyJiVpr1bUlzJF0k\naYMay06U1Caprb29vcgwzcxKrdBEEBGrImIsMAoYL2kX4GvAjsBewDDgzBrLTo6I1ohobWlpKTJM\nM7NSa8hZQxHxGjADODgiFkXmLeAKYHwjYjAzs84VedZQi6ShaXgQcCDwtKQRaZqAI4G5RcVgZmZd\nK/KsoRHAFEn9yBLODRExXdL9kloAAbOBLxUYg5mZdaHIs4bmAHt0Mn3/orZpZmbd5yuLzcxKzonA\nzKzknAjMzErOicDMrOScCMzMSs6JwMys5JwIzMxKzonAzKzknAjMzErOicDMrOScCMzMSs6JwMys\n5JwIzMxKzonAzKzknAjMzErOicDMrOSKfFTlQEkPS3pC0lOSzk/Tt5Y0S9KzkqZKWr+oGMzMrGtF\n1gjeAvaPiN2BscDBkvYGvgdcFBFjgFeBEwqMwczMulBYIojM8jQ6IL0C2B+YlqZPIXuAvZmZNUmh\nfQSS+kmaDSwB7gH+B3gtIlamIguAkTWWnSipTVJbe3t7kWGamZVaoYkgIlZFxFhgFDAe2KmzYjWW\nnRwRrRHR2tLSUmSYZmal1pCzhiLiNWAGsDcwVFL/NGsUsLARMZiZWeeKPGuoRdLQNDwIOBCYBzwA\nHJ2KTQBuLSoGMzPrWv+ui/TYCGCKpH5kCeeGiJgu6ffA9ZK+BTwOXFZgDGZm1oXCEkFEzAH26GT6\nfLL+AjMzWwv4ymIzs5JzIjAzKzknAjOzknMiMDMrOScCM7OScyIwMys5JwIzs5JzIjAzKzknAjOz\nksuVCCTtUnQgZmbWHHlrBJekx07+n44byZmZ2bohVyKIiH2BzwFbAm2SrpX0yUIjMzOzhsjdRxAR\nzwLnAGcCHwd+LOlpSX9fVHBmZla8vH0Eu0m6iOx5AvsDh0fETmn4ogLjMzOzguW9DfV/Aj8Dzo6I\nFR0TI2KhpHMKiczMzBoib9PQp4BrO5KApPUkbQgQEVd1toCkLSU9IGmepKckfTlNnyTpJUmz0+tT\nvbEjZmbWM3kTwb3AoIrxDdO0elYCp6cmpL2BkyXtnOZdFBFj0+uObkVsZma9Km/T0MCIWN4xEhHL\nO2oEtUTEImBRGl4maR4wsseRmplZIfLWCN6QNK5jRNKewIo65dcgaTTZYytnpUmnSJoj6XJJm+Rd\nj5mZ9b68ieBU4EZJv5X0W2AqcEqeBSUNBm4CTo2IpcBPgW2BsWQ1hh/UWG6ipDZJbe3t7TnDNDOz\n7srVNBQRj0jaEdgBEPB0RLzT1XKSBpAlgWsi4pdpXYsr5v8MmF5jm5OByQCtra2RJ04zM+u+vH0E\nAHsBo9Mye0giIn5Rq7AkAZcB8yLiworpI1L/AcBRwNxuR21mZr0mVyKQdBVZc85sYFWaHEDNRADs\nAxwHPClpdpp2NnCspLFp+eeBL3Y/bDMz6y15awStwM4RkbuJJiJmkjUjVfPpomZma5G8ncVzgc2L\nDMTMzJojb41gOPB7SQ8Db3VMjIhPFxKVmZk1TN5EMKnIIMzMrHnynj76oKS/AcZExL3pquJ+xYZm\nZmaNkPc21CcB04BL06SRwC1FBWVmZo2Tt7P4ZLLTQZfCuw+p2bSooMzMrHHyJoK3IuLtjhFJ/cmu\nAzAzsz4ubyJ4UNLZwKD0rOIbgduLC8vMzBolbyI4C2gHniS7EvgOsucXm5lZH5f3rKHVZI+q/Fmx\n4ZiZWaPlvdfQc3TSJxAR2/R6RGZm1lDduddQh4HAPwDDej8cMzNrtFx9BBHxl4rXSxHxQ2D/gmMz\nM7MGyNs0NK5idD2yGsKQQiIyM7OGyts0VPk4yZVkzxH4x16PxszMGi7vWUOfKDoQMzNrjrxNQ1+p\nN7/yUZQVy2xJ9gSzzYHVwOSI+JGkYcBUssdePg/8Y0S82r2wzcyst+S9oKwV+Beym82NBL4E7EzW\nT1Crr2AlcHpE7ATsDZwsaWeyi9Pui4gxwH1p3MzMmqQ7D6YZFxHLACRNAm6MiBNrLZAeUL8oDS+T\nNI8siRwB7JeKTQFmAGf2IHYzM+sFeWsEWwFvV4y/Tda0k4uk0cAewCxgs5QkOpJFp3cxlTRRUpuk\ntvb29rybMjOzbspbI7gKeFjSzWRXGB9F1v7fJUmDgZuAUyNiqdTZ8+zfLyImA5MBWltbfadTM7OC\n5D1r6NuS7gT+Lk36fEQ83tVykgaQJYFrIuKXafJiSSMiYpGkEcCSngRuZma9I2/TEMCGwNKI+BGw\nQNLW9Qor++l/GTCv6qyi24AJaXgCcGs3YjAzs16W9/TRr5OdObQDcAUwALia7KlltewDHAc8KWl2\nmnY2cAFwg6QTgBfJ7ltkZmZNkreP4Ciyzt7HACJioaS6t5iIiJlArQ6BA3JHaGZmhcrbNPR2RATp\nVtSSPlRcSGZm1kh5E8ENki4Fhko6CbgXP6TGzGydkPesof9IzypeStZPcF5E3FNoZGZm1hBdJgJJ\n/YC7IuJAwF/+ZmbrmC6bhiJiFfCmpI0bEI+ZmTVY3rOG/kp2Gug9wBsdEyPiXwuJyszMGiZvIvhV\nepmZ2TqmbiKQtFVEvBgRUxoVkJmZNVZXfQS3dAxIuqngWMzMrAm6SgSVVwZvU2QgZmbWHF0lgqgx\nbGZm64iuOot3l7SUrGYwKA2TxiMiNio0OjMzK1zdRBAR/RoViJmZNUd3nkdgZmbrICcCM7OScyIw\nMyu5whKBpMslLZE0t2LaJEkvSZqdXp8qavtmZpZPkTWCK4GDO5l+UUSMTa87Cty+mZnlUFgiiIjf\nAK8UtX4zM+sdzegjOEXSnNR0tEmtQpImSmqT1Nbe3t7I+MzMSqXRieCnwLbAWGAR8INaBSNickS0\nRkRrS0tLo+IzMyudhiaCiFgcEasiYjXZM4/HN3L7Zmb2fg1NBJJGVIweBcytVdbMzBoj74Npuk3S\ndcB+wHBJC4CvA/tJGkt2A7vngS8WtX0zM8unsEQQEcd2MvmyorZnZmY94yuLzcxKzonAzKzknAjM\nzErOicDMrOScCMzMSs6JwMys5JwIzMxKzonAzKzknAjMzErOicDMrOScCMzMSs6JwMys5JwIzMxK\nzonAzKzknAjMzEqusESQHk6/RNLcimnDJN0j6dn0t+bD683MrDGKrBFcCRxcNe0s4L6IGAPcl8bN\nzKyJCksEEfEb4JWqyUcAU9LwFODIorZvZmb5NLqPYLOIWASQ/m5aq6CkiZLaJLW1t7c3LEAzs7JZ\nazuLI2JyRLRGRGtLS0uzwzEzW2c1OhEsljQCIP1d0uDtm5lZlUYngtuACWl4AnBrg7dvZmZVijx9\n9Drgd8AOkhZIOgG4APikpGeBT6ZxMzNrov5FrTgijq0x64CitmlmZt231nYWm5lZYzgRmJmVnBOB\nmVnJORGYmZWcE4GZWck5EZiZlZwTgZlZyTkRmJmVnBOBmVnJORGYmZWcE4GZWck5EZiZlZwTgZlZ\nyTkRmJmVnBOBmVnJORGYmZVcYQ+mqUfS88AyYBWwMiJamxGHmZk1KREkn4iIl5u4fTMzw01DZmal\n16xEEMDdkh6VNLGzApImSmqT1Nbe3t7g8MzMyqNZiWCfiBgHHAKcLOlj1QUiYnJEtEZEa0tLS+Mj\nNDMriaYkgohYmP4uAW4GxjcjDjMza0IikPQhSUM6hoGDgLmNjsPMzDLNOGtoM+BmSR3bvzYift2E\nOMzMjCYkgoiYD+ze6O2amVnnfPqomVnJORGYmZWcE4GZWck5EZiZlZwTgZlZyTkRmJmVnBOBmVnJ\nORGYmZWcE4GZWck5EZiZlZwTgZlZyTkRmJmVnBOBmVnJORGYmZWcE4GZWck5EZiZlVxTEoGkgyX9\nQdIfJZ3VjBjMzCzTjGcW9wP+H3AIsDNwrKSdGx2HmZllmlEjGA/8MSLmR8TbwPXAEU2Iw8zMaM7D\n60cCf6oYXwD8bXUhSROBiWl0uaQ/NCC2ZhkOvNyojel7jdpSKTT02HG+GrapkmjsZ+/4hh+/v8lT\nqBmJoLN3It43IWIyMLn4cJpPUltEtDY7Dus+H7u+zccv04ymoQXAlhXjo4CFTYjDzMxoTiJ4BBgj\naWtJ6wOfBW5rQhxmZkYTmoYiYqWkU4C7gH7A5RHxVKPjWMuUoglsHeVj17f5+AGKeF/zvJmZlYiv\nLDYzKzknAjOzknMi6CWS/q+kpyTNkTRb0vuujagoe7ykLRoZX9lJmiHpf1VNO1XS5ZKm9WB9P+/q\ninhJX5L0v7u7bsvUOWY/+YDr/YakA3uw3H6Spn+Qba+tmnEdwTpH0keBw4BxEfGWpOHA+nUWOR6Y\ni0+bbaTryM5Qu6ti2meBMyLit9WFJfWPiJW1VhYRJ3a1wYi4pCeB2rtqHrOuFpQksj7Q1dXzIuK8\nXouwfgx1/4fWJq4R9I4RwMsR8RZARLwcEQsl7SnpQUmPSrpL0ghJRwOtwDWp5jBI0gGSHpf0ZPqF\nugGApAsk/T7VMv4jTTtc0qxU/l5JmzVtr/uWacBhFe/taGALYIGkuWna8ZJulHQ7cLek9ST9JNX0\npku6Ix2/jl+rrWl4uaRvS3pC0kMdx0TSJEn/loa3S8frCUmPSdpW0mBJ96XxJyX5VitrqnXMZko6\nQ9Ij6bNxfsd8SfNSjeExYEtJV0qam97f01K5KyuO416S/jsdl4clDZE0UNIVaZnHJX2iOjBJwyTd\nkrb/kKTd0vRJkiZLuhv4RfFvUS+JCL8+4AsYDMwGngF+AnwcGAD8N9CSyhxDdqoswAygNQ0PJLvl\nxvZp/BfAqcAw4A+8d2bX0PR3k4ppJwI/aPb+95UX8CvgiDR8FvDvwGhgbpp2PNkFj8PS+NHAHWQ/\nmDYHXgWO7uQYBnB4Gv4+cE4angT8WxqeBRxVccw3JKuRb5SmDQf+2HFs/ap7zA4iO+1T6dhMBz6W\njuVqYO9Ufk/gnop1dXyGrkzHdn1gPrBXmr5ROianA1ekaTsCL6Zjth8wPU2/GPh6Gt4fmF1xzB8F\nBjX7vevOyzWCXhARy8n+6SYC7cBU4IvALsA9kmYD55BdRV1tB+C5iHgmjU8h+6deCvwV+Lmkvwfe\nTPNHAXdJepKsivyRQnZq3dTR1ED6e10nZe6JiFfS8L7AjRGxOiL+DDxQY71vk30ZQfYlMLpypqQh\nwMiIuBkgIv4aEW+SfZF9R9Ic4F6y+3C5hremzo7ZQen1ONkv/x2BManMCxHxUBqeD2wj6WJJB5N9\npirtACyKiEcAImJpZE05+wJXpWlPAy8A21ctW1nmfuDDkjZO826LiBUfaK8bzImgl0TEqoiYERFf\nB04BPgM8FRFj02vXiDiok0U7vQtV+occD9wEHAn8Os26GPjPiNiVLNkM7O19WYfdAhwgaRzZL7bH\nOinzRsVw3juEvRPp5yCwivf3vdVaz+eAFmDPiBgLLMbHs1pnx0zAdys+W9tFxGWp/LvHLyJeBXYn\nq72dDPy8at2ik/ucke+417tn2hudzFurORH0Akk7SBpTMWksMA9oSR3JSBogqePX+zJgSBp+Ghgt\nabs0fhzwoKTBwMYRcQdZU9HYNH9j4KU0PKGQHVpHpZrbDOByOq8NVJsJfCb1FWxG1jTQk+0uJeuL\nOBJA0gaSNiQ7lksi4p3UDp3rTpFlUuOY3QV8IX1GkDRS0qbVy6aTNtaLiJuAc4FxVUWeBraQtFcq\nP0RSf+A3ZEkaSdsDW5E101aqLLMfWR9hdY2jz/BZQ71jMHCxpKHASrK23olk7Zg/TlXG/sAPgafI\n2igvkbQC+CjweeDG9E/4CHAJWR/BrZIGkv36OC1ta1Iq+xLwELB1I3ZwHXId8Evea26o5ybgALIz\nvJ4ha+d/vYfbPQ64VNI3gHeAfwCuAW6X1EbWx/R0D9e9rlvjmEXE3ZJ2An4nCWA58M9ktbFKI4Er\nJHX84P1a5cyIeFvSMWSf3UHACuBAsn6+S1Lz60rg+MjOBqxcfFJa9xyyZts+/aPMt5gwq0PS4IhY\nLunDwMPAPqm/wGyd4RqBWX3TU01vfeCbTgK2LnKNwMys5NxZbGZWck4EZmYl50RgZlZyTgRmFSQt\n70bZd+8lVMT6zRrFicDMrOScCMy60MUdX3eXdL+kZyWdVLHM++6Oaba2ciIw69pMsjta7gFcD3y1\nYt5uwKFkV4ifJ2kLSQeR3QRtPNmtQfaU9LEGx2yWmy8oM+vaKGCqpBFkF5Y9VzHv1nSnyRWSHiD7\n8t+X9+6OCdktSMaQ3Z/GbK3jRGDWtYuBCyPitnSDsUkV86qvyAzeuzvmpY0Jz+yDcdOQWdfq3fH1\niPREqw+T3Z30EXLeHdNsbeEagdmaNpS0oGL8Qurf8fVhsqdobUV2L6KFwMIad8dcUnz4Zt3new2Z\nmZWcm4bMzErOicDMrOScCMzMSs6JwMys5JwIzMxKzonAzKzknAjMzEru/wO5uWujAJ4mSgAAAABJ\nRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "count_classes = pd.value_counts(iris_training.select('label').toPandas()['label'], sort = True)\n", + "count_classes.plot(kind = 'bar', rot=0)\n", + "plt.title(\"Training Class distribution\")\n", + "plt.xticks(range(n_classes), LABELS)\n", + "plt.xlabel(\"Label\")\n", + "plt.ylabel(\"Frequency\");" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGktJREFUeJzt3XmcnFWB7vHfAwEBCWtaZAsBQdRR\nRGxcRgdUwAFl0RlcGEVQMc4d93ED9Ur0XhxXQPF6MQKyiCiLC4IOiwoOI1vYQYJ6WUMQGlFD2BOe\n+8d7GipFL5VOV1WS83w/n/r0u/U5p963u546533rLdkmIiLqtUq/GxAREf2VIIiIqFyCICKicgmC\niIjKJQgiIiqXIIiIqFyCIJaKpBmSLGlKmf+FpAM62XYCdX1K0jHL0t6JKu3euk91v0rSvJb5GyS9\napLKfpukc1vmJ/V5SlooaavJKi96I0FQGUnnSPr8CMv3kfSnpX3Rtr2H7RMmoV1LvPiVsr9g+6Bl\nLXuU+jaWdKykuyTdL2mupM9Jeno36lsWtv/O9gVjbdNp6No+2fZrJ6Ndki6QtMTxsb227Zsno/zo\nnQRBfY4H9pektuX7AyfbXtT7JvWWpA2Ai4E1gZfbngrsBqwHPKufbeumifbMogK286joQfPi9zdg\np5Zl6wMPAy8s868HrgIWAHcAs1q2nQEYmFLmLwAOKtOrAl8F7gVuBt7Xtu07gRuB+8v695blTwce\nAh4HFpbHJsAs4Hstde8N3AD8tdT73JZ1twIfA64tz++HwBqj7IP/DVwHrDLGfjKwdQf7Yw3ge8Cf\nS7suBzYq6w4sz/N+4BbgbWMck+OBvwC/Az4OzGt7bruW6ZcAc0pb7gYOL8tvL20e3n8vL/X/N3AE\ncF953gcCF7U9zw+Wdt4LfGV4v4yw/5849sBhwGKav5uFwDdH2G/rAicCQ8BtwGdayj4QuIjm7+Uv\nZf/s0e//j1of6RFUxvZDwKnAO1oWvxmYa/uaMv9AWb8ezYvg/5D0hg6Kfw+wJ/AiYBDYt239PWX9\nOjShcISkHWw/AOwBzHcztLC27fmtvyjp2cApwIeBAeDnwM8krd72PHYHtgS2o3mxGcmuwI9sP97B\nc4Kx98cBNC94mwMbAv8KPFSGmL5B8+I2Ffh74OpRyj+UpifyLOAfS5mj+TrwddvrlO1PLct3Kj/X\nK/vv4jL/UpoX+WfQvHiP5I00x2sHYB/gXWPUD4DtTwP/Bby/1Pf+ETY7imbfbAXsTLMP39my/qXA\nTcA04MvAsSP0VKMHEgR1OgF4k6Q1y/w7yjIAbF9g+zrbj9u+luYFeOcOyn0zcKTtO2zfB/xH60rb\nZ9v+f25cCJwL/EOHbX4LcLbt82w/RvNOck2aF9hh37A9v9T9M2D7UcraELirw3rH2x+PlfK2tr3Y\n9hW2F5R1jwPPl7Sm7bts3zBKFW8GDrN9n+07aAJkNI8BW0uaZnuh7UvGaf5820fZXlTeBIzkS6Xu\n24Ejgf3GKXNcklalOWaH2L7f9q3A12iGIIfdZvs7thfT/P1tDGy0rHXH0ksQVMj2RTTd9X3KFR47\nAt8fXi/ppZJ+LWlI0t9o3uVO66DoTWiGTobd1rpS0h6SLpF0n6S/Aq/rsNzhsp8or7ybvwPYtGWb\nP7VMPwisPUpZf6Z50enIOPvjJOAc4AeS5kv6sqTVSi/nLWXbuySdLek5Yzy3Ufdbm3cDzwbmSrpc\n0p7jNP+Ocda3b3Nbac+ymgaszpLP5TZGOV62HyyTox2z6KIEQb1OpOkJ7A+ca/vulnXfB84ENre9\nLnA00EmX/S6aIZJh04cnJD0NOIPmnfxGttejGd4ZLne82+DOB7ZoKU+lrjs7aFe784E3Sur073/U\n/WH7Mdufs/08mt7JnpRhN9vn2N6NJnTmAt8ZpfxR91s723+wvR/NUM+XgNPLMNRo+6+T2wu31z08\nLPcAsFbLumcuRdn30vRetmhZNp2JHa/osgRBvU6kGSt/Dy3DQsVU4D7bD0t6CfAvHZZ5KvBBSZtJ\nWh84uGXd6sDTaHoiiyTtAbRexng3sKGkdcco+/WSdpG0GvBR4BHgtx22rdXhNOcpTpC0BYCkTSUd\nLmm7EbYfdX9IerWkF5ShkAU0L36LJW0kae/yIv0IzQnVxWM8t0MkrS9pM+ADozVc0tslDZQe0V/L\n4sU0+/VxmvH4pfXxUvfmwIdoTrRDc05jJ0nTy3E5pO337h6tvjLccypwmKSpZT//O82J9VjOJAgq\nVcZsf0tzxc6Zbav/Dfi8pPuBz/LkCcnxfIdmmOQa4ErgRy313U9zdcqpNFeJ/Etrvbbn0oy93yzp\nr5KWGJ6wfRPwdpoTkPcCewF72X60w7a1lnUfzbv3x4BLy/P8Jc3VRn8c4VfG2h/PBE6nCYEbgQtp\nXuxWoQmr+TRX7OxcyhnJ52iGTW6hOW9y0hjN3x24QdJCmhPHb7X9cBlaOQz477L/XjbmTljST4Er\naF74zwaOBbB9Hk0oXFvWn9X2e18H9pX0F0kjndf4AE2v4maaK4S+Dxy3FO2KHpGdL6aJiKhZegQR\nEZVLEEREVC5BEBFRuQRBRETlVoibUE2bNs0zZszodzMiIlYoV1xxxb22B8bbboUIghkzZjBnzpx+\nNyMiYoUiaaxPqT8hQ0MREZVLEEREVC5BEBFRuQRBRETlEgQREZVLEEREVK5rQSDpOEn3SLp+hHUf\nk2RJnX4pSUREdEk3ewTH09wydwnlnue70XzZdkRE9FnXgsD2b2juw97uCOATdPbNSRER0WU9/WSx\npL2BO21f03zT4JjbzgRmAkyfPuo393XFjIPP7ml9vXbrF1/f7yZExHKkZyeLJa0FfJrmG57GZXu2\n7UHbgwMD494qIyIiJqiXVw09C9gSuEbSrcBmwJWS2r8QOyIieqhnQ0O2rwOeMTxfwmDQ9r29akNE\nRDxVNy8fPQW4GNhW0jxJ7+5WXRERMXFd6xHY3m+c9TO6VXdERHQunyyOiKhcgiAionIJgoiIyiUI\nIiIqlyCIiKhcgiAionIJgoiIyiUIIiIqlyCIiKhcgiAionIJgoiIyiUIIiIqlyCIiKhcgiAionIJ\ngoiIyiUIIiIqlyCIiKhcgiAionIJgoiIyiUIIiIq17UgkHScpHskXd+y7CuS5kq6VtKPJa3Xrfoj\nIqIz3ewRHA/s3rbsPOD5trcDfg8c0sX6IyKiA10LAtu/Ae5rW3au7UVl9hJgs27VHxERnennOYJ3\nAb8YbaWkmZLmSJozNDTUw2ZFRNSlL0Eg6dPAIuDk0baxPdv2oO3BgYGB3jUuIqIyU3pdoaQDgD2B\nXWy71/VHRMSSehoEknYHPgnsbPvBXtYdEREj6+blo6cAFwPbSpon6d3AN4GpwHmSrpZ0dLfqj4iI\nznStR2B7vxEWH9ut+iIiYmLyyeKIiMolCCIiKpcgiIioXIIgIqJyCYKIiMolCCIiKpcgiIioXIIg\nIqJyCYKIiMolCCIiKpcgiIioXIIgIqJyCYKIiMolCCIiKpcgiIioXIIgIqJyCYKIiMolCCIiKpcg\niIioXIIgIqJyCYKIiMp1LQgkHSfpHknXtyzbQNJ5kv5Qfq7frfojIqIz3ewRHA/s3rbsYOCXtrcB\nflnmIyKij7oWBLZ/A9zXtngf4IQyfQLwhm7VHxERnen1OYKNbN8FUH4+Y7QNJc2UNEfSnKGhoZ41\nMCKiNsvtyWLbs20P2h4cGBjod3MiIlZavQ6CuyVtDFB+3tPj+iMiok2vg+BM4IAyfQDw0x7XHxER\nbbp5+egpwMXAtpLmSXo38EVgN0l/AHYr8xER0UdTulWw7f1GWbVLt+qMiIilt9yeLI6IiN5IEERE\nVC5BEBFRuQRBRETlEgQREZVLEEREVC5BEBFRuQRBRETlEgQREZXrKAgkPb/bDYmIiP7otEdwtKTL\nJP2bpPW62qKIiOipjoLA9iuBtwGbA3MkfV/Sbl1tWURE9ETH5whs/wH4DPBJYGfgG5LmSvqnbjUu\nIiK6r9NzBNtJOgK4EXgNsJft55bpI7rYvoiI6LJOb0P9TeA7wKdsPzS80PZ8SZ/pSssiIqInOg2C\n1wEP2V4MIGkVYA3bD9o+qWuti4iIruv0HMH5wJot82uVZRERsYLrNAjWsL1weKZMr9WdJkVERC91\nGgQPSNpheEbSi4GHxtg+IiJWEJ2eI/gwcJqk+WV+Y+At3WlSRET0UkdBYPtySc8BtgUEzLX92EQr\nlfQR4CDAwHXAO20/PNHyIiJi4pbmpnM7AtsBLwL2k/SOiVQoaVPgg8Cg7ecDqwJvnUhZERGx7Drq\nEUg6CXgWcDWwuCw2cOIy1LumpMdoTjrPH2f7iIjokk7PEQwCz7PtZa3Q9p2SvgrcTnPC+Vzb57Zv\nJ2kmMBNg+vTpy1pt1GLWuv1uQXfN+lu/W9BVLzjhBf1uQlddd8B1/W7CiDodGroeeOZkVChpfWAf\nYEtgE+Dpkt7evp3t2bYHbQ8ODAxMRtURETGCTnsE04DfSboMeGR4oe29J1DnrsAttocAJP0I+Hvg\nexMoKyIillGnQTBrEuu8HXiZpLVohoZ2AeZMYvkREbEUOr189EJJWwDb2D6/vIivOpEKbV8q6XTg\nSmARcBUweyJlRUTEsuv0qqH30Jy43YDm6qFNgaNp3s0vNduHAodO5HcjImJydXqy+H3AK4AF8MSX\n1DyjW42KiIje6TQIHrH96PCMpCk0nyOIiIgVXKdBcKGkT9F8CGw34DTgZ91rVkRE9EqnQXAwMERz\nX6D3Aj+n+f7iiIhYwXV61dDjNF9V+Z3uNiciInqt06uGbmGEcwK2t5r0FkVERE8tzb2Ghq0BvInm\nUtKIiFjBdXSOwPafWx532j4SeE2X2xYRET3Q6dDQDi2zq9D0EKZ2pUUREdFTnQ4Nfa1lehFwK/Dm\nSW9NRET0XKdXDb262w2JiIj+6HRo6N/HWm/78MlpTkRE9NrSXDW0I3Bmmd8L+A1wRzcaFRERvbM0\nX0yzg+37ASTNAk6zfVC3GhYREb3R6S0mpgOPtsw/CsyY9NZERETPddojOAm4TNKPaT5h/EbgxK61\nKiIieqbTq4YOk/QL4B/Konfavqp7zYqIiF7pdGgIYC1gge2vA/MkbdmlNkVERA91FASSDgU+CRxS\nFq0GfK9bjYqIiN7ptEfwRmBv4AEA2/PJLSYiIlYKnQbBo7ZNuRW1pKd3r0kREdFLnQbBqZK+Dawn\n6T3A+SzDl9RIWk/S6ZLmSrpR0ssnWlZERCybTq8a+mr5ruIFwLbAZ22ftwz1fh34T9v7Slqd5kR0\nRET0wbhBIGlV4BzbuwLL8uI/XN46wE7AgQC2H2XJD6tFREQPjTs0ZHsx8KCkdSepzq2AIeC7kq6S\ndMxI5xwkzZQ0R9KcoaGhSao6IiLadXqO4GHgOknHSvrG8GOCdU4BdgD+r+0X0VyJdHD7RrZn2x60\nPTgwMDDBqiIiYjyd3mLi7PKYDPOAebYvLfOnM0IQREREb4wZBJKm277d9gmTVaHtP0m6Q9K2tm8C\ndgF+N1nlR0TE0hlvaOgnwxOSzpjEej8AnCzpWmB74AuTWHZERCyF8YaG1DK91WRVavtqmi+7iYiI\nPhuvR+BRpiMiYiUxXo/ghZIW0PQM1izTlHnbXqerrYuIiK4bMwhsr9qrhkRERH8szfcRRETESihB\nEBFRuQRBRETlEgQREZVLEEREVC5BEBFRuQRBRETlEgQREZVLEEREVC5BEBFRuQRBRETlEgQREZVL\nEEREVC5BEBFRuQRBRETlEgQREZVLEEREVK5vQSBpVUlXSTqrX22IiIj+9gg+BNzYx/ojIoI+BYGk\nzYDXA8f0o/6IiHhSv3oERwKfAB7vU/0REVH0PAgk7QncY/uKcbabKWmOpDlDQ0M9al1ERH360SN4\nBbC3pFuBHwCvkfS99o1sz7Y9aHtwYGCg122MiKhGz4PA9iG2N7M9A3gr8Cvbb+91OyIiopHPEURE\nVG5KPyu3fQFwQT/bEBFRu/QIIiIqlyCIiKhcgiAionIJgoiIyiUIIiIqlyCIiKhcgiAionIJgoiI\nyiUIIiIqlyCIiKhcgiAionIJgoiIyiUIIiIqlyCIiKhcgiAionIJgoiIyiUIIiIqlyCIiKhcgiAi\nonIJgoiIyiUIIiIq1/MgkLS5pF9LulHSDZI+1Os2RETEk6b0oc5FwEdtXylpKnCFpPNs/64PbYmI\nqF7PewS277J9ZZm+H7gR2LTX7YiIiEZfzxFImgG8CLh0hHUzJc2RNGdoaKjXTYuIqEbfgkDS2sAZ\nwIdtL2hfb3u27UHbgwMDA71vYEREJfoSBJJWowmBk23/qB9tiIiIRj+uGhJwLHCj7cN7XX9ERCyp\nHz2CVwD7A6+RdHV5vK4P7YiICPpw+ajtiwD1ut6IiBhZPlkcEVG5BEFEROUSBBERlUsQRERULkEQ\nEVG5BEFEROUSBBERlUsQRERULkEQEVG5BEFEROUSBBERlUsQRERULkEQEVG5BEFEROUSBBERlUsQ\nRERULkEQEVG5BEFEROUSBBERlUsQRERULkEQEVG5vgSBpN0l3STpj5IO7kcbIiKi0fMgkLQq8H+A\nPYDnAftJel6v2xEREY1+9AheAvzR9s22HwV+AOzTh3ZERAQwpQ91bgrc0TI/D3hp+0aSZgIzy+xC\nSTf1oG39Mg24t1eV6Uu9qqkKPT12fE49q6oSvf3fO7Dnx2+LTjbqRxCMtCf8lAX2bGB295vTf5Lm\n2B7sdzti6eXYrdhy/Br9GBqaB2zeMr8ZML8P7YiICPoTBJcD20jaUtLqwFuBM/vQjoiIoA9DQ7YX\nSXo/cA6wKnCc7Rt63Y7lTBVDYCupHLsVW44fIPspw/MREVGRfLI4IqJyCYKIiMolCCaJpE9LukHS\ntZKulvSUz0a0bHugpE162b7aSbpA0j+2LfuwpOMknT6B8o4Z7xPxkv5V0juWtuxojHHMvrWM5X5e\n0q4T+L1XSTprWepeXvXjcwQrHUkvB/YEdrD9iKRpwOpj/MqBwPXkstleOoXmCrVzWpa9Ffi47f9q\n31jSFNuLRivM9kHjVWj76Ik0NJ4w6jEb7xclieYc6OPt62x/dtJaOHYbxvwbWp6kRzA5Ngbutf0I\ngO17bc+X9GJJF0q6QtI5kjaWtC8wCJxceg5rStpF0lWSrivvUJ8GIOmLkn5XehlfLcv2knRp2f58\nSRv17VmvWE4H9mzZtzOATYB5kq4vyw6UdJqknwHnSlpF0rdKT+8sST8vx2/43epgmV4o6TBJ10i6\nZPiYSJol6WNleutyvK6RdKWkZ0laW9Ivy/x1knKrlSWNdswukvRxSZeX/43PDa+XdGPpMVwJbC7p\neEnXl/37kbLd8S3HcUdJvy3H5TJJUyWtIem75XeukvTq9oZJ2kDST0r9l0jariyfJWm2pHOBE7u/\niyaJ7TyW8QGsDVwN/B74FrAzsBrwW2CgbPMWmktlAS4ABsv0GjS33Hh2mT8R+DCwAXATT17ZtV75\nuX7LsoOAr/X7+a8oD+BsYJ8yfTDwFWAGcH1ZdiDNBx43KPP7Aj+necP0TOAvwL4jHEMDe5XpLwOf\nKdOzgI+V6UuBN7Yc87VoeuTrlGXTgD8OH9s8xjxmr6W57FPl2JwF7FSO5ePAy8r2LwbOaylr+H/o\n+HJsVwduBnYsy9cpx+SjwHfLsucAt5dj9irgrLL8KODQMv0a4OqWY34FsGa/993SPNIjmAS2F9L8\n0c0EhoAfAu8Fng+cJ+lq4DM0n6Juty1wi+3fl/kTaP6oFwAPA8dI+ifgwbJ+M+AcSdfRdJH/ritP\nauU0PNRA+XnKCNucZ/u+Mv1K4DTbj9v+E/DrUcp9lObFCJoXgRmtKyVNBTa1/WMA2w/bfpDmhewL\nkq4Fzqe5D1d6eEsa6Zi9tjyuonnn/xxgm7LNbbYvKdM3A1tJOkrS7jT/U622Be6yfTmA7QVuhnJe\nCZxUls0FbgOe3fa7rdv8CthQ0rpl3Zm2H1qmZ91jCYJJYnux7QtsHwq8H/hn4Abb25fHC2y/doRf\nHfEuVOUP8iXAGcAbgP8sq44Cvmn7BTRhs8ZkP5eV2E+AXSTtQPOO7coRtnmgZbrTO4Q95vJ2EFjM\nU8+9jVbO24AB4MW2twfuJsez3UjHTMB/tPxvbW372LL9E8fP9l+AF9L03t4HHNNWthjhPmd0dtzH\numfaAyOsW64lCCaBpG0lbdOyaHvgRmCgnEhG0mqSht+93w9MLdNzgRmSti7z+wMXSlobWNf2z2mG\nirYv69cF7izTB3TlCa2kSs/tAuA4Ru4NtLsI+OdyrmAjmqGBidS7gOZcxBsAJD1N0lo0x/Ie24+V\nceiO7hRZk1GO2TnAu8r/CJI2lfSM9t8tF22sYvsM4H8CO7RtMhfYRNKOZfupkqYAv6EJaSQ9G5hO\nM0zbqnWbV9GcI2zvcawwctXQ5FgbOErSesAimrHemTTjmN8oXcYpwJHADTRjlEdLegh4OfBO4LTy\nR3g5cDTNOYKfSlqD5t3HR0pds8q2dwKXAFv24gmuRE4BfsSTww1jOQPYheYKr9/TjPP/bYL17g98\nW9LngceANwEnAz+TNIfmHNPcCZa9slvimNk+V9JzgYslASwE3k7TG2u1KfBdScNveA9pXWn7UUlv\nofnfXRN4CNiV5jzf0WX4dRFwoJurAVt/fVYp+1qaYdsV+k1ZbjERMQZJa9teKGlD4DLgFeV8QcRK\nIz2CiLGdVXp6qwP/KyEQK6P0CCIiKpeTxRERlUsQRERULkEQEVG5BEFEC0kLl2LbJ+4l1I3yI3ol\nQRARUbkEQcQ4xrnj6wsl/UrSHyS9p+V3nnJ3zIjlVYIgYnwX0dzR8kXAD4BPtKzbDng9zSfEPytp\nE0mvpbkJ2ktobg3yYkk79bjNER3LB8oixrcZ8ENJG9N8sOyWlnU/LXeafEjSr2le/F/Jk3fHhOYW\nJNvQ3J8mYrmTIIgY31HA4bbPLDcYm9Wyrv0TmebJu2N+uzfNi1g2GRqKGN9Yd3zdp3yj1YY0dye9\nnA7vjhmxvEiPIGJJa0ma1zJ/OGPf8fUymm/Rmk5zL6L5wPxR7o55T/ebH7H0cq+hiIjKZWgoIqJy\nCYKIiMolCCIiKpcgiIioXIIgIqJyCYKIiMolCCIiKvf/AXMR2CdFUGRDAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "count_classes = pd.value_counts(iris_test.select('label').toPandas()['label'], sort = True)\n", + "count_classes.plot(kind = 'bar', rot=0)\n", + "plt.title(\"Validation Class distribution\")\n", + "plt.xticks(range(n_classes), LABELS)\n", + "plt.xlabel(\"Label\")\n", + "plt.ylabel(\"Frequency\");" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Examining the Data\n", + "\n", + "Let's eyeball the data and see what it looks like.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[[6.4, 2.8, 5.6, 2.2, 3.0],\n", + " [5.0, 2.3, 3.3, 1.0, 2.0],\n", + " [4.9, 2.5, 4.5, 1.7, 3.0],\n", + " [4.9, 3.1, 1.5, 0.1, 1.0],\n", + " [5.7, 3.8, 1.7, 0.3, 1.0],\n", + " [4.4, 3.2, 1.3, 0.2, 1.0],\n", + " [5.4, 3.4, 1.5, 0.4, 1.0],\n", + " [6.9, 3.1, 5.1, 2.3, 3.0],\n", + " [6.7, 3.1, 4.4, 1.4, 2.0],\n", + " [5.1, 3.7, 1.5, 0.4, 1.0],\n", + " [5.2, 2.7, 3.9, 1.4, 2.0],\n", + " [6.9, 3.1, 4.9, 1.5, 2.0],\n", + " [5.8, 4.0, 1.2, 0.2, 1.0],\n", + " [5.4, 3.9, 1.7, 0.4, 1.0],\n", + " [7.7, 3.8, 6.7, 2.2, 3.0],\n", + " [6.3, 3.3, 4.7, 1.6, 2.0],\n", + " [6.8, 3.2, 5.9, 2.3, 3.0],\n", + " [7.6, 3.0, 6.6, 2.1, 3.0],\n", + " [6.4, 3.2, 5.3, 2.3, 3.0],\n", + " [5.7, 4.4, 1.5, 0.4, 1.0],\n", + " [6.7, 3.3, 5.7, 2.1, 3.0],\n", + " [6.4, 2.8, 5.6, 2.1, 3.0],\n", + " [5.4, 3.9, 1.3, 0.4, 1.0],\n", + " [6.1, 2.6, 5.6, 1.4, 3.0],\n", + " [7.2, 3.0, 5.8, 1.6, 3.0],\n", + " [5.2, 3.5, 1.5, 0.2, 1.0],\n", + " [5.8, 2.6, 4.0, 1.2, 2.0],\n", + " [5.9, 3.0, 5.1, 1.8, 3.0],\n", + " [5.4, 3.0, 4.5, 1.5, 2.0]]" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "iris_k_train.take(29)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Converting to RDD of Sample\n", + "\n", + "BigDL requires all input data to be in its native Sample format (for the RDD API). The Pipeline api, shown in feedforward-iris-pipeline -- allows data to be in Spark Dataframes.\n", + "\n", + "To convert the data into type Sample, we will use a helper method here to convert. The sample function requires two numpy arrays:\n", + "\n", + "1. Feature array (of all the features in double type)\n", + "2. Label array: the label value (usually just a single number in the array).\n", + "\n", + "Then we convert these to RDDs of type Sample." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training Count: 120\n", + "Test Count: 30\n" + ] + } + ], + "source": [ + "#convert ndarray data into RDD[Sample]\n", + "def array2rdd(ds):\n", + " #build Sample from ndarrays\n", + " def build_sample(c0,c1,c2,c3,prediction):\n", + " feature = np.array([c0,c1, c2, c3]).flatten()\n", + " label = np.array(prediction)\n", + " return Sample.from_ndarray(feature, label)\n", + " rdd = ds.map(lambda (c0,c1,c2,c3,prediction): build_sample(c0,c1,c2,c3,prediction))\n", + " return rdd\n", + "\n", + "iris_rdd_train = array2rdd(iris_k_train)\n", + "iris_rdd_train.cache()\n", + "print(\"Training Count: \" + str(iris_rdd_train.count()))\n", + "\n", + "iris_rdd_test = array2rdd(iris_k_test)\n", + "iris_rdd_test.cache()\n", + "print (\"Test Count: \" + str(iris_rdd_test.count()))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setting up our network\n", + "\n", + "Here we are going to actually set up our network. It will be single hidden layer network, with input layer (based on our 4-inputs), and output layer (softmax with 3 classes). \n", + "\n", + "The hidden layer has been set up to 3, and we will use ReLU activation for that hidden layer. ReLU is currently preferred as a activation layer for these types of networks as it is linear in the postive direction, avoiding the vanishing gradient problem.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "creating: createSequential\n", + "creating: createLinear\n", + "creating: createReLU\n", + "creating: createLinear\n", + "creating: createLogSoftMax\n" + ] + } + ], + "source": [ + "# Create model\n", + "\n", + "def multilayer_perceptron(n_hidden_1, n_input, n_classes):\n", + " # Initialize a sequential container\n", + " model = Sequential()\n", + " # Hidden layer with ReLu activation\n", + " model.add(Linear(n_input, n_hidden_1).set_name('mlp_fc1'))\n", + " model.add(ReLU())\n", + " # output layer\n", + " model.add(Linear(n_hidden_1, n_classes).set_name('mlp_fc4'))\n", + " model.add(LogSoftMax())\n", + " return model\n", + "\n", + "model = multilayer_perceptron(n_hidden_1, n_input, n_classes)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Training the Model\n", + "\n", + "Now we need to set up our training. We are going to use the following:\n", + "\n", + "* Loss Function: ClassNLLCriterion\n", + "* Optimization: AdaGrad \n", + "\n", + "For validation, we will validating against top1Accuracy for every epoch." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "creating: createClassNLLCriterion\n", + "creating: createAdagrad\n", + "creating: createMaxEpoch\n", + "creating: createOptimizer\n", + "creating: createEveryEpoch\n", + "creating: createTop1Accuracy\n", + "creating: createTrainSummary\n", + "creating: createSeveralIteration\n", + "creating: createValidationSummary\n", + "('saving logs to ', 'iris-20171205-221139')\n" + ] + } + ], + "source": [ + "# Create an Optimizer\n", + "optimizer = Optimizer(\n", + " model=model,\n", + " training_rdd=iris_rdd_train,\n", + " criterion=ClassNLLCriterion(),\n", + " optim_method=Adagrad(learningrate=learning_rate, learningrate_decay=0.0002),\n", + " end_trigger=MaxEpoch(training_epochs),\n", + " batch_size=batch_size)\n", + "\n", + "# Set the validation logic\n", + "optimizer.set_validation(\n", + " batch_size=batch_size,\n", + " val_rdd=iris_rdd_test,\n", + " trigger=EveryEpoch(),\n", + " val_method=[Top1Accuracy()]\n", + ")\n", + "\n", + "app_name='iris-'+dt.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n", + "train_summary = TrainSummary(log_dir='/tmp/bigdl_summaries',\n", + " app_name=app_name)\n", + "train_summary.set_summary_trigger(\"Parameters\", SeveralIteration(50))\n", + "val_summary = ValidationSummary(log_dir='/tmp/bigdl_summaries',\n", + " app_name=app_name)\n", + "optimizer.set_train_summary(train_summary)\n", + "optimizer.set_val_summary(val_summary)\n", + "print(\"saving logs to \",app_name)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Optimization Done.\n", + "CPU times: user 0 ns, sys: 20 ms, total: 20 ms\n", + "Wall time: 11.4 s\n" + ] + } + ], + "source": [ + "%%time\n", + "# Boot training process\n", + "trained_model = optimizer.optimize()\n", + "print(\"Optimization Done.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Let's look at the training graphs\n", + "\n", + "We're going to plot the Loss and Top1 Accuracy by iteration to see how our training went.\n", + "\n", + "Note there are two graphs: Loss, and top1 Accuracy" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsYAAAK7CAYAAADx1EmqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3Xd8m+d5L/zfjb0IgCBBikuD2rIs\nybFsyyumY7ex0yRu2iRNOrLrtyejzek6aZumbZq8TZPT9nSkJ6874vbkJK6TNqmt2pbt2IyHLFuS\nLcnWljhEigMkOLD3/f7x4AFBDAIEQRIQf9/PJ5+awAPg4W02+fHidV+3kFKCiIiIiGit06z2DRAR\nERER1QIGYyIiIiIiMBgTEREREQFgMCYiIiIiAsBgTEREREQEgMGYiIiIiAgAgzER0aoTQgwIIe5d\n7fsgIlrrGIyJiIiIiMBgTEREREQEgMGYiKhmCCGMQoj/JYQYSf/nfwkhjOnnmoUQB4UQM0KIKSHE\ni0IITfq5/yGEuCqE8Ashzgsh7lnd74SIqD7pVvsGiIgo4w8AHACwD4AE8J8AvgjgDwH8FoBhAO70\ntQcASCHEdgCfBXCTlHJECLERgHZlb5uI6NrAijERUe34JQBfllJ6pJQTAP4EwK+kn4sDaAOwQUoZ\nl1K+KKWUAJIAjAB2CSH0UsoBKeXlVbl7IqI6x2BMRFQ72gEMZn09mH4MAL4B4BKAp4UQfUKILwCA\nlPISgM8D+GMAHiHEI0KIdhAR0aIxGBMR1Y4RABuyvl6ffgxSSr+U8reklN0A3gPgN9VeYinld6WU\nd6RfKwH8+creNhHRtYHBmIiodnwPwBeFEG4hRDOALwH4DgAIId4thNgihBAAfFBaKJJCiO1CiHek\nN+lFAITTzxER0SIxGBMR1Y6vADgG4BSANwG8nn4MALYCeBZAAMArAP5eStkLpb/4awAmAYwBaAHw\n+yt610RE1wih7N0gIiIiIlrbWDEmIiIiIgKDMRERERERAAZjIiIiIiIADMZERERERABW8Uhop9Mp\nt2zZslofX9eCwSCsVutq30Zd4tpVjmtXGa5b5bh2lePaVY5rV5laX7fjx49PSindpa5btWDc2tqK\nY8eOrdbH17Xe3l709PSs9m3UJa5d5bh2leG6VY5rVzmuXeW4dpWp9XUTQgyWvoqtFEREREREABiM\niYiIiIgAMBgTEREREQFgMCYiIiIiAsBgTEREREQEgMGYiIiIiAgAgzEREREREQAGYyIiIiIiAAzG\nREREREQAGIyJiIiIiACUEYyFEP8shPAIId4qcd1NQoikEOL9ldxIMiUreRkRERERUVWUUzF+GMB9\nC10ghNAC+HMAhyq5iW8+fwkff/goEslUJS8nIiIiIlqyksFYSvkCgKkSl30OwL8D8FRyEy6rAS9c\nmMDXD52v5OVEREREREumW+obCCE6ALwPwDsA3FTJe3z45vU4M+LDQy/0YVebHT97Q8dSb4uIiIiI\naFGElKV7e4UQGwEclFLuLvDc9wH8hZTyiBDi4fR1PyjyPg8CeBAA3G73jY8++mjmuURK4htHI+ib\nTeH3bzFhk0NbwbezNgQCAdhsttW+jbrEtasc164yXLfKce0qx7WrHNeuMrW+bnffffdxKeX+UtdV\nIxj3AxDpL5sBhAA8KKX80ULvuX37dnn+/PzWCW8givf+3ctISYnHPnsH3A3Gkve2FvX29qKnp2e1\nb6Muce0qx7WrDNetcly7ynHtKse1q0ytr5sQoqxgvORxbVLKTVLKjVLKjQB+AODTpUJxMU02Ix76\nyI2YDsXw+z98c6m3RkRERERUtpI9xkKI7wHoAdAshBgG8EcA9AAgpfxWtW/ounYH7rtuHY4NTlf7\nrYmIiIiIiioZjKWUHy73zaSUH1vS3aTZzXr4I4lqvBURERERUVlq8uS7BpMOgWgC5fQ/ExERERFV\nQ40GYz2SKYlQLLnat0JEREREa0SNBmOlw4PtFERERES0Umo0GOsBAP5IfJXvhIiIiIjWihoNxkrF\n2MeKMRERERGtkJoMxvZMKwUrxkRERES0MmoyGM+1UrBiTEREREQro0aDMTffEREREdHKqtFgzM13\nRERERLSyajIYWw1aaAQrxkRERES0cmoyGAshYDPqWDEmIiIiohVTk8EYUNopWDEmIiIiopVSw8FY\nxznGRERERLRiajYY2016tlIQERER0Yqp2WDcYNKxlYKIiIiIVkxtB+MoK8ZEREREtDJqOBhz8x0R\nERERrZwaDsZKK4WUcrVvhYiIiIjWgBoOxnokUxLheHK1b4WIiIiI1oAaDsY6ADz9joiIiIhWRh0E\nY27AIyIiIqLlV7PB2G7SAwAP+SAiIiKiFVGzwZitFERERES0kmo4GCsVY7ZSEBEREdFKqOFgzIox\nEREREa2cOgjGrBgTERER0fKr2WBsNeggBCvGRERERLQyajYYazQCDUYdgzERERERrYiaDcaAsgHP\nx1YKIiIiIloBNR6MWTEmIiIiopVR08HYbtJz8x0RERERrYiaDsasGBMRERHRSmEwJiIiIiJCzQdj\ntlIQERER0cqo8WCsVIyllKt9K0RERER0jSsZjIUQ/yyE8Agh3iry/C8JIU6l/3NYCLG3WjfXYNIj\nkZKIxFPVeksiIiIiooLKqRg/DOC+BZ7vB3CXlHIPgD8F8FAV7gsAj4UmIiIiopVTMhhLKV8AMLXA\n84ellNPpL48A6KzSvWWCsY8b8IiIiIhomYly+neFEBsBHJRS7i5x3W8D2CGl/FSR5x8E8CAAuN3u\nGx999NEFP/fkRAJ/dTyKPzxgwmantuR9rhWBQAA2m221b6Muce0qx7WrDNetcly7ynHtKse1q0yt\nr9vdd999XEq5v9R1ump9oBDibgCfBHBHsWuklA8h3Wqxfft22dPTs+B72gam8FfHX8HWXXvw9m3u\nat1q3evt7UWptaPCuHaV49pVhutWOa5d5bh2lePaVeZaWbeqBGMhxB4A/wjgfimltxrvCSib7wBw\nljERERERLbslj2sTQqwH8B8AfkVKeWHptzSHm++IiIiIaKWUrBgLIb4HoAdAsxBiGMAfAdADgJTy\nWwC+BKAJwN8LIQAgUU4PRznmgjErxkRERES0vEoGYynlh0s8/ykABTfbLZXVoIMQrBgTERER0fKr\n6ZPvNBoBm1HHcW1EREREtOxqOhgDgN2kZysFERERES27mg/GDSYdWymIiIiIaNnVSTBmxZiIiIiI\nllcdBGM9/FFWjImIiIhoedVBMGbFmIiIiIiWH4MxERERERHqIhjr4Y/EIaVc7VshIiIiomtYHQRj\nHeJJiWgitdq3QkRERETXsDoIxnoAgI8j24iIiIhoGdV8MLablFOr2WdMRERERMup5oNxA4MxERER\nEa2AOgjGSivFYk6/Oz0yi8lAdLluiYiIiIiuQXUQjBdXMQ5GE/jAt17BXz5zYTlvi4iIiIiuMXUQ\njBdXMX7unAehWBKXxgPLeVtEREREdI2pg2C8uIrxwVMjAIC+yeCy3RMRERERXXtqPhjbDDoIAfjC\npSvGgWgCz5+fgMWgxWQgyhFvRERERFS2mg/GGo2AzaCDr4yK8Y/PjiOWSOGXD2wAAPRPsGpMRERE\nROWp+WAMKO0U5bRSPH5yFOvsJrz/xk4AQD/bKYiIiIioTHUSjPUlN9/5InG8cGEC77q+DRuaLNAI\noG+CG/CIiIiIqDx1Eox1mC3RY/zsmXHEkim8e28bjDotOhst3IBHRERERGWri2C8u8OB169MY9wX\nKXrNwVOj6HCacUOXEwDQ7baylYKIiIiIylYXwfhjt21EIiXxL4cHCj4/G4rjxYsTeNf16yCEAABs\nalaCsZRyBe+UiIiIiOpVXQTjjc1W/PSuVvzfV68gFMvfhPf0mTHEkxLv3tOeeay72YpQLIlxH4+G\nJiIiIqLS6iIYA8Cv3tmN2XAcPzg+PO9xKSV++MZVdLnM2NPpyDze7bYBAPomuQGPiIiIiEqrm2B8\n44ZG7Oty4p9e6kcyNdce8a2f9OHwZS8+cmBjpo0CUFopAKCPs4yJiIiIqAx1E4yFEPjVO7sx6A3h\nmTPjAIBDp8fw9UPn8J697fjUnZvmXb/OboJJr+EGPCIiIiIqS90EYwB453Wt6Gw04x9f7MPpkVl8\n/pET2NPpxDfev2detRhQTszb1GxbcjCOJ1P49sv9iMSTS3ofIiIiIqptdRWMdVoNPnH7JhwbnMYv\n/+OrcFr0+IdfuREmvbbg9d3N1iUf8vHypUn8yeNn8OLFySW9DxERERHVtroKxgDwwZu60GDSIRJP\n4R8/uh8tdlPRazc1WzE0HUYskar48y55lGA94ed0CyIiIqJrmW61b2CxbEYdvv2xm2DUaXFdu2PB\na7vdViRTEkPTIWxOT6lYLAZjIiIiorWh7oIxAOzf6CrrOnUyRf9EsOJgfDndijERKH7qHhERERHV\nv7prpViM7ualzzJmxZiIiIhobbimg7HDokeT1VDxZApvIIrpUBwAgzERERHRte6aDsaA0k5R6SEf\narXYZTVgIsBgTERERHQtKxmMhRD/LITwCCHeKvK8EEL8jRDikhDilBDibdW/zcptarair8KK8aV0\nf/GBbhcm/FFIKUu8goiIiIjqVTkV44cB3LfA8/cD2Jr+z4MA/vfSb6t6NrmtmPBH4Y/Ei15TLPBe\n9gRh1muxp9OJSDyFQDSxXLdJRERERKusZDCWUr4AYGqBSx4A8K9ScQSAUwjRVq0bXCp1A97AZKjg\n87PhOPb+ydM4dHos77lLEwFsbrGi1W4EwD5jIiIiomtZNca1dQAYyvp6OP3YaO6FQogHoVSV4Xa7\n0dvbW4WPX5jXrxzucfCFo/C253+7F6eT8EUSePjHJ2CcmH9YyOkrIWxr1GDk8nkAwDMvvortrsKn\n7K2kQCCwImt3LeLaVY5rVxmuW+W4dpXj2lWOa1eZa2XdqhGMRYHHCvYmSCkfAvAQAGzfvl329PRU\n4eMXFokn8cXDT8HkXo+enm15z0+9Pgy8ehJ9AR3uuusuCKF8O8FoAt6nDuG23Ztx73Xr8I1jL6Bj\ny0707Glf8POklAjHk7AYlm9EdG9vL1Zi7a5FXLvKce0qw3WrHNeucly7ynHtKnOtrFs1plIMA+jK\n+roTwEgV3rcqTHotOhvNRUe2DXiVFotxX3TeJj11ksWWFhvcDeW3Ujz11hhu+sqz8C3Q00xERERE\ntacawfgxAB9JT6c4AGBWSpnXRrGaNjXbih7yMehVNtgBwOFLk5nHL034ASjB2GnWQ6cRZQXjs2N+\nBGNJjM3ypDwiIiKielLOuLbvAXgFwHYhxLAQ4pNCiF8TQvxa+pInAPQBuATgHwB8etnutkLdzVb0\nTwQLTp8Y9IZww3onOpxmHL7szTx+2ROEViOwockKjUag2WYsKxh7fEog9gZi1fsGiIiIiGjZlWyE\nlVJ+uMTzEsBnqnZHy6DbbUUwlsSEP4oW+/wNdoPeIO6/vg3tTjOePTuOVEpCoxG45AlgQ5MFBp3y\nu4O7wVjWIR/j6WA8HWIwJiIiIqon1/zJd4ByyAcAXM45AW82HMd0KI4NLgtu39KEmVAcZ8d8AJRR\nbVvctsy17obyKsbjPuUab5DBmIiIiKierKlgnLsB70p6492GJitu7W4GABy+5EU8mcLAZBCbW7KC\ncbmtFH6lYjzFVgoiIiKiurImgnG7wwyjToP+nA14g1NKUN7QZME6hwndbisOX57EoDeERErmVYy9\nwRiSqeLHQseTKUymA/FUkIeBEBEREdWTNRGMNRqBTc3WzAg21WCmYmwBANy2uQmv9U/h/NjcRAqV\nu8GIZEou2DucXVGeCnFcGxEREVE9WRPBGFDaKXJbKQYmg2hpMGYO47h9czOCsSR++MYwAMxvpShj\nlrG68Q5gxZiIiIio3qyZYNzttuLKVAjxZCrz2OBUKFMtBoAD3U0AgB+f86DNYYLNODe0o7xgrDzX\n4TRzXBsRERFRnVkzwXhTsw2JlMTQVCjz2KA3iA1N1szXjVYDdrXZIeX8NgpA2XwHLByM1Y13O9sa\nMMWpFERERER1ZQ0F4/mTKcKxJMZ9UWxwWeZdd/sWpWq82Z0TjNWK8QKzjMd9Eeg0ApvdNkyHYgUP\nFCEiIiKi2rRmgvFm9/xgfCVdOd7QbJ133W2blbFtm3MqxlajDhaDtmQrRUuDEc02I+JJCX80UbX7\nJyIiIqLltWaCsdNiQKNFnznkY8Cr/N+NTbkV42Z85u7NeNfudXnvUeqQj3FfBC12E1xWAwDOMiYi\nIiKqJ2smGAPqZApllnHmcA/X/IqxQafB77xzB5rSPcXZSh3yMe6LoNVuhMumBGOefkdERERUP9ZU\nMO522zKtFAPeIJwWPRwWfdmvdzcYS/QYR9FqN8FlSVeMqxCMz435cHJoZsnvQ0REREQLW1PBeFOz\nFeO+KALRBK5MhfI23pWyUCtFJJ7EbDiuBON0K8V0FYLxVw6exRd/9NaS34eIiIiIFramgnF3eqPd\nwGQQAzmj2srhthkxG44jmkjmPedJzzBuaTCiqYqtFGO+CLwLVKmJiIiIqDrWVDDelJ5McX7Mj6vT\n4XmHe5RDHdk2WWBT3Xh6hnGr3QSLQQeTXlOV0+8m/FFMLXAMNRERERFVx5oKxhubrBACePHiBFIS\ni68YL3D6nXocdKvdBABoshqXXDFW2zMi8RTCsfwqNRERERFVz5oKxia9Fu0OM35yYQJA/qi2UhYO\nxspjrXblmkarfsmb7yazWiimWTUmIiIiWlZrKhgDQLfbiulQHACwvorB2OOLwKDTwGFWply4rMYl\nb77L/hweMU1ERES0vNZeME5vwLMYtHAXmFW8kCbrwq0UrXYjhBDpaw1LbqXw+FkxJiIiIlopay4Y\nb0oH4/UuSybElsug06DRosdEIJL33LgvitYGU+Zrl9Ww5CrvxLxgHF/SexERERHRwtZeMHbbACgb\n8SpRbJbxuD+S2XgHKME4FEsiEq9809y8YMxWCiIiIqJlteaCsdpKsaF5cf3FqmLB2OOLosU+15qh\nHvKxlKrxRCAKh1kPIdhjTERERLTc1lww7nCa8ZFbN+A9e9orer3bln8sdCCaQCCayKsYA0sLtB5f\nFG0OE+wmPWbYY0xERES0rHSrfQMrTaMR+PIDuyt+vVoxllJmepQ9mRnGcxXjJuvST7+bCEThbjAi\nmkhhij3GRERERMtqzVWMl8rdYEQknkIgmsg8lplh3FCoYlz56XeTfiUYN1r07DEmIiIiWmYMxotU\naJaxJ30cdEuBVgpvgeOjyyGlxEQ6GFdjwgURERERLYzBeJG6m5WpFkf6pjKPjRdopbCb9NBqRMXz\nh2fDccSSKbhtRjgtBvYYExERES0zBuNF2tPpwK42Ox4+3A8pJQBgbDYKi0ELm3GuZVujEWi0VF7p\nVSvSLXaTUjFmMCYiIiJaVgzGiySEwMdv34gL4wG8fMkLYG6Gce6BIU1WQ8WtFGowdtuMaLQYEImn\nEI6Vnok8G44jmqh8djIRERHRWsVgXIH37G1Hs82Ab7/cD0CZStHSkH+89FJ6g9XjoNXNd0B5x0J/\n4FuH8ZdPX6joM4mIiIjWMgbjCpj0WvziLRvw3HkP+ieDynHQWRvvVEtpgZjIDsZlzkSWUqJ/MogL\n4/6KPpOIiIhoLWMwrtAvH1gPnUbgXw4PYNwXmbfxTrWUivFEIAqjTgO7SZeZcFGqYhyIJhBPysz4\nOCIiIiIqH4NxhVoaTHjPnnZ877UriCZSRSvGM6E4EsnUot9fHdUmhLKJDyhdMVafV6dkEBEREVH5\nGIyX4OO3b0I0oYTelgLBuMmmVnoXf2qdxx/JzExWe4xnSryPGoy9wRhiicWHcSIiIqK1jMF4Ca7v\ndGD/hkYAwLoCwbicSm80kcRsOD/wTvijmQ19DrMeQpSuGGe3WqiHjhARERFRecoKxkKI+4QQ54UQ\nl4QQXyjw/HohxPNCiDeEEKeEEO+q/q3Wps/cvQUNJh263da855rK2DT39afO491/+2JmJrJKbaUA\nAJ1WA4dZX7LHeCo4F7DZZ0xERES0OCWDsRBCC+CbAO4HsAvAh4UQu3Iu+yKAR6WUNwD4EIC/r/aN\n1qq7d7Tg1B/9NJptBTbf2UoH4+fPeTA0FcbwdDjzWCyRwnQoDrdtrgrdaDGUbMmYCmYdU80+YyIi\nIqJFKadifDOAS1LKPillDMAjAB7IuUYCsKf/2QFgpHq3WPtyD/ZQuTIV48LVW48vgr7JIADg9SvT\nmccnA+qpd3Nhu9Gix3TJzXdzwXmMwZiIiIhoUXSlL0EHgKGsr4cB3JJzzR8DeFoI8TkAVgD3Fnoj\nIcSDAB4EALfbjd7e3kXebn1JpJT2iOOnL6ArOpD3/JHRROafD77yFhwzFwEAfbPKyXVjAxfQG+oD\nAKTCEVyZlujt7UUgECi4dmcuR+EwCgRiEkffuohN8cEqf0f1r9jaUWlcu8pw3SrHtasc165yXLvK\nXCvrVk4wLlQOlTlffxjAw1LKvxBC3Arg/wghdksp541GkFI+BOAhANi+fbvs6emp4Jbri/2FQ3C4\n29HTszvvuad/+CYajCPY2mrDpAR6em4HACTOjAOvHMM7bt2PvV1OAMB/TZzES5cm0dPTg97eXhRa\nu+8MHkNbIgR/JAGj04Wenn3L+r3Vo2JrR6Vx7SrDdasc165yXLvKce0qc62sWzmtFMMAurK+7kR+\nq8QnATwKAFLKVwCYADRX4wbrXZPNCG+RFogjfV7ctMmF/RtdOD3iQzShVIqzj4NWNVoNJTffTYdi\ncFkNaLUbOcuYiIiIaJHKCcZHAWwVQmwSQhigbK57LOeaKwDuAQAhxE4owXiimjdar4qdfufxR9A3\nEcQtm1zY1+VELJHC2VHlKGf1OOjsDX2NFgMi8RTCsWTRz5oOxtBoNWCdw8RgTERERLRIJYOxlDIB\n4LMADgE4C2X6xGkhxJeFEO9NX/ZbAH5VCHESwPcAfEzmzh9bo4oF41f7pgAAB7qbsC/dLnEivQFv\nIhBBo0UPg06T9T7KIR9TC1SNp0IxuCwGtDSYOK6NiIiIaJHK6TGGlPIJAE/kPPalrH8+A+D26t7a\ntcFlMeDE0Eze40f6vLAZdbiu3Q6tRqDVbsxclz3DWKUeFlJsMkUimcJMKA6X1QCzQYtANIFANAGb\nsax/xURERERrHk++W2YumwHTwRhSqfkF9CN9Xty0sRE6rQZCCOzrcmaCsccfRUvD/JP0Gq3q8dKF\ng/FM+vQ8tccYANspiIiIiBaBwXiZ7WqzI5GS+MHx4cxjE/4oLk8EcUt3U+axfV2NGPCGMBWMLVgx\nLnZYiFpJbrQa0JoO1QzGREREROVjMF5mP3N9G27e5MJXnzib2VT3ar8XgNJfrLphvdJnfHJopmAw\nVg8LKdZKoQbmJqsBrQ4lGHvYZ0xERERUNgbjZabRCPy/77se4VgSf3rwDACljcJq0GJ3uz1z3fUd\nDmgE8MLFCUQTKbTkBGOHWQ8hUPRYaDUYN1oMaLUrwZin3xERERGVj8F4BWxpseHTd2/GYydH0Hve\ng1f7pnDTJhd02rnltxp12NbagKdPjwNAXsVYqxFwmPVFe4zVaRUuqwE2ow42o46tFERERESLwGC8\nQv5bz2ZsdlvxhX9/Exc9AdyyqSnvmhvWN+LqTBgA4LYZ8553WQqPfgOye4yVsW4tPOSDiIiIaFEY\njFeIUafFn/3cnkx7w4FuV941N6TnGQP5FWNg4dPvvMEYbEYdjDotAKCVs4yJiIiIFoXBeAXdvMmF\nXz6wHs02I3Z3OPKe37d+LhjnjmsDgEaLHtPBwj3Gyql3+szX6xwmjM2yYkxERERULgbjFfbl9+5G\n7+/0QK/NX/otbhsajDoYtBrYzfkHczRaileMp0JxuNIj3QCllcLjj4AHEBIRERGVh8F4hWk0ouhp\ndBqNwJ4uB9wNRggh8p4vdrw0oFaM54LxOrsJ8aQsOsWCiIiIiObjecE15vfu34mJQOHe4EarAdFE\nCtFkfhV4KhjD1lZb5uvMyLbZSGYGMhEREREVx2BcYwr1HqsaLUoPcSBWOBhnt1KowXjcH8Eu2POu\nJyIiIqL52EpRR9Rjof05wTgcSyIcT85rpWi1K1MtxldxA543EMXBUyOr9vlEREREi8FgXEfUlohA\nfH4wns463EOlTrVYzZFt3355AJ/97hvwcJ4yERER1QEG4zriTFeMAzn779QNednB2KDToMlqwLh/\n9ULpiaEZAMClicCq3QMRERFRuRiM64gafP05FeNCwRgAWuymVWulSKUkTg4rwfjyRHBV7oGIiIho\nMRiM64jDrIcQ+Zvv1FaKRsv8YLzObly1inHfZBD+SAIAcNnDijERERHVPgbjOqLVCDjN+rweY7Vi\n3JRTMW61mzA2uzo9xifTbRQNJh0us5WCiIiI6gCDcZ1ptBjyplJMBWPQCMBu1s97vMVugjcYRTyZ\nWslbBACcHJ6B1aBFz/YW9LGVgoiIiOoAg3GdcTcYMRnOD8ZOiwFazfzT8tbZTZASmPCvfNX4xNAM\n9nQ6sa3FhqszYYRjyRW/ByIiIqLFYDCuM2/b0IhBXwqhWCLz2HQoljn8I1tmlvEKj0uLxJM4O+rD\n3i4nNrcop/H1TbKdgoiIiGobg3GduWWTC0kJvD44k3nMG4ihyWrMuzZz+t0KzzI+O+pDPCmxr8uB\nzW4lGHMyBREREdU6BuM6s3+jCxoBHOnzZh6bDsXQaC1UMVaD8cpWjNX5xfu6GrGhyQKN4GQKIiIi\nqn0MxnXGZtRho12DV/vngvFUMJ43wxhQplTotQIXxv0reYs4OTSDVrsR6xwmmPRadLksnExBRERE\nNY/BuA7tcGlxYmgG4VgSUsp0j3F+MNZoBB7Y14FHjw2hf3LlWhlODs9ib6cz83V3s5WtFERERFTz\nGIzr0HaXBvGkxBtXpuELJ5BMyYIVYwD43fu2w6jT4k8PnlmRe5sJxdA/GcS+9XPBeLPbhv7JAFIp\nucAriYiIiFYXg3Ed2taozfQZT4UKHwetamkw4dfv2YLnznnw/DnPst/byeFZAMC+rIrx5hYbIvEU\nRmbDy/75RERERJViMK5DZp3W8bUIAAAgAElEQVTA7g4HjvRPZU69aywSjAHgY7dtQnezFV8+eAax\nxPIe9nFyaAZCANd3OjKPcTIFERER1QMG4zp1oLsJJ67MYDRdhXUV6DFWGXQafOk9u9A/GcS3X+5f\n1vs6MTSDLW4bGkxzUzI2u60AOJmCiIiIahuDcZ26ZZMLsWQKz51V2iOKtVKoera34J4dLfibH19c\ntvFtUkqcHJrB3i7nvMddVgOcFj0nUxAREVFNYzCuU+o842fOjgMoHYwB4A/fvQvxlMR7/vYl/Nep\nUUhZ3c1ww9NheIOxvGAshMBmt43BmIiIiGoag3Gdcpj12NVuhz+SgEGngcWgLfmajc1W/ODXboW7\nwYjPfPd1fPzhoxiaClXtnjIHe3Q6857jyDYiIiKqdQzGdezApiYAykEeQoiyXrOn04n//Mzt+NK7\nd+Fo/xR+6q9+gt945A386I2rmY18lTo+OA2zXosdbQ15z21usWHCH4UvEl/SZxAREREtFwbjOnZL\ntxKMCx3usRCdVoNP3LEJz/7WXfjZfR146eIkPv9vJ3DjV57Bz//vwxVXkY8NTmFflxN6bf6PlTqZ\nom+BqnEqJRFNJCv6bCIiIqKlKisYCyHuE0KcF0JcEkJ8ocg1HxRCnBFCnBZCfLe6t0mF3LzRBSHK\n6y8upM1hxtd+fg+O/sG9+M/P3I7fuGcr3ro6i2/95PKi3ysYTeDsqB/7NzYWfL6cyRR/9uRZvOdv\nX+JBIERERLQqSgZjIYQWwDcB3A9gF4APCyF25VyzFcDvAbhdSnkdgM8vw71SDodFj7u2ubEna2Zw\nJTQagb1dTnz+3m141/VteOzECMKxxVVuTwzNIJmSuHFD4WDc5bJArxULbsA7O+rHhfEAXr48uajP\nJiIiIqqGcirGNwO4JKXsk1LGADwC4IGca34VwDellNMAIKVc/iPWCADw8Mdvxu/et6Nq7/fB/V3w\nRxN48q3RRb3u2MA0hADeViQY67UabGiyLhiMx9Jj5B55bWhRn01ERERUDeUE4w4A2UllOP1Ytm0A\ntgkhXhZCHBFC3FetG6SVdaDbhQ1NFvzb0cWF02ODU9je2gB71sEeuTa7F55MMTYbgVYj8PSZMXgD\n0UV9PhEREdFSiVKzbIUQHwDwTinlp9Jf/wqAm6WUn8u65iCAOIAPAugE8CKA3VLKmZz3ehDAgwDg\ndrtvfPTRR6v4rawdgUAANptt2d7/8csx/PvFOP78TjNaraV/d0pJiU8/G8Kt7Tp89Dpj0eu+fz6G\npwbi+IeftkCTM0UjnJD4b8+GcEeHDi9dTeAXthtw/6biIbtSy7121zKuXWW4bpXj2lWOa1c5rl1l\nan3d7r777uNSyv2lrtOV8V7DALqyvu4EMFLgmiNSyjiAfiHEeQBbARzNvkhK+RCAhwBg+/btsqen\np4yPp1y9vb1YzrXbcUMEP/zaj3FF145f6CndpnFmxIfIoRfxwG270XND7h8T5lw1D+K/+t/Czrcd\nQJvDPO+5S54A8OxP8P47diNyZBBHvTF87WN3lT2GrlzLvXbXMq5dZbhulePaVY5rVzmuXWWulXUr\np5XiKICtQohNQggDgA8BeCznmh8BuBsAhBDNUFor+qp5o7Ry1jlMuGubGz84PoxEMlXy+uODUwBQ\ndOOdqsOphOGr0+G859Rjqtc5TPjQzevRNxnEq/1Ti711IiIiooqVDMZSygSAzwI4BOAsgEellKeF\nEF8WQrw3fdkhAF4hxBkAzwP4HSmld7lumpbfL9zUhXFfFC9cnCh57dGBabTajehsNC94XSYYz+QH\n49HZdDC2m/Az17ehwaTDI69dqeDOiYiIiCpT1hxjKeUTUsptUsrNUsqvph/7kpTysfQ/Synlb0op\nd0kpr5dSPrKcN03L7x07WtFkNeDRo8Mlrz0+OI39G1wl2x7aFwjG2RVjs0GL993QgSfeGsNMaGmn\n8RERERGViyffUUEGnQbvu6EDz54dx+QCEyJGZ8O4OhMu2UYBAFajDk6LHiMFgvHYbAROix4mvRYA\n8KGb1iOWSOGHb1yt/JsgIiIiWgQGYyrqQzevR1JKPPRC8XbxYwPTAFD0xLtcHU5zwR7j0dkI1tlN\nma93tduxp9OBx07m7vMsLhBNrJkjpf/1lQH8x+ulq/lERERUPgZjKmpLiw0//7ZOPPzyAIamQgWv\nOT44DYtBi11t9rLes8NpLtpK0ZoVjAFgb6cTfQvMPc71C//fK/jzJ8+XfX09+9dXBvGvrwyu9m0Q\nERFdUxiMaUG/9dPboNEA3zhUOHAeG5zCvi4ndNryfpTa0xXj3PnZY74I2hzzg3GXy4zZcBy+SLzk\n+0opcdETwNlRX1n3Ue+mgjEMTxf+ZYWIiIgqw2BMC2pzmPGpO7rx2MkRnByad14LgtEEzo76sb+M\n/mJVZ6MZwVgSvnAi81g8mcJkIJpXMe5qtABA0Wp1Nl84gVgiVbAafa1JpiRmQjFMBmIIxRKlX0BE\nRERlYTCmkv6fu7rRZDXgq0+czVR6kyml9ziZkrhxo6vs9yo0mcLjj0JKZSJFti6XGoxLh92JgDLV\nYnQ2jFRq4dMc650vHIf6LRbq1yYiIqLKMBhTSQ0mPT7/U9vwWv8Unj3rwZkRH37u71/GX//4Iu7d\n2Ypbu5vKfq9Cs4zHZudGtWVTK8bltAx4fMrkjHhSYmKBKRrLZWgqhN/7j1OIJUofiLJUU1kj7IbY\nTkFERFQ1DMZUlg/d1IVutxX/499P4b1/9xKuzoTxtx++Af/wkRth0JX/Y9TRqJ5+NxfoxrIO98jm\nsOjRYNKV1UqRHYaHV6GK+tjJEXzvtSGcH/Mv+2dNBbOCcRnVdCIiIioPgzGVRa/V4Is/sxMzoZgy\n3/g378J79raXPNQjV5PVAKNOg5F0GAaUjXdAfjAGlKrxUBlBV60YA4UPEFlup0dm05+9/BXc+cGY\nFWMiIqJq0a32DVD9eMeOVrz1J++ExVD5j40QIm+W8bgvAqNOA6dFn3d9l8tc1si2iUAUOo1AIiUL\nHiCy3E6PKNMwilWrkymJUCyBBlP+97hY0+lgbDPq2EpBRERURawY06IsJRSr2nNmGY/ORrDOYSpY\nfe5qtGC4wHi3XB5fBG1OE+wm3YpvSPNF4hj0KgG1WDD+zpFB3Pn156tyAIk3HYyv73CwlYKIiKiK\nGIxpxeUe8jE+m3+4h6rLZUE4nsRkIFbweZXHH0VLgwkdjZYVb6U4OzI3O7nYZ58ansVMKL6oA0uK\nmQ7GYDFosaXFxooxERFRFTEY04prd5ox4Y8iEleqp4UO91B1uZTNeqUC4IQ/CrfNiA6nqeJWCo8/\nUrIyXYjaRrG7w160Wn1lSgnEF8aXvjlvKhRDo8WALpcZ/kgCs+HSB6AQERFRaQzGtOLUyRRjs0oQ\nHfNFCm68A8o/5MPjj6LFbszrXy6XxxfBbX/2HP7ymQuLfu3pER+abQbc0NVYtGKstlpUY2rFdDAG\nl9WwqANQiIiIqDQGY1px2bOMp0NxxBKpoq0UnZlZxsXDbjSRxGw4rlSMG83wRxdfRb3oCSCRkvi7\n5y/h8KXJRb329MgsdrU70NmoHGHtzznCOhxLwuNXpmZUpWKsBmNX+XOeyyGlxBtXpvGVg2fwwDdf\nxiXP8o+eIyIiqiWcSkErLjsYN1oMAFC0lcJs0KLZZlywKjqRDp0tdiNsRmXqw8hMGA5z+RMgBrxK\nq0NLgxGf/7cTeOrzb4fLaij5umgiiUueAO7e0TI3o3kmjB3r5j77SvreTXoNzlWhYjwViqHbbUNn\n+vOWugFPSom/772M7756BVdnwpnpHq9c9mJLS8OS75eIiKhesGJMK06ZQKEcZzzmU0Jda5FgDCh9\nxlfKCMbuBiPancr7LLad4oo3BINOg3/66E2YCcXxO98/WVa/8cVxpdJ8Xbt9LvDnfPZgOnTfudWN\n4ekwAtHEou4t13QwjkaLAQ6zHg1VGNl2dtSPbxw6j45GM/7nB/bi2BfvhV4r5s2aJiIiWgsYjGnF\nGXQatDQYcXUmjLFZJdQWqxgD6iEfxcOf2qagTKXIP3K6HAPeINa7LNjd4cDvvWsHfnzOg4cPD5R8\nnXqwx3XtjqJtH2p/8U/tagUAXFxCO0U0kUQgmkCTzQAhBDpdliX3GJ8YmgEAfP3n9+D9N3bCaTGg\n1W7CaJE1HJ4O4QPfOgzvKhy9TUREtJwYjGlVdDjNGJkJY8wXgUYAbpux6LVdLjNGZiJIJFMFn8+u\nGDdbjTDoNIueTDHoDWFDumf3Y7dtxD07WvBnT5zD5YnAgq87PeKDzajDBpcFzTblVL/cUD44FYTd\npMMtm1wACvcZ/82PL+IX/+FIyfucDir9y2oLSlejuayTARdyYmgajRY9NjRZMo+1O8xFK8ZH+qZw\ndGAap4Znl/S5REREtYbBmFaFOm94bDaMZpsROm3xH8WuRguSKYnRIkHN449CCOW4aY1GoN1hwnBO\nOJVS4nd/cBJnvfkHbEgplWDcZAWgnM73lfftRiyZwnNnPQt+H6dHfNjZ1gCNRhQ81Q9A5r27Gi0w\n6TU4P5Yftn904ipe6fNmRtgVox4H7bIqPcxdLguGp0MVjZlTnRiawd4u57wDVtqcJozOFg7c6vdX\n7N9HvTs76kMyVfl6EhFR/WIwplXR7jRhdCaC0dniM4xV6vSFYu0UE/4omqyGTLjuaMwPp5c8ATx6\nbBiHR/L7eyf8UYTjSWxsnquYtjnM6HCacWJ4puh9JVMSZ0d9uK7dkXmso9GcF8qvTIWwvskCjUZg\nW2tDXsV4ZCaMvokgpAT6Jxc+AGQ6pATj7IpxJJ4qeQBKMf5IHBc9Aezrcs57fJ3DhLHZCFIFAqI6\nBWOsSHCuZyMzYdz/1y/iL54+v9q3QkREq4DBmFZFp9OMWDKF0yO+oqPaVOq83uEi0xcm/BG4G+be\nQ23TyPZKn1d5D39+O8Zgukd3vcsy7/F9XU6cHCoejAe8QYRiSexqt2ce62w042pWgI8nU7g6HcbG\ndJvC9tYGnM8Jxi9ljYcr1bqhHgfdZEsH4xK/NJTy5vAspEReMG53mBFPSkwG8/uI1VaRMd/yVIxn\nQ/Gi1erlNp7+nh56oa8qo/WIiKi+MBjTqmhPT3CYCsawrkTFuM1pgkYsXDF2N8z1KHc4LfD4o4gm\n5toSDl9KB+NAKu/P5APpKu3GdCuFam+XA8PTYUwW2WSmnnh3XVYw7nCaMRmIZVoiRmbCSKQkNriU\n996+rgET/mimJQIAXro4CZfVACGAy54SFePg/Ipx5xIP+XgjHfxzg7FaxR+dyQ+/w8vcSvHHj5/G\n/X/9YtF1X07q/OuklPiDH75ZsGJORETXLgZjWhXq9AgAJYOxXqtBm8NcNPx5/FG0ZAVjdWSbGupS\nKYkj/V7YjDrEU3Mzi1WD3hC0GjHvngBgb6cSFotVjU+PzEKvFdiaNes3dyqGOpFifbpivK1VuVat\nRqZSEi9fmsTbtzajw2kuWTGeCsYgBODMBGPl8xY6AGUhJ4dmsKnZmnk/lfqLS27lNpmSmWr82DIF\n47OjPsyE4vjy42eW5f0XogbjB+/sxtGBaXz/+NCK3wMREa0eBmNaFWrwAlD0OOhsXa7C0xdSKYnJ\nQE7FOB0W1QB3bsyPmVAcH9zfpXw9Ov9P5INTIXQ4zdDnbAC8vtMBjSgejM+M+LCttQEG3dzrcke2\nqW0a6sSH7evmB+NzY354gzHcsdWNzW5bWcHYadZDq1E2ylmNOjRZDRVVjKWUODE0k1ctBrIqxjnh\n1+OPIJGSMOu1y9JKkUxJ9E8G0WjR47GTI3ju3HjVP2MhajD+5J2bcPNGF/7syXMcS0dEtIYwGNOq\nsJv0aDApBy+WFYwbC8/rnQnHEU/KeRXjTmc6nKaDsdpf/NHbNkAjgHNjvnnvMegNzhtVprIYdNjW\n2oATBcaSSSlxesQ3r40CQN4hH1e8QRh0GrSme6BbGoxwmPU4nz4B7+V0f/EdW5ozwXihP99PhWJo\nzDmRr9O18JznYkZnI/D4owWDscuqjJ7LDcZq4N/X5YQ/kljyYSW5RmbCiCZS+O8/tQ1bW2z44g/f\nqvpnLGQmpARjp9mAr75vN4LRBL76xNmK309KiViS7RhERPWCwZhWjRoiS7VSAMomM48/mjfOLHuG\nsSr7ZD0AeOXyJDY2WbChyYp1VoGzORXjgclgXn+xSt2AlzsObcwXwVQwNm8iBQC02k3QaQSuzihB\nVZ2PrElXeIUQ2J41meLFS5PY0mLDOocJW1psiMRTGFlg49l0MAZXTttDV6O5olaKE0X6i9X7bHOY\n8jYxqmu6f2MjgOq3U6gV8x3r7Pjaz+/BqC+C/3lo5SZEzIbjsBi0MOg02NragAff3o3/eP0q3qxw\nZvOjx4bw+edD8EfiVb5TIiJaDgzGtGoWE4zViRG5AdDjV4JZS9ZUCvVkvZGZMJIpiVf7p3Dr5iYA\nQJdNM69iPBOKwRdJFKwYA8DeLidmw/FMr7Dq2MA0AGB3x/yKsVYjsM5hmqsYT4Xy3nvbOhvOj/kR\nTSTxWr8Xd2xpBgBsdivh/PJE8Q14U8H8inGXy5L5XhfjxNAMDDoNdrbZCz7f5jAXqBgr63DjhuUK\nxsr3vtltxY0bGvHRWzfiX14ZwPHB6ap+TjGz4TicZn3m64/euhEA8PqVyj7/R2+MIJQAD0MhIqoT\nDMa0ara02tBqN8Ji0JW8tsulhOjcloFCFWNACd1XZ8I4PTILfySBA93pYNygwfB0GL50BW/Aq/YA\nF64YqxvwTuT0GX//+DDaHCbs62rMe01nuoKrHhyy3jX/vbe3NsAXSeCJN0cRiafmgnGLDQBw2VO8\nz3gqGENTbjButCCelIvu+T1xZQbXtdvn9Uhna3PmHws9PB1Gs82QqbBXu8+4byIAh1kPV/p7/O13\nbkeb3YQ/f/JcVT+nmJlQHPasYOxuMMJu0uGiZ/Gj22ZDcbw2MAUg/+eHiIhqE4MxrZpff8dW/PDT\nt5d17dws4/nB2JMOxi25wTh9st7hy0p/sVox7mxQfuTVHt9BrzqqrXDFeFurDWa9dl6wGZoK4cWL\nE/jA/q7MJrh5n+1UPls9OCSvYpyeTPFPL/VDqxG4pVs5KrrJaoDDrC+6AU9KiekCPcaZXxoWsQEv\nkUzhzauzBdsoVO0OM8b90XmV6KszYXQ0WjJV/mof8nF5IoDNbmvmFD6bUYd3723HiaEZxBKFjwSv\nJl84DqdlLhgLIbC1tQEXxxfeFFlI7wUPkikJgwZ4o8KKMxERrSwGY1o1VqNu3nSKhbgbjDDqNHmT\nKSb8UVgMWliN86vO6sl6hy97saXFlmm16EoH43OjSjuF2iLR5SocjHVaDa7vcOBk1gl43z+mjPD6\n4P7Ogq/paDRj3BfBpXTAXZ8TjNXJFG9d9eGGLicaTEoQE0Jgs9uKS0UqxoFoAvGkzOsxrmSW8flx\nP8Lx5ILBuM1pQjIlM+0qgFIx7mw0w6TXotGir/os476JILrdtnmP7etyIpZM5W2aXA4z4RgcWRVj\nANjaUnpaSCHPnvWg2WbE/nU6vHElv0+diIhqD4Mx1QUhBDobzbjiza8Y51aLgbmT9Q5fmsSt6TYK\nAHCZBOwmHc6mK8YD3iDaHCaY9Nqin723y4HTIz7EEikkkik8emwYb9/qzgTSvM9uNCMlgVf7lD+j\n527sc1oMaLUr93x7uo1CtaXFVrTHWD0UxGXNnTmsbDYsNM6uGLUCfkOBVpDM+zrUsXfpedBS4up0\nGJ2Z3nBz5qS4avBH4vD4o+h25x60svA86WqaDcfzgvGWFhsmA7F5h7KUEkuk0Hveg3t2tGCLUwNv\nMFbxrGkiIlo5DMZUN3a22XFscAqJ5Nyf1JXjoPODsTrLOJGSmTYKQAnYO9rs8yrGxTbeqfZ2ORFL\npHB+zI/e8xMY80Xw4ZvXF71eDY6vXPZCI+Y2GWZT2ynu3Do/GG922zAZiGI2lD/FoFgwNuq0aHeY\ncWxgquyq5IkrM3BZDZk2jELa1INS0u0SvqhELJnKHCqyzm6sasW4L7Pxbn7FuN1hQrPNiBNDy7+B\nbSYUzzvsZGv631WxSn4hRwem4I8kcO+uVmx2Kv81W+kGPiIiWjkMxlQ3fub6NkwGYni1fyrzmFIx\nzp9q0eGcC7sHsirGALBzXQPOj/mRSsn0OLXCG+9UcxvwpvHI0Stothlxz86WoterofyNoWm0O80F\nN7fdsL4RzTZDphqqUkPh5cn8EDYdSh8HnROMAeATd2zC4ctefOfVKwt+Lyr1YA+1l7eQNnv69Lt0\nxXgyrIRutVK+zmGu6lSKvvT3vDmnYiyEwL4uB04MLW+wjMSTiCZSBSvGABa1Ae+ZM+Mw6jS4Y0sz\nOm0amPQavHGFG/CIiGodgzHVjbt3tMBq0OLgqZHMYxP+aMGKsXos9I51DXkV1h1tdgRjSZwb82My\nEMWG5oUrxp2NZjTbDDh0ehzPnfPgA/s7807Jy9bmMEMIIJ6URavRn717C57573flvc9Ckym8gXTF\n2JIfjD9+20bcubUZXzl4BhfHFw5wgWgClyYCmcBfjN2sg8WgzcxVnowowVgN/m0OE7zBGKKJZNH3\nKOS5c+P43PfeyKtuX/YEodWIvCkegPLLyeWJYGaayHLwpU+9yw3G7Q4TrAZt2RvwpJR49uw47tjS\nDLNBC61GYE+nk5MpiIjqQFnBWAhxnxDivBDikhDiCwtc934hhBRC7K/eLRIpTHot7t3ViiffGkM8\nmUIknoQ/kigYjBtMeqx3WXDvzta853akN78dOj0GIL8HOJcQAns7nXjp0iRSEvjQTV0LXp990l2h\nkKdeU6jy29VohkGryWzcy6ZWjF22/NdpNAJ/8cG9sBl1+Nz33sg7CCXbW1dnIaXSO70Q9ZCPuYqx\n0sKSmT+dPrHQ41vckcnPnPHg8ZMjODM6fzNd32QA612WghV2tbJe6UEb5ZgpEoyFEOne7/KC8flx\nP4anw7h319zP3g1dTpwZ8S36lwiqDf2TQez4wyfx1lXOoya61pUMxkIILYBvArgfwC4AHxZC7Cpw\nXQOAXwfwarVvkkj17j3tmAnF8fKlyaIzjFVP/sad+Py9W/Me376uAULMBeP1RSZSZFOD2W2bm4rO\nPM6mVlVL9S/n0mk12NhswWVP/ga8qWAcBq0GVkPhjYItDSZ8/f17cG7Mj68/Vfy0ODVcXt+xcDAG\ngHanGaO+uVaKRos+MwFEHdm22D5jT/r9njvrmff4ZU8wr41CVWyedDXNpoNx9rg21ZaW8ke2/Tj9\nfd2zY67d5ob1ymSNMyPLP1mjntXq5I7X+r2IxFN4KX2EOxFdu8qpGN8M4JKUsk9KGQPwCIAHClz3\npwC+DqC685uIsrx9WzMaTDocPDWadepd4WBsNeqgK9DyYDHosLHJinPpyRTlhNf96ZPefvGW4pvu\nsqlV1WLzkRey2W1DX6GKcTCGRqt+wb7ge3a24qO3bsA/v9yPFy5MFLzmzauz6HCa0WQrvG7ZlIqx\n0krhDct5kzja1FnGi5xMoV7/7Lm5YJxMSfR780e1qRwWPbqbrcs6mWImVLhiDCh9xmO+SFmtHM+c\nGcfeTgda7HO97+pBMOwzLu4bh87hZ/7mpZoMx+ovNOX+xeL44BQe+ObLCMf4FwKielP6yDGgA8BQ\n1tfDAG7JvkAIcQOALinlQSHEbxd7IyHEgwAeBAC3243e3t5F3zABgUBgTa/dHhfwXyeH0RxXgt/g\n+TfRO1p83Fo2de2atBH0A7AbgONHXi75OiklvnTABKv3PHp7L5S8PuVX2h48fWfQO1m8eluILhTD\ngDeOZ597HrqsA0QuXInAIGXJf/e32yQeNwn81cHjSL0tf2PiqxdD6GzQlPUzFJ2OYcKv3IsnmECX\ndu5nLxRXAszLr5+Gfbr0mqiGJoPQCGX82o8OPQenUQNPKIVYIoX41DB6e8cLvq7VEMFrl0N4/vnn\nF/zl4CfDcXTYNNjiLO9nQvXqVSX0njv1OqYuzf+FKuJJAAAefeqFBd93JprCiaEwfm6rPrNOgUAA\n5944ApdJ4NCx8+hODGauT0mJaBIw64p/P2uBlBKPvBKGNyLx7f98Dt3pNa6V/6575Zzyy+Frl8bK\nup+Dl2M4ORTHvz3Zi42Oxf0cVkutrF094tpV5lpZt3KCcaH/xs78Si+E0AD4KwAfK/VGUsqHADwE\nANu3b5c9PT1l3STN19vbi7W8drLNg49/+yjOROwAJvCuu+8o2k6RS127k4mLODZ+AVvWOdHTU97p\ne3cv5ibbPDjjO4MP3HcnzEVaH4qZdgzj8b6T2Lh7P7a0NGQe/9uzh7HerkFPz4GS7/Hs9EkcOj2O\nt7/9LmiywvVsOI7xp57GR+7cgp6eLSXfZ9x6Bf95+U1s33cLpp99Hu/Zuh49PUonlZQS1hcPwdLc\njp6e68r63uLJFHxPPYl3XteKQ6fHEXVtQc9N6/H8eQ/wwlG8644bcdNGV8HXDuj78crjZ7DjbQfQ\n5ig8Zi6RTOHBPzqEe3e24FM/e2NZ96S6/FI/8OYZvLPnTjhy2ik2eYP469d7Ye/Yhp4Fesz/4/Vh\nACfxyftvwXXtSquK+jN34OpxvHl1NvP/u1JKfPr/vo5Tw7N46X/0LBj2V0IimcIfP34an7h9U9HK\n/XI5P+aH99ALAIARXRs+kf4Zq4X/rpNS4nO9T0OvlZgIS+y7+ba8kX65en2ngYsDaN60Ez172lfo\nTnPuoQbWrl5x7SpzraxbOa0UwwCy/5egE8BI1tcNAHYD6BVCDAA4AOAxbsCj5XLHlmY4LXq8cGEC\nGpE/17ccO9qUwFlq412lera34Lnf7ll0KAbmRrblzs2dDsbQWOJ/kFUHupswG47jbM5pcafTm4f2\ndJbuLwaQCaBvXZ1FLIXMDGNA2ZS2zmFa1CEf6hHePdtb0O4w4dl0P646hSN3hnG2cg76GPAGEUuk\n0D9Z/imAqtlQDEIADUELOGsAACAASURBVKb8ekFnowVGnabkyLYBbwhCAFuzfqFR7etyYmgqjMmA\nsgb/+GI/nnxrDFdnwjVx+EffZBDfOXIFf/FM+dX/avnxOeWvBHu7nDh4ahSpVO20UwxPh+GPJPBT\n6c2Ub5axAU/d/zDoXfzPIRGtrnKC8VEAW4UQm4QQBgAfAvCY+qSUclZK2Syl3Cil3AjgCID3SimP\nLcsd05qn12pw/+51AIBmmxFazeIrbTvX2QHkH9dcCzKzjHNOwPMGY2X/EnBLenazevqe6tTV8jfe\nAXNj744OKDOEO3JO+2tzmBe1+U4N0evsJtyzsxUvXZxEJJ5E32QQTot+we9vV7sdeq1Y8KCPs6NK\ncB30BhfdqzobjsNu0s+rsKu0GoHNblvJQz6uTofR2mAqOrsaUA5XOTowha89dQ4725Sfw2Jh6+K4\nH/91ahSBaGJR30sl1H83T6XD+kp6/pwHuzvs+MTtGzHmi+B4DR2GcjY9PeUD+5X60OKCceFTLCtR\ni73XRNeiksFYSpkA8FkAhwCcBfColPK0EOLLQoj3LvcNEhXy7vSfJ8ttocjV5TLjyw9ch18oMXpt\nNViNOrQ5TPNmGSeSKcyG42UH4w6nGetdFhzp8857/M3hWax3WUr+KVilVoyPDigBO7tiDACtdtOi\nDvkYT1/bYlcOSQnHk3ilz4vLngC6mxeu3ht1Wuxqsy940Me5dIU8FEtmwkm5ZgocB51tS4sNF0sF\n45lQZiJJrt3tDug0As+eHcdnv/s6uhrN+M4nb4ZeK4qGrT/40Vv4zHdfx41/+gx+7f8cx8FTIwuO\n4luK8fTYvWRK4l8ODyzLZxQyE4rh+OA03rG9BffsbIVRp8HjJ0dKv3CFnBn1QQjg5o0ubGiylLUB\nbyL9V4GBKlWMv/Dvp/DJf2GtiWgllDXHWEr5hJRym5Rys5Tyq+nHviSlfKzAtT2sFtNyu2WTC802\nY2YywmIJIfCRWzcW7VVdbZvdNpwe8WWqROqM3cW0jRzoduHV/ql5f5Y+dXUG15fZRgEoId1u0mVm\nDueGvjaHCR5/FMky//SdXTE+0N0Ei0GLH58dR99kcME2CtXeLifeHJ4t+nnnRudaHfonF1etmw3H\nC45qU21tsWF4OoxQrHj1dng6XPAIcAAwG7TY0daAR44OYSYUxzd/6W1oshmxfV1DwbAVTSRxYmgG\n77yuFR+6qQvHr0zjs999A7/56IlFfV/lUqe83LuzBd977cqKVKkB4CcXJpCSygE+NqMO9+xswRNv\njs47+n01nR31YWOTFVajDtd3OBZVMb5SpWB8pM+LFy5MILhC/06I1jKefEd1SafV4OGP34Tff9fO\n1b6VZfHT17Xi/Lgfhy8rFd/pYPo46DIrvQBwyyalz1gdSzcdjGFoKlx2G4Wq3WlGMiVh0QF20/zg\nuM5hQjIlM32zpYz5otBrBRotBpj0WtyxpRlPvjmGCX+0rA1fezudCMaSRQ/bODfmx750L/Ji+ztn\nQgtXjLe2qqcSFg7cyZTE2GykaMUYAG5Ij2378gPXZTbnqWEr90/lb131IZZI4X03dOBPHtiNI793\nD37ubR34yfmJsn8RyeXxRRBLFA6cHl8UDcb/n737jm+rOh8//jmSvPe2k3iPOHsnJCQhYUMLgVKg\nQEuhlEL3ooP223750d3STceXltFCoWV0AGWEEJIASSB723Gm48R7ynvo/P7QvbJsy7ak2JHtPO/X\nixe2dCWfHF/bzz33Oc9j47Or87C3d/Pc9lMejxtp64uqSIgIdtWq/uDsSQNavwfSoXI704w9CbOn\nxFBW30ad8fPoSWtnN80d3USF2Khoaj/rkm3tXT2U1rXS7dDsOOlfismPXjnEZqnBLIRXJDAW49bM\nyTHnfPf8uXLTwnRSo0P51brDaK2pNf4QJ/iwYrwkx1nd4b3jzuDaXOma7WNgbK7KJ4YN/HVhdr/z\nNs+4qqmd5KhQVx7vpdNSXP+2wZp7uDM34Hlq9NHY1sXphjYuKUwmyKo47mN+Z1NbF9FDplI4g6PB\nNuBVNrXT7dAD0k3c3XNRDr+6eS43LexN4Zk5OYbGti5O1fXN691upK8syHR+H60WxbLcxCEvDIbS\n0d3DJb/YyOPvHvf4fJW9neToEOZlxLEgM47H3z3hdwDure4eBxsPV7NqarLrnFg9NZnwfq3fA8Xe\n3kVpXatrT8JM42dnqFVjc7V4vlH7vLTu7FaNj9e0YH4btvRLjfLGmYY2/m/TMZ7Zdm4udIQY7yQw\nFmIMCg2y8pnVuWw7Uc/mo7W9K8Y+BMZT4sJJjw9z5Rmbf8xn+BgYpxrpJglhAzelmd3vKhq926xV\n0dROSnRvXvjqwmTMKmXeXOTkJEYQFWrzWJmi2FgZnzk5hvT4cE74k0oxRGCcmRBOkFUNmmdsblgb\nLJUCnN+T6+ZN7lOabfZko911v2Br24l6shMj+uTRm6vhu/1oFHK0qgV7e/eg469s6iDZaGV+1/Js\nSutaWXfIc03pkbLrVAMNrV1c7NYlMCzYymVG6/fuAFenMO+2TJ/UNzAeqjW0GRgvynIGxme7Ac/c\n8BkXHjRgz4A3thh3nQ6VT8yui6/sK2fzUVkNFyNHAmMhxij3VWNzVdXX0nQXZCe48oz3lTWSnRgx\nZLqAJ5NcK8YDA2NX9zsvV4wrmtpdwTQ4N0/OmRKLzaK86kBosSjmZcR5DBDMP/zT0qLJSojwaeOT\n1nrYzXdBVgtZCRGDVqY4bZRcG2rF2JOC1MgBG/C01uw4WccCY9XRZF4Y7PKjA2BxpXN+zgxScaLS\n7aLl8ukpTIkL49G3Pa8ue3Kkys4jm476lJu8vqgKm0WxoiCxz+Nm6/eDtYHtHOd+ToEzlSgnMYK9\nZYPPv1mScKFRj/tsS7aVVDVjUXDD/CnsLWv0OffbXGU+Vt08ahs3A6XHobn/n/v43VtHAj0UMYFI\nYCzEGOW+avzfveUAQ24O82RJTgINrV0UV9rZd7rR5/xigDRjBdRTKkV8RDDBVgvlXtYyrnJblTR9\nZlUud6/MIchD+25PLipI4mh1C6f63aIuqmgiNjyIlOgQshIifCrZ1tLZQ49DDzu/+SmDl2wrq3eO\nZ9IQK8aehNisFKZGs+90b7B1tLqF+tYu16qjyWJRzE2P9ZhKMhxz9dNTYKy1psreQYqRGmOzWrhj\nWRbvn6gbNq+1qb2L7718kCt/9TY/fKWID/zmbXZ5WW7traIqFmXFD8hdN1u/v1ce+MA4Jiyozybf\nmZNjhqxMYa4Y5yVHEhMWxMm6s1sxPlrVTHp8OKumJtPj0K4UG29ordlytJbIEBsODYcrh67DPd7s\nP93obFrU5FsFGiGGIoGxEGOYuWq85VgtUSE2Qmy+NQxZku1ctXplXzmnG9q8buzhzkwNSPKwYqyU\nIiUmxKsV4+YO56ak1H6VRC6fkco3riz0ejyrpiYBsOFwdZ/HD5XbKUyNQilFdmI4rZ09rtU7d28V\nV1Hbb7NgQ6tzRX641fS85ChO1rZ4XHk73dBGQkQw4cHeNBTtywy2zEB+x0ln8LPQQxfAuemxFFc0\nDVkdwxMz1eRMY/uAC4bGti46ux190jY+sjiDlOgQvv2vfR437GmteXbbKS5+aAOPvXucGxem8+jH\nF9Ldo/nwH7fw8PqSIXOUTze0UVRh75NGYQqxWbl0Wgp7a7oDWr/3oLHxrk/qy5QYzjS2D7rhtNre\ngdXi3GCalRA+AivGdvKTI1mQGUeQVfmUZ3yqro3TDW2uspQTLZ3i7RLn7wBfmgwJMRwJjIUYw8xV\nY/Atv9iUHh/OlLgwntx6EvC+sYe7xdnx/OSGWcxO8hyUp0WHeRUYu5dqOxs5iRGkx4exoajK9ZjD\noSmusFNobJLKNDoa9i/ZVmVv587Ht/FEvzq9jUY5vJiwoec4PzkSh4Zj1QNXAcvq24asSDGU2VNi\naGrvdm3U2nainviIYI+1neemx+LQeFVP193hCjtKQWe3w5WaYzIvIFLcvjeRITZ+eP0siirsPOzh\nVvUPXznE11/YS2ZCBC99bjk/+tAsLpmWwitfXMHVs9J4aO1hbn/svUHLrq03vn+rPQTG4Dzv7J2+\nl90bKT0OTXFFE9PT+v7MzBpmA16VvZ2EiGCsFkVGQsRZBcbdPQ6O17SQmxxJWLCVuemxbD3m/Yqx\nmXt786J0IoKtrgY4E8XbJc5/n7292+cLRSEGI4GxEGOcuWqcGOl7YAzO9tANrV0o5fvGO3BWQ7h5\nUQa2QToMpsaEUuHFio17c4+zoZRiVUEym4/WulZuS+taaevqcZXVyjYCyv4bn7Ydd97iP9Yv2Gps\nNQPjoVeMC1Od72/m67o73TB4DePh9A+2tp9w5he7r1Sa5g5RmWMwjW1dnGlsZ57x2v7pFOZFS3K/\nhjmXTEvh+nmT+f1bRzhwpjcQ/NOmY/zp7ePcvjST5+9d6tqUBs45/M1H5vKNKwt590jtoOPcWFxN\nenzYoNVIzDSS7ScC0wXveE0L7V0O1zllmjE5BqUGvzCptne4zvGshHDK6lsHLZE3nJN1rXT1aFeL\n8aU5Cew/3Yi9vcur128+WktSVAj5yZFMTY1y1SOfCFo6utlZWu+60JZ0CjFSJDAWYowLDbLyxCcW\n8f3rZvn1+guM9tA5iRFEhvh+m384qTGhlHu4Pd9fpX1kVowBVhcm0dbV4+rIZ3a8M1eMJ8WGOUu2\n1fRdrXvfKF3XP2DuXTEeOjDOTowgxGbh4Jm+AYbWmtNDNPcYTkFKFMFWC/tON1Jt7+BEbeuA/GJT\nQmQI6fFhPgXGZm6pmbYwMDAeuGJs+t9rphMbHszXnttLV4+D/+w+zQ9eOcTVs1L532tmeAzelVLc\nsjgdi+pd1XPX1eNgy9EaVuYneXw9OJvcRAb1dl081/pvvDNFhtjISYwYdMW4urmDpEhnYJwRH45D\n43eLbTOfPS/ZWbHlgpwEI894+IsFrTVbjtWyLDcBpRTT0qI5VN406qkp9vYun/Kg/fXe8Vq6ejTX\nz58MSDqFGDkSGAsxDhSmRrtKRvnKzDOebTRQGGmp0aF0djuobx16FauicfDgy1dLcxIJtlnYUOzM\nMTxUbseinAEmOFe5PZVse98IKE7WtPYJEMzAeLjNdzarhUIPK281zZ10dDt8rkhhCrZZKExzdsAz\n84vN+sWezE2P8ykwNjferZpqBsZ9gwiz652n1fzY8GC+f91MDpY38cW/7+K+5/awJDueX9w0F+sg\ndxHM182eEuvKA3W3q7SBls4eVuQnDfp6pRT5cVa2+9nU4pV95QMuYHxxqLwJm0W5Gru4mz0ldsgV\nYzNXO2uQOxfe6h8Yz8+MI9hq8SrPuLxFU23vYKlxYTwtLRp7e7ffQbo3DlfaWfPwu3z4j1vOau69\n8XZJDSE2Cx+YlQZIYCxGjgTGQkxw6fHhfGplDrcszhiV9zdX1B57Z+jSXpVN7USF2IgYgVXrsGAr\nS7LjeavYmadaVNFEVmIEYcG9edDZCRGccAtIGlu7KKpoIjEyBHtHd5/uZQ1erhiDs6btwTN9V95c\nNYzjhi85N5iZRge894/XE2KzMHPy4BdCc9NjKW9s9zoYKK5oIirUxoxJ0YQFWQesGJtd7wbbOHjl\nzFQ+ODuNV/ZVkJsUySO3LyQ0aPiNoCvyE9l9qsF14WF6u6Qaq0WxNDdhyNfnx1k4XtPidWdFk9aa\n+57bw+2Pved1KcH+DpY3kZcc6XHD68zJMVQ0tbsuKEzOLpCdrsA4M955PvibZ3ykqplJMaGuOz2h\nQVbmZsR6Vc/4UJ0zzWhZrrMUnvlzOlp5xi/tOcN1v3uXpnZn2tZo18B+u6SGxdnxZBhlHiUwFiNF\nAmMhzgPfunoai7MHX4E8G0tzE7h5YTq/23DE4+qgqbKp/azzi92tnprMMaNs26Fyu6s7mSkr0bnx\nyQxgt5+sQ2u4wbj16l7nuLGtiyCrIjx4+GBvelo09a1dffKqzRrG/qZSgLMjob29m5f2nmFOeuyQ\nFUjmpjtzer1dNS6usDM1xVldIS02lDP9GrKYXe+G8r01M7n3olz+8onFXtfCXpGfhEPDln4NGDaV\n1DA3PXbY9ymIdc6Br3nGNc2dtHb2UNPcyaf/toOObt/Lvh0qbxqQRmGaady96b8qWt/aSY9Du0oS\nJkWFEBZk9TswLqmyk5vcd8XazDNuGibP+FBtD5Njw0iPd56TzootI1+Zoseh+d7LB/n8M7uYlhbN\nf7+wgrnpsbw5ioFxeWMbR6qaWZmfZFzQWSXHWIwYCYyFEGftgWtnkJcUyZf/sZuqQVZu+jf3OFtm\n2baX95ZTWtfq2hhnykoIp62rx/UH8/3jdQRbLVw7dxLQ9/Z2Q6uzucdg+a7upnsIik43OAMff6tS\nQG9XtWp7BwszPecXm2ZMisFmUV4FxlpriirsTDXmZ3Js2IBUikoP9aX7i4sI5ptXFfqUCjMvI5aI\nYGufPOOG1k72ljWwIj9xiFc6ZcZYCLZZfM5ZPWXUlP7wginsKm3gey8f9On1tc0dVDZ1DNh4Z8o3\nUnb617Q2axibK8ZKORvX+JNK4XBojla1uDbemS7IScChYdvxwefE4dAcquthqZFfDBARYiMzPnzE\nA+OX9pzh0XeO8/GlmTxz9wWkRIdy6bQU9pQ1Dvq74GyZ59Py/ERnycjoUFkxFiNGAmMhxFkLC7by\nu9vm09zRzZf+sdtj/dqqpo4RyS82ZSdGkBEfzuPvOlM4CtMGrhgDrnSK90/UMXtKDHnJkVhU3xXj\npmG63rmbmhqNUn0D47L6NqJCbT53FXRXkBJFsM35K3mRh/rF7kKDrExLi/aqNXRFUzv29m7XhcOk\nmDCPVSlSRnA13xRktbA0N7FPYPzukVq0Zsj8YtfrLYq5U2LZ5mOesdn85Z6VOdxzUQ5PbS3lue2n\nvH79O0ec4+3fedAUHxFMQkQwR6v7BsZV/QJjcLYSP1nn+4rxmcY22rp6XPnFpnkZsQTbLDy/o4wd\nJ+tdNbjdFVXYaemCZf1SVcwNeCNpfVEViZHB/O81M1zn7yXTkl3PDaWisZ1PP7WDW/+01adNge+U\n1JAYGeI6p5OjQqiSFWMxQiQwFkKMiIKUKB5cM5PNR2t5eH3furcOhzaCr5ELjJVSrJqa5ApGBq4Y\nG4FxTQutnd3sK2tkcXY8ITYrk2LD+q4Yt3V6HdRGhtjISojoswHvbCpSmIJtFqYZt7vnZwy9YgzO\nPOO9ZQ1DNtGA3o13U41Uk7TYUKrsHa70ArPrXfIIfm/crSxIpLSu1TXfb5dUExVqY46XzWYWZsVx\n4HSjT3VqzcB4Slw4X7t8KstyE/j2v/f3KTk3lDcOVpIYGcLc9MG/D3nJkZRUel4xTu4TGEdQWteK\nY5jvU38lxmp0/81/oUFWVhUk8er+Cm74w2bmPvgGC773Bp9+agdrD1TQ2e1w1S/un8M9LS2ak3Wt\ntPjYVnowPQ7NppJqVhYkYXHbiDk1JYrJsWGsO+Q5MHY4NE9uOcGlv9jIq/sr2Hy01utNgQ6H5t0j\nNawwVovBuaHXm5KRIyGQDWfEuSGBsRBixNy4YArXz5vMr9883GdVsq61k26HHpFSbe5WG1UWIkNs\nAypCTIoNI9hq4XhtC7tKG+h2aFeedVZCxIAcY19We6enRfcNjBva/K5I4e4Ds9O4ckYqMV60/p6b\nHktLZ8+gLapNZse7qcbtf7NldaVRJcTsete/hvFIWZ7nTJnYVFKD1pq3S2q4MDcRm5ctwBdlxdPt\n0D5V4ThV10ZiZAhhwVZsVgu/vWUeQRbFM++XDvvaju4eNhRXc9n05CGrbuQlR1JS1dwnUDID48TI\nvivGnd2OPoFbS0c3207UDejA6O6oWZEiaWBVjD98dAEbv7aKx+5YyLevnsbFhclsO1HHp57cwZIf\nruOxd46TEq5Ii+l7Tk5Li0br3oslk7/B3t6yBhpau7iooO/qv1KKS6cl886R6gFdIk/VtfLhP27m\nO/85wNz0WH5/23yAYVuPmw6WN1Hb0uk6r8BZMrKyafiSkWfrSFUzsx5Y6/W5uO1EHbf+aas0Hxln\nJDAWQowYpRSfXZ2LQ/e9jWpWBhjp2/UX5CQ4S52lRg3ID3aWbAvjZE0r7x2vw6J6b433z/tsbOsi\nNtz7BirTJ0VzsrbVtQFqJFaMAT61Mpc/fHSBV8fOzTAbfQwdUBRX2EmNDnUF2+Y4zQ14nrrejaTs\nxAgmx4bxTkk1x2paON3QxoqC4fOLTfMz4lDKtw14pXWtZMT3fj8SIkMoTIvmcOXQFxEAW47W0tzR\nzWXTU4Y8Lj85ksa2LqrdgttqewcRwdY+lVcy4/um9Git+dzTO7nxj1tY8P11LPz+G9z6p62ulCBT\nSWUziZHBHjteWi2KzIQILi5M4e6VOfzsxjlsuf8SHr9jERfmJVLT0sm85IEVRsycafd0ihM1LSz6\nwTpe218x3NQMsPFwNUrBSg9pMZdMS6G9q3f1GpwrzJ9/ZhclVc38/MY5PHnXYi6fnkJ4sJWdXgbG\nZpqLe456clQIHd0OmtpGNwB9/3gdzR3dA75Xg1l3sJLNR2tdZSXF+CCBsRBiROUmRZIRH94nMDY3\nxox08BUWbOUbVxZy1/Jsj89nJzpLtm07XseMSTFEhTqDw6yECBpau1z5mebmO29NN/KZi8rtNLZ1\nYe/oPquNd/7IToggOtQ27OqV+8Y76F0xNlf0B+t6N1KUUqwsSGTzkVreMs6JFXnD5xebYsKDmJoS\n5VOjj1P1raTH9y2dV5ASxeFK+7CrimsPVhIebHWVORuMawOeW7BdZW/vk18Mzosw6C3Z9vdtp3ir\nuJp7Lsrhfz7gXO2ta+nk/710kNcP9AanR6qbyfWwWjyYIKuF1YXJPHzrfPY/cAU3TR14Pk+ODSM6\n1OYKjHscmq8+t4ea5k6/mnJsKK5mzpRYj8H7kpx4IoKtfdIpntp6kt2nGvjempncsGAKSilsVgtz\n02PZUTp8YLyvrJEnt5ykMDWqT+qP+XtltNMpzEZCr+6r6FPucTBmOsyrflx0iMCRwFgIMaKUUlxc\nmMy7R2po63TeRh2qs9rZumt5NlcZRf77yzRqGe8sre+zoc09WOlxaOzt3b4Fxq7KFI2U1ffms55L\nFotiQWYc7w1RnaCrx8HRquY++ddpRmWQ3sB4dFeMwbnRzt7RzZ/fPk5mQrir9qy3FmbFsat0+Hxq\ncP6bzzS0kdEvMJ6aEklDa98V3v4cDs0bBytZNTVp2DrN+camuCNuG/Cq7QOre5hdGE/WtlJa28r3\nXj7I8rxEvnFFIZ9ckcNPPzyHlz6/nBmTovn2v/ZR19KJ1pqSSrvH5iLeCLZZsAzSkbDQbQPeI5uO\nseOks272CR8rZ9S3dLKnrMFVHaa/EJuVFflJrD9UhdaaMw1t/PS1IlYWJLHGqAxjWpAZx6Fy+6C5\nzw6H5pFNR/nQH97FoTU//FDfLqBmtZvRrkxRVG4nJTqEzh4HL+woG/Z4s+Pk+kOVA1JKxNglgbEQ\nYsRdMi2ZDrdNQBVN7SjFgNW00ZaVGEF7l4OObkefOs7uFSuafGjuYUqOCiEhIpiD5U0jUsPYX8ty\nEzlW3TJoE4sTNS109jj6rBiHBllJiAjmjPGaobrejdw4E7Ao53ngTZm2/hZlxdPc0e1asRtKeUM7\nDg3p/S5UCow5OFwxeDrFnrIGqu0dXD49ddivkxQVQlSorc8GvOrmjgHnuNWiSI8L51h1M195djdW\ni+KnH57dZ7NakNXCz2+aQ2NbF9/5936qmztoau/2mF98tqanRVNUYefAmUZ+8UYxV89KddYEr/Et\nMN5UUo3WDMgvdnfJtGQqmto5cKaJ7/5nPz1a84PrZg5Ie1qQGUePQ7OnbODdj2p7Bx9//H1++EoR\nlxSm8OoXVwzYnJoSNfqBsdaaQxVNXDY9hQWZcTzzfumQdx9aO7spq29jcVY8LZ09bDos6RTjhQTG\nQogRtzjbeRv1TePWeVVTO4mRIQR5ueFqpGQblSkAFmX1/jHNcOtI5m07aHdKKWcHvPImt6535z4w\nNqsObDlW4/H53ooUfSt2TIrtLdk2XNe7kWC2hwbvyrT1Z+aGe5NnbNYwnhLf9/thtgsvrhy889va\ng5XYLMq1qXMoSinykyMpqep9v+qmgYExOO9QrDtUyfaT9Ty4ZoYrncVdYWo0X7q0gP/uK+eXb5QA\nvekaI2laWhStnT3c/ZftxIQF8/3rZpGdFEFpbSvdPQ6v32fj4WriwoOGbDW/ujAZpeA7/9nPukNV\nfOWyggEpLgDzjEDXU57xV5/bw/vH6/jB9TP5w0fne9wLYF7UjWZgfKbRLHsYzW1LMjhW0zJka+6j\nVc4LjY8tzSQmLEjSKcYRCYyFECOu/23UilGqkzscM2UiPzmSBLdKAaFBVtJiQjlR2+JTO2h309Oi\nOVzRzMnaVkKDLCR4yLMcbdPTookND+LdI57/QB+utGO1qAG1cNNiQnsDY3s7Sefge3NJYTJhQdZh\n20B7Mjk2jLSYUN4uqR42R7jUKNXWP5UiMdK5yn+4YojA+EAFF+QkeFUVBCA/OcpVFaStswd7R/cg\ngXEEDg1XzUzlurmTB32/e1bmMCc91lU9o//3bSSY3fzONLbzkxtmER8RTHZCBN0O7VPJtE2Ha1iR\nnzRk5Y7EyBDmpceyq7SBGZOi+cSFnvcCxIQFUZASOaAyxZmGNt4uqebei3K5bUnmoA14QoOsxIQF\njWr3uyIj/WRaWhRXz0ojJiyIp98bvMqJmUYxLS2ay6ensO5gpV8dGEdDtb2Dx945zq1/2ior2R5I\nYCyEGBUXG7dRD5Y3UdnUMeKl2rwxKTaMiGArF+QMDMaclSl6V4x9DownRdPZ42Dj4Womx4Z51TVv\npFksiqU5CWw5WusxYCyqsJOdGDGgvfSk2DDKje53lU0drlvRo+mei3J586sXER3qexMUpRTXz5vM\nukNV/PKNw0MGk1ofHAAAIABJREFUx6fqWrFZBpYqA2MDXpXnwPhodTNHq1uGrUbhLj8lkprmTupb\nOqlpHtjcw7Q8L5FZk2P4voc0Anc2q4Wf3zibYJuFqFDbqGyILEiJIjLExkcWpXPJNOe/NTvJeWfF\n23SKg+VN1DR3DJlGYbpqZhpWi+LHH5o9ZIm+BZlx7DhZ36fe8792nTbauE8Z9uukDtL9rr2rx6vc\n9OGYd18KUqIIDbJyw/wpvH6gwvV976+kqpkgq7Pz4dWz0rB3dPPuEc93ds6VjYeruf2x91nyw3U8\n+PJBdpys5zv/2U+XD3cKzgcSGAshRoV5O3r9oSoqm9pHrYHEUKwWxXP3LuO+y6cOeC4rIYKTtS1+\npVIAzDA24B2vaWHyOd54525ZbgKnG9pcK6UmrTWHypsGpFGAcwXW3tFNU3vXqHW96y/YZvGYQuCt\n+y6fys0L0/nN+iP8cl3JoMeV1rUyOS7M40pmQUokhys8V6Z442AlgE+Bca7bBjwzV9tTYHzp9BRe\n+vzyPnctBpOXHMVDN87h8xfnjcrFVmiQlfX3XcQPru/dwJad2NsMxxsbjVXGlV4ExndcmMWmr69m\n1jANXeZnxNHU3u3qJqi15vkdZSzOjvdqs2ZydAiV9r5Bqtaay3+5iYfWFg/7elNtc4fH8+NgeRPp\n8WGuyja3Lkmnq0fz3HbPm/BKKu3kJEYSZLWwLC+BqFAbr+4LXDqFvb2LT/5lG0cq7XxmVR7rvrKS\nP350ASdrW/m7h/rexRV2Lvzxenac9L1ayXgngbEQYlQkRYUwJz2WV/c7SxsFYsUYnCu7nm6NZyZE\nUNPc6do8F+3jinF2YiShQc5foYHYeGdaapQV23y0bzrFztJ6yurbWOphtTwttrcyxWh2vRtJFovi\nRx+axU0Lp/CbN0v45RuHPR53qr5twMY7U0FqFC2dPR5TBtYeqGDW5BifgnezMkVJZbOruUeSF8Hv\ncK6dM4lPrcw96/cZTHJUaJ8Lh4SIYKJCbBwfJDBu7ezuEyxuKK5i5uRorzbTBlktXv18mHnkZjrF\nztIGjte08GEvVovBWVWlst8m1BO1rZTWtfLc9jKv8qfL6ltZ+qP1bCkfmPJQVN5EYWpv2/m85CgW\nZ8fzzPulHrsallQ1k2dUFQmxWblsWgprD1YGbHV2+4l6uno0P7txDvddMZW85ChWTU1icXY8v37z\nSJ+KIO1dPXzhmV2cbmhj7YHKgIw3kCQwFkKMmksKk10d4gKRYzyULGMVaq+xE97XVAqrRbnaLI9E\n1zt/5SZFkBwVMiAwfmLzSaJCbVw/b2BOqxn8HSpvGtWudyPNYtySv3HBFH79Zgl/fvvYgGPK6lpJ\nj/f8/TC7/x3utwGvqqmdXacauNyH1WKASTFhhAdbKamy97aDHmPnuTeUUmQnRXgMjI9VO7u9zX5g\nLTf+cTPf+fd+dpY2eJVG4YvsxAjiI4JdgfELO8sIDbJw1azhK4SA8/dLdXNHn7QJczNfTXPHkBvl\nTOsOVtLZ4+C98r5l49q7ejhe08K0fndfbl2cQWld64AazG2dPZyqb6Uguff4K2em0tjWxZajw49j\nNGw9VkuQVfWp6KGU4ptXFVLT3MGj7/Q2LfnRK4corrSTFBUyZDnIiUoCYyHEqLm4sHd3/2jWyfVH\nplGxYs+pBsKCrAPycL1hNvoIZGCslOLCvES2HK1xrepVNrXz6r5yblyQ3qcLm8lcwdtd6rwoGA8r\nxiaLRfGTG2azNCeBJzaf6LOS2dLRTW1Lp8fKB9Bb5aF/B7x1h6rQGi6f4V0Q5j6W3KRIjlQ5V4wt\nChIixl9gDM7UIk+B8eajtfQ4NJfPSMWh4Z87y3BozRU+ztVwlHIGbTtK62nv6uGlPWe4amaaK3Vh\nOKnRofQ4NLUtvekUu07VExliIyrExn92nxn2PdYbHeoO1va4arCD846AQ0NhWnSf41cXOtuGb+zX\n2e5odTNa06cO9cqCJCKCrby6vxytNfb2Lk7UtFA1yrWXTVuP1TIvPY6w4L6/5+ZnxHHFjBQe2XSM\n2uYO3jxUyV+2nOSu5dncuGAK+083nnctrSUwFkKMmhmTol0rxWYR/rHCrFhxprHd5/xik9noI5Cp\nFOAs21bT3OkK+P72Xik9WnP70kyPxydGhmCzKHYZXfNSxsmKscliUVw9K5Wy+jZXRznoLdU2WCpF\nTFgQqdGhAypTrD1YQUZ8OAV+NNTIT3YGxlX2DuIjQoas0jCWZSdGcLqhbUAjil2lDSRGBvPQjbN5\n4dPL2PfAFez538uHLNPmrwWZcRyrbuG57aewt3d7tenOZF7cVTa6BcalDcxNj+XKmam8tr9iyCYb\nLR3dbD1ay/S0aLocva2nAQ4Z9bML+60Yx4QFMTc9lk0lfQNjs4RfvltVkdAgKxdPS+HZ7WVM/c5r\nzHpgLase2sAlv9g46Aa+kWJv72Lf6UYuyIn3+PzXriiktbObB18+yNee38u0tGi+fuVUFmfH0+3Q\n7CodWF/64JkmLvzxeldVlolEAmMhxKhxdsFz3p4+F5UPfBERYnPlSPqaRmH64Kw0PrMqlznpIx8k\n+GKZUQJt89EaOrsdPP1eKaunJrsamfRntShSY0I5eMb5B388rRibzHrIb7sFJafqnLnD/Uu1uStI\njepTy7i5o5vNR2q5fHqKX5vd8lIiKW9s51hNy7hJSfEkJykCrZ1VPdztKq1nbnqca24sFuVXZRFv\nmHnGD609TFpMqE+l/cw7UmZlitbObooq7MzLiGXN3Mk0d3S7WpJ78u6RGjp7HHz9yqmE2eDNQ725\ntUXldsKCrK67TO5W5iex73RjnxbRJZXN2CxqwM/fZ1blcuOCKdy5LItvXV3Ig2tm0NrZw2/eHHwz\nqbsdJ+v9SsXYfqIeh8ZjdR5wlgW8eVE6/9l9htbObn57y1xCbFYWZMZhUfC+h3SKf+4s43RDG3/Z\nfMLn8Yx1EhgLIUbVZ1bl8uCaGcQFoM7vcMw8Y1833pniIoL5+pWF57xxSX9T4sLJiA9n89FaXtlX\nTk1zBx9fljXkaybFhtFt5GOOx4AuMyGc9PgwNpX0ruyZlTkGS6UAZ2voI1XNrlzUjcXVdPY4fE6j\nMOUbeaS7SxvOeWfHkZSVMLBkW31LJ8dqWpifeW4u/GZPicFmUTS2dXH9vMk+rb6bm3srjeoge8sa\n6XFo5mfEsTQ3gcTIkCHTKd4qriIyxMay3ERmJVpZd6jKtamuqKKJgtQoj+NZWZCI1n0v0A5XNpOd\nGDHg98K0tGh+fMNs7r96Gp9amcvtS7O4ZXE6T79XyrHqoVde95Y1cOuftvKxR99js49l37YeqyXY\nanE1UvHki5cUkJMYwffWzCTPOKejQoOYPil6QGCsteb1g84KG//adXrQVt7jlQTGQohRlR4fzu1L\nswI9DI/MFaBYPwPjseTCvAS2Hqvl8XePk5MYwYq8oVsvm+kfUSE2j3nIY51SihX5SWw5Wuva6X+q\nrpWIYCtxQ6TG5KdE0dHtcAXRaw9WEB8R7Fqt9JXZhKOzxzG+A2NjddM9z3i3kWozL92/ufFVaJCV\nGZOdZd1uWOB9GgVAYmQwSuGqTLHT2BA3Nz0Wq0VxzZw01hdXucozutNas76oihX5iQTbLMxNtlHT\n3MGesgZX2cP+G+9Ms6fEEhsexKbDvcHqkSq7q9PicL54SQEhNgs/fW3wknIVje3c/dftJEaGkJUY\nwb1P7eDIIPW4PdlyrJa56bED8ovdpcaEsv6+Vdy4ML3P44uy4tl1qp7O7t5qGgfLmzhV18aNC6bQ\n3NHNi3uGz98eTyQwFkKctzKNlUV/UynGkqW5idjbu9lT1sjtSzOxDLPalmbkfJ+LrnejZWV+Is0d\n3a4A7lRdK+nx4UOmRJiVKYor7HR2O1hfVMWl05L9zg1Ojwsj2Ob8UzqeA+OYsCASIoL71DLeVVqP\nRcGc9KFrEI+k2xZncOuSDHKTfMv3tlktJEaGuLrf7SptICcxwnWn6to5k+jsdvD6gYG1hA+ccTYh\nMjcLz060YrUo3jxURZW9g/rWrgH5xSarRbE8L9HVlbG9q4eTda1edy1Migrhnotyee1ABdtPDExZ\naOvs4e6/bqe5vZtH71jI43csIthm4c4ntnmVm9zU3sX+IfKLh7MkO572Lgf7Tje6Hnv9QCUWBd+4\nqpDC1Cie2npy2I6U44kExkKI81amsUrm7+a7scSsVxwRbPVqtc0s2TbWcr99sTQ3EYuCt42GE6fq\nW4dMo4DeSgGHK+28d7wWe3s3l0/3v8KCzWohxziPxmNKirvsxIg+qRQ7SxsoTI0mPPjc3VG4aVE6\nP3RrPuKLlOgQKu3taK2dudEZvSkgc9NjyYgP50UP6RRm7vEqoylRZLBiUVYc6w5VcsgoN9m/IoW7\nlQVJVNk7KKqwe6xIMZxPrsgmOSqEH75yqE+A6XBovvrcbvafaeQ3t8yjMDWa9Phw/nT7QqqaOrj7\nr9uH3FAIsP1EnTO/2I9W7AALs5wB9Ta3oH3tgQoWZsWTGBnCbUsyOHCmiT1ljYO9xbgjgbEQ4rxl\n5hhPhBXjpKgQLp2WzKdW5npV4spMpRiPdXdNvVUBnKXqTtW1DbnxDiA82EZGfDiHK+28cbCSsCAr\ny/OHTjsZjlkGbjyvGIMzMDZXjHscmt2nGpiXEdiNpb5IjQ6lorGdsvo2apo7B9TsXTN3EpuP1ri6\nFJrWF1cxZ0pMn+/fpdNSKKqws87YhDfYijE4N+ABbDpc7arS4G0qBTjPya9eXsDO0gb+vu0Ubx6q\n5NfrSrj9sfd5ZV8F919V6GrfDTAvI45f3TyXXaUNfOOFvUOu1m49Vkew1dJnLnyRGBlCTlIE24w8\n4xM1LRRV2F3l+q6bN5nwYCt/23rSr/cfiyQwFkKct3KTIkmOCunT0Wo8+/PHF/HFS/O9OtbsfjfW\n6kv7akV+EnvLGjha3UJbVw/pXtSULkiJpLjCztoDlawsSCQ0yPca1u7yjNv+I9H1LpCyEiOosnfQ\n3OFszdzc0e13QBUIydGhVNk7XPnF/YP6NXMn4dDwq3Ulrs2Xtc0d7D7V4KqeYzID0We3l5EWE0ps\n+OCbh1NjQpmaEsWmkmoOV9qxWpRrM6O3PrwgnYKUSO7/5z7u+st2fvXmYc40tPGFS/K5e0XOgOOv\nmpXGVy8r4D+7z/CPbacGfd+tx2qZmxF7Vuf4kux4tp2ow+HQrlQUsxlOVGgQa+ZO5qW9Z2jpmhjp\nFF4FxkqpK5VSxUqpI0qpb3p4/itKqYNKqb1KqTeVUp6LZwohxBgSEWLj/W9fyqU+djybCKbEhRMe\nbHUFdePVyoJEHBr+sa0UgIyEoVeMwbmaV1LVTEVT+1mlUZiW5MQTFmQlO8m3YGisMVNCTtS0uLrG\njacV45SoUOpaOnnveB3hwVZXPrkpLzmKO5Zl8fR7pdzx+PvUt3Syobgarfs2IwLn6nluUgSd3Y4h\nV4tNKwsS2Xa8nr1ljWQlhLvyzr1ltSh+f9t8vrdmBs/du5R9D1zB+vtW8ZXLCgbNmf/M6jyW5yXy\nvy8eoLhi4Ga83vxi/9IoTIuy4mlq76a40s7rByqYMSm6T8rSbUsyaO9y8O7piVGdYtjvnFLKCvwO\nuAqYDtyilJre77BdwEKt9WzgeeCnIz1QIYQQIycyxMamr6/2eff/WDNnSixRITae31EGDN7cw91U\nI9CxWtSAgMgfF+QkcPDBK0gex/na0Lcyxa7SBmLDg8gepBb2WJQa41yxX3ug0ln6zUMZxQeuncGP\nPzSL947Vcc3D7/D0+6UkR4UwY9LAu0bmBfO0IfKLTSsLkujscfDOkRqf0ijc5SVH8bGlWSzKiifS\ni0oxVoviFzfPISo0iM89vXNAhzpXfrGfG+9Mi7Odr3957xl2ljYM6Ho4c3IMc9JjeetU14TYhOfN\nJc1i4IjW+pjWuhP4O7DG/QCt9Vtaa7Mq+FZgfP+mFUKI80Bi5Pjt1GayWS0sy0ugvtVZhmuKF4Gx\nWXt4cVb8iNXX9qc5yFhj3v4/XtPCztJ65qXHjqt/l9mopqa5Y8iavR9ZnMGz9y6lx6HZcbKe1VOT\nPVZxMQPA2VOGr8qxKCue0CCLc+OdlxUpRkJyVCi/unkuR6qbeeDFA32eO9v8YtOUuHAmxYTy6DvH\nATy2A//okgzKW7Srm+Z4poaL7pVSHwau1Fp/0vj8Y8ASrfXnBjn+YaBCa/19D899CvgUQFJS0oJn\nn332LId/fmpubiYycnzf/gwUmTv/ydz5R+bNf97O3frSLv56sJOYEMWvVw8fGHc5NPe/3caH8oNZ\nNmn81XD2hr/n3Vc2tJIRZWF3dQ/X5wWxJm/sNeYZTGlTD9/d7NxY98X5IcxLHvp729Sh+c/RTi7O\nCGJyZO86ofvcnWjsISPagsWLC4Sfb29nX00Pn54TwpK0c3tevXC4k5eOdbEwxUqQFXocUFTnIC1C\ncf+Ss29Z/8c97Wwt7yElXPHjFWEDLpg6ejQlVS3MSI0YsxdTq1ev3qG1Xjjccd585zz9Cz1G00qp\njwILgYs8Pa+1fgR4BGDq1Kl61apVXnx50d+GDRuQufOPzJ3/ZO78I/PmP2/nLqe2lb8efIvclBhW\nrbrQq/e+7OKzHNwY5+95V1iy1VWa64aL5p91xY5zqba5g+9uXgfAx65eQaIXmyGv9fCYv3N3POg4\n+146yPUXL/E7ncJfy1c40P/Yzc6T9disFoKsiknxFj6zOo9Vcyad9fufDjvJ1n/t57pF2axePc3j\nMSET5HedN4FxGeDeCmUKMKAQoFLqUuDbwEVa6+GrTgshhBAjICMhnMLUKGZMOneNKCaq7MQINh+t\nRZ3jxh4jIT4imCCrIi0mzKugeKTduiSDzITwcx4UgzOl6OFb54/a+19cmExhahQ3jvM9Cd7wJjDe\nBuQrpbKB08BHgFvdD1BKzQP+D2fKRdWIj1IIIYQYwgufXobNOjZv4Y4n5ma7guQor+phjyVKKXKT\nIpmbHphKGiE264CybxNFWkwYr31pZaCHcU4MGxhrrbuVUp8DXgeswGNa6wNKqQeB7VrrF4GfAZHA\nc0ZuSanW2tMdCiGEEGLERXixi18MzwyMx1OZNnfP3H0BIUHSokH4z6vfJFrrV4BX+j32XbePLx3h\ncQkhhBDiHJuaGoVFcda1bwNlpKqMiPOXXGILIYQQAnCW5lr3lYt87twmxEQhgbEQQgghXHLGeTdE\nIc6GJOIIIYQQQgiBBMZCCCGEEEIAEhgLIYQQQggBSGAshBBCCCEEIIGxEEIIIYQQgATGQgghhBBC\nABIYCyGEEEIIAUhgLIQQQgghBCCBsRBCCCGEEIAExkIIIYQQQgASGAshhBBCCAGA0loH5gsrZQeK\nA/LFx79EoCbQgxinZO78J3PnH5k3/8nc+U/mzn8yd/4Z6/OWqbVOGu4g27kYySCKtdYLA/j1xy2l\n1HaZO//I3PlP5s4/Mm/+k7nzn8yd/2Tu/DNR5k1SKYQQQgghhEACYyGEEEIIIYDABsaPBPBrj3cy\nd/6TufOfzJ1/ZN78J3PnP5k7/8nc+WdCzFvANt8JIYQQQggxlkgqhRBCCCGEEEhgLIQQQgghBBCg\nwFgpdaVSqlgpdUQp9c1AjGE8UEqlK6XeUkodUkodUEp90Xg8Xin1hlKqxPh/XKDHOlYppaxKqV1K\nqZeNz7OVUu8Zc/cPpVRwoMc4FimlYpVSzyuliozzb6mcd95RSn3Z+Hndr5R6RikVKuedZ0qpx5RS\nVUqp/W6PeTzPlNNvjL8be5VS8wM38sAaZN5+Zvy87lVK/UspFev23P3GvBUrpa4IzKjHBk9z5/bc\nfUoprZRKND6Xc87NYHOnlPq8cW4dUEr91O3xcXnenfPAWCllBX4HXAVMB25RSk0/1+MYJ7qBr2qt\npwEXAJ815uqbwJta63zgTeNz4dkXgUNun/8E+KUxd/XAXQEZ1dj3a+A1rXUhMAfnHMp5Nwyl1GTg\nC8BCrfVMwAp8BDnvBvMEcGW/xwY7z64C8o3/PgX84RyNcSx6goHz9gYwU2s9GzgM3A9g/M34CDDD\neM3vjb/D56snGDh3KKXSgcuAUreH5Zzr6wn6zZ1SajWwBpittZ4BPGQ8Pm7Pu0CsGC8Gjmitj2mt\nO4G/45xU0Y/WulxrvdP42I4zOJmMc77+Yhz2F+C6wIxwbFNKTQE+APzZ+FwBFwPPG4fI3HmglIoG\nVgKPAmitO7XWDch55y0bEKaUsgHhQDly3nmktd4E1PV7eLDzbA3wV+20FYhVSqWdm5GOLZ7mTWu9\nVmvdbXy6FZhifLwG+LvWukNrfRw4gvPv8HlpkHMO4JfA1wH3igRyzrkZZO4+DfxYa91hHFNlPD5u\nz7tABMaTgVNun5cZj4khKKWygHnAe0CK1rocnMEzkBy4kY1pv8L5i85hfJ4ANLj98ZBzz7McoBp4\n3EhD+bNSKgI574altT6Nc8WkFGdA3AjsQM47Xwx2nsnfDu99AnjV+FjmbRhKqWuB01rrPf2ekrkb\nXgGwwkgV26iUWmQ8Pm7nLhCBsfLwmNSMG4JSKhJ4AfiS1rop0OMZD5RSHwSqtNY73B/2cKicewPZ\ngPnAH7TW84AWJG3CK0Y+7BogG5gEROC8HdufnHe+k59fLyilvo0zDe9v5kMeDpN5MyilwoFvA9/1\n9LSHx2Tu+rIBcTjTPb8GPGvcnR23cxeIwLgMSHf7fApwJgDjGBeUUkE4g+K/aa3/aTxcad7OMf5f\nNdjrz2MXAtcqpU7gTNe5GOcKcqxxixvk3BtMGVCmtX7P+Px5nIGynHfDuxQ4rrWu1lp3Af8EliHn\nnS8GO8/kb8cwlFIfBz4I3KZ7mxTIvA0tF+eF7B7j78UUYKdSKhWZO2+UAf800k3ex3mHNpFxPHeB\nCIy3AfnGLu1gnMnZLwZgHGOecdX1KHBIa/0Lt6deBD5ufPxx4D/nemxjndb6fq31FK11Fs5zbL3W\n+jbgLeDDxmEydx5orSuAU0qpqcZDlwAHkfPOG6XABUqpcOPn15w7Oe+8N9h59iJwu1Ep4AKg0Uy5\nEM5qT8A3gGu11q1uT70IfEQpFaKUysa5kez9QIxxLNJa79NaJ2uts4y/F2XAfOP3oJxzw/s3zoUn\nlFIFQDBQw3g+77TW5/w/4Gqcu2aPAt8OxBjGw3/Acpy3HvYCu43/rsaZK/smUGL8Pz7QYx3L/wGr\ngJeNj3Nw/nAeAZ4DQgI9vrH4HzAX2G6ce//GeatMzjvv5u7/AUXAfuBJIETOu0Hn6hmcudhdOAOS\nuwY7z3Demv2d8XdjH87KHwH/N4yheTuCM6fT/FvxR7fjv23MWzFwVaDHP9bmrt/zJ4BE42M554aZ\nO5yB8FPG77udwMVux4/L805aQgshhBBCCIF0vhNCCCGEEAKQwFgIIYQQQghAAmMhhBBCCCEACYyF\nEEIIIYQAJDAWQgghhBACkMBYCCGEEEIIQAJjIYQQQgghAAmMhRBCCCGEACQwFkIIIYQQApDAWAgh\nhBBCCEACYyGEEEIIIQAJjIUQQgghhAAkMBZCCCGEEAKQwFgIIYQQQghAAmMhhBBCCCEACYyFEEII\nIYQAJDAWQgghhBACkMBYCCGEEEIIQAJjIYQQQgghAAmMhRBCCCGEACQwFkIIIYQQApDAWAghhBBC\nCEACYyGEEEIIIQAJjIUQQgghhAAkMBZCCCGEEAKQwFgIIYQQQghAAmMhhBBCCCEACYyFEEIIIYQA\nJDAWQggAlFInlFKXjtB7BSulnjfeUyulVo3E+wohhBhdEhgLIcToeAf4KFAR6IEMRyllDfQYhBBi\nLJDAWAhx3lNKPQlkAC8ppZqVUl83Hr9WKXVAKdWglNqglJrm9poTSqn7lVIHlVL1SqnHlVKhAFrr\nTq31r7TW7wA9Xnz9O5VSh5RSdqXUMaXUPf2eX6OU2q2UalJKHVVKXWk8Hm983TPGGP5tPH6HUuqd\nfu+hlVJ5xsdPKKX+oJR6RSnVAqxWSn1AKbXL+BqnlFIP9Hv9cqXUZmMuThlfY5FSqlIpZXM77gal\n1G4fpl8IIcYMCYyFEOc9rfXHgFLgGq11pNb6p0qpAuAZ4EtAEvAKzsA52O2ltwFXALlAAfA/fg6h\nCvggEA3cCfxSKTUfQCm1GPgr8DUgFlgJnDBe9yQQDswAkoFf+vA1bwV+AEThXN1uAW43vsYHgE8r\npa4zxpABvAr8FudczAV2a623AbXAZW7v+1FjXEIIMe5IYCyEEJ7dDPxXa/2G1roLeAgIA5a5HfOw\n1vqU1roOZ5B5iz9fSGv9X631Ue20EVgLrDCevgt4zBiHQ2t9WmtdpJRKA64C7tVa12utu4zXeus/\nWut3jfds11pv0FrvMz7fi/Oi4CLj2NuAdVrrZ4yvU6u1NleF/4IzGEYpFY/zQuFpf+ZBCCECTQJj\nIYTwbBJw0vxEa+0ATgGT3Y455fbxSeM1PlNKXaWU2qqUqlNKNQBXA4nG0+nAUQ8vSwfqtNb1/nxN\n+o4dpdQSpdRbSqlqpVQjcK8XYwB4CrhGKRUJ3AS8rbUu93NMQggRUBIYCyGEk+73+Rkg0/xEKaVw\nBoin3Y5Jd/s4w3iNT5RSIcALOFekU7TWsTjTNpRxyCmcqRr9nQLilVKxHp5rwZliYX6NVA/H9P/3\nPg28CKRrrWOAP3oxBrTWp4EtwPXAx5A0CiHEOCaBsRBCOFUCOW6fPwt8QCl1iVIqCPgq0AFsdjvm\ns0qpKUYKwbeAf5hPKKVCzM14QLBSKtQIrvsLBkKAaqBbKXUVcLnb848CdxrjsCilJiulCo1V2VeB\n3yul4pRSQUqplcZr9gAzlFJzjTE84MW/PwrnCnS7kdd8q9tzfwMuVUrdpJSyKaUSlFJz3Z7/K/B1\nYBbwLy++lhBCjEkSGAshhNOPgP8xqi7cp7Uuxpk7+1ugBrgG5+a8TrfXPI0zH/iY8d/33Z4rBtpw\npl68bnzDPbwgAAAgAElEQVScST9aazvwBZyBeD3OgPRFt+ffx9iQBzQCG93e52NAF1CEcwPfl4zX\nHAYeBNYBJTg31w3nM8CDSik78F1jPOYYSnGmd3wVqAN2A3PcXvsvY0z/0lq3ePG1hBBiTFJa97+b\nJoQQYjhKqRPAJ7XW6wI9lrFAKXUUuEfmQwgxnsmKsRBCiLOilLoBZ87y+kCPRQghzoZt+EOEEEII\nz5RSG4DpwMeMyh1CCDFuSSqFEEIIIYQQSCqFEEIIIYQQQABTKWJjY3VeXl6gvvy41tLSQkRERKCH\nMS7J3PlP5s4/Mm/+k7nzn8yd/2Tu/DPW523Hjh01Wuuk4Y4LWGCckpLC9u3bA/Xlx7UNGzawatWq\nQA9jXJK585/MnX9k3vwnc+c/mTv/ydz5Z6zPm1Lq5PBHSSqFEEIIIYQQgATGQgghhBBCABIYCyGE\nEEIIAUhgLIQQQgghBCCBsRBCCCGEEIAExkIIIYQQQgASGAshhBBCCAFIYCyEEEIIIQQggbEQQggh\nhBCABMZCCCGEEEIAEhgLIYQQQggBgC3QAxBCjC6tNdf9fjM3LZzCbUsyAz0ccZ5YX1TJp5/aSY9D\n+/0eWmvU2ldGcFTnj8I4C6mFTRSmRgd6KOeE1pq1Byv56WtFnKxtPev3kvPOd1prXp9uJy85KtBD\nOSsSGAsxwZ1uaGPPqQYqGtu4aWE6QVa5USRG36bDNSgF91yU4/d7nDxZSmZmxgiO6vzQ3uXg7+8d\n5+pfv80tizP4ymUFJESGBHpYo+bgmSa+9/JBthyrJS85kk+tzEEp/99Pzjv/nDxZSmx4cKCHcdYk\nMBZigjtcaQegsqmDV/aVs2bu5ACPSJwPDlfamZoazdeuKPT7PTZsqGDVKv9ffz6bF1zJ9vZkntx6\nkhd3n+ELl+Tz8WVZBNsmzoVxtb2DX7xRzN+3nSImLIgH18zg1sUZ2M7y4l/OO/9s2FBB4gS4AJs4\nPyFCCI+KKpyB8ZS4MB595zha+39rWwhvFVfYKUwZ37dUx7PIYMUD187g9S+tYGFWHD945RCX/3Ij\naw9UjPvfAR3dPfxx41FWP7SB57aXceeybDbet5rbl2addVAshJxBQkxwxRV2JsWEcs9Fuewta2Rn\naX2ghyQmuGp7B7UtnUxNlcA40PKSo3j8zsU8cecibFYLn3pyB7f9+T0OlTcFemg+01rz2v4KLvvF\nJn78ahFLsuN5/csr+e4104kJDwr08MQEIYGxEBNccYWdqalR3DB/MjFhQTz2zolAD0lMcMXGXYpC\nCYzHjFVTk3n1iyt4cM0MDpY38YHfvM39/9xHTXNHoIfmlf2nG/nII1u596kdhAZZePKuxTx6xyJy\nkyIDPTQxwUiOsRATWFePg6PVzayamkx4sI1bFmfwyKajlNW3MiUuPNDDExNUUYVzNVJWjMeWIKuF\n25dmsWbOZH79Zgl/3XKCl/ec4XMX53HHhVmE2KyBHuIAVfZ2fv76YZ7dcYq48GC+f91MPrIoXVIm\nxKiRM0uICexYdQtdPdq1cnf70kyUUvx1y8kAj0xMZMUVdhIjQyZ0JYTxLCY8iO9eM53Xv7ySRdnx\n/OjVIi77xSZe2z928o/bu3r4/YYjrP7ZBl7YWcZdF2bz1n2r+OgFmRIUi1ElZ5cQE1j/lbtJsWFc\nNTOVZ94vpaWjO5BDExNYcaWdqalyi3usy02K5LE7FvGXTywmxGbh3qd2cMuftnLgTGPAxqS15pV9\n5Vz6i4389LViluYmsvbLK/mfD04nJkzyiMXok8BYiAmsuMKO1aLISYpwPfaJ5dnY27t5YWdZAEcm\nJqoeh3aWaks5PxpLTAQXFSTx6hdX8L01MyiusPPB377DN1/YS7X93OYf7z/dyM3/t5XP/G0nEcE2\nnrprCX/++EJyJI9YnEOSYyzEBFZcYScnMaJP7uD8jDjmpsfy+Lsn+OiSTCyWs6iEL0Q/pXWttHc5\nZOPdOGOzWvjY0iyunTOZ364v4YnNJ3h5bzmfXZ3HnRdmERo0evnHVU3t/Oz1Yp7fWUZceDA/uH4m\nNy+UPGIRGHLWCTGBOW9pDwxQ7lqezfGaFt4qrgrAqMREZlakkI1341NMeBD/88HprP3ySi7ISeAn\nrxVx2S838uq+8hHPP27v6uF3bx1h9UMb+Pfu09y9IocNX1vFbUskj1gEjpx5QkxQzR3dlNW3eVy5\nu3JmKmkxoTz27vEAjExMZMUVdpSCAmnuMa7lJEXy548v5Km7lhAeZOPTf9vJzY9sZf/ps88/1lrz\n373lXPLzjfzs9WIuzEvkjS9fxLeunkZ0qOQRi8CSwFiICap35W5grqdZtundI7WuDXpCjITiyiYy\n48MJCx57pb+E75bnJ/LfLyznB9fP5EhVM9c8/A5ff34PVU3tfr3f3rIGbvq/LXz26Z1EhwXx9N1L\neOT2hWQlRgz/YiHOAQmMhZighmuycMvidMKCrDwuDT/ECCqq8Jy+I8Yvm9XCbUsyeeu+Vdy9Iod/\n7TrN6oc28Lu3jtDe1ePVe1Q2tfPVZ/dw7cPvcrymhR9/aBYvf345y3ITR3n0QvhGAmMhJqjiiiYi\ngq1Mjg3z+HxseDA3LJjMv3afpnacdL8SY1t7Vw8nalo83qUQ419MWBDfunoab3z5Ii7MS+Rnrxdz\nyc838t+9g+cft3f18Ns3S1j90AZe2nOGey/K5a37VvGRxRlYZeOvGIMkMBZigiqqsFOQGjVk1Yk7\nlmXT2e3gb++VnsORiYnqSFUzDi2toCe6rMQIHrl9IU9/cglRoTY++/RObvq/Lewta3Ado7XmxT1n\nuOTnG/n5G4dZmZ/EG19ZyTevKiRK8ojFGCbl2oSYgLTWFFfauWpm6pDH5SVHsmpqEk9uPck9F+WM\nyZaw491Y6SR2LhQZ6Tuy8e78sCwvkf9+YQX/2HaKn68t5tqH3+WG+VO4Zk4av11/hB0n65meFs1D\nN85haW5CoIcrhFdkxViICajK3kFDa5dXAconLsym2t7Bf/eWn4ORnV/+sOEo921s8zoPc7wrrmgi\n2GYhKyE80EMR54jVorh1SQZvfW0V91yUw0t7znDH49s4WdvKT26YxUufXy5BsRhXZMVYiAmoyIda\nsivyE8lPjuTRd45z/bzJKCV5fyPhhR1l/OS1IgAOljcxPyMuwCMafUUVdvKTI6UG7XkoOjSI+6+a\nxq2LM3j/eB1XzkyVlAkxLslvLyEmoMOuihTDb4JSSvGJ5dkcONPE+8frRnto54V3j9TwjRf2Mmty\nDAAHRqD263hweJCGMuL8kZkQwY0L0yUoFuOWBMZCTEBFFXaSokKIjwj26vjr500mLjxIGn6MgKKK\nJu59cge5SZE89cklRAbB/tMTv1Z0Q2snlU0dsvFOCDGuSWAsxARUXNnkU4ASGmTl1iUZrD1YSWlt\n6yiObGIrb2zjzse3ER5i5fE7FxETFkRmtIUD5RN/xbhoiIYyQggxXkhgLMQE0+PQlFQ2M9XHygAf\nuyALq1L8ZcuJURnXRGdv7+LOx7dhb+/m8TsWM8moH50ZbaW4wk5ntyPAIxxdwzWUEUKI8UACYyEm\nmBO1LXR0O3zO9UyNCeWDs9P4x7ZT2Nu7Rml0E1NXj4PP/G0nR6qa+f1t85k+qXfVNCvaQleP5nCl\nPYAjHH1FFXZiw4NIjgoJ9FCEEMJvEhgLMcEU+7Dxrr9PLM+muaOb57aXjfSwJiytNff/cx9vl9Tw\nww/NYmVBUp/nM6Odv2YPnJnY6RTFFU1MTYmSqiZCiHFNAmMhJpiiCjtKQX5KpM+vnT0lloWZcTyx\n+QQ9jvOnMcXZ+NW6Ep7fUcYXL8nnpoXpA55PCldEhdgm9AY8rTWHK5ulIoUQYtyTwFiICaa4ooms\nhAhCg/zrYveJ5dmU1rXy5qHKER7ZxPPs9lP8+s0SPrxgCl+6NN/jMRalmDYpmv0TeMW4rL6N5o5u\nCYyFEOOeBMZCTDDFFXafN965u3x6CpNjw6R02zA2Ha7mW//cx4r8RH70oVlDphDMnPT/27vz+Liq\n8/7jnzOjXRrJli1rLNnG+8iWALOTAMGQkAAJdps2KTQbhIRXmtJ0SdIfWZqStGnaJk1/bUqTH1kg\ne0rSNDaFsITgEAgGDNjgReNF3uWRLcmWRrtm5vz+mBlbtiVrNJqZO8v3/Xr5hWZ0de/D8ZXnmTPP\neU4NO470Egrn5wI8LbwTkXyhxFgkjwyOhNnfPTCtmbsit4vb37iQjW3deV8Xm6zt7b189IevsHRO\nFf/5nospnmSnt5bGaoZGI7R19mcowszyxxYWJrIFuYhINlNiLJJHdh0NYu30Z+7efdl8KkrcfOfZ\nfakJLI+0nxjkjgdfxFNWxIN3XJ7QDl8t8R3w8vSNhj8QpHFGuXY7E5Gcp8RYJI+c2mRheolxTXkx\n77pkHg9vaedocCgVoeWFnsFor+KB4TAP3HEZ3pqyhH5u8exKyopdebsAzx8IqoxCRPKCEmORPOIP\nBCkrdnHerMppn+v2qxYxEo7ww40HUhBZ7hsJRfiTH7xMW2cf33jfJVNqh1fkdrFibjVbD+ffjPFI\nKMKeY+pIISL5IaHE2BhzozHGb4zZbYy5Z5zvn2eMecoY85oxZoMxZl7qQxWRyfgDQZbN8eB2Tb+X\n7KLZlby5aQ4/fGE/Q6PhFESXu6y13PPfr/G7PV380x9cwFVLZ0/5HC0NNWxv7yWSZ23w2jr7CEWs\nEmMRyQuTJsbGGDdwH3ATsBK4zRiz8ozDvgJ8z1p7AfAF4EupDlREJtcaCKY0Qbnz6kV09o2wfkt7\nys6Zi7765E5+/uphPn7Dct55cXLv+1saqwkOhzjQPZDi6Jw1nQ1lRESyTSIzxpcDu621bdbaEeAn\nwNozjlkJPBX7+ulxvi8iadbVN0xn33BKaz3fsGQWTV4P33l2L9bm10xnon784gG+9uvd3HrZfO6+\nfmnS52luiC7Ay7d+xq2BIEUuw6LZ0y/fERFxWiKJcSNwcMzjQ7HnxtoC/EHs698HPMaYWdMPT0QS\nFZ+5S2XLLGMMH7xqEa2BIK3d+dmD91ye9h/ls7/YyrXL6/i732uZ1nbHy+qrKHabvFuA5w8EWVJX\nRUmRlqyISO4rSuCY8V4Jzpw6+gTwH8aY24FngMNA6KwTGXMXcBdAXV0dGzZsmEqsEtPX16exS1I+\nj90T+0YB6G57nQ3tqUtSZoQtnhJ4dM8gK/J07MazryfMl14cYl6Vi1sX9PPcb59J6jxj77mGSsOz\nW/exoTyQwkidtWXfAMtmutLye5XPv6/pprFLnsYuOfkybokkxoeA+WMezwNOKzi01rYD7wQwxlQB\nf2CtPevzQmvt/cD9AD6fz65evTq5qAvchg0b0NglJ5/H7pc/e42ZFQHWvu26ac1sjueOyE6+9tQu\nFrZcxsIC+Mj80PEBPvmfv2O2p5yHPvpG6qsTa8s2nrH33JWdr/Hkjg6uvfbalP8dOaF3aJSux57g\ngxcsZfXq5MtMJpLPv6/pprFLnsYuOfkybolMK70ELDPGLDLGlAC3AuvHHmCMmW2MiZ/rU8B3Uhum\niEzG3xFdeJeOhOu9Vy7AZeDB3+1L+bmzTc/AKLc/8BJDo9FexdNJis/U0lhNd/8IR3ryozf0rg5t\nBS0i+WXSxNhaGwLuBh4HdgAPWWu3GWO+YIxZEztsNeA3xuwE6oEvpileERlHJGLZ2RFMW2eAOZ4y\nrpxbxEObDtIzOJqWa2SD4VCYu76/if1d/dz/vktTvsVxc2wHvHzpZ5yqDWVERLJFQoWI1tpHrbXL\nrbVLrLVfjD33OWvt+tjXP7PWLosd8yFr7XA6gxaR0x06PsjASDitCcoN5xUxMBLmp5sOTn5wDopE\nLJ/86Wu8sLebr7zrQt6wJPXrh1d4q3EZ2NqeHwvw/IEgntIiGmeUOx2KiEhKaBmxSB5oDUQTrXQm\nxgtr3Fy+qJYHnttHKJx/HSq+/ISf9Vva+esbfaxddWbjndQoL3GzdE4V2/Joxnh5msp3REScoMRY\nJA+ko1XbeO68ehGHTwzy5PaOtF4n036wcT9f37CHP75iAX9y7ZK0Xqu5oSYvehlba/GneEMZERGn\nKTEWyQOtHUHm15ZTVZpIo5nkvWVFPfNry/nOc3vTep1MempHB59bt5Xrm+bwhTXNaZ/9bG6opqN3\nmGPB3K446+gdpmdwFF+a34yJiGSSEmORPOAPBDOSoLhdhtvfuIiX9h3ntUMn0n69dNty8AR3/+hV\nmhtq+NptF1HkTv8/iS2xBXjbcnzWOBPlOyIimabEWCTHDYfC7O3sz1iC8u5L51FVWsQDz+3LyPXS\n5WD3AHd+9yVmVZXw7dsvpTLNs+1xKxuinUO25fgCvHj5jlq1iUg+UWIskuN2H+0jHLH40tSq7Uye\nsmLefel8Ht7STkdvbvbjPTEwwgceeJHRsOXBOy5njid1vYonU11WzMJZFTnfss0fCFJfXcqMihKn\nQxERSRklxiI5bqcDmyzc/saFhK3l+8/vz9g1U2VoNMyHv7eJQ92DfPP9l7J0TlXGY2huzP0FeNEN\nZTLzZkxEJFOUGIvkuNZAkGK3YVEGt2peMKuCG1bU88MX9jM0Gs7YdacrErF8/KdbeGnfcf7l3Rdy\n+aJaR+JoaajhYPcgPQO5uVlKKBxh19E+lVGISN5RYiyS4/yBIEvqqijOwMKxsT549SKOD4zyi1cP\nZ/S60/GPj7XyyGtH+PTNTdxyYYNjcbQ0xuuMc3PWeF/XACOhiDpSiEjeUWIskuP8gaAjM3dXLKpl\n5dxqvvPcXqy1Gb/+VH33d/u4/5k23v+G8/jwNYsdjaW5IbY1dI4mxn5tBS0ieUqJsUgO6xkY5UjP\nkCO1nsYY7rx6ETs7+nh2d2fGrz8VT2wL8PmHt/GWFfX87S3p71U8mdrKEhpqyth6ODc7U/gDvbgM\njtRni4ikkxJjyRhrbU7MLOYSvwML78Z6x4VzmV1Vyneezd4NP149cJyP/eRVzp83g6/ddhFuV3Zs\nX9zcWJOzpRStgSALZ1dSVux2OhQRkZRSYiwZ8/2N+7n6n54mHFFynCr+2CYLyx1KjEuL3LzvyvN4\n2n+MPcf6HIlhIiOhCN98po33f/tF5njK+PYHLqW8JHsSuZaGGto6++kfDjkdypT5O5wp3xERSTcl\nxpIxj20NcPjEIPu7+p0OJW+0BoJ4yopoqMlcH94zvefKBZS4XTyYJRt+WGt5YluAt/7rb/jiozu4\nZOFMfvihK5hdVep0aKdpaazGWthxJLfKKQZGQhzoHsBXr1ZtIpJ/lBhLRgyHwry8/zhwqu+uTF98\nK2gna2ZnV5WydlUDP3v5ECcGRhyLA6JJ5nu+9QJ3ff9litwuHrjjMh6843Lm11Y4Gtd44ltD59pG\nHzs7+rBWC+9EJD8pMZaM2HKwh+FQBIjOcsr0WWtjmyw4n6B88OpFDI6G+clLBx25fmffMJ/6+eu8\n/d9/y/YjvXx+TTO//PNruM43x5F4EjHHU8rsqlK25tjW0Du1FbSI5LEipwOQwrCxrQtjorOLfiXG\nKXGkZ4jgUCgrEpQVc6t545JZfPd3+7jz6kUZ66k8HArz4HP7+NqvdzM0GuYDb1zIn795WU5sU2yM\noaWxOudmjFsDQcqL3SzIwll4EZHpUmIsGbGxrYuVc6uZP7NCiXGKnOolmx21nh+8ahEf+t4mHt8W\n4B0XpHfzDGstj2/r4B8e3cGB7gGub5rDp29ekXPtw1oaavjtrk6GRsM50+HB39HL8voqXFnS3UNE\nJJVUSiFpF68vvnLxLHxeD/u6+nNqG+FsFS9JyZbdx65vmsPCWRV8O82t27a193DbNzfykR+8TGmR\ni+9+8HK+c/tlOZcUAzQ3VBOO2Jx6s+gPZEf5johIOigxlrTbfOAEw6EIVy6eRZPXQ8TCro7sau2V\ni/yBXubWlFFTUex0KAC4XIY7rlrEqwdO8MqB4yk//7HgMPf892u842vP4g8E+bu10Tria5fXpfxa\nmRJfgLctR+qMO/uG6ewbyZpPKUREUk2lFJJ2G9u6MQYuX1hLZ/8wAK2BXs6fV+NwZLmtNQtn7v7w\nknl85Qk/Dzy3j4sXzEzJOYdGwzzw3D7uezpaR3znVYv4s+uXZc0bgumYN7Oc6rKinNka2p9ln1KI\niKSaEmNJu3h9cU1FMVVlRZQWuXLqo+NsNBqOsOdYX9bNllaWFnHrZfP5znP7+PTNTcytKU/6XNZa\nHtsa4B9+uYOD3YO8ZUU9n765icV1uVcyMZHoArwatuXIAryT5TtZ9oZMRCRVVEohaTU0GuaVA9H6\nYgC3y7CsvurkVsaSnL2d/YyGbVYmKO9/w0KstXzv+f1Jn2Pr4R7+6P6N/MkPX6GiuIgf3HkF3/rA\npXmVFMe1NNawIxBkNBxxOpRJ+QO9zKosoc6TXZuliIikihJjSastB0/VF8f56qs1YzxN2TxzN7+2\ngrc1e/nRCwcYGJnadsdHg0P89c+2cMt/PMvuo3188fdbeORjV3P1stlpitZ5zQ3VjIQi7D6a/XX3\nWngnIvlOibGk1dj64rgmr4ejwWGO9zu7S1ou2xkI4naZrO3EcOfVi+gZHOXnrxxO6Pih0TD3Pb2b\n6768gf959TAfvmYxT39iNe+54jyKMtQT2Sm5sgNeJGLZ2dGnxFhE8ppqjCWtnm/rpLmh+rSFUvEX\n1tZAkDcsmTXRj8o5tAaCLJpdSWlRdva+veS8mVwwr4YHntvLH1++YMKet9ZaHn09wJd+uYNDxwd5\n68p6Pn3zChbOrsxwxM5ZNKuSyhI329p7eZfTwZzDweMDDI6Gs2JDGRGRdMnvqRhxVLS++ARXLjo9\n+Y2/sPoDudGiKhv5O3qzeubOGMMHr1rEnmP9PLPr2LjHvH6oh3f/v+f50x+9QlVpET/60BXc//5L\nCyophmibuxVzs38HvNYs21BGRCQdlBhL2mw+eIKRM+qLAeo8pcysKNYCvCT1DYc42D1IU5a3zLr5\n/LnM8ZSeteFHR+8Qn/jpFtbc9yxtx/r50jvP55GPXcMbl+ZvHfFkWhpr2H6kl0jEOh3KhPyBIMbA\n8vrsLN8REUkFlVJI2mxs68IYuGxR7WnPG2PweT0nZ6BkanZ2ZO/Cu7FKilx84I0L+fLjfnZ2BFlQ\nW8G3ftvGf27YQyhsuetNi/nT65ZSXZb7/Yinq7mhmoGRMHu7+lmSpZ03/IHo32FFiV42RCR/6V84\nSZuNbV3R+uLysxMfX72Hn718iEjETlh/KuOLd/RoyoGPtG+7fAH//tQu/uYXWzl0fJDDJwa5sdnL\np25u4rxZhVUycS5jF+Bla2LcGuhleZZ/SiEiMl0qpZC0mKi+OM7nraZ/JMzhE4MZjiz3+QNBKkrc\nzJuZ/OYZmVJbWcI7L27khb3dVJcX8+MPX8k33neJkuIzLJ1TRUmRK2u3hh4aDbOva0AL70Qk72nG\nWNJiovriuLGdKebXVmQytJzXGuhlWb0nZ2baP3XzCt660subltfhzpGYM63Y7WKF15O1C/B2H+0j\nHMnODWVERFJJM8aSFhvbunCNU18cF3+B3akFeFNircUfCGb9wruxqsuKua5pjpLiSTQ31rD1cA/W\nZt8CvFPlO7lz34mIJEOJsaTF83u6aG6oGbe+GKCqtIh5M8u1AG+KjvUNc3xgVDN3eailoYbeoRCH\njmdfeZG/I0hJkYuFKoERkTynxFhSbmg0zKsHT3Dl4vFni+OavB71Mp4izdzlr+aG6GLKbCyn8AeC\nLK2ryvtdCEVE9K+cpNyrB85dXxzn83poO9bPSCiSochynz+QG63aZOp8Xg9ul8nKBXj+QFBvxkSk\nICgxlpSL1xdfuvDcM8Y+bzWhiGXPsb4MRZb7WgNBZleVMquq1OlQJMXKit0sm1PF1vbsmjHuGRgl\n0DukN2MiUhCUGEvKRfsXT1xfHHdqa2jVGSdKM3f5rSULF+C1xsqdlBiLSCFQYiwplWh9McCi2ZUU\nu40W4CUoHLHs7AgqQcljLQ3VdPaNcDQ47HQoJ/lzZKdFEZFUUGIsKZVofTFEe7cuqavSArwE7e/q\nZzgUwZdDrdpkasbugJctWgNBqsuK8FaXOR2KiEjaKTGWlJqsf/GZfF6PSikSpIV3+W/F3GqMga2H\ns+fNYrR8pxpj1IdaRPKfEmNJqefbumhprKG67Nz1xXE+r4f2niF6h0bTHFnuaw0EMQaWa8Y4b1WW\nFrF4dmXWLMCz1rIzoPIdESkcSowlZYZGw2w+cCKhMoq4+EKynZo1npQ/EOS82grKS9xOhyJp1NxQ\nw7YsKaU4fGKQ4HBIibGIFAwlxpIyrxw4zkg4ktDCuzifN7qpgRbgTU4L7wpDS2M17T1DdPePOB3K\nyS3b1QlFRAqFEmNJmY1t3Qn1Lx6roaYMT1mR6ownMTQaZl9X/8k3EpK/WhqiC/C2ZUE5RfwN63Il\nxiJSIBJKjI0xNxpj/MaY3caYe8b5/gJjzNPGmFeNMa8ZY25OfaiS7TZOsb4YwBiDr14L8Cazq6OP\niNXMXSFoboh3pnB+AZ4/EKRxRvmUfqdFRHLZpImxMcYN3AfcBKwEbjPGrDzjsM8CD1lrLwJuBf4z\n1YFKdkumvjjO5/XQGujNqk0Nso02WSgcNRXFzK8tz4oFeH4tvBORApPIjPHlwG5rbZu1dgT4CbD2\njGMsEP+MtwZoT12IkguSqS+O83k99A6FCPQOpSGy/OAPBCkpcrFwVqXToUgGtGTBArzRcIQ9x/qU\nGItIQUkkMW4EDo55fCj23Fj3Au81xhwCHgX+LCXRSc7YuKdryvXFcfENK7QAb2L+jiDL5lThdqmX\nbCFoaaxhX9eAo20M2471Mxq22lBGRApKUQLHjPdKfOZn3rcBD1pr/8UY8wbg+8aYFmtt5LQTGXMX\ncBdAXV0dGzZsSCJk6evry7qxe+zVQc6rdvHKxuem/LP9o9Hb6ZfPbcYcKUl1aKfJxrFLxGsHBmiZ\n5aeLls8AACAASURBVHY09lwdO6clM27hzhAAP3r0GZpqnWnPt7E9GkPwkJ8NPbsciUH3XPI0dsnT\n2CUnX8YtkcT4EDB/zON5nF0qcSdwI4C19nljTBkwGzg69iBr7f3A/QA+n8+uXr06uagL3IYNG8im\nsRscCbPvySe446qFrF69Iqlz/N1LTxGqnMXq1atSHN3psm3sEtHdP0LPY09y7aqlrH7TEsfiyMWx\nywbJjFtzcJivvvwriuoWsfqaxekJbBIvPtZKkauNP7ppNSVFzjQw0j2XPI1d8jR2ycmXcUvkX7uX\ngGXGmEXGmBKii+vWn3HMAeDNAMaYFUAZcCyVgUr2evVkffHUF97FRRfgqZRiPKcW3qlVW6Go85RS\nX13K9nbnOlP4A0EW11U6lhSLiDhh0n/xrLUh4G7gcWAH0e4T24wxXzDGrIkd9nHgw8aYLcCPgdut\nWgwUjI1t8frimUmfo8nrYfexPkLhyOQHF5j4roBq1VZYWhpqHO1M4e8I6s2YiBScREopsNY+SnRR\n3djnPjfm6+3AVakNTXLFxrZuzm+swTONXqc+r4eRUIR9Xf0snaMEcCx/R5AZFcXM8ZQ6HYpkUHNj\nDU/7jzI4Es74NuB9wyEOHR/ktssXZPS6IiJO02dkMi2DI2E2H0yuf/FY8ZZQKqc4W2sgiK/egzHq\nSFFIWhqqiVjYEch8OUV8wx11pBCRQqPEWKYlFfXFAEvqoq3ItAPe6SIRy85AUGUUBailMbY1tAP9\njE8mxrrvRKTAKDGWaXm+rQu3y0yrvhigrNjNwlkVmjE+w+ETg/SPhFXrWYDm1pRRW1niyNbQ/kAv\nlSVu5s0sz/i1RUScpMRYpmVjWxct06wvjmvyVmvG+AytJ2fuqhyORDLNGENzQ7UjC/BaA0GWe1W+\nIyKFR4mxJO1UffHUd7sbj8/r4UD3AP3DoZScLx/4Y/Wly1XrWZBaGmvY2RFkJJS5bi3WWvwdKt8R\nkcKkxFiS9sqB44yG7bTri+Pi9Yy7jval5Hz5oDUQpHFGeUpm5CX3NDdUMxq27OzI3CcpR4PDnBgY\n1cI7ESlISowlaRvj9cXnTa++OC4+Q+V3YBV+tvJr4V1Ba2mILcDLYDnFqfId1bWLSOFRYixJS2V9\nMcD8mRVUlLi1AC9mOBRmb2e/OgMUsAW1FXhKizK6AE8byohIIVNiLElJdX0xgMtlWFbv0QK8mLZj\n/YQiVolxAXO5DCszvACvNRBkjqeUmZUlGbumiEi2UGIsSXl5f2rri+OalBif5D85c6ePtAtZS2MN\nO470Zmy7dH9Hr96MiUjBUmIsSYnXF1+2MHUzxhBdgNfVP8Kx4HBKz5uLWgNBit2GxXWVTociDmpp\nrGZoNEJbZ3/arxWOWHZ19KmMQkQKlhJjScrGti7Ob6yhqrQopef1nVyAp1ljf6CXJXVVFLv1a1rI\n4gvwtmZgB7x9Xf0MhyJaeCciBUuvuDJlAyMhthw6kfIyCjiVGLeqMwX+QFD9i4XFdVWUFbsysgDv\n5FbQuu9EpEApMZYpe2X/iVh9cWrLKABmV5Uyu6oko31bs1HP4CjtPUOq9RTcLsOKudUZadnWGgji\nMrCsXjstikhhUmIsU3ayf3GK64vjfF4twIu/MVCtp0C0nGJ7ey+RiE3rdfyBXhbOqqSs2J3W64iI\nZCslxjJl6aovjvPVV7Ozoy/tSUA2O7XJghJjiS7ACw6HONA9kNbr+ANB3XMiUtCUGMuUpLO+OK7J\n62FwNJz2JCCb7QwE8ZQW0Tij3OlQJAs0xxfgpbGcYmAkxP7uASXGIlLQlBjLlJzqX5yeMgoYuwCv\ncMsp/IEgy70ejDFOhyJZYHm9h2K3SesCvN1H+7BW5TsiUtiUGMuUpLu+GKJJgDGF27LNWktrQJss\nyCklRS58Xk9aF+CdKt9RqzYRKVxKjGVKNrZ1c8G89NUXA5SXuDmvtgJ/R2G2bAv0DtE7FNLMnZym\npaGGrYd7sDY9tff+QJCyYhcLaivScn4RkVygxFgSNjASYsvB9NYXxy2v9xRsKUWresnKOJobazg+\nEG3jlw7xvtlul8p3RKRwKTGWhL28/zihiM1IYtzk9bCvs5+h0XDar5Vt/OpIIeNoboiWOGxL0w54\nrdpQRkREibEk7mR98Xkz034tn7eaiI0uCCo0/kCQ+upSZlSUOB2KZJEV3mpcBra2p77EqKtvmM6+\nYZXviEjBU2IsCYvXF1emsb44Lj5bWogL8FoDQS2AkrOUl7hZOqcqLTPG+pRCRCRKibEkJJP1xQAL\nZ1VQUuTCX2BbQ4+GI+w52qeZOxlXS0NNWnoZa0MZEZEoJcaSkE37MldfDFDkdrFsTlXBLcDb19nP\nSDiihXcyrubGGjp6hzkaTO0CPH8gSG1lCXVVpSk9r4hIrlFiLAnZ2NZFUYbqi+N8Xg/+QGG1bIvP\nkGvmTsbTEl+Al+I6Y39HEF+9NpQREVFiLAnZ2NaVsfriuCavh47eYU4MjGTsmk7zB4K4XYalc6qc\nDkWy0Mo0dKaIRCw7O4J6MyYighJjSUD/cIjXDvVkrIwiLr4ArZDKKVoDQRbOqqCs2O10KJKFPGXF\nLJpdmdKtoQ8dH2RgJKy6dhERlBhLAjLZv3ispgLsTOEPBGlSRwo5h5UN1Ww7kroZ49ZYuZJmjEVE\nlBhLAuL1xZdksL4YYI6nlJry4oKZMe4fDnGge0CbLMg5tTTUcLB7kJ6B0ZScL/7Gc5nuOxERJcYy\nOSfqiwGMMfi8HnYWSMu2nVp4JwloaYwvwEvNrHFrR5D5teVUZfj3W0QkGykxlnNyqr44rsnrYWcg\niLXWketnUnzmTrWeci7NDTUAKetn7A8E8dWrfEdEBJQYyyQ2OVRfHOfzeggOhzh8YtCR62dSayBI\nebGbBbUVTociWay2soTGGeUpWYA3HAqzt7Nfb8ZERGKUGMs5OVVfHFdIC/D8gSDL66twudRLVs6t\nuaE6JTPGu4/2EY5Yle+IiMQoMZZz2tjWxYXzZ2S8vjguvhCtEBbgqZesJKqlsYa9nf30DYemdZ54\nXbtmjEVEopQYy4RO1RfXOhaDp6yYxhnleT9jfCw4TFf/yMnezSLn0tJYjbWw48j0yilaA0FK3C4W\nzq5MUWQiIrlNibFMaNP+44QdrC+Oa/J68j4x1sI7mYqW2AK86e6A5w8EWTKnimK3XgpERECJsZyD\n0/XFcT6vhz3H+hgJRRyNI520yYJMxZzqMmZXlbK1fXozxtENZXTPiYjEKTGWCcXriytKnO1v6vN6\nCEUsbZ19jsaRTv5AkFmVJcyuKnU6FMkRLY3VbJ3GjHHPwChHeoa0oYyIyBhKjGVcfVlQXxznK4DO\nFH4tvJMpammoYdfRPoZGw0n9vF8L70REzqLEWMa1aV93VtQXAyyeXUWRy+RtYhyOWHWkkClraawm\nHLFJ/174Vb4jInIWJcYyro1t3RS7na8vBigpcrGkripvE+MD3QMMjUY0cydTMt0d8FoDQTxlRcyt\nKUtlWCIiOU2JsYxrY1sXF85zvr44zuf15G0v41Mzd2rVJombN7OcmvLipHfAiy+8M0YbyoiIxCkx\nlrP0DYd4/XBPVpRRxPm8Hg6fGCQ4NOp0KCnnD/RhDCyvr3I6FMkhxhhaGqvZlsSMsbVWde0iIuNI\nKDE2xtxojPEbY3YbY+4Z5/v/aozZHPuz0xhzIvWhSqZkU31xXLzMIL5TVz7xd/SyoLYia2bnJXe0\nNNTQGggyGp5aK8MjPUMEh0L6lEJE5AyTJsbGGDdwH3ATsBK4zRizcuwx1tq/tNaustauAr4G/Dwd\nwUpmxOuLLz5vhtOhnBSf2crHcorWQBCfWmZJElY2VDMSirD76NRaGWpDGRGR8SUyY3w5sNta22at\nHQF+Aqw9x/G3AT9ORXDijGyrLwZonFFOVWlR3i3AGxoNs6+zXwmKJKWlMbYAb4r9jONvMNXDWETk\ndIkkxo3AwTGPD8WeO4sx5jxgEfDr6YcmTsjG+mKI1lMur6/Kuxnj3Uf7iFhYrsRYkrBoViWVJW62\nTXEHPH+gl7k1ZdSUF6cpMhGR3JTIlOB4S5btBMfeCvzMWjtux3ljzF3AXQB1dXVs2LAhkRjlDH19\nfWkbu9eOhQhHLOXBQ2zYcCQt10hWdWSYl46EePrpp5NeSZ/OsUvGs4ejiwl7D7SyoXunw9GcW7aN\nXa5I97g1Vlqe236ADdXHEv6Zl/cMUldqsv7vU/dc8jR2ydPYJSdfxi2RxPgQMH/M43lA+wTH3gr8\n6UQnstbeD9wP4PP57OrVqxOLUk6zYcMG0jV2z/9yB8XuvXxwzWrKS9xpuUay9pfsY8P6bay85A3U\nVyfXezWdY5eM5x7ZTknRft5902qK3NndJCbbxi5XpHvcNvRu46FNB7nmTdfidk3+hnE0HCHw5GPc\nfPFCVq9ekba4UkH3XPI0dsnT2CUnX8YtkVfil4BlxphFxpgSosnv+jMPMsb4gJnA86kNUTJpY1s3\nq+bPyLqkGPJzAV5rIMjSuqqsT4ole7U01jAwEmZvZ39Cx+/t7Gc0bFXXLiIyjklfja21IeBu4HFg\nB/CQtXabMeYLxpg1Yw69DfiJtXaiMgvJcsGhUbZmYX1xXPyFPL4hRj6Ib7IgkqyWxmjLtUT7Gcff\nWPrq1apNRORMCbUdsNY+Cjx6xnOfO+PxvakLS5ywaf/xrOtfPNaMihLqq0vzZsb4eP8IR4PD2mRB\npmVpXRWlRS62tfeydtW466JP4w/04nYZlsypzEB0IiK5RZ/fykkb27qi/YsXzHQ6lAn5vNV507LN\nH9usRImxTEeR20WT15NwyzZ/oI/FsyspLcq+cikREacpMZaTNu7pytr64rgmr4ddR/sITXGnr2x0\napMFfaQt09PcWMPWwz0kUsnm7+jVmzERkQkoMRYgWl+cjf2Lz+Sr9zASirCva8DpUKatNRCkpryY\n+upSp0ORHNfSUEPvUIhDxwfPeVzfcIiD3YOqaxcRmYASYwFg077jRCzZnxifXICX++UU/kAvvnpP\n0j2ZReLiC/AmK6fY2aEd70REzkWJsQC5UV8MsHROFS5zqj43V1lr2dnRp4+0JSWW13sochm2TtKZ\nQuU7IiLnpsRYgGhifNH8mVldXwxQVuxm4ezKnG/Zduj4IH3DISXGkhJlxW6W1XvYevjcvxf+QJCK\nEjfzZpZnKDIRkdyixFjG1BfXOh1KQpq8npwvpTg1c6fEWFKjpaGabe3nXoDXGuhleb0HVwI75ImI\nFCIlxpIz9cVxvvpq9ncPMDAScjqUpMVLQZYrMZYUaWmsobMv2ht7PNZabSgjIjIJJcbCxrYuStwu\nLsry+uI4n9eDtbCro8/pUJLmDwRpnFFOdVmx06FInmhuOPcCvGPBYY4PjKp8R0TkHJQYC8+3ZX//\n4rGa8qAzhT8QVIIiKbVibjXGMGGdsTaUERGZnBLjAtc7NMrWHKovBlhQW0F5sTtnt4YeCUXYc0wd\nKSS1KkuLWDy7csLOFOpIISIyOSXGBW7Tvu6cqi8GcLkMy+ur8HfkZmeKts4+QhGrWk9JuZbGGrZN\nUErRGghS5ymltrIkw1GJiOQOJcYFbmNbNyVuFxeflxv1xXHL63O3M0U8bm2yIKnW0lBDe88QXX1n\nL8DzB4L4dM+JiJyTEuMCt7Gti1ULZlBWnBv1xXE+r4fOvhE6x0kAsl1rIEiRy7CkrsrpUCTPNMd2\nwNvWfvqnKeGIZWeH6tpFRCajxLiAnaovzp0yirh4neTOHJw19geCLK6rpKRIv36SWs0NNcDZifH+\nrn6GQxElxiIik9ArcwE7VV+cOwvv4uIv8Lm4AC/akUILoCT1asqLWVBbcdYCPG0oIyKSGCXGBez5\nPdH+xRfnSP/iseo8pcyqLMm5OuPeoVEOnxhUgiJp09xQfdYCvNZAEGNg2RzddyIi56LEuIBtbOvO\nyfriOJ/XQ2tHbiXG8dIPLYKSdGlprGFf1wC9Q6Mnn/MHgiycVZkzvcpFRJyixLhA9QyOsq09N+uL\n43xeD7s6gkQi1ulQEqZNFiTd4jvgbR9TZ7yzQx0pREQSocS4QOVyfXFck9fDwEiYg8cHnA4lYf5A\nkKrSIubNLHc6FMlT8QV48a2hh0bD7Ovq15sxEZEEKDEuUBvbcre+OC6+gC2XFuC1BoIsr6/CGON0\nKJKn6jyleKvLTnam2NXRR8Rq4Z2ISCKUGBeojW3dXJTD9cUAy+ZE+wDnygI8a22sI4USFEmvlsbq\nkzPGrYFogrxc952IyKSUGBegfKgvBqgsLWJBbcXJut1s19E7TM/gqGo9Je2aG2rYc6yPwZEw/kCQ\n0iIXC2dVOh2WiEjWU2JcgE7VF+d2YgzRRWy5MmMcn7lTD2NJt5bGGiIWdgR68XcEWVZfhdul8h0R\nkckoMS5Az+/poqTIxUULZjgdyrQ1eT3s7exnOBR2OpRJaZMFyZSW+NbQh3toDQTx1evNmIhIIpQY\nF6CNe7u4aH5u1xfH+bwewhHL7qN9TocyKX8gyBxPKTMrS5wORfKct7qM2soSntnVybHgsN6MiYgk\nSIlxgYnWF/fmRRkFnJp9zYVyilYtvJMMMcbQ3FDN061HAfXNFhFJlBLjAvPS3m5sntQXAyycVUlJ\nkSvrE+NQOMLuY32auZOMaWmsIRTb/Eb3nYhIYpQYF5iNbflTXwxQ5HaxtK4q63sZ7+saYCQU0cI7\nyZiW2EYfMyuKqfOUOhyNiEhuUGJcYDbu7eLiHO9ffKamHOhMoYV3kmnxBXg+r0cbyoiIJEiJcQE5\nFhzOq/riuOVeD4HeIXoGRp0OZUK/2tFBidvF0timJCLptqC2gvrqUi7K4d0tRUQyrcjpACRzHnmt\nHWvh7efPdTqUlIovLPJ3BLl8Ua3D0ZztlQPH+Z9XD/Mnq5fk1Uy9ZDdjDI9+7BoqS/XPvIhIojRj\nXEDWbWmnyethWZ7tvHaqM0Wvw5GcLRKxfH79NuqrS7n7uqVOhyMFZlZVqd6MiYhMgRLjAnGga4BX\nD5xg7apGp0NJOW91GdVlRVm5AO9nrxxiy6Ee7rmpSTN3IiIiWU6JcYFYv+UwALdcmF9lFBD9yLjJ\nW511C/B6h0b558dauXjBDH4vD9+QiIiI5BslxgXAWsu6ze1ctnAm82ZWOB1OWvi8HvwdQay1Tody\n0r//ahdd/SN8fk2LugKIiIjkACXGBaA1EGTX0T7WXNjgdChp4/N6CA6FaO8ZcjoUAHYf7ePB3+3j\njy6dz/nzapwOR0RERBKgxLgArNvcjttluDnPulGMlU0L8Ky1fOF/t1Ne4uYTb/M5HY6IiIgkSIlx\nnotELA9vaeeaZbOZVZW/u18tjyXG2bAA76kdR3lm5zH+4i3LmZ3HYy4iIpJvlBjnuZcPHOfwiUHW\nrsrfMgqA6rJiGmrKHF+ANzQa5gv/u52lc6p4/xvOczQWERERmRolxnlu/eZ2yopd3LDS63QoaefL\ngq2hv/3sXg50D/C3t6yk2K1fLxERkVyiV+48NhqO8MjrR3jzinqqCqCHrs9bzZ5jfYyGI45cP9Az\nxH1P7+atK+u5ZlmdIzGIiIhI8pQY57Fnd3fS3T/C2jzuRjFWk9fDaNiyt7Pfkev/4y93EIpYPvv2\nlY5cX0RERKZHiXEeW7+5neqyIq71Fcbspc/BBXib9nXzi83t3HXNYhbMys9e0SIiIvlOiXGeGhwJ\n8/i2ADefP5fSIrfT4WTEkroqilwm4y3bwhHLvQ9vw1tdxkevW5LRa4uIiEjqJJQYG2NuNMb4jTG7\njTH3THDMu40x240x24wxP0ptmDJVT7V2MDASzutNPc5UUuRicV1lxhfgPbTpIFsP9/Kpm5uoKMn/\nWm4REZF8NemruDHGDdwH3AAcAl4yxqy31m4fc8wy4FPAVdba48aYOekKWBKzbnM7czylXLF4ltOh\nZJTPW82rB45n7Ho9g6N8+XE/ly2cWVBvQkRERPJRIjPGlwO7rbVt1toR4CfA2jOO+TBwn7X2OIC1\n9mhqw5Sp6BkYZYP/KLdc2IDbZZwOJ6OavB4OHR+kbziUkev931/t5MTACPeuacaYwhprERGRfGOs\ntec+wJg/BG601n4o9vh9wBXW2rvHHPMLYCdwFeAG7rXWPjbOue4C7gKoq6u75KGHHkrV/0dB6evr\no6qqasLv/+bgKA9sG+Fv31DGoprCqC+Oe/VoiH97ZZjPXlHG0pln/79PNnZTcTgY4W9+N8ib5hVx\ne3P+73CXyrErJBq35GnskqexS57GLjnZPm7XXXfdy9baSyc7LpGCyPGmwc7MpouAZcBqYB7wW2NM\ni7X2xGk/ZO39wP0APp/Prl69OoHLy5k2bNjAucbu/m9uZNHsIW5fc23BzWIu6R7g3155msrGZay+\nfMFZ359s7BJlreW9336BqtJRvnr7ddRWlkz7nNkuVWNXaDRuydPYJU9jlzyNXXLyZdwSKaU4BMwf\n83ge0D7OMeustaPW2r2An2iiLBnW0TvE821d3HJhQ8ElxQCNM8qpLHGnfQHe49s6eG53F391w/KC\nSIpFREQKQSKJ8UvAMmPMImNMCXArsP6MY34BXAdgjJkNLAfaUhmoJObhLe1YS8EuBHO5DMu9HlrT\n2LJtaDTM3z+yneX1Vbz3yvPSdh0RERHJrEkTY2ttCLgbeBzYATxkrd1mjPmCMWZN7LDHgS5jzHbg\naeCT1tqudAUtE1u/pZ2WxmqWzsneOp90a/J68AeCTFY/n6xvPtPGoeOD3HtLM0VutQIXERHJFwk1\nXbXWPgo8esZznxvztQX+KvZHHLK3s5/XDvXwmZtXOB2Ko3z1Hn784kGOBYeZU12W0nO3nxjkvg27\nuanFyxuXzk7puUVERMRZmu7KI+s3t2MMvOPCuU6H4iiftxpIz9bQX/plK9bCpwv8zYeIiEg+UmKc\nJ6y1rNtymMsX1jK3ptzpcBzV5PUApHwB3gttXTy8pZ2PXLuE+bUVKT23iIiIOE+JcZ7Y1t5L27F+\n1q5qdDoUx82sLGGOpzSlM8bhiOXeh7fTUFPGR65dkrLzioiISPZQYpwn1m0+TLHbcFOL1+lQsoLP\n62FnR+oS4x+/eIAdR3r5zNtXUl5SWJumiIiIFAolxnkgErE8vOUI1y6vY6Z66gLRBXg7O4KEI9Pv\nTHFiYISvPOHnikW13Hy+3niIiIjkKyXGeeDFfd0Eeoe4pUB7F4/H5/UwHIqwv6t/2uf66pM76R0c\n5d41zQW5aYqIiEihUGKcB9Ztbqe82M0NK+udDiVrNMU6U0x3AV5roJcfbNzPe688jxVzq1MRmoiI\niGQpJcY5biQU4dHXj/DW5noqShJqS10QltVX4TLTa9lmreXe9duoLi/mr25YnsLoREREJBspMc5x\nz+w8Rs/gKGtXqYxirLJiNwtnVU5rxviXWwNsbOvm42/1MaNCtdsiIiL5Tolxjlu/pZ2ZFcVcs6zO\n6VCyjs/rwZ9kZ4rBkTBffGQHTV4Pf3z5ghRHJiIiItlIiXEO6x8O8eT2Dm46fy7Fbv1Vnsnn9bCv\nq5/BkfCUf/Ybv9nD4ROD3LumGbdLC+5EREQKgbKpHParHR0MjoZZq24U42ryerAWdh2d2qzxoeMD\nfOM3e3jHBXO5cvGsNEUnIiIi2UaJcQ5bt7mduTVlXLaw1ulQspIvyc4U//DoDoyBT9+8Ih1hiYiI\nSJZSYpyjjveP8MzOY6y5sAGXPuof14LaCsqKXVNKjH+3p5NHXw/w0dVLaZhRnsboREREJNsoMc5R\nj249QihitanHObhdhmVzEl+AFwpH+Pz67cybWc5db1qc5uhEREQk2ygxzlHrNrezpK6S5gZtOnEu\nPq8n4V7GP3zhAP6OIJ99+wrKit1pjkxERESyjRLjHNQ1GOHFvd2sXdWoLYon0eT1cCw4THf/yDmP\n6+4f4atP7uSqpbN4W7M3Q9GJiIhINlFinINeCIQAWKMyikn5vB4gurXzufzLE376hkP87S3NerMh\nIiJSoJQY56AXjoS5cP4MFs6udDqUrBdPjM+1AG9bew8/evEA77vyPJbXezIVmoiIiGQZJcY5ZvfR\nIPt7I5otTlBdVSm1lSUTJsbWWj6/fjszyov5y7csz3B0IiIikk2UGOeY9ZvbMcAtF8x1OpScYIzB\nVz/xAryHXzvCi/u6+eTbmqipKM5wdCIiIpJNlBjnEGst67a0s2KWiznVZU6HkzN8Xg+7OoJEIva0\n5wdGQnzp0R00N1TzR5fNdyg6ERERyRZKjHPIa4d62N81wJVzi5wOJaf4vB76R8IcPjF42vNf37CH\nIz1D3LumGbc2SRERESl4SoxzyLrN7ZS4XVxSr8R4Kk51pjhVTnGga4D/90wba1c1aEttERERAZQY\n54xwxPLwa+2s9tVRWazZzamId5rwj2nZ9sVHt+M2hntuanIqLBEREckySoxzxMa2Lo4Fh1m7qtHp\nUHJOVWkR82vLT84Y/3bXMR7f1sHd1y9lbk25w9GJiIhItlBinCPWbT5MZYmbN6+Y43QoOclXX40/\nECQUsXz+4e0sqK3gzqsXOR2WiIiIZBElxjlgOBTml1sDvK3FS1mx2+lwclKT10NbZz9P7Btl99E+\nPvv2FRpLEREROY0S4xywwX+M4FBIm3pMg8/rIRyx/PeuUa5ZNpsbVtY7HZKIiIhkmYJMjB966SBd\nfcNOh5Gw9ZvbmVVZwlVLZzsdSs5q8p7a6vlvb1mJMVrAKCIiIqcruMT4YPcAf7NuKx/63iYGR8JO\nhzOp4NAov9rRwdsvmEuxu+D+ulJm4exKaitLeNvCYpbO8Uz+AyIiIlJwCi7Tml9bwb/duorNB0/w\nF//1KuEzdkPLNk9u72A4FGHtKpVRTEex28Wz/+c63rVc2z6LiIjI+AouMQa4sWUuf/P2lTy+rYO/\nf2S70+Gc07rN7cybWc7FC2Y6HUrOqygpUgmFiIiITKhgt1D74NWLOHxikG8/u5fGGeV86JrFo3tu\nvAAADVRJREFUTod0ls6+YZ7d3cldb1qshE5EREQkzQo2MQb4zM0raD8xyBcf3UHDjHJuPn+u0yGd\n5tHXjxCOWJVRiIiIiGRAQZZSxLlchn/9o1VcvGAmf/Ffm9m0r9vpkE6zfnM7vnoPTd5qp0MRERER\nyXsFnRgDlBW7+eb7L42WU3xvE23H+pwOCYh2z9i0/zhrNFssIiIikhEFnxgD1FaW8OAdl+E2htsf\neInOLOhx/PBr7QDa1ENEREQkQ5QYx5w3q5JvfeBSjgaHuPPBlxgYCTkaz/rN7Vy8YAbzayscjUNE\nRESkUCgxHuOiBTP591sv4vXDPXzsx5sd63HsDwRpDQRZu6rRkeuLiIiIFCIlxmd4a7OXe9c086sd\nHXz+4W1Ym/nkeP2Ww7hdJuu6ZIiIiIjks4Ju1zaR979hIYeOD3L/M23Mm1nOXW9akrFrW2tZt7md\nq5bOps5TmrHrioiIiBQ6zRhP4J4bm3j7BXP5h0dbeXhLe8au+8qBExw6PqhFdyIiIiIZphnjCbhc\nhn9514Uc6x3m4w9tob66jMsX1ab9uus3H6akyMXbmuvTfi0REREROUUzxudQVuzm/vdfwrzacj78\nvU3sPhpM6/VC4QiPvH6Et6yYg6esOK3XEhEREZHTJZQYG2NuNMb4jTG7jTH3jPP9240xx4wxm2N/\nPpT6UJ0xo6KE795xOcXuaI/jo8GhtF3rd3u66OwbYc2F6kYhIiIikmmTJsbGGDdwH3ATsBK4zRiz\ncpxD/8tauyr251spjtNR82sr+M7tl9HVN8KdD25KW4/jdZvb8ZQWsdpXl5bzi4iIiMjEEpkxvhzY\nba1ts9aOAD8B1qY3rOxzwbwZ/McfX8S29h7u/tGrhMKRlJ5/aDTM49sC3NjipazYndJzi4iIiMjk\nzGR9eo0xfwjcaK39UOzx+4ArrLV3jznmduBLwDFgJ/CX1tqD45zrLuAugLq6ukseeuihFP1vZM6v\nD4zyve0jXDe/iPevLMEYk5LzvhQIcd/mYT55aRnNs8+dGPf19VFVVZWS6xYajV3yNHbJ0bglT2OX\nPI1d8jR2ycn2cbvuuutettZeOtlxiXSlGC/zOzObfhj4sbV22BjzEeC7wPVn/ZC19wP3A/h8Prt6\n9eoELp9dVgOVj7Xy9Q17uKx5CR9dvTQl5/3J919mdtVxPvLO63G7zp1sb9iwgVwcu2ygsUuexi45\nGrfkaeySp7FLnsYuOfkybomUUhwC5o95PA84rbGvtbbLWjsce/hN4JLUhJedPvlWH2subOCfH/Oz\nbvPhaZ+vZ3CUX/uPcsuFcydNikVEREQkPRJJjF8ClhljFhljSoBbgfVjDzDGjN27eA2wI3UhZh+X\ny/Dld13AFYtq+cRPt/D8nq5pne/xbQFGQhFt6iEiIiLioEkTY2ttCLgbeJxowvuQtXabMeYLxpg1\nscM+ZozZZozZAnwMuD1dAWeL0iI397/vUhbOquSu729iZ0fyPY7Xb25nQW0Fq+bPSGGEIiIiIjIV\nCfUxttY+aq1dbq1dYq39Yuy5z1lr18e+/pS1ttlae6G19jprbWs6g84WNRXFPHDHZZQVu7njgZc4\n2jv1HsdHg0P8bk8na1c1pGwhn4iIiIhMnXa+m6Z5Myt44PbLOD4wwh0PvkTf8NR6HD/y2hEiFtau\nUhmFiIiIiJOUGKdAS2MN973nYloDQf70h69Mqcfxus3trJhbzdI5njRGKCIiIiKTUWKcItf55vD3\nv9fCb3Ye47O/2Mpk/aEB9nf1s/ngCc0Wi4iIiGSBRPoYS4Juu3wBh48P8h9P72bezHLuvn7ZOY9f\nvzna9e4WdaMQERERcZwS4xT7+FuX035ikK88sZOGGeW88+J54x5nrWXdlnYuX1hL44zyDEcpIiIi\nImdSKUWKGWP4xz+4gDcumcVf/+w1ntvdOe5xO44E2X20jzUqoxARERHJCkqM06CkyMU33ncJS+qq\n+Mj3X8YfOLvH8bothylyGW4+f+44ZxARERGRTFNinCbVZdEexxWlbm5/4EUCPad6HEciloc3t3PN\nstnUVpY4GKWIiIiIxCkxTqOGGeV85/bL6B0c5Y4HXyI4NArAyweO094zxNpVjQ5HKCIiIiJxSozT\nrLmhhq+/9xJ2dgT56A9fYTQcYd3mw5QVu7hhZb3T4YmIiIhIjBLjDHjT8jq+9M7z+e2uTu7579d5\n5LUj3LDSS2WpmoKIiIiIZAtlZhny7kvnc/j4IP/21C4A1qh3sYiIiEhWUWKcQX/xlmUc6xvm+T1d\nvGn5bKfDEREREZExlBhnkDGGf/j984lELC6XcTocERERERlDNcYOUFIsIiIikn2UGIuIiIiIoMRY\nRERERARQYiwiIiIiAigxFhEREREBlBiLiIiIiABKjEVEREREACXGIiIiIiKAEmMREREREUCJsYiI\niIgIoMRYRERERARQYiwiIiIiAoCx1jpzYWOCgN+Ri+e+2UCn00HkKI1d8jR2ydG4JU9jlzyNXfI0\ndsnJ9nE7z1pbN9lBRZmIZAJ+a+2lDl4/ZxljNmnskqOxS57GLjkat+Rp7JKnsUuexi45+TJuKqUQ\nEREREUGJsYiIiIgI4GxifL+D1851GrvkaeySp7FLjsYteRq75GnskqexS05ejJtji+9ERERERLKJ\nSilERERERFBiLCIiIiICOJQYG2NuNMb4jTG7jTH3OBFDLjDGzDfGPG2M2WGM2WaM+fPY87XGmCeN\nMbti/53pdKzZyhjjNsa8aoz539jjRcaYF2Jj91/GmBKnY8xGxpgZxpifGWNaY/ffG3TfJcYY85ex\n39etxpgfG2PKdN+NzxjzHWPMUWPM1jHPjXufmah/j71uvGaMudi5yJ01wbh9Ofb7+pox5n+MMTPG\nfO9TsXHzG2Pe5kzU2WG8sRvzvU8YY6wxZnbsse65MSYaO2PMn8XurW3GmH8e83xO3ncZT4yNMW7g\nPuAmYCVwmzFmZabjyBEh4OPW2hXAlcCfxsbqHuApa+0y4KnYYxnfnwM7xjz+J+BfY2N3HLjTkaiy\n378Bj1lrm4ALiY6h7rtJGGMagY8Bl1prWwA3cCu67ybyIHDjGc9NdJ/dBCyL/bkL+HqGYsxGD3L2\nuD0JtFhrLwB2Ap8CiL1m3Ao0x37mP2Ovw4XqQc4eO4wx84EbgANjntY9d7oHOWPsjDHXAWuBC6y1\nzcBXYs/n7H3nxIzx5cBua22btXYE+AnRQZUzWGuPWGtfiX0dJJqcNBIdr+/GDvsu8HvORJjdjDHz\ngLcD34o9NsD1wM9ih2jsxmGMqQbeBHwbwFo7Yq09ge67RBUB5caYIqACOILuu3FZa58Bus94eqL7\nbC3wPRu1EZhhjJmbmUizy3jjZq19wlobij3cCMyLfb0W+Im1dthauxfYTfR1uCBNcM8B/Cvw18DY\njgS658aYYOz+BPhHa+1w7Jijsedz9r5zIjFuBA6OeXwo9pycgzFmIXAR8AJQb609AtHkGZjjXGRZ\n7f8S/YcuEns8Czgx5sVD9974FgPHgAdiZSjfMsZUovtuUtbaw0RnTA4QTYh7gJfRfTcVE91neu1I\n3AeBX8a+1rhNwhizBjhsrd1yxrc0dpNbDlwTKxX7jTHmstjzOTt2TiTGZpzn1DPuHIwxVcB/A39h\nre11Op5cYIx5B3DUWvvy2KfHOVT33tmKgIuBr1trLwL6UdlEQmL1sGuBRUADUEn049gz6b6bOv3+\nJsAY8xmiZXg/jD81zmEatxhjTAXwGeBz4317nOc0dqcrAmYSLff8JPBQ7NPZnB07JxLjQ8D8MY/n\nAe0OxJETjDHFRJPiH1prfx57uiP+cU7sv0cn+vkCdhWwxhizj2i5zvVEZ5BnxD7iBt17EzkEHLLW\nvhB7/DOiibLuu8m9BdhrrT1mrR0Ffg68Ed13UzHRfabXjkkYYz4AvAN4jz21SYHG7dyWEH0juyX2\nejEPeMUY40Vjl4hDwM9j5SYvEv2EdjY5PHZOJMYvActiq7RLiBZnr3cgjqwXe9f1bWCHtfarY761\nHvhA7OsPAOsyHVu2s9Z+ylo7z1q7kOg99mtr7XuAp4E/jB2msRuHtTYAHDTG+GJPvRnYju67RBwA\nrjTGVMR+f+Njp/sucRPdZ+uB98c6BVwJ9MRLLiTa7Qn4P8Aaa+3AmG+tB241xpQaYxYRXUj2ohMx\nZiNr7evW2jnW2oWx14tDwMWxfwd1z03uF0QnnjDGLAdKgE5y+b6z1mb8D3Az0VWze4DPOBFDLvwB\nrib60cNrwObYn5uJ1so+BeyK/bfW6Viz+Q+wGvjf2NeLif5y7gZ+CpQ6HV82/gFWAZti994viH5U\npvsusbH7PNAKbAW+D5TqvptwrH5MtBZ7lGhCcudE9xnRj2bvi71uvE6084fj/w9ZNG67idZ0xl8r\nvjHm+M/Exs0P3OR0/Nk2dmd8fx8wO/a17rlJxo5oIvyD2L93rwDXjzk+J+87bQktIiIiIoJ2vhMR\nERERAZQYi4iIiIgASoxFRERERAAlxiIiIiIigBJjERERERFAibGIiIiICKDEWEREREQEgP8PMA0/\nU+oYPkgAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "loss = np.array(train_summary.read_scalar(\"Loss\"))\n", + "top1 = np.array(val_summary.read_scalar(\"Top1Accuracy\"))\n", + "\n", + "plt.figure(figsize = (12,12))\n", + "plt.subplot(2,1,1)\n", + "plt.plot(loss[:,0],loss[:,1],label='loss')\n", + "plt.xlim(0,loss.shape[0]+10)\n", + "plt.grid(True)\n", + "plt.title(\"loss\")\n", + "plt.subplot(2,1,2)\n", + "plt.plot(top1[:,0],top1[:,1],label='top1')\n", + "plt.xlim(0,loss.shape[0]+10)\n", + "plt.title(\"top1 accuracy\")\n", + "plt.grid(True)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## See visualizations on tensorboard as well\n", + "\n", + "We can run tensorboard at the command prompt and see visualizations too -- much prettier ones in fact than is available here.\n", + "\n", + "To do, run at the command line:\n", + "\n", + "$ tensorboard --logdir=\"/tmp/bigdl_summaries\"\n", + "\n", + "And then, you can go to your browser:\n", + "\n", + "http://YOURHOSTNAME:6006" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Show Confusion Matrix and Accuracy score on Validation Data\n", + "\n", + "Here, we are going to run predictions on our validation data to determine our accuracy on validation data and our confusion matrix.\n", + "\n", + "Nobody really cares what our error (loss) is on training data. It's validation data that really matters. So, calculating accuracy.\n", + "\n", + "As Iris is a 3-class output, we will be doing a confusion matrix to see how that went." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "predictions = trained_model.predict(iris_rdd_test).collect()\n", + "\n", + "def map_predict_label(l):\n", + " return np.array(l).argmax()\n", + "def map_groundtruth_label(l):\n", + " return l.to_ndarray()[0] - 1\n", + "\n", + "y_pred = np.array([ map_predict_label(s) for s in predictions])\n", + "\n", + "y_true = np.array([map_groundtruth_label(s.label) for s in iris_rdd_test.collect()])" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The prediction accuracy is 83.33%\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiYAAAHWCAYAAABDtELCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAG51JREFUeJzt3X2wZVV5J+Df202rMKBEMdDdoEgw\naNQRNOJHKgwKESR8GDEqNQnRWPaQqAP5NtHR0ooxGasSpbRC2mhAExFKM4pCGBISgolBaS00QBPl\na9INLRpREG2hu++aP/pKbi7d9zbN3Z6zdp7H2uX52GefdWEX563fu/ba1VoLAMA0WDbpAQAAfJ/C\nBACYGgoTAGBqKEwAgKmhMAEApobCBACYGgoTAGC3VNWZVXVtVV1XVWft4P2qqrOr6saq+lJVPWOx\nYypMAIAHraqemuQ1SY5M8vQkJ1bVE+ft9qIkT5zd1iT548WOqzABAHbHk5Nc1Vr7bmtta5K/T/Iz\n8/Y5JckH23ZXJdm3qlYudFCFCQCwO65NclRVPaaq9kpyQpKD5u2zOsmGOc83zr62U3ss6RB3YPOV\n51rzniW1z7FvnPQQABa09b7b6gf1XVv+7eZBfmcf9tgf+R/Z3n75vrWttbXff9JaW19Vf5Dkr5Pc\nk+SLSbbOO8yO/jksON7BCxMAoD+zRcjaRfZ5f5L3J0lV/V62JyJzbcx/TFEOTHL7QsdUmABAz2a2\nTeyrq+qHW2tfq6rHJXlJkufO2+WiJK+rqo8keXaSu1prmxY6psIEANhdH6uqxyTZkuS1rbVvVtUZ\nSdJaOyfJJdk+9+TGJN9N8qrFDqgwAYCetZnJfXVrP7mD186Z87glee2DOaarcgCAqSExAYCezUwu\nMRmCwgQAOtYm2MoZglYOADA1JCYA0LORtXIkJgDA1JCYAEDPRjbHRGECAD2b4MqvQ9DKAQCmhsQE\nAHo2slaOxAQAmBoSEwDo2cguF1aYAEDHrPwKADAQiQkA9GxkrRyJCQAwNSQmANAzc0wAAIYhMQGA\nno1sSXqFCQD0TCsHAGAYEhMA6JnLhQEAhiExAYCejWyOicIEAHqmlQMAMAyJCQB0rLVxrWMiMQEA\npobEBAB6ZvIrADA1TH4FABiGxAQAejayVo7EBACYGhITAOjZzLguF1aYAEDPtHIAAIYhMQGAnrlc\nGABgGBITAOiZOSYAAMOQmABAz0Y2x0RhAgA9G1lhopUDAEwNiQkAdKy1ca38KjEBAKaGxAQAejay\nOSYKEwDomXVMAACGITEBgJ6NrJUjMQEApobEBAB6NrI5JgoTAOiZVg4AwDAkJgDQs5G1ciQmAMDU\nkJgAQM/MMQEAGIbEBAB6NrLERGECAD0z+RUAYBgSEwDo2chaORITAGBqSEwAoGfmmLDUPvTXn8tL\n3vy+nPqW9+UNaz+ee7dsnfSQ6NxxLzw61117ZW64/h/ym7/x2kkPhxFwTk2xmZlhtglRmEzYHd/8\nds6/fF0+/KZX5mNvfU22zbRc+rnrJz0sOrZs2bKc/e6358STfi5Pe/rz8/KXvzhPfvITJz0sOuac\nYkeq6rCqumbOdndVnTVvn6Or6q45+7x5seMu2sqpqiclOSXJ6iQtye1JLmqtrd/Nv4V5ts3M5N4t\nW7PH8uX53n1b8th99570kOjYkc86IjfddGtuueVfkyQXXviJnHzScVm//isTHhm9ck5NuQm1clpr\n/5Lk8CSpquVJbkvyf3aw66dbayfu6nEXTEyq6reSfCRJJflckqtnH59fVW/Y1S9h5/b/oX1y+guf\nneN/6735qV8/O3vv+fA87ymHTHpYdGzV6gOyYePt9z/feNumrFp1wARHRO+cU+yCY5Lc1Fr7fw/1\nQIu1cl6d5Fmttd9vrf357Pb7SY6cfW+HqmpNVa2rqnXvv+iKhzrGUbv7O5tzxTVfycXv+OVc9s7X\nZ/N9W3LxVddOelh0rKoe8FprbQIjYSycU1NuoDkmc3/LZ7c1C4ziFUnO38l7z62qL1bVX1XVUxb7\ncxZr5cwkWZVkfgW0cva9HWqtrU2yNkk2X3mus3cBV62/Nav3e1Qevc9eSZJjjjgs19y0MT/9nKdO\neGT06raNm3LQgavuf37g6pXZtOmOCY6I3jmnptxAE1Xn/pYvpKoeluTkJL+9g7e/kOTxrbV7quqE\nJB9PsuAEpcUSk7OSXD5b5ayd3S5NcnmSMxcbLItb+ehH5ks3357N925Jay2fveHWHHLAfpMeFh27\net01OfTQJ+Tggw/KihUr8rKXnZJPfuqySQ+LjjmnWMSLknyhtfaAarW1dndr7Z7Zx5ckWVFVC/7I\nLZiYtNYuraofzfbWzepsn1+yMcnVrbVtu/kHMMfTDlmdY595WE773Q9k+bJledLj9s+pRx0+6WHR\nsW3btuXMs96USy7+cJYvW5Zzz7sg11//5UkPi445p6bc5Ntqp2UnbZyqOiDJHa21VlVHZnsg8o2F\nDlZD9wm1clhq+xz7xkkPAWBBW++77YETcway+YK3DvI7u+fL37Lo31BVeyXZkOSQ1tpds6+dkSSt\ntXOq6nVJfinJ1iSbk/xqa+0zCx3Tyq8A0LMJLobWWvtuksfMe+2cOY/fk+Q9D+aYFlgDAKaGxAQA\nejayuwsrTACgZ27iBwAwDIkJAPRsZK0ciQkAMDUkJgDQs8kvsLakFCYA0DOtHACAYUhMAKBnEhMA\ngGFITACgZyNbYE1hAgAdazPjuipHKwcAmBoSEwDomcmvAADDkJgAQM9GNvlVYgIATA2JCQD0bGRX\n5ShMAKBnJr8CAAxDYgIAPZOYAAAMQ2ICAD1rJr8CANNCKwcAYBgSEwDo2cjWMZGYAABTQ2ICAD0b\n2b1yFCYA0DOtHACAYUhMAKBjzeXCAADDkJgAQM/MMQEAGIbEBAB65nJhAGBqaOUAAAxDYgIAPXO5\nMADAMCQmANCzkc0xUZgAQM9GdlWOVg4AMDUkJgDQs5G1ciQmAMDUkJgAQMfGdndhhQkA9EwrBwBg\nGBITAOiZxAQAYBgSEwDomQXWAACGITEBgJ6NbI6JwgQAOtZGVpho5QAAU0NiAgA9k5gAAAxDYgIA\nPXOvHABgamjlAAAMQ2ICAD2TmAAADENiAgAda01iAgBMi5k2zLYLqmrfqvpoVd1QVeur6rnz3q+q\nOruqbqyqL1XVMxY7psQEANhd705yaWvtpVX1sCR7zXv/RUmeOLs9O8kfz/7/TilMAKBnE5r8WlWP\nTHJUklcmSWvtviT3zdvtlCQfbNv7TVfNJiwrW2ubdnbcwQuTfY5949BfwX8ym2//9KSHwIhseP4Z\nkx4C9OqQJF9P8mdV9fQkn09yZmvtO3P2WZ1kw5znG2df22lhYo4JAHSszbRBtqpaU1Xr5mxr5n31\nHkmekeSPW2tHJPlOkjfM26d2NOSF/h6tHADgAVpra5OsXWCXjUk2ttY+O/v8o3lgYbIxyUFznh+Y\n5PaFvldiAgA9m9BVOa21rybZUFWHzb50TJLr5+12UZLTZ6/OeU6SuxaaX5JITACgb5O9h9/rk/zF\n7BU5Nyd5VVWdkSSttXOSXJLkhCQ3JvluklctdkCFCQCwW1pr1yT58XkvnzPn/ZbktQ/mmAoTAOhY\nc68cAIBhSEwAoGcjS0wUJgDQs8lOfl1yWjkAwNSQmABAx0x+BQAYiMQEAHo2sjkmChMA6JhWDgDA\nQCQmANCzkbVyJCYAwNSQmABAx9rIEhOFCQD0bGSFiVYOADA1JCYA0LGxtXIkJgDA1JCYAEDPJCYA\nAMOQmABAx8Y2x0RhAgAdG1thopUDAEwNiQkAdExiAgAwEIkJAPSs1aRHsKQUJgDQMa0cAICBSEwA\noGNtZlytHIkJADA1JCYA0LGxzTFRmABAx9rIrsrRygEApobEBAA6NrZWjsQEAJgaEhMA6JjLhQEA\nBiIxAYCOtTbpESwthQkAdEwrBwBgIBITAOiYxAQAYCASEwDomMmvAMDU0MoBABiIxAQAOubuwgAA\nA5GYAEDHxnZ3YYUJAHRsRisHAGAYEhMA6JjJrwAAA5GYAEDHLLAGADAQiQkAdMy9cgCAqaGVAwAw\nEIkJAHTMAmsAAAORmABAx8a2wJrCBAA6NrarcrRyAICpITEBgI6Z/AoAMBCJyRQ47oVH5w//8G1Z\nvmxZPvBn5+d/v/O9kx4SnfvQhR/Pxy66NK21vPTk4/PzL/+ZSQ+Jzh106QfTvrs5bdtMsm1bbnvF\n6yY9JGaZ/MqSWrZsWc5+99tz/AmnZePGTbnqny7JJz91Wdav/8qkh0anvnLzrfnYRZfm/D99V1bs\nsSJn/NqbctTzjszjD1o96aHRudt/8Tcy8627Jz0M5pn05NeqWp5kXZLbWmsnznvvlUnemeS22Zfe\n01r704WOp5UzYUc+64jcdNOtueWWf82WLVty4YWfyMknHTfpYdGxm2/dkP/6lCdlz0c8InvssTw/\nfvjTcvmVn5n0sIDxOjPJ+gXev6C1dvjstmBRkjyEwqSqXrW7n+XfrVp9QDZsvP3+5xtv25RVqw6Y\n4Ijo3aGHPD6f/+K1+dZdd2fz976XT//T1fnqHV+f9LDoXUtW/sk7svqC92afl54w6dEwx0yrQbZd\nUVUHJvnpJIsWHLvqobRy3prkz3b0RlWtSbImSWr5o7Js2X95CF8zblUP/JffJp3L0bUfOfhx+cX/\n/rN5zVm/k7323DM/eughWb58+aSHReduP/2sbPv6nVn26H2zcu07suWWDfne5/950sNiQHN/y2et\nba2tnbfbu5L8ZpJ9FjjUqVV1VJIvJ/mV1tqGhb53wcKkqr60s7eS7L+zz80OfG2S7PGw1X5lF3Db\nxk056MBV9z8/cPXKbNp0xwRHxBicetJxOXW2Jfiuc87NAT+834RHRO+2ff3OJMnMnd/Kdy//TB7+\n1MMUJlNiqMmvc3/Ld6SqTkzytdba56vq6J3s9skk57fW7q2qM5Kcl+QFC33vYq2c/ZOcnuSkHWzf\nWOSz7IKr112TQw99Qg4++KCsWLEiL3vZKfnkpy6b9LDo3De++a0kyaavfi2X//0/5kXH/rcJj4ie\n1Z6PSO215/2P93zeM3LfjbdOdlBMg59IcnJV3ZrkI0leUFV/PneH1to3Wmv3zj59X5JnLnbQxVo5\nn0qyd2vtmvlvVNUVuzBoFrFt27acedabcsnFH87yZcty7nkX5PrrvzzpYdG5X/md38237r47e+yx\nR974a7+cRz1yoZQVFrb8Mftm/3e9JUlSy5fnnkv+Lpv/cd2ER8X3TWqBtdbabyf57SSZTUx+vbX2\nc3P3qaqVrbVNs09PzsKTZLd/Zuj5DFo5LLXNt3960kNgRDY8/4xJD4EROuSfL/uBVQtXrXrJIL+z\nz7n9L3f5b5hTmJxYVW9Lsq61dlFVvSPbC5KtSe5M8kuttRsWOpZ1TACAh6S1dkWSK2Yfv3nO6/en\nKrtKYQIAHXOvHACAgUhMAKBj7pUDAEyNmUkPYIlp5QAAU0NiAgAdaxlXK0diAgBMDYkJAHRsZmTL\nmCpMAKBjM1o5AADDkJgAQMdMfgUAGIjEBAA6ZoE1AICBSEwAoGNjm2OiMAGAjmnlAAAMRGICAB2T\nmAAADERiAgAdM/kVAJgaM+OqS7RyAIDpITEBgI65uzAAwEAkJgDQsTbpASwxhQkAdMw6JgAAA5GY\nAEDHZsrkVwCAQUhMAKBjY5v8KjEBAKaGxAQAOja2q3IUJgDQMffKAQAYiMQEADrmXjkAAAORmABA\nx8Z2ubDCBAA6ZvIrAMBAJCYA0LGxrWMiMQEApobEBAA6ZvIrADA1TH4FABiIxAQAOmbyKwDAQCQm\nANAxiQkAwEAkJgDQsTayq3IUJgDQMa0cAICBSEwAoGMSEwCAgUhMAKBj7pUDAEwN98oBABiIxAQA\nOmbyKwDAQCQmANCxsSUmChMA6NjYrsrRygEApobEBAA65nJhAICBKEwAoGMzA22LqapHVNXnquqL\nVXVdVb11B/s8vKouqKobq+qzVXXwYsdVmAAAu+PeJC9orT09yeFJjq+q58zb59VJvtlaOzTJHyX5\ng8UOqjABgI61gbZFv3e7e2afrpjd5n/0lCTnzT7+aJJjqmrBWTEmv9KdPVf95KSHwIh8+2/ePukh\nwEMyM8ELhqtqeZLPJzk0yXtba5+dt8vqJBuSpLW2taruSvKYJP+2s2NKTACAB6iqNVW1bs62Zv4+\nrbVtrbXDkxyY5Miqeur8w+zg0AtWUhITAOjYUCu/ttbWJlm7i/t+q6quSHJ8kmvnvLUxyUFJNlbV\nHkkeleTOhY4lMQEAHrSqemxV7Tv7eM8kxya5Yd5uFyX5hdnHL03yt601iQkAjNUEl6RfmeS82Xkm\ny5Jc2Fr7VFW9Lcm61tpFSd6f5ENVdWO2JyWvWOygChMA6NikbuLXWvtSkiN28Pqb5zz+XpKffTDH\n1coBAKaGxAQAOuZeOQAAA5GYAEDHJrnA2hAUJgDQsXGVJVo5AMAUkZgAQMcmdbnwUCQmAMDUkJgA\nQMdMfgUApsa4yhKtHABgikhMAKBjJr8CAAxEYgIAHRvb5FeJCQAwNSQmANCxceUlChMA6JrJrwAA\nA5GYAEDH2siaORITAGBqSEwAoGNjm2OiMAGAjlnHBABgIBITAOjYuPISiQkAMEUkJgDQsbHNMVGY\nAEDHxnZVjlYOADA1JCYA0DErvwIADERiAgAdM8cEAGAgEhMA6NjY5pgoTACgY1o5AAADkZgAQMdm\n2rhaORITAGBqSEwAoGPjyksUJgDQtbHdxE8rBwCYGhITAOjY2NYxkZgAAFNDYgIAHRvbAmsKEwDo\nmMmvAAADkZgAQMdMfgUAGIjEBAA6NrbJrxITAGBqSEwAoGNtZHcXVpgAQMdcLgwAMBCJCQB0zORX\nAICBSEwAoGNjW2BNYQIAHTP5FQBgIBITAOjY2NYxkZgAAFNDYgIAHRvb5cIKEwDo2NiuytHKAQCm\nhsQEADrmcmGW3HEvPDrXXXtlbrj+H/Kbv/HaSQ+HEXBOsdQ+9Nefy0ve/L6c+pb35Q1rP557t2yd\n9JAYKYXJhC1btixnv/vtOfGkn8vTnv78vPzlL86Tn/zESQ+LjjmnWGp3fPPbOf/ydfnwm16Zj731\nNdk203Lp566f9LCY1VobZFtMVX2gqr5WVdfu5P2jq+quqrpmdnvzrvw9ixYmVfWkqjqmqvae9/rx\nu/IFLOzIZx2Rm266Nbfc8q/ZsmVLLrzwEzn5pOMmPSw65pxiCNtmZnLvlq3Zum0m37tvSx67796L\nf4ixOzfJYrXAp1trh89ub9uVgy5YmFTV/0zyiSSvT3JtVZ0y5+3f25UvYGGrVh+QDRtvv//5xts2\nZdWqAyY4InrnnGKp7f9D++T0Fz47x//We/NTv3529t7z4XneUw6Z9LCYNZM2yLaY1tqVSe5c6r9n\nscTkNUme2Vp7cZKjk/yvqjpz9r3a2Yeqak1VrauqdTMz31makY5U1QP/MY5tFT9+sJxTLLW7v7M5\nV1zzlVz8jl/OZe98fTbftyUXX7XD9J4JaAP9b+5v+ey2ZjeG99yq+mJV/VVVPWVXPrDYVTnLW2v3\nJElr7daqOjrJR6vq8VmgMGmtrU2yNkn2eNhq/0VcwG0bN+WgA1fd//zA1SuzadMdExwRvXNOsdSu\nWn9rVu/3qDx6n72SJMcccViuuWljfvo5T53wyBjS3N/y3fSFJI9vrd1TVSck+XiSRSe8LZaYfLWq\nDp8zyHuSnJhkvyRPewiDZdbV667JoYc+IQcffFBWrFiRl73slHzyU5dNelh0zDnFUlv56EfmSzff\nns33bklrLZ+94dYccsB+kx4Ws2ZaG2R7qFprd88JNy5JsqKqFj1xFktMTk/yH64Ja61tTXJ6Vf3J\n7g6Wf7dt27acedabcsnFH87yZcty7nkX5PrrvzzpYdEx5xRL7WmHrM6xzzwsp/3uB7J82bI86XH7\n59SjDl/8g/ynVlUHJLmjtdaq6shsD0O+sejnhu49a+UA0+zbf/P2SQ+BEdrzqFfudLrDUvvJ1ccM\n8jv76dsuX/BvqKrzs33+6X5J7kjyliQrkqS1dk5VvS7JL2V7wLE5ya+21j6z2Pda+RUAOjaplV9b\na6ct8v57krznwR7XAmsAwNSQmABAx9wrBwBgIBITAOjY2BZQVJgAQMe0cgAABiIxAYCONYkJAMAw\nJCYA0LGxTX6VmAAAU0NiAgAdG9tVOQoTAOiYVg4AwEAkJgDQsbG1ciQmAMDUkJgAQMfGtsCawgQA\nOjZj8isAwDAkJgDQsbG1ciQmAMDUkJgAQMfGNsdEYQIAHdPKAQAYiMQEADo2tlaOxAQAmBoSEwDo\nmDkmAAADkZgAQMfGNsdEYQIAHdPKAQAYiMQEADrW2sykh7CkJCYAwNSQmABAx2ZGNsdEYQIAHWsj\nuypHKwcAmBoSEwDo2NhaORITAGBqSEwAoGNjm2OiMAGAjo1tSXqtHABgakhMAKBj7pUDADAQiQkA\ndGxsk18lJgDA1JCYAEDHxrbAmsIEADqmlQMAMBCJCQB0zAJrAAADkZgAQMfGNsdEYQIAHRvbVTla\nOQDA1JCYAEDHxtbKkZgAAFNDYgIAHRvb5cIKEwDoWDP5FQBgGBITAOjY2Fo5EhMAYGpITACgYy4X\nBgAYiMQEADo2tqtyFCYA0DGtHACAJFV1fFX9S1XdWFVv2MH7D6+qC2bf/2xVHbzYMRUmANCx1tog\n22KqanmS9yZ5UZIfS3JaVf3YvN1eneSbrbVDk/xRkj9Y7LgKEwBgdxyZ5MbW2s2ttfuSfCTJKfP2\nOSXJebOPP5rkmKqqhQ6qMAGAjrWBtl2wOsmGOc83zr62w31aa1uT3JXkMQsddPDJr1vvu23Byoh/\nV1VrWmtrJz0OxsH5xFJzTk2noX5nq2pNkjVzXlo779//jr53fk2zK/v8BxKT6bJm8V1glzmfWGrO\nqf9EWmtrW2s/PmebX5RuTHLQnOcHJrl9Z/tU1R5JHpXkzoW+V2ECAOyOq5M8saqeUFUPS/KKJBfN\n2+eiJL8w+/ilSf62LTKz1jomAMCD1lrbWlWvS/J/kyxP8oHW2nVV9bYk61prFyV5f5IPVdWN2Z6U\nvGKx49bYFmbpmf4tS8n5xFJzTvGDoDABAKaGOSYAwNRQmEyBxZb0hQejqj5QVV+rqmsnPRbGoaoO\nqqq/q6r1VXVdVZ056TExXlo5Eza7pO+Xk/xUtl9WdXWS01pr1090YHSrqo5Kck+SD7bWnjrp8dC/\nqlqZZGVr7QtVtU+Szyd5sf9OMQSJyeTtypK+sMtaa1dmkXUC4MForW1qrX1h9vG3k6zPA1f4hCWh\nMJm8XVnSF2AqzN4d9ogkn53sSBgrhcnkPejlegEmoar2TvKxJGe11u6e9HgYJ4XJ5O3Kkr4AE1VV\nK7K9KPmL1tpfTno8jJfCZPJ2ZUlfgImZvU39+5Osb6394aTHw7gpTCZs9jbQ31/Sd32SC1tr1012\nVPSsqs5P8k9JDquqjVX16kmPie79RJKfT/KCqrpmdjth0oNinFwuDABMDYkJADA1FCYAwNRQmAAA\nU0NhAgBMDYUJADA1FCYAwNRQmAAAU0NhAgBMjf8PZDwUd80FXpAAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "acc = accuracy_score(y_true, y_pred)\n", + "print(\"The prediction accuracy is %.2f%%\"%(acc*100))\n", + "\n", + "cm = confusion_matrix(y_true, y_pred)\n", + "cm.shape\n", + "df_cm = pd.DataFrame(cm)\n", + "plt.figure(figsize = (10,8))\n", + "sn.heatmap(df_cm, annot=True,fmt='d');" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "def map_predict_label(outputs, threshold):\n", + " p = map(lambda x: np.exp(x), outputs)\n", + " if (np.max(p) < threshold):\n", + " return len(outputs) + 1\n", + " return np.argmax(outputs) + 1\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "true_positive_rate = list()\n", + "false_positive_rate = list()\n", + " \n", + "for threshold in np.linspace(0.0, 1.0, num = 100):\n", + " y_pred = np.array([map_predict_label(s, threshold) for s in predictions])\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/lstm-20news.ipynb b/notebooks/lstm-20news.ipynb new file mode 100644 index 0000000..04cb4be --- /dev/null +++ b/notebooks/lstm-20news.ipynb @@ -0,0 +1,312 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# LSTM Example: 20 Newsgroup Classification\n", + "\n", + "This shows how to do an LSTM classification on the 20 newsgroup collection" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import itertools\n", + "import re\n", + "from optparse import OptionParser\n", + "\n", + "from bigdl.dataset import news20\n", + "from bigdl.nn.layer import *\n", + "from bigdl.nn.criterion import *\n", + "from bigdl.optim.optimizer import *\n", + "from bigdl.util.common import *\n", + "from bigdl.util.common import Sample\n", + "import datetime as dt\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "def text_to_words(review_text):\n", + " letters_only = re.sub(\"[^a-zA-Z]\", \" \", review_text)\n", + " words = letters_only.lower().split()\n", + " return words\n", + "\n", + "\n", + "def analyze_texts(data_rdd):\n", + " def index(w_c_i):\n", + " ((w, c), i) = w_c_i\n", + " return (w, (i + 1, c))\n", + " return data_rdd.flatMap(lambda text_label: text_to_words(text_label[0])) \\\n", + " .map(lambda word: (word, 1)).reduceByKey(lambda a, b: a + b) \\\n", + " .sortBy(lambda w_c: - w_c[1]).zipWithIndex() \\\n", + " .map(lambda w_c_i: index(w_c_i)).collect()\n", + "\n", + "\n", + "# pad([1, 2, 3, 4, 5], 0, 6)\n", + "def pad(l, fill_value, width):\n", + " if len(l) >= width:\n", + " return l[0: width]\n", + " else:\n", + " l.extend([fill_value] * (width - len(l)))\n", + " return l\n", + "\n", + "\n", + "def to_vec(token, b_w2v, embedding_dim):\n", + " if token in b_w2v:\n", + " return b_w2v[token]\n", + " else:\n", + " return pad([], 0, embedding_dim)\n", + "\n", + "\n", + "def to_sample(vectors, label, embedding_dim):\n", + " # flatten nested list\n", + " flatten_features = list(itertools.chain(*vectors))\n", + " features = np.array(flatten_features, dtype='float').reshape(\n", + " [sequence_len, embedding_dim])\n", + "\n", + " return Sample.from_ndarray(features, np.array(label))\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The Model: An LSTM\n", + "\n", + "Notice that this is an LSTM model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "def build_model(class_num):\n", + " model = Sequential()\n", + "\n", + " model.add(Recurrent().add(LSTM(embedding_dim, 256, p)))\n", + " model.add(Select(2, -1))\n", + " model.add(Linear(256, 128)) \\\n", + " .add(Dropout(0.2)) \\\n", + " .add(ReLU()) \\\n", + " .add(Linear(128, class_num)) \\\n", + " .add(LogSoftMax())\n", + "\n", + " return model\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "action = \"train\"\n", + "learning_rate = 0.05\n", + "batch_size = 128\n", + "embedding_dim = 300\n", + "max_epoch = 30\n", + "model_type = \"lstm\"\n", + "p = 0.0\n", + "data_path=\"/tmp/news20\"\n", + "sequence_len = 500\n", + "max_words = 5000\n", + "training_split = 0.8" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "redire_spark_logs()\n", + "show_bigdl_info_logs()\n", + "init_engine()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Processing text dataset\n", + "Found 19997 texts.\n", + "creating: createSequential\n", + "creating: createRecurrent\n", + "creating: createLSTM\n", + "creating: createSelect\n", + "creating: createLinear\n", + "creating: createDropout\n", + "creating: createReLU\n", + "creating: createLinear\n", + "creating: createLogSoftMax\n", + "creating: createClassNLLCriterion\n", + "creating: createMaxEpoch\n", + "creating: createAdagrad\n", + "creating: createOptimizer\n", + "creating: createEveryEpoch\n", + "creating: createTop1Accuracy\n", + "creating: createTrainSummary\n", + "creating: createSeveralIteration\n", + "creating: createValidationSummary\n" + ] + }, + { + "ename": "Py4JJavaError", + "evalue": "An error occurred while calling o601.optimize.\n: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 31.0 failed 1 times, most recent failure: Lost task 0.0 in stage 31.0 (TID 38, localhost, executor driver): java.lang.OutOfMemoryError: Java heap space\n\tat java.nio.HeapByteBuffer.(HeapByteBuffer.java:57)\n\tat java.nio.ByteBuffer.allocate(ByteBuffer.java:335)\n\tat org.apache.spark.storage.ShuffleBlockFetcherIterator$$anonfun$5.apply(ShuffleBlockFetcherIterator.scala:390)\n\tat org.apache.spark.storage.ShuffleBlockFetcherIterator$$anonfun$5.apply(ShuffleBlockFetcherIterator.scala:390)\n\tat org.apache.spark.util.io.ChunkedByteBufferOutputStream.allocateNewChunkIfNeeded(ChunkedByteBufferOutputStream.scala:87)\n\tat org.apache.spark.util.io.ChunkedByteBufferOutputStream.write(ChunkedByteBufferOutputStream.scala:75)\n\tat org.apache.spark.util.Utils$$anonfun$copyStream$1.apply$mcJ$sp(Utils.scala:342)\n\tat org.apache.spark.util.Utils$$anonfun$copyStream$1.apply(Utils.scala:327)\n\tat org.apache.spark.util.Utils$$anonfun$copyStream$1.apply(Utils.scala:327)\n\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1337)\n\tat org.apache.spark.util.Utils$.copyStream(Utils.scala:348)\n\tat org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:395)\n\tat org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:59)\n\tat scala.collection.Iterator$$anon$12.nextCur(Iterator.scala:434)\n\tat scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:440)\n\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)\n\tat org.apache.spark.util.CompletionIterator.hasNext(CompletionIterator.scala:32)\n\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n\tat scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:438)\n\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)\n\tat scala.collection.Iterator$class.foreach(Iterator.scala:893)\n\tat scala.collection.AbstractIterator.foreach(Iterator.scala:1336)\n\tat scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:104)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:48)\n\tat scala.collection.TraversableOnce$class.to(TraversableOnce.scala:310)\n\tat scala.collection.AbstractIterator.to(Iterator.scala:1336)\n\tat scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:302)\n\tat scala.collection.AbstractIterator.toBuffer(Iterator.scala:1336)\n\tat scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:289)\n\tat scala.collection.AbstractIterator.toArray(Iterator.scala:1336)\n\tat com.intel.analytics.bigdl.dataset.DataSet$$anonfun$5.apply(DataSet.scala:360)\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1499)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1487)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1486)\n\tat scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1486)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:814)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:814)\n\tat scala.Option.foreach(Option.scala:257)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:814)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1714)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1669)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1658)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)\n\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:630)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2022)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2043)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2062)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2087)\n\tat org.apache.spark.rdd.RDD.count(RDD.scala:1158)\n\tat com.intel.analytics.bigdl.dataset.DistributedDataSet$$anon$5.cache(DataSet.scala:188)\n\tat com.intel.analytics.bigdl.optim.DistriOptimizer.prepareInput(DistriOptimizer.scala:757)\n\tat com.intel.analytics.bigdl.optim.DistriOptimizer.optimize(DistriOptimizer.scala:777)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n\tat py4j.Gateway.invoke(Gateway.java:280)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:214)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: java.lang.OutOfMemoryError: Java heap space\n\tat java.nio.HeapByteBuffer.(HeapByteBuffer.java:57)\n\tat java.nio.ByteBuffer.allocate(ByteBuffer.java:335)\n\tat org.apache.spark.storage.ShuffleBlockFetcherIterator$$anonfun$5.apply(ShuffleBlockFetcherIterator.scala:390)\n\tat org.apache.spark.storage.ShuffleBlockFetcherIterator$$anonfun$5.apply(ShuffleBlockFetcherIterator.scala:390)\n\tat org.apache.spark.util.io.ChunkedByteBufferOutputStream.allocateNewChunkIfNeeded(ChunkedByteBufferOutputStream.scala:87)\n\tat org.apache.spark.util.io.ChunkedByteBufferOutputStream.write(ChunkedByteBufferOutputStream.scala:75)\n\tat org.apache.spark.util.Utils$$anonfun$copyStream$1.apply$mcJ$sp(Utils.scala:342)\n\tat org.apache.spark.util.Utils$$anonfun$copyStream$1.apply(Utils.scala:327)\n\tat org.apache.spark.util.Utils$$anonfun$copyStream$1.apply(Utils.scala:327)\n\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1337)\n\tat org.apache.spark.util.Utils$.copyStream(Utils.scala:348)\n\tat org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:395)\n\tat org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:59)\n\tat scala.collection.Iterator$$anon$12.nextCur(Iterator.scala:434)\n\tat scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:440)\n\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)\n\tat org.apache.spark.util.CompletionIterator.hasNext(CompletionIterator.scala:32)\n\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n\tat scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:438)\n\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)\n\tat scala.collection.Iterator$class.foreach(Iterator.scala:893)\n\tat scala.collection.AbstractIterator.foreach(Iterator.scala:1336)\n\tat scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:104)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:48)\n\tat scala.collection.TraversableOnce$class.to(TraversableOnce.scala:310)\n\tat scala.collection.AbstractIterator.to(Iterator.scala:1336)\n\tat scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:302)\n\tat scala.collection.AbstractIterator.toBuffer(Iterator.scala:1336)\n\tat scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:289)\n\tat scala.collection.AbstractIterator.toArray(Iterator.scala:1336)\n\tat com.intel.analytics.bigdl.dataset.DataSet$$anonfun$5.apply(DataSet.scala:360)\n", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mPy4JJavaError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 54\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_val_summary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_summary\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 55\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m \u001b[0mtrain_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 57\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 58\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/tmp/spark-47eec23e-bb73-40dd-a253-27977bf72df2/userFiles-13c2c15a-3eb6-499c-a25c-bec2ce8ab961/bigdl-0.3.0-python-api.zip/bigdl/optim/optimizer.py\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 596\u001b[0m \u001b[0mDo\u001b[0m \u001b[0man\u001b[0m \u001b[0moptimization\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 597\u001b[0m \"\"\"\n\u001b[0;32m--> 598\u001b[0;31m \u001b[0mjmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcallJavaFunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mget_spark_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 599\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mbigdl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLayer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 600\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mLayer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mof\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/tmp/spark-47eec23e-bb73-40dd-a253-27977bf72df2/userFiles-13c2c15a-3eb6-499c-a25c-bec2ce8ab961/bigdl-0.3.0-python-api.zip/bigdl/util/common.py\u001b[0m in \u001b[0;36mcallJavaFunc\u001b[0;34m(sc, func, *args)\u001b[0m\n\u001b[1;32m 491\u001b[0m \u001b[0;34m\"\"\" Call Java Function \"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 492\u001b[0m \u001b[0margs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0m_py2java\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 494\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_java2py\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 495\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/spark/python/lib/py4j-0.10.4-src.zip/py4j/java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m 1131\u001b[0m \u001b[0manswer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1132\u001b[0m return_value = get_return_value(\n\u001b[0;32m-> 1133\u001b[0;31m answer, self.gateway_client, self.target_id, self.name)\n\u001b[0m\u001b[1;32m 1134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1135\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtemp_arg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtemp_args\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/spark/python/pyspark/sql/utils.py\u001b[0m in \u001b[0;36mdeco\u001b[0;34m(*a, **kw)\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdeco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 64\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mpy4j\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPy4JJavaError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 65\u001b[0m \u001b[0ms\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjava_exception\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoString\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/spark/python/lib/py4j-0.10.4-src.zip/py4j/protocol.py\u001b[0m in \u001b[0;36mget_return_value\u001b[0;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[1;32m 317\u001b[0m raise Py4JJavaError(\n\u001b[1;32m 318\u001b[0m \u001b[0;34m\"An error occurred while calling {0}{1}{2}.\\n\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 319\u001b[0;31m format(target_id, \".\", name), value)\n\u001b[0m\u001b[1;32m 320\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 321\u001b[0m raise Py4JError(\n", + "\u001b[0;31mPy4JJavaError\u001b[0m: An error occurred while calling o601.optimize.\n: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 31.0 failed 1 times, most recent failure: Lost task 0.0 in stage 31.0 (TID 38, localhost, executor driver): java.lang.OutOfMemoryError: Java heap space\n\tat java.nio.HeapByteBuffer.(HeapByteBuffer.java:57)\n\tat java.nio.ByteBuffer.allocate(ByteBuffer.java:335)\n\tat org.apache.spark.storage.ShuffleBlockFetcherIterator$$anonfun$5.apply(ShuffleBlockFetcherIterator.scala:390)\n\tat org.apache.spark.storage.ShuffleBlockFetcherIterator$$anonfun$5.apply(ShuffleBlockFetcherIterator.scala:390)\n\tat org.apache.spark.util.io.ChunkedByteBufferOutputStream.allocateNewChunkIfNeeded(ChunkedByteBufferOutputStream.scala:87)\n\tat org.apache.spark.util.io.ChunkedByteBufferOutputStream.write(ChunkedByteBufferOutputStream.scala:75)\n\tat org.apache.spark.util.Utils$$anonfun$copyStream$1.apply$mcJ$sp(Utils.scala:342)\n\tat org.apache.spark.util.Utils$$anonfun$copyStream$1.apply(Utils.scala:327)\n\tat org.apache.spark.util.Utils$$anonfun$copyStream$1.apply(Utils.scala:327)\n\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1337)\n\tat org.apache.spark.util.Utils$.copyStream(Utils.scala:348)\n\tat org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:395)\n\tat org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:59)\n\tat scala.collection.Iterator$$anon$12.nextCur(Iterator.scala:434)\n\tat scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:440)\n\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)\n\tat org.apache.spark.util.CompletionIterator.hasNext(CompletionIterator.scala:32)\n\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n\tat scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:438)\n\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)\n\tat scala.collection.Iterator$class.foreach(Iterator.scala:893)\n\tat scala.collection.AbstractIterator.foreach(Iterator.scala:1336)\n\tat scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:104)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:48)\n\tat scala.collection.TraversableOnce$class.to(TraversableOnce.scala:310)\n\tat scala.collection.AbstractIterator.to(Iterator.scala:1336)\n\tat scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:302)\n\tat scala.collection.AbstractIterator.toBuffer(Iterator.scala:1336)\n\tat scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:289)\n\tat scala.collection.AbstractIterator.toArray(Iterator.scala:1336)\n\tat com.intel.analytics.bigdl.dataset.DataSet$$anonfun$5.apply(DataSet.scala:360)\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1499)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1487)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1486)\n\tat scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1486)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:814)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:814)\n\tat scala.Option.foreach(Option.scala:257)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:814)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1714)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1669)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1658)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)\n\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:630)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2022)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2043)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2062)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2087)\n\tat org.apache.spark.rdd.RDD.count(RDD.scala:1158)\n\tat com.intel.analytics.bigdl.dataset.DistributedDataSet$$anon$5.cache(DataSet.scala:188)\n\tat com.intel.analytics.bigdl.optim.DistriOptimizer.prepareInput(DistriOptimizer.scala:757)\n\tat com.intel.analytics.bigdl.optim.DistriOptimizer.optimize(DistriOptimizer.scala:777)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n\tat py4j.Gateway.invoke(Gateway.java:280)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:214)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: java.lang.OutOfMemoryError: Java heap space\n\tat java.nio.HeapByteBuffer.(HeapByteBuffer.java:57)\n\tat java.nio.ByteBuffer.allocate(ByteBuffer.java:335)\n\tat org.apache.spark.storage.ShuffleBlockFetcherIterator$$anonfun$5.apply(ShuffleBlockFetcherIterator.scala:390)\n\tat org.apache.spark.storage.ShuffleBlockFetcherIterator$$anonfun$5.apply(ShuffleBlockFetcherIterator.scala:390)\n\tat org.apache.spark.util.io.ChunkedByteBufferOutputStream.allocateNewChunkIfNeeded(ChunkedByteBufferOutputStream.scala:87)\n\tat org.apache.spark.util.io.ChunkedByteBufferOutputStream.write(ChunkedByteBufferOutputStream.scala:75)\n\tat org.apache.spark.util.Utils$$anonfun$copyStream$1.apply$mcJ$sp(Utils.scala:342)\n\tat org.apache.spark.util.Utils$$anonfun$copyStream$1.apply(Utils.scala:327)\n\tat org.apache.spark.util.Utils$$anonfun$copyStream$1.apply(Utils.scala:327)\n\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1337)\n\tat org.apache.spark.util.Utils$.copyStream(Utils.scala:348)\n\tat org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:395)\n\tat org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:59)\n\tat scala.collection.Iterator$$anon$12.nextCur(Iterator.scala:434)\n\tat scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:440)\n\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)\n\tat org.apache.spark.util.CompletionIterator.hasNext(CompletionIterator.scala:32)\n\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n\tat scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:438)\n\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)\n\tat scala.collection.Iterator$class.foreach(Iterator.scala:893)\n\tat scala.collection.AbstractIterator.foreach(Iterator.scala:1336)\n\tat scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:104)\n\tat scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:48)\n\tat scala.collection.TraversableOnce$class.to(TraversableOnce.scala:310)\n\tat scala.collection.AbstractIterator.to(Iterator.scala:1336)\n\tat scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:302)\n\tat scala.collection.AbstractIterator.toBuffer(Iterator.scala:1336)\n\tat scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:289)\n\tat scala.collection.AbstractIterator.toArray(Iterator.scala:1336)\n\tat com.intel.analytics.bigdl.dataset.DataSet$$anonfun$5.apply(DataSet.scala:360)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "----------------------------------------\n", + "Exception happened during processing of request from ('127.0.0.1', 41902)\n", + "----------------------------------------\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Traceback (most recent call last):\n", + " File \"/opt/conda/envs/py27/lib/python2.7/SocketServer.py\", line 290, in _handle_request_noblock\n", + " self.process_request(request, client_address)\n", + " File \"/opt/conda/envs/py27/lib/python2.7/SocketServer.py\", line 318, in process_request\n", + " self.finish_request(request, client_address)\n", + " File \"/opt/conda/envs/py27/lib/python2.7/SocketServer.py\", line 331, in finish_request\n", + " self.RequestHandlerClass(request, client_address, self)\n", + " File \"/opt/conda/envs/py27/lib/python2.7/SocketServer.py\", line 652, in __init__\n", + " self.handle()\n", + " File \"/usr/local/spark/python/pyspark/accumulators.py\", line 235, in handle\n", + " num_updates = read_int(self.rfile)\n", + " File \"/usr/local/spark/python/pyspark/serializers.py\", line 577, in read_int\n", + " raise EOFError\n", + "EOFError\n" + ] + } + ], + "source": [ + "#train(sc, data_path, batch_size, sequence_len, max_words, embedding_dim, training_split)\n", + "print('Processing text dataset')\n", + "texts = news20.get_news20(source_dir=data_path)\n", + "data_rdd = sc.parallelize(texts, 2)\n", + "\n", + "word_to_ic = analyze_texts(data_rdd)\n", + "\n", + "# Only take the top wc between [10, sequence_len]\n", + "word_to_ic = dict(word_to_ic[10: max_words])\n", + "bword_to_ic = sc.broadcast(word_to_ic)\n", + "\n", + "w2v = news20.get_glove_w2v(dim=embedding_dim)\n", + "filtered_w2v = dict((w, v) for w, v in w2v.items() if w in word_to_ic)\n", + "bfiltered_w2v = sc.broadcast(filtered_w2v)\n", + "\n", + "tokens_rdd = data_rdd.map(lambda text_label:\n", + " ([w for w in text_to_words(text_label[0]) if\n", + " w in bword_to_ic.value], text_label[1]))\n", + "\n", + "padded_tokens_rdd = tokens_rdd.map(\n", + " lambda tokens_label: (pad(tokens_label[0], \"##\", sequence_len), tokens_label[1]))\n", + "\n", + "vector_rdd = padded_tokens_rdd.map(lambda tokens_label:\n", + " ([to_vec(w, bfiltered_w2v.value,\n", + " embedding_dim) for w in tokens_label[0]], tokens_label[1]))\n", + "sample_rdd = vector_rdd.map(\n", + " lambda vectors_label: to_sample(vectors_label[0], vectors_label[1], embedding_dim))\n", + "\n", + "train_rdd, val_rdd = sample_rdd.randomSplit(\n", + " [training_split, 1-training_split])\n", + "\n", + "optimizer = Optimizer(\n", + " model=build_model(news20.CLASS_NUM),\n", + " training_rdd=train_rdd,\n", + " criterion=ClassNLLCriterion(),\n", + " end_trigger=MaxEpoch(max_epoch),\n", + " batch_size=batch_size,\n", + " optim_method=Adagrad(learningrate=learning_rate, learningrate_decay=0.001))\n", + "\n", + "optimizer.set_validation(\n", + " batch_size=batch_size,\n", + " val_rdd=val_rdd,\n", + " trigger=EveryEpoch(),\n", + " val_method=[Top1Accuracy()]\n", + ")\n", + "\n", + "logdir = '/tmp/.bigdl/'\n", + "app_name = 'adam-' + dt.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n", + "\n", + "train_summary = TrainSummary(log_dir=logdir, app_name=app_name)\n", + "train_summary.set_summary_trigger(\"Parameters\", SeveralIteration(50))\n", + "val_summary = ValidationSummary(log_dir=logdir, app_name=app_name)\n", + "optimizer.set_train_summary(train_summary)\n", + "optimizer.set_val_summary(val_summary)\n", + " \n", + "train_model = optimizer.optimize()\n", + "\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/lstm-stocks.ipynb b/notebooks/lstm-stocks.ipynb new file mode 100644 index 0000000..506fe16 --- /dev/null +++ b/notebooks/lstm-stocks.ipynb @@ -0,0 +1,245 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# LSTM Stock Market Example\n", + "\n", + "Wouldn't you like to predict the stock market? (Don't try this at home). Here's an example of how we can use LSTMs to predict patterns in the S&P 500." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "from bigdl.nn.layer import *\n", + "from bigdl.nn.criterion import * \n", + "from bigdl.optim.optimizer import *\n", + "from bigdl.util.common import *\n", + "from bigdl.dataset.transformer import *\n", + "from bigdl.dataset import mnist\n", + "from bigdl.dataset.base import *\n", + "from bigdl.util.common import *\n", + "\n", + "import numpy as np\n", + "from numpy import newaxis\n", + "import lstm, time #helper libraries" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "learning_rate = 0.1\n", + "training_epochs = 100\n", + "batch_size = 10\n", + "init_engine()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading the Data\n", + "\n", + "We are loading the S&P 500 Data. It is simply a sequence of the s&P 500 per line. For example\n", + "\n", + "```\n", + "1095.23\n", + "1100.53\n", + "1103.74\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "#Step 1 Load Data\n", + "X_train, y_train, X_test, y_test = lstm.load_data('sp500.csv', 50, True)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "X_train_rdd = sc.parallelize(X_train).map(lambda x: x.flatten())\n", + "y_train_rdd = sc.parallelize(y_train)\n", + "X_test_rdd = sc.parallelize(X_test).map(lambda x: x.flatten())\n", + "y_test_rdd = sc.parallelize(y_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "x_train_sample = X_train_rdd.zip(y_train_rdd).map(lambda x:Sample.from_ndarray(x[0], x[1]))\n", + "X_test_sample = X_test_rdd.zip(y_test_rdd).map(lambda x:Sample.from_ndarray(x[0], x[1]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Building the Model\n", + "\n", + "The model is a type of Recurrent Neural Network called an LSTM.\n", + "\n", + "It takes an input of 50 (because we are going to use 50 as the sequence), and an ouptut as 1." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "creating: createSequential\n", + "creating: createRecurrent\n", + "creating: createLSTM\n", + "creating: createInferReshape\n", + "creating: createSelect\n", + "creating: createLinear\n" + ] + } + ], + "source": [ + "def build_model(input_size, hidden_size, output_size):\n", + " model = Sequential()\n", + " recurrent = Recurrent()\n", + " recurrent.add(LSTM(input_size, hidden_size))\n", + " model.add(InferReshape([-1, input_size], True))\n", + " model.add(recurrent)\n", + " model.add(Select(2, -1))\n", + " model.add(Linear(hidden_size, output_size))\n", + " return model\n", + "model = build_model(50, 50, 1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Training The Model\n", + "\n", + "Notice that the model is a regression, not classificaiton model" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "creating: createMSECriterion\n", + "creating: createDefault\n", + "creating: createSGD\n", + "creating: createMaxEpoch\n", + "creating: createOptimizer\n" + ] + } + ], + "source": [ + "# Create an Optimizer\n", + "optimizer = Optimizer(\n", + " model=model,\n", + " training_rdd=x_train_sample,\n", + " criterion=MSECriterion(),\n", + " optim_method=SGD(learningrate=learning_rate),\n", + " end_trigger=MaxEpoch(training_epochs),\n", + " batch_size=batch_size)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Making Predictions\n", + "\n", + "Here we are going to make predictions based on the model. We will use sliding windows of length 50." + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "\n", + "#Predict sequence of 50 steps before shifting prediction run forward by 50 steps\n", + "prediction_seqs = []\n", + "data = X_test\n", + "prediction_len = 50\n", + "window_size=50\n", + "for i in range(len(data)//prediction_len):\n", + " curr_frame = data[i*int(prediction_len)]\n", + " predicted = []\n", + " for j in range(int(prediction_len)):\n", + "\n", + " curr_data = curr_frame[newaxis,:,:]\n", + "\n", + " curr_data_rdd = sc.parallelize(curr_data).map(lambda x: Sample.from_ndarray(x,[0]))\n", + "\n", + " preds = model.predict(curr_data_rdd).collect()[0][0][0]\n", + " predicted.append(preds)\n", + " curr_frame = curr_frame[1:]\n", + " curr_frame = np.insert(curr_frame, [window_size-1], predicted[-1], axis=0)\n", + " prediction_seqs.append(predicted)\n", + "predictions = prediction_seqs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# Step 5 -- Plot predictions\n", + "\n", + "lstm.plot_results_multiple(predictions, y_test, 50)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/scripts/imagenet.py b/scripts/imagenet.py new file mode 100644 index 0000000..37ba9e6 --- /dev/null +++ b/scripts/imagenet.py @@ -0,0 +1,126 @@ + +# +# Licensed to Intel Corporation under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# Intel Corporation licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Part of the code originally from Tensorflow + +import numpy as np +from os import listdir +from os.path import join, basename +import struct +from scipy import misc +from transformer import Resize + + +def read_local_path(folder, has_label=True): + # read directory, create map + dirs = listdir(folder) + # create image path and label list + image_paths = [] + if has_label: + dirs.sort() + for d in dirs: + for f in listdir(join(folder, d)): + image_paths.append((join(join(folder, d), f), dirs.index(d) + 1)) + else: + for f in dirs: + image_paths.append((join(folder, f), -1)) + return image_paths + + +def read_local(sc, folder, normalize=255.0, has_label=True): + """ + Read images from local directory + :param sc: spark context + :param folder: local directory + :param normalize: normalization value + :param has_label: whether the image folder contains label + :return: RDD of sample + """ + # read directory, create image paths list + image_paths = read_local_path(folder, has_label) + # create rdd + image_paths_rdd = sc.parallelize(image_paths) + feature_label_rdd = image_paths_rdd.map(lambda path_label: (misc.imread(path_label[0]), np.array(path_label[1]))) \ + .map(lambda img_label: + (Resize(256, 256)(img_label[0]), img_label[1])) \ + .map(lambda feature_label: + (((feature_label[0] & 0xff) / normalize).astype("float32"), feature_label[1])) + return feature_label_rdd + + +def read_local_with_name(sc, folder, normalize=255.0, has_label=True): + """ + Read images from local directory + :param sc: spark context + :param folder: local directory + :param normalize: normalization value + :param has_label: whether the image folder contains label + :return: RDD of sample + """ + # read directory, create image paths list + image_paths = read_local_path(folder, has_label) + # create rdd + image_paths_rdd = sc.parallelize(image_paths) + features_label_name_rdd = image_paths_rdd.map(lambda path_label: (misc.imread(path_label[0]), np.array(path_label[1]), basename(path_label[0]))) \ + .map(lambda img_label_name: + (Resize(256, 256)(img_label_name[0]), img_label_name[1], img_label_name[2])) \ + .map(lambda features_label_name: + (((features_label_name[0] & 0xff) / normalize).astype("float32"), features_label_name[1], features_label_name[2])) + return features_label_name_rdd + + +def read_seq_file(sc, path, normalize=255.0, has_name=False): + """ + Read images from sequence file + :param sc: spark context + :param path: location of sequence file + :param normalize: normalize index for the image + :param has_name: whether the sequence file includes image name + :return: RDD of sample image + """ + raw = sc.sequenceFile(path, "org.apache.hadoop.io.Text", "org.apache.hadoop.io.BytesWritable") + + def parse(data): + img = data[1] + length = len(img)-8 + metrics = struct.unpack('>ii', img[0:8]) + width = metrics[0] + height = metrics[1] + features = np.array(img[8:], dtype="int8") + normalized_features = (features & 0xff) / normalize + if has_name: + key = data[0].split('\n') + name = key[0] + label = key[1] + features = np.array(normalized_features, dtype="float32").reshape((height, width, length / width / height)) + return features, np.array(int(label)), name + else: + label = data[0] + features = np.array(normalized_features, dtype="float32").reshape((height, width, length / width / height)) + return features, np.array(int(label)) + return raw.map(parse) + + +def load_mean_file(mean_file): + """ + Read mean file which contains means for every pixel + :param mean_file: + :return: + """ + mean_array = np.load(mean_file).transpose(1,2,0) + return mean_array / 255.0 + diff --git a/scripts/inception_transfer.py b/scripts/inception_transfer.py new file mode 100644 index 0000000..723eb5b --- /dev/null +++ b/scripts/inception_transfer.py @@ -0,0 +1,660 @@ + +# Verify if the spark context was initialized + +#Import all the required packages + +import numpy as np +import pandas as pd + +from os import listdir +from os.path import join, basename +import struct +import json +from scipy import misc +import datetime as dt + +from bigdl.nn.layer import * +from optparse import OptionParser +from bigdl.nn.criterion import * +from bigdl.optim.optimizer import * +from bigdl.util.common import * +from bigdl.dataset.transformer import * +from bigdl.nn.initialization_method import * +from transformer import * +from imagenet import * +from transformer import Resize + +# if you want to train on whole imagenet +#from bigdl.dataset import imagenet + +from pyspark import SparkConf, SparkContext + +conf = SparkConf().setAppName("inception") +sc = SparkContext(conf=conf) + +sc + + +# helper func to read the files from disk +def read_local_path(folder, has_label=True): + """ + :param folder: local directory (str) + :param has_label: does image have label (bool) + :return: list of (image path , label) tuples + """ + # read directory, create map + dirs = listdir(folder) + print "local path: ", folder + print "listdir: ", dirs + # create a list of (image path , label) tuples + image_paths = [] + #append image path to the label (ex: ) + if has_label: + dirs.sort() + for d in dirs: + for f in listdir(join(folder, d)): + image_paths.append((join(join(folder, d), f), dirs.index(d) + 1)) + else: + for f in dirs: + image_paths.append((join(folder, f), -1)) + return image_paths + +# helper func to read the files from disk +def read_local(sc, folder, normalize=255.0, has_label=True): + """ + Read images from local directory + :param sc: spark context + :param folder: local directory + :param normalize: normalization value + :param has_label: whether the image folder contains label + :return: RDD of sample + """ + # read directory, create image paths list + image_paths = read_local_path(folder, has_label) + # print "BEFORE PARALLELIZATION: ", image_paths + # create rdd + image_paths_rdd = sc.parallelize(image_paths) + # print image_paths_rdd + feature_label_rdd = image_paths_rdd.map(lambda path_label: (misc.imread(path_label[0]), np.array(path_label[1]))) \ + .map(lambda img_label: + (Resize(256, 256)(img_label[0]), img_label[1])) \ + .map(lambda feature_label: + (((feature_label[0] & 0xff) / normalize).astype("float32"), feature_label[1])) + return feature_label_rdd + +def scala_T(input_T): + """ + Helper function for building Inception layers. Transforms a list of numbers to a dictionary with ascending keys + and 0 appended to the front. Ignores dictionary inputs. + + :param input_T: either list or dict + :return: dictionary with ascending keys and 0 appended to front {0: 0, 1: realdata_1, 2: realdata_2, ...} + """ + if type(input_T) is list: + # insert 0 into first index spot, such that the real data starts from index 1 + temp = [0] + temp.extend(input_T) + return dict(enumerate(temp)) + # if dictionary, return it back + return input_T + +# Question: What is config? +def Inception_Layer_v1(input_size, config, name_prefix=""): + """ + Builds the inception-v1 submodule, a local network, that is stacked in the entire architecture when building + the full model. + + :param input_size: dimensions of input coming into the local network + :param config: ? + :param name_prefix: string naming the layers of the particular local network + :return: concat container object with all of the Sequential layers' ouput concatenated depthwise + """ + + ''' + Concat is a container who concatenates the output of it's submodules along the provided dimension: all submodules + take the same inputs, and their output is concatenated. + ''' + concat = Concat(2) + + """ + In the above code, we first create a container Sequential. Then add the layers into the container one by one. The + order of the layers in the model is same with the insertion order. + + """ + conv1 = Sequential() + + #Adding layes to the conv1 model we jus created + + #SpatialConvolution is a module that applies a 2D convolution over an input image. + conv1.add(SpatialConvolution(input_size, config[1][1], 1, 1, 1, 1).set_name(name_prefix + "1x1")) + conv1.add(ReLU(True).set_name(name_prefix + "relu_1x1")) + concat.add(conv1) + + conv3 = Sequential() + conv3.add(SpatialConvolution(input_size, config[2][1], 1, 1, 1, 1).set_name(name_prefix + "3x3_reduce")) + conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3_reduce")) + conv3.add(SpatialConvolution(config[2][1], config[2][2], 3, 3, 1, 1, 1, 1).set_name(name_prefix + "3x3")) + conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3")) + concat.add(conv3) + + + conv5 = Sequential() + conv5.add(SpatialConvolution(input_size,config[3][1], 1, 1, 1, 1).set_name(name_prefix + "5x5_reduce")) + conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5_reduce")) + conv5.add(SpatialConvolution(config[3][1], config[3][2], 5, 5, 1, 1, 2, 2).set_name(name_prefix + "5x5")) + conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5")) + concat.add(conv5) + + + pool = Sequential() + pool.add(SpatialMaxPooling(3, 3, 1, 1, 1, 1, to_ceil=True).set_name(name_prefix + "pool")) + pool.add(SpatialConvolution(input_size, config[4][1], 1, 1, 1, 1).set_name(name_prefix + "pool_proj")) + pool.add(ReLU(True).set_name(name_prefix + "relu_pool_proj")) + concat.add(pool).set_name(name_prefix + "output") + return concat + +def Inception_v1_NoAuxClassifier(class_num): + model = Sequential() + model.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False).set_name("conv1/7x7_s2")) + model.add(ReLU(True).set_name("conv1/relu_7x7")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool1/3x3_s2")) + model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("pool1/norm1")) + model.add(SpatialConvolution(64, 64, 1, 1, 1, 1).set_name("conv2/3x3_reduce")) + model.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) + model.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1).set_name("conv2/3x3")) + model.add(ReLU(True).set_name("conv2/relu_3x3")) + model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) + model.add(Inception_Layer_v1(192, scala_T([scala_T([64]), scala_T( + [96, 128]), scala_T([16, 32]), scala_T([32])]), "inception_3a/")) + model.add(Inception_Layer_v1(256, scala_T([scala_T([128]), scala_T( + [128, 192]), scala_T([32, 96]), scala_T([64])]), "inception_3b/")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) + model.add(Inception_Layer_v1(480, scala_T([scala_T([192]), scala_T( + [96, 208]), scala_T([16, 48]), scala_T([64])]), "inception_4a/")) + model.add(Inception_Layer_v1(512, scala_T([scala_T([160]), scala_T( + [112, 224]), scala_T([24, 64]), scala_T([64])]), "inception_4b/")) + model.add(Inception_Layer_v1(512, scala_T([scala_T([128]), scala_T( + [128, 256]), scala_T([24, 64]), scala_T([64])]), "inception_4c/")) + model.add(Inception_Layer_v1(512, scala_T([scala_T([112]), scala_T( + [144, 288]), scala_T([32, 64]), scala_T([64])]), "inception_4d/")) + model.add(Inception_Layer_v1(528, scala_T([scala_T([256]), scala_T( + [160, 320]), scala_T([32, 128]), scala_T([128])]), "inception_4e/")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) + model.add(Inception_Layer_v1(832, scala_T([scala_T([256]), scala_T( + [160, 320]), scala_T([32, 128]), scala_T([128])]), "inception_5a/")) + model.add(Inception_Layer_v1(832, scala_T([scala_T([384]), scala_T( + [192, 384]), scala_T([48, 128]), scala_T([128])]), "inception_5b/")) + model.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) + model.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) + model.add(View([1024], num_input_dims=3)) + model.add(Linear(1024, class_num).set_name("loss3/classifier_flowers")) + model.add(LogSoftMax().set_name("loss3/loss3")) + model.reset() + return model + +def Inception_v1(class_num): + """ + Builds the entire network using Inception architecture + + :param class_num: number of categories of classification + :return: entire model architecture + """ + #contains first 3 inception modules + feature1 = Sequential() + + feature1.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False).set_name("conv1/7x7_s2")) + feature1.add(ReLU(True).set_name("conv1/relu_7x7")) + feature1.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool1/3x3_s2")) + feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("pool1/norm1")) + feature1.add(SpatialConvolution(64, 64, 1, 1, 1, 1).set_name("conv2/3x3_reduce")) + feature1.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) + feature1.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1).set_name("conv2/3x3")) + feature1.add(ReLU(True).set_name("conv2/relu_3x3")) + feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) + feature1.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) + feature1.add(Inception_Layer_v1(192,scala_T([scala_T([64]), scala_T([96, 128]),scala_T([16, 32]), scala_T([32])]), + "inception_3a/")) + feature1.add(Inception_Layer_v1(256, scala_T([scala_T([128]), scala_T([128, 192]), scala_T([32, 96]), scala_T([64])]), + "inception_3b/")) + feature1.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool3/3x3_s2")) + feature1.add(Inception_Layer_v1(480, scala_T([scala_T([192]), scala_T([96, 208]), scala_T([16, 48]), scala_T([64])]), + "inception_4a/")) + # 1st classification ouput after 3 inception subnetworks + output1 = Sequential() + output1.add(SpatialAveragePooling(5, 5, 3, 3, ceil_mode=True).set_name("loss1/ave_pool")) + output1.add(SpatialConvolution(512, 128, 1, 1, 1, 1).set_name("loss1/conv")) + output1.add(ReLU(True).set_name("loss1/relu_conv")) + output1.add(View([128 * 4 * 4], num_input_dims = 3)) + output1.add(Linear(128 * 4 * 4, 1024).set_name("loss1/fc")) + output1.add(ReLU(True).set_name("loss1/relu_fc")) + output1.add(Dropout(0.7).set_name("loss1/drop_fc")) + output1.add(Linear(1024, class_num).set_name("loss1/classifier_5classes")) + output1.add(LogSoftMax().set_name("loss1/loss")) + + # contains next 3 inception submodules + feature2 = Sequential() + feature2.add(Inception_Layer_v1(512, scala_T([scala_T([160]), scala_T([112, 224]),scala_T([24, 64]), scala_T([64])]), + "inception_4b/")) + feature2.add(Inception_Layer_v1(512, scala_T([scala_T([128]), scala_T([128, 256]),scala_T([24, 64]), scala_T([64])]), + "inception_4c/")) + feature2.add(Inception_Layer_v1(512, scala_T([scala_T([112]), scala_T([144, 288]), scala_T([32, 64]), scala_T([64])]), + "inception_4d/")) + # 2nd classification output after 3 more inception subnetworks + output2 = Sequential() + output2.add(SpatialAveragePooling(5, 5, 3, 3).set_name("loss2/ave_pool")) + output2.add(SpatialConvolution(528, 128, 1, 1, 1, 1).set_name("loss2/conv")) + output2.add(ReLU(True).set_name("loss2/relu_conv")) + output2.add(View([128 * 4 * 4], num_input_dims=3)) + output2.add(Linear(128 * 4 * 4, 1024).set_name("loss2/fc")) + output2.add(ReLU(True).set_name("loss2/relu_fc")) + output2.add(Dropout(0.7).set_name("loss2/drop_fc")) + output2.add(Linear(1024, class_num).set_name("loss2/classifier_5classes")) + output2.add(LogSoftMax().set_name("loss2/loss")) + + # final 3 inception submodules followed by linear/softmax + output3 = Sequential() + output3.add(Inception_Layer_v1(528, scala_T([scala_T([256]), scala_T([160, 320]), scala_T([32, 128]), scala_T([128])]), + "inception_4e/")) + output3.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool4/3x3_s2")) + output3.add(Inception_Layer_v1(832, scala_T([scala_T([256]), scala_T([160, 320]), scala_T([32, 128]), scala_T([128])]), + "inception_5a/")) + output3.add(Inception_Layer_v1(832,scala_T([scala_T([384]), scala_T([192, 384]),scala_T([48, 128]), scala_T([128])]), + "inception_5b/")) + output3.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) + output3.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) + output3.add(View([1024], num_input_dims=3)) + output3.add(Linear(1024, class_num).set_name("loss3/classifier_5classes")) + output3.add(LogSoftMax().set_name("loss3/loss3")) + + # Attach the separate Sequential layers to create the whole model + split2 = Concat(2).set_name("split2") + split2.add(output3) + split2.add(output2) + + #create a branch starting from feature2 upwards + mainBranch = Sequential() + mainBranch.add(feature2) + mainBranch.add(split2) + + #concatenate the mainBranch with output1 + split1 = Concat(2).set_name("split1") + split1.add(mainBranch) + split1.add(output1) + + #Attach feature1 to the rest of the model + model = Sequential() + + model.add(feature1) + model.add(split1) + + model.reset() + return model + +def get_inception_data(folder, file_type="image", data_type="train", normalize=255.0): + """ + Builds the entire network using Inception architecture + + :param class_num: number of categories of classification + :return: entire model architecture + """ + #Getting the path of our data + path = os.path.join(folder, data_type) + if "seq" == file_type: + return read_seq_file(sc, path, normalize) + elif "image" == file_type: + return read_local(sc, path, normalize) + +# initializing BigDL engine +init_engine() + +# paths for datasets, saving checkpoints + +DATA_PATH = "./sample_images/" +checkpoint_path = "./sample_images/checkpoints" + +#providing the no of classes in the dataset to model (5 for flowers) +classNum = 5 + +# Instantiating the model the model +inception_model = Inception_v1_NoAuxClassifier(classNum) + +# path, names of the downlaoded pre-trained caffe models +caffe_prototxt = 'bvlc_googlenet.prototxt' +caffe_model = 'bvlc_googlenet.caffemodel' + +# loading the weights to the BigDL inception model, EXCEPT the weights for the last fc layer (classification layer) +model = Model.load_caffe(inception_model, caffe_prototxt, caffe_model, match_all=False, bigdl_type="float") + +# if we want to export the whole caffe model including definition, this can be used. +#model = Model.load_caffe_model(inception_model, caffe_prototxt, caffe_model, match_all=True) + +# Get the flower categories +from os import listdir +from os.path import isfile, join +labels = listdir("./sample_images/flower_photos") +print "labels: ", labels + +''' +Helper function to make sure image width or height is no smaller than 224x224 +''' +def adjust_dimensions(input_img): + + dimensions = input_img.getbbox() + if dimensions[2] < 224: + input_img = input_img.resize((224,dimensions[3])) + dimensions = input_img.getbbox() + # print "1", dimensions + if dimensions[3] < 224: + input_img = input_img.resize((dimensions[2],224)) + # print "2", input_img.getbbox() + return input_img + +''' +Convert the test image to an Image object +Note: Images in vegnonveg-sample are large and maybe need to be cropped/resized before being trained on. +''' +from PIL import Image # for seeing image +import cv2 # converting img to numpy array (RGB to BGR) + +sample_images_path = "./sample_images/flower_photos/sunflowers/" +input_str = '1008566138_6927679c8a.jpg' +input_img = Image.open(sample_images_path + input_str) +input_img = adjust_dimensions(input_img) +print "accepted image dimensions: ", input_img.getbbox() +# DISPLAY IMAGE HERE +input_img.show() + +''' +Convert Image object into a 3d numpy array representing BGR of each pixel (for bigdl) +''' +# img = cv2.imread(sample_images_path + input_str) +# # print img +img = np.array(input_img) +img = img[:,:,::-1].copy() #invert RGB representation to BGR for each pixel in img +img.shape + +''' +Normalize, crop, finish pre-processing of image so it can be fed to rdd[sample] for bigdl. +''' +# defining the transformer, which we will use to pre-process our test image +img_rows = 224 +img_cols = 224 + + +transform_input = Transformer([Crop(img_rows, img_cols, "center"), + ChannelNormalizer(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + TransposeToTensor(False) + ]) +# pre-processing the img, feature transformation decreases training time +img_tranx = transform_input(img) + +''' +Converting the image to 'Sample' format which BigDL expects. +''' +label = np.array(1) #label of dandelion +img_to_model = Sample.from_ndarray(img_tranx, label) + +''' +Converting image from 'Sample' format into RDD format +''' +img_data_rdd = sc.parallelize([img_to_model]) + +# predicting the image using our model +predict_result = model.predict_class(img_data_rdd) +pred_index = predict_result.collect()[0] +print pred_index + +# printing out the category +if pred_index > classNum - 1 : + pred_index = pred_index % classNum + +class_predicted = str(labels[pred_index - 1]) +print (class_predicted) + +''' +GOAL: Predicts the first 20 images in a specified flower folder using the pre-trained model +''' +from PIL import Image # for seeing image +import cv2 # converting img to numpy array (RGB to BGR) + +# get local path of each image +flower = "sunflowers" +dand_path = "./sample_images/flower_photos/" + flower + "/" +imgs = listdir(dand_path) + +# predict first 20 images +for img in imgs[0:19]: + input_img = Image.open(dand_path + img) + input_img = adjust_dimensions(input_img) + + # convert img to np.array form + img_bgr = np.array(input_img) + img_bgr = img_bgr[:,:,::-1].copy() + img_tranx = transform_input(img_bgr) + + #get label of flower + label = np.array(labels.index(flower)) + + # converting to 'Sample' format which BigDL expects. + img_to_model = Sample.from_ndarray(img_tranx, label) + + # converting from 'Sample' format into RDD format + img_data_rdd = sc.parallelize([img_to_model]) + + # predicting the image using our model + predict_result = model.predict_class(img_data_rdd) + pred_index = predict_result.collect()[0] + + # printing out the category + if pred_index > classNum: + print pred_index + pred_index = pred_index % classNum + class_predicted = str(labels[pred_index - 1]) + print (class_predicted) + + + +# the image size expected by the model +image_size = 224 + +# image transformer, used for pre-processing the train images +train_transformer = Transformer([Crop(image_size, image_size), + Flip(0.5), + ChannelNormalizer(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + TransposeToTensor(False)]) + +# reading the traning + +''' +Goal: Predict first 20 images in the "dandelion" folder using un-trained model. +ERROR: py4j.Py4JException: Method modelPredictClass([class com.intel.analytics.bigdl.nn.Sequential, class java.util.ArrayList]) does not exist +''' +from PIL import Image # for seeing image +import cv2 # converting img to numpy array (RGB to BGR) + +def map_groundtruth_label(l): + return l.to_ndarray()[0] - 1 +# defining the tranformer, which we will use to pre-process our test image + +img_rows = 224 +img_cols = 224 + + +transform_input = Transformer([Crop(img_rows, img_cols, "center"), + Flip(0.5), + ChannelNormalizer(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + TransposeToTensor(False) + ]) +dand_path = "./sample_images/flower_photos/dandelion/" +imgs = listdir(dand_path) + +#get paths +img_paths = [] +for img in imgs[0:20]: + img_paths.append((dand_path+img, 1)) + +print ("img_paths:") +print (img_paths) + +#turn img_paths into labelled rdds +image_paths_rdd = sc.parallelize(img_paths) + +feature_label_rdd = image_paths_rdd.map(lambda path_label: (misc.imread(path_label[0]), np.array(path_label[1]))) \ + .map(lambda img_label: + (Resize(256, 256)(img_label[0]), img_label[1])) \ + .map(lambda feature_label: + (((feature_label[0] & 0xff) / 255.0).astype("float32"), feature_label[1])) + +print("Feature label:") +print(feature_label_rdd.take(10)) + +#turn to sample form for predictions +img_data = feature_label_rdd.map( + lambda features_label: (train_transformer(features_label[0]), features_label[1])).map( + lambda features_label: Sample.from_ndarray(features_label[0], features_label[1] + 1)) + +# predicting the image using our model +print "Predictions: " +res = model.predict_class(img_data) +print res.collect() + +print "True Labels: " +print ', '.join(str(map_groundtruth_label(s.label)) for s in img_data.take(10)) + + +''' +Goal: Predict first 20 images in the "dandelion" folder using un-trained model. +ERROR: py4j.Py4JException: Method modelPredictClass([class com.intel.analytics.bigdl.nn.Sequential, class java.util.ArrayList]) does not exist +''' +from PIL import Image # for seeing image +import cv2 # converting img to numpy array (RGB to BGR) + +# defining the tranformer, which we will use to pre-process our test image + +img_rows = 224 +img_cols = 224 + + +transform_input = Transformer([Crop(img_rows, img_cols, "center"), + Flip(0.5), + ChannelNormalizer(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + TransposeToTensor(False) + ]) +dand_path = "./sample_images/flower_photos/dandelion/" +imgs = listdir(dand_path) + + + + +#get paths +img_paths = [] +for img in imgs[0:20]: + img_paths.append((dand_path+img, 1)) + + +img_paths + +''' +Reading the training and validation data and perform pre-processing +''' + + +# the image size expected by the model +image_size = 224 + +# image transformer, used for pre-processing the train images +train_transformer = Transformer([Crop(image_size, image_size), + Flip(0.5), + ChannelNormalizer(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + TransposeToTensor(False)]) + +# reading the traning data +train_data = get_inception_data(DATA_PATH, "image", "train").map( + lambda features_label: (train_transformer(features_label[0]), features_label[1])).map( + lambda features_label: Sample.from_ndarray(features_label[0], features_label[1] + 1)) + +# validation data transformer +val_transformer = Transformer([Crop(image_size, image_size, "center"), + Flip(0.5), + ChannelNormalizer(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + TransposeToTensor(False)]) + +#reading the validation data +val_data = get_inception_data(DATA_PATH, "image", "val").map( + lambda features_label: (val_transformer(features_label[0]), features_label[1])).map( + lambda features_label: Sample.from_ndarray(features_label[0], features_label[1] + 1)) + + + +# training the model +# parameters for +batch_size = 16 +no_epochs = 2 + +# Optimizer +optimizer = Optimizer( + model=model, + training_rdd=train_data.filter(lambda l: l.label.to_ndarray()[0] <= 5), + #optim_method=Adam(learningrate=0.002), + optim_method = SGD(learningrate=0.01, learningrate_decay=0.0002), + criterion=ClassNLLCriterion(), + end_trigger=MaxEpoch(no_epochs), + batch_size=batch_size + ) + +# setting checkpoints +optimizer.set_checkpoint(EveryEpoch(), checkpoint_path, isOverWrite=False) + +# setting validation parameters +optimizer.set_validation( batch_size=batch_size, + val_rdd=val_data, + trigger=EveryEpoch(), + val_method=[Top1Accuracy()]) + + + +# Log the training process to measure loss/accuracy, can be +app_name= 'inception-' + dt.datetime.now().strftime("%Y%m%d-%H%M%S") +train_summary = TrainSummary(log_dir='/tmp/inception_summaries', + app_name=app_name) +train_summary.set_summary_trigger("Parameters", SeveralIteration(50)) +val_summary = ValidationSummary(log_dir='/tmp/inception_summaries', + app_name=app_name) +optimizer.set_train_summary(train_summary) +optimizer.set_val_summary(val_summary) +print "saving logs to ",app_name + + +# Boot training process +trained_model = optimizer.optimize() +print "Optimization Done." + + +# image transformer, used for pre-processing the validation images +test_transformer = Transformer([Crop(image_size, image_size, "center"), + ChannelNormalizer(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + TransposeToTensor(False)]) + +# reading val data +# get_inception_data() returns a PythonRDD +test_data = get_inception_data(DATA_PATH, "image", "test").map( + lambda features_label: (test_transformer(features_label[0]), features_label[1])).map( + lambda features_label: Sample.from_ndarray(features_label[0], features_label[1] + 1)) + + +print "Predictions: " +res = trained_model.predict_class(test_data) +print res.collect() + +print "True Labels: " +print ', '.join(str(map_groundtruth_label(s.label)) for s in test_data.take(8)) +# testing the trained model +results = trained_model.evaluate(test_data, batch_size, [Top1Accuracy()]) + +# Output results +for i in results: + print (i) + + + diff --git a/scripts/run_transfer.sh b/scripts/run_transfer.sh new file mode 100644 index 0000000..bced9b6 --- /dev/null +++ b/scripts/run_transfer.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +## Usage +# BIGDL_HOME=~/apps/BigDL SPARK_HOME=~/apps/spark ./run-bigdl.sh + +export PYSPARK_PYTHON=$(which python) +export BIGDL_HOME=/usr/local/BigDL + +# Check environment variables +if [ -z "${BIGDL_HOME}" ]; then + echo "Please set BIGDL_HOME environment variable" + exit 1 +fi + +if [ -z "${SPARK_HOME}" ]; then + echo "Please set SPARK_HOME environment variable" + exit 1 +fi + +# activate py35 environment +#source activate py35 +# activate py27 environment +source activate py27 +conda info -e + + +#setup paths +export BIGDL_JAR_NAME=`ls ${BIGDL_HOME}/lib/ | grep jar-with-dependencies.jar` +export BIGDL_JAR="${BIGDL_HOME}/lib/$BIGDL_JAR_NAME" +export BIGDL_PY_ZIP_NAME=`ls ${BIGDL_HOME}/lib/ | grep python-api.zip` +export BIGDL_PY_ZIP="${BIGDL_HOME}/lib/$BIGDL_PY_ZIP_NAME" +export BIGDL_CONF=${BIGDL_HOME}/conf/spark-bigdl.conf + +# Check files +if [ ! -f ${BIGDL_CONF} ]; then + echo "Cannot find ${BIGDL_CONF}" + exit 1 +fi + +if [ ! -f ${BIGDL_PY_ZIP} ]; then + echo ${BIGDL_PY_ZIP} + echo "Cannot find ${BIGDL_PY_ZIP}" + exit 1 +fi + +if [ ! -f $BIGDL_JAR ]; then + echo "Cannot find $BIGDL_JAR" + exit 1 +fi + +#cd work/notebooks/inception_v1 + +pwd + +${SPARK_HOME}/bin/spark-submit \ + --master local[4] \ + --driver-memory 16g \ + --properties-file ${BIGDL_CONF} \ + --py-files ${BIGDL_PY_ZIP} \ + --jars ${BIGDL_JAR} \ + --conf spark.driver.extraClassPath=${BIGDL_JAR} \ + --conf spark.executor.extraClassPath=${BIGDL_JAR} \ + --conf spark.sql.catalogImplementation='in-memory' \ + ./inception_transfer.py + diff --git a/scripts/transformer.py b/scripts/transformer.py new file mode 100644 index 0000000..9f759af --- /dev/null +++ b/scripts/transformer.py @@ -0,0 +1,173 @@ + +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import random +import numpy as np +from scipy import misc + + +class Transformer(object): + """Composes several transforms together. + Args: + transforms (List[Transform]): list of transforms to compose. + Example: + >>> Transformer([ + >>> Crop(10, 10), + >>> Normalizer(0.3, 0.8) + >>> ]) + """ + + def __init__(self, transforms): + self.transforms = transforms + + def __call__(self, img): + for t in self.transforms: + img = t(img) + return img + + +class Crop(object): + """ + Crop image which is an numpy array to specified width and height. The shape of image is (height, width, channel). + :param crop_width: width after cropped + :param crop_height: height after cropped + :param crop_method: crop method, should be random or center + :return: cropped image. The shape of image is (height, width, channel) + """ + + def __init__(self, crop_width, crop_height, crop_method='random'): + self.crop_width = crop_width + self.crop_height = crop_height + self.crop_method = crop_method + + def __call__(self, img): + h = img.shape[0] + w = img.shape[1] + if self.crop_method == 'random': + x1 = random.randint(0, w - self.crop_width) + y1 = random.randint(0, h - self.crop_height) + elif self.crop_method == 'center': + x1 = (w - self.crop_width) / 2 + y1 = (h - self.crop_height) / 2 + cropped = img[y1:y1 + self.crop_height, x1:x1 + self.crop_width] + return cropped + + +class Normalizer(object): + """ + Normalize image which is an numpy array by mean and standard deviation The shape of image is (height, width, channel) + :param mean: mean + :param std: standard deviation + :return: normalized image. The shape of image is (height, width, channel) + """ + + def __init__(self, mean, std): + self.mean = mean + self.std = std + + def __call__(self, img): + return (img - self.mean) / self.std + + +class PixelNormalizer(object): + """ + Normalize image which is an numpy array by means of every pixel. The shape of image is (height, width, channel) + :param mean: mean + :param std: standard deviation + :return: normalized image. The shape of image is (height, width, channel) + """ + + def __init__(self, mean): + self.mean = mean + + def __call__(self, img): + return img - self.mean + + +class ChannelNormalizer(object): + """ + Normalize image which is an numpy array by means and std of each channel. The shape of image is (height, width, channel) + :param mean_r: mean for red channel + :param mean_g: mean for green channel + :param mean_b: mean for blue channel + :param std_r: std for red channel + :param std_g: std for green channel + :param std_b: std for blue channel + :return: normalized image. The shape of image is (height, width, channel) + """ + def __init__(self, mean_r, mean_g, mean_b, std_r, std_g, std_b): + self.mean_r = mean_r + self.mean_g = mean_g + self.mean_b = mean_b + self.std_r = std_r + self.std_g = std_g + self.std_b = std_b + + def __call__(self, img): + mean = np.array([self.mean_b, self.mean_g, self.mean_r]) + std = np.array([self.std_b, self.std_g, self.std_r]) + mean_sub = img[:, :] - mean + return mean_sub[:, :] / std + + +class Flip(object): + """ + Flip image which is an numpy array horizontally. The shape of image is (height, width, channel) + :param threshold: if random number is over the threshold, we need to flip image, otherwise, do not filp + :return: flipped image or original image. The shape of image is (height, width, channel) + """ + def __init__(self, threshold): + self.threshold = threshold + + def __call__(self, img): + if random.random() > self.threshold: + # flip with axis 1 which is horizontal flip + return np.flip(img, 1) + else: + return img + + +class TransposeToTensor(object): + """ + Transpose the shape of image which is an numpy array from (height, width, channel) to (channel, height, width) + :param to_rgb: whether need to change channel from bgr to rgb + :return: transposed image + """ + def __init__(self, to_rgb=True): + self.to_rgb = to_rgb + + def __call__(self, img): + if self.to_rgb: + return img.transpose(2, 0, 1)[(2, 1, 0), :, :] + else: + return img.transpose(2, 0, 1) + + +class Resize(object): + """ + Resize image to specified width and height. The type of image should be uint8. + The shape of image is (height, width, channel) + :param resize_width: the width resized to + :param resize_height: the height resized to + :return: resized image. + """ + def __init__(self, resize_width, resize_height): + self.resize_width = resize_width + self.resize_height = resize_height + + def __call__(self, img): + return misc.imresize(img, (self.resize_width, self.resize_height)) From 9c0b02192f1d8efe460fb3e48530081a1335690b Mon Sep 17 00:00:00 2001 From: "Sujee Maniyam @ sujee-mac2" Date: Fri, 19 Jan 2018 14:51:21 -0800 Subject: [PATCH 13/19] move assets into elephantscale folder --- {docker => elephantscale/docker}/Dockerfile | 6 +++--- {docker => elephantscale/docker}/README.md | 0 {docker => elephantscale/docker}/run-bigdl.sh | 0 {docker => elephantscale/docker}/start-notebook.sh | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename {docker => elephantscale/docker}/Dockerfile (95%) rename {docker => elephantscale/docker}/README.md (100%) rename {docker => elephantscale/docker}/run-bigdl.sh (100%) rename {docker => elephantscale/docker}/start-notebook.sh (100%) diff --git a/docker/Dockerfile b/elephantscale/docker/Dockerfile similarity index 95% rename from docker/Dockerfile rename to elephantscale/docker/Dockerfile index 9dfc189..0e9a549 100644 --- a/docker/Dockerfile +++ b/elephantscale/docker/Dockerfile @@ -3,7 +3,7 @@ FROM jupyter/pyspark-notebook@sha256:64420e4c348ab48fb806f42332109cbc205ae74cda6 ## or latest #FROM jupyter/pyspark-notebook@latest -MAINTAINER Intel BigDL Project +MAINTAINER Elephant Scale ## --- CONFIG ARG MAVEN_VERSION=3.5.2 @@ -101,10 +101,10 @@ USER $NB_USER RUN conda install -y nltk ## python 3.5 env -RUN conda create -y -n py35 python=3.5 numpy scipy pandas scikit-learn matplotlib seaborn jupyter nltk tensorflow +RUN conda create -y -n py35 python=3.5 numpy scipy pandas scikit-learn matplotlib seaborn jupyter nltk tensorflow opencv pillow ## python 2.7 env -RUN conda create -y -n py27 python=2.7 numpy scipy pandas scikit-learn matplotlib seaborn jupyter nltk tensorflow +RUN conda create -y -n py27 python=2.7 numpy scipy pandas scikit-learn matplotlib seaborn jupyter nltk tensorflow opencv pillow # list envs RUN conda info -e diff --git a/docker/README.md b/elephantscale/docker/README.md similarity index 100% rename from docker/README.md rename to elephantscale/docker/README.md diff --git a/docker/run-bigdl.sh b/elephantscale/docker/run-bigdl.sh similarity index 100% rename from docker/run-bigdl.sh rename to elephantscale/docker/run-bigdl.sh diff --git a/docker/start-notebook.sh b/elephantscale/docker/start-notebook.sh similarity index 100% rename from docker/start-notebook.sh rename to elephantscale/docker/start-notebook.sh From 46e0ce3281ee2fac9c48ce27fd31ca8ba98fbaf4 Mon Sep 17 00:00:00 2001 From: Tim Fox Date: Mon, 22 Jan 2018 20:24:07 -0500 Subject: [PATCH 14/19] Moved everything to elephantscale directory --- .../notebooks}/Testing-123.ipynb | 0 ...forward-credit-card-default-pipeline.ipynb | 0 .../feedforward-credit-card-default.ipynb | 0 ...edforward-credit-card-fraud-pipeline.ipynb | 0 .../feedforward-credit-card-fraud.ipynb | 0 .../feedforward-iris-pipeline.ipynb | 0 .../notebooks}/feedforward-iris.ipynb | 0 .../notebooks}/lstm-20news.ipynb | 0 .../notebooks}/lstm-stocks.ipynb | 0 elephantscale/notebooks/lstm.py | 97 +++++++++++++++++++ .../scripts}/imagenet.py | 0 .../scripts}/inception_transfer.py | 0 .../scripts}/run_transfer.sh | 0 .../scripts}/transformer.py | 0 14 files changed, 97 insertions(+) rename {notebooks => elephantscale/notebooks}/Testing-123.ipynb (100%) rename {notebooks => elephantscale/notebooks}/feedforward-credit-card-default-pipeline.ipynb (100%) rename {notebooks => elephantscale/notebooks}/feedforward-credit-card-default.ipynb (100%) rename {notebooks => elephantscale/notebooks}/feedforward-credit-card-fraud-pipeline.ipynb (100%) rename {notebooks => elephantscale/notebooks}/feedforward-credit-card-fraud.ipynb (100%) rename {notebooks => elephantscale/notebooks}/feedforward-iris-pipeline.ipynb (100%) rename {notebooks => elephantscale/notebooks}/feedforward-iris.ipynb (100%) rename {notebooks => elephantscale/notebooks}/lstm-20news.ipynb (100%) rename {notebooks => elephantscale/notebooks}/lstm-stocks.ipynb (100%) create mode 100644 elephantscale/notebooks/lstm.py rename {scripts => elephantscale/scripts}/imagenet.py (100%) rename {scripts => elephantscale/scripts}/inception_transfer.py (100%) rename {scripts => elephantscale/scripts}/run_transfer.sh (100%) rename {scripts => elephantscale/scripts}/transformer.py (100%) diff --git a/notebooks/Testing-123.ipynb b/elephantscale/notebooks/Testing-123.ipynb similarity index 100% rename from notebooks/Testing-123.ipynb rename to elephantscale/notebooks/Testing-123.ipynb diff --git a/notebooks/feedforward-credit-card-default-pipeline.ipynb b/elephantscale/notebooks/feedforward-credit-card-default-pipeline.ipynb similarity index 100% rename from notebooks/feedforward-credit-card-default-pipeline.ipynb rename to elephantscale/notebooks/feedforward-credit-card-default-pipeline.ipynb diff --git a/notebooks/feedforward-credit-card-default.ipynb b/elephantscale/notebooks/feedforward-credit-card-default.ipynb similarity index 100% rename from notebooks/feedforward-credit-card-default.ipynb rename to elephantscale/notebooks/feedforward-credit-card-default.ipynb diff --git a/notebooks/feedforward-credit-card-fraud-pipeline.ipynb b/elephantscale/notebooks/feedforward-credit-card-fraud-pipeline.ipynb similarity index 100% rename from notebooks/feedforward-credit-card-fraud-pipeline.ipynb rename to elephantscale/notebooks/feedforward-credit-card-fraud-pipeline.ipynb diff --git a/notebooks/feedforward-credit-card-fraud.ipynb b/elephantscale/notebooks/feedforward-credit-card-fraud.ipynb similarity index 100% rename from notebooks/feedforward-credit-card-fraud.ipynb rename to elephantscale/notebooks/feedforward-credit-card-fraud.ipynb diff --git a/notebooks/feedforward-iris-pipeline.ipynb b/elephantscale/notebooks/feedforward-iris-pipeline.ipynb similarity index 100% rename from notebooks/feedforward-iris-pipeline.ipynb rename to elephantscale/notebooks/feedforward-iris-pipeline.ipynb diff --git a/notebooks/feedforward-iris.ipynb b/elephantscale/notebooks/feedforward-iris.ipynb similarity index 100% rename from notebooks/feedforward-iris.ipynb rename to elephantscale/notebooks/feedforward-iris.ipynb diff --git a/notebooks/lstm-20news.ipynb b/elephantscale/notebooks/lstm-20news.ipynb similarity index 100% rename from notebooks/lstm-20news.ipynb rename to elephantscale/notebooks/lstm-20news.ipynb diff --git a/notebooks/lstm-stocks.ipynb b/elephantscale/notebooks/lstm-stocks.ipynb similarity index 100% rename from notebooks/lstm-stocks.ipynb rename to elephantscale/notebooks/lstm-stocks.ipynb diff --git a/elephantscale/notebooks/lstm.py b/elephantscale/notebooks/lstm.py new file mode 100644 index 0000000..727e65d --- /dev/null +++ b/elephantscale/notebooks/lstm.py @@ -0,0 +1,97 @@ +import time +import warnings +import numpy as np +from numpy import newaxis +import matplotlib.pyplot as plt + +from bigdl.nn.layer import * +from bigdl.nn.criterion import * +from bigdl.optim.optimizer import * +from bigdl.util.common import * +from bigdl.dataset.transformer import * +from bigdl.dataset import mnist +from bigdl.dataset.base import * +from bigdl.util.common import * + + + + +warnings.filterwarnings("ignore") + +#convert ndarray data into RDD[Sample] + + +def plot_results_multiple(predicted_data, true_data, prediction_len): + fig = plt.figure(facecolor='white') + ax = fig.add_subplot(111) + ax.plot(true_data, label='True Data') + #Pad the list of predictions to shift it in the graph to it's correct start + for i, data in enumerate(predicted_data): + padding = [None for p in range(i * prediction_len)] + plt.plot(padding + data, label='Prediction') + plt.legend() + plt.show() + +def load_data(filename, seq_len, normalise_window): + f = open(filename, 'r').read() + data = f.split('\n') + + sequence_length = seq_len + 1 + result = [] + for index in range(len(data) - sequence_length): + result.append(data[index: index + sequence_length]) + + if normalise_window: + result = normalise_windows(result) + + result = np.array(result) + + row = round(0.9 * result.shape[0]) + train = result[:int(row), :] + np.random.shuffle(train) + x_train = train[:, :-1] + y_train = train[:, -1] + x_test = result[int(row):, :-1] + y_test = result[int(row):, -1] + + x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1)) + x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1)) + + return [x_train, y_train, x_test, y_test] + +def normalise_windows(window_data): + normalised_data = [] + for window in window_data: + normalised_window = [((float(p) / float(window[0])) - 1) for p in window] + normalised_data.append(normalised_window) + return normalised_data + +def predict_point_by_point(model, data): + #Predict each timestep given the last sequence of true data, in effect only predicting 1 step ahead each time + predicted = model.predict(data) + predicted = np.reshape(predicted, (predicted.size,)) + return predicted + +def predict_sequence_full(model, data, window_size): + #Shift the window by 1 new prediction each time, re-run predictions on new window + curr_frame = data[0] + predicted = [] + for i in range(len(data)): + predicted.append(model.predict(curr_frame[newaxis,:,:])[0,0]) + curr_frame = curr_frame[1:] + curr_frame = np.insert(curr_frame, [window_size-1], predicted[-1], axis=0) + return predicted + +def predict_sequences_multiple(model, data, window_size, prediction_len, sc): + #Predict sequence of 50 steps before shifting prediction run forward by 50 steps + prediction_seqs = [] + for i in range(len(data)//prediction_len): + curr_frame = data[i*int(prediction_len)] + predicted = [] + for j in range(int(prediction_len)): + + predicted.append(model.predict(curr_frame[newaxis,:,:])[0,0]) + curr_frame = curr_frame[1:] + curr_frame = np.insert(curr_frame, [window_size-1], predicted[-1], axis=0) + prediction_seqs.append(predicted) + return prediction_seqs diff --git a/scripts/imagenet.py b/elephantscale/scripts/imagenet.py similarity index 100% rename from scripts/imagenet.py rename to elephantscale/scripts/imagenet.py diff --git a/scripts/inception_transfer.py b/elephantscale/scripts/inception_transfer.py similarity index 100% rename from scripts/inception_transfer.py rename to elephantscale/scripts/inception_transfer.py diff --git a/scripts/run_transfer.sh b/elephantscale/scripts/run_transfer.sh similarity index 100% rename from scripts/run_transfer.sh rename to elephantscale/scripts/run_transfer.sh diff --git a/scripts/transformer.py b/elephantscale/scripts/transformer.py similarity index 100% rename from scripts/transformer.py rename to elephantscale/scripts/transformer.py From 851e95b46ba1511acd8d7d27dca094923eb56d4d Mon Sep 17 00:00:00 2001 From: Tim Fox Date: Fri, 2 Feb 2018 16:20:32 -0500 Subject: [PATCH 15/19] Updated some text. Changed learning rate to 0.01 --- .../notebooks/feedforward-iris-pipeline.ipynb | 2 +- elephantscale/notebooks/lstm-stocks.ipynb | 61 +- elephantscale/notebooks/sp500.csv | 4171 +++++++++++++++++ 3 files changed, 4222 insertions(+), 12 deletions(-) create mode 100644 elephantscale/notebooks/sp500.csv diff --git a/elephantscale/notebooks/feedforward-iris-pipeline.ipynb b/elephantscale/notebooks/feedforward-iris-pipeline.ipynb index b481de8..dd6f003 100644 --- a/elephantscale/notebooks/feedforward-iris-pipeline.ipynb +++ b/elephantscale/notebooks/feedforward-iris-pipeline.ipynb @@ -106,7 +106,7 @@ "metadata": {}, "outputs": [], "source": [ - "learning_rate = 0.1\n", + "learning_rate = 0.01\n", "training_epochs = 100\n", "batch_size = 16\n", "display_step = 1\n", diff --git a/elephantscale/notebooks/lstm-stocks.ipynb b/elephantscale/notebooks/lstm-stocks.ipynb index 506fe16..ba97dea 100644 --- a/elephantscale/notebooks/lstm-stocks.ipynb +++ b/elephantscale/notebooks/lstm-stocks.ipynb @@ -6,13 +6,17 @@ "source": [ "# LSTM Stock Market Example\n", "\n", - "Wouldn't you like to predict the stock market? (Don't try this at home). Here's an example of how we can use LSTMs to predict patterns in the S&P 500." + "Wouldn't you like to predict the stock market? (Don't try this at home). Here's an example of how we can use LSTMs to predict patterns in the S&P 500.\n", + "\n", + "What is an LSTM? An LSTM (Long Short Term Memory) is a recurrent neural network that will help us to predict patterns in time-series data." ] }, { "cell_type": "code", "execution_count": 3, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "\n", @@ -33,7 +37,9 @@ { "cell_type": "code", "execution_count": 3, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "learning_rate = 0.1\n", @@ -60,7 +66,9 @@ { "cell_type": "code", "execution_count": 8, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "#Step 1 Load Data\n", @@ -70,7 +78,9 @@ { "cell_type": "code", "execution_count": 9, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "X_train_rdd = sc.parallelize(X_train).map(lambda x: x.flatten())\n", @@ -82,7 +92,9 @@ { "cell_type": "code", "execution_count": 11, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "x_train_sample = X_train_rdd.zip(y_train_rdd).map(lambda x:Sample.from_ndarray(x[0], x[1]))\n", @@ -139,7 +151,7 @@ "source": [ "# Training The Model\n", "\n", - "Notice that the model is a regression, not classificaiton model" + "Notice that the model is a regression, not a classification model. This is why we are using the MSECriterion. What we are trying to do is predict the next value of the S&P500 given the input sequence of S&P 500 values." ] }, { @@ -178,13 +190,16 @@ "source": [ "# Making Predictions\n", "\n", - "Here we are going to make predictions based on the model. We will use sliding windows of length 50." + "Here we are going to make predictions based on the model. We will use sliding windows of length 50.\n", + "\n", + "Here's how this works: We're going to Take sequences of length 50, and then use those 50 steps as inputs to the model. Then we will get the output of the model for the next step, and continue on making predictions." ] }, { "cell_type": "code", "execution_count": 43, "metadata": { + "collapsed": true, "scrolled": false }, "outputs": [], @@ -212,10 +227,20 @@ "predictions = prediction_seqs" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Plots\n", + "\n", + "Let us go ahead and plot some of our predictions." + ] + }, { "cell_type": "code", "execution_count": null, "metadata": { + "collapsed": true, "scrolled": false }, "outputs": [], @@ -228,16 +253,30 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 2", + "display_name": "Python 3", "language": "python", - "name": "python2" + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.4" } }, "nbformat": 4, diff --git a/elephantscale/notebooks/sp500.csv b/elephantscale/notebooks/sp500.csv new file mode 100644 index 0000000..f770a07 --- /dev/null +++ b/elephantscale/notebooks/sp500.csv @@ -0,0 +1,4171 @@ +1455.219971 +1399.420044 +1402.109985 +1403.449951 +1441.469971 +1457.599976 +1438.560059 +1432.25 +1449.680054 +1465.150024 +1455.140015 +1455.900024 +1445.569946 +1441.359985 +1401.530029 +1410.030029 +1404.089966 +1398.560059 +1360.160034 +1394.459961 +1409.280029 +1409.119995 +1424.969971 +1424.369995 +1424.23999 +1441.719971 +1411.709961 +1416.829956 +1387.119995 +1389.939941 +1402.050049 +1387.670044 +1388.26001 +1346.089966 +1352.170044 +1360.689941 +1353.430054 +1333.359985 +1348.050049 +1366.420044 +1379.189941 +1381.76001 +1409.170044 +1391.280029 +1355.619995 +1366.699951 +1401.689941 +1395.069946 +1383.619995 +1359.150024 +1392.140015 +1458.469971 +1464.469971 +1456.630005 +1493.869995 +1500.640015 +1527.349976 +1527.459961 +1523.859985 +1507.72998 +1508.52002 +1487.920044 +1498.579956 +1505.969971 +1494.72998 +1487.369995 +1501.339966 +1516.349976 +1504.459961 +1500.589966 +1467.170044 +1440.51001 +1356.560059 +1401.439941 +1441.609985 +1427.469971 +1434.540039 +1429.859985 +1477.439941 +1460.98999 +1464.920044 +1452.430054 +1468.25 +1446.290039 +1415.099976 +1409.569946 +1432.630005 +1424.170044 +1412.140015 +1383.050049 +1407.810059 +1420.959961 +1452.359985 +1466.040039 +1447.800049 +1437.209961 +1406.949951 +1400.719971 +1373.859985 +1399.050049 +1381.52002 +1378.02002 +1422.449951 +1420.599976 +1448.810059 +1477.26001 +1467.630005 +1457.839966 +1471.359985 +1461.670044 +1456.949951 +1446 +1469.439941 +1470.540039 +1478.72998 +1464.459961 +1486 +1475.949951 +1479.130005 +1452.180054 +1441.47998 +1455.310059 +1450.550049 +1454.819946 +1442.390015 +1454.599976 +1469.540039 +1446.22998 +1456.670044 +1478.900024 +1475.619995 +1480.880005 +1492.920044 +1495.839966 +1509.97998 +1510.48999 +1493.73999 +1481.959961 +1495.569946 +1480.189941 +1464.290039 +1474.469971 +1452.420044 +1449.619995 +1419.890015 +1430.829956 +1438.099976 +1438.699951 +1452.560059 +1462.930054 +1479.319946 +1482.800049 +1472.869995 +1460.25 +1471.839966 +1491.560059 +1484.430054 +1479.849976 +1496.069946 +1491.719971 +1499.47998 +1498.130005 +1505.969971 +1508.310059 +1506.449951 +1514.089966 +1509.839966 +1502.589966 +1517.680054 +1520.77002 +1507.079956 +1492.25 +1502.51001 +1494.5 +1489.26001 +1481.98999 +1484.910034 +1480.869995 +1465.810059 +1444.51001 +1459.900024 +1451.339966 +1449.050049 +1448.719971 +1439.030029 +1427.209961 +1426.569946 +1458.290039 +1436.51001 +1436.22998 +1426.459961 +1434.319946 +1436.280029 +1408.98999 +1402.030029 +1387.02002 +1364.589966 +1329.780029 +1374.170044 +1374.619995 +1349.969971 +1342.130005 +1388.76001 +1396.930054 +1395.780029 +1398.130005 +1364.900024 +1364.439941 +1379.579956 +1398.660034 +1429.400024 +1421.219971 +1428.319946 +1426.689941 +1432.189941 +1431.869995 +1409.280029 +1400.140015 +1365.97998 +1351.26001 +1382.949951 +1389.810059 +1372.319946 +1367.719971 +1342.619995 +1347.349976 +1322.359985 +1341.77002 +1348.969971 +1336.089966 +1341.930054 +1314.949951 +1315.22998 +1324.969971 +1376.540039 +1351.459961 +1343.550049 +1369.890015 +1380.199951 +1371.180054 +1359.98999 +1340.930054 +1312.150024 +1322.73999 +1305.599976 +1264.73999 +1274.859985 +1305.949951 +1315.189941 +1328.920044 +1334.219971 +1320.280029 +1283.27002 +1347.560059 +1333.339966 +1298.349976 +1295.859985 +1300.800049 +1313.27002 +1326.819946 +1318.550049 +1326.650024 +1329.469971 +1347.969971 +1342.540039 +1342.900024 +1360.400024 +1364.300049 +1357.51001 +1354.949951 +1364.170044 +1373.72998 +1366.01001 +1373.469971 +1349.469971 +1354.310059 +1352.26001 +1340.890015 +1332.530029 +1314.76001 +1330.310059 +1318.800049 +1315.920044 +1326.609985 +1301.530029 +1278.939941 +1255.27002 +1252.819946 +1245.859985 +1267.650024 +1257.939941 +1239.939941 +1241.22998 +1234.180054 +1241.410034 +1253.800049 +1261.890015 +1264.73999 +1233.420044 +1180.160034 +1197.660034 +1166.709961 +1173.560059 +1150.530029 +1170.810059 +1142.619995 +1122.140015 +1117.579956 +1139.829956 +1152.689941 +1182.170044 +1153.290039 +1147.949951 +1160.329956 +1145.869995 +1106.459961 +1103.25 +1151.439941 +1128.430054 +1137.589966 +1168.380005 +1165.890015 +1183.5 +1179.680054 +1191.810059 +1238.160034 +1253.689941 +1242.97998 +1224.359985 +1209.469971 +1228.75 +1234.52002 +1253.050049 +1249.459961 +1266.439941 +1267.430054 +1248.579956 +1266.609985 +1263.51001 +1261.199951 +1255.540039 +1255.180054 +1245.670044 +1248.920044 +1249.439941 +1284.98999 +1288.48999 +1291.959961 +1312.829956 +1309.380005 +1289.050049 +1293.170044 +1277.890015 +1267.930054 +1248.079956 +1255.819946 +1260.670044 +1267.109985 +1283.569946 +1270.030029 +1276.959961 +1264.959961 +1254.390015 +1255.849976 +1241.599976 +1219.869995 +1214.359985 +1208.430054 +1212.579956 +1223.140015 +1237.040039 +1225.349976 +1218.599976 +1216.76001 +1211.069946 +1226.199951 +1224.380005 +1236.719971 +1234.449951 +1219.23999 +1190.589966 +1198.780029 +1181.52002 +1180.180054 +1208.140015 +1215.680054 +1202.449951 +1214.439941 +1207.709961 +1215.02002 +1210.849976 +1191.030029 +1171.650024 +1190.48999 +1202.930054 +1205.819946 +1204.52002 +1211.22998 +1215.930054 +1220.75 +1214.349976 +1200.47998 +1204.400024 +1183.530029 +1183.430054 +1190.160034 +1191.290039 +1186.72998 +1178.02002 +1181.660034 +1161.969971 +1171.410034 +1157.26001 +1165.310059 +1162.089966 +1184.930054 +1179.209961 +1161.51001 +1148.560059 +1129.030029 +1133.579956 +1132.939941 +1131.73999 +1106.400024 +1085.780029 +1092.540039 +1038.77002 +1032.73999 +1016.099976 +984.539978 +965.799988 +1003.450012 +1012.27002 +1007.039978 +1018.609985 +1040.939941 +1038.550049 +1051.329956 +1072.280029 +1069.630005 +1071.380005 +1062.439941 +1056.75 +1080.98999 +1097.430054 +1091.650024 +1089.97998 +1097.540039 +1077.089966 +1068.609985 +1073.47998 +1089.900024 +1084.780029 +1085.199951 +1100.089966 +1104.609985 +1078.300049 +1059.790039 +1059.780029 +1084.099976 +1087.199951 +1102.839966 +1118.859985 +1115.800049 +1118.540039 +1120.310059 +1118.329956 +1139.089966 +1141.209961 +1142.23999 +1138.650024 +1151.060059 +1142.660034 +1137.030029 +1150.339966 +1157.420044 +1149.5 +1128.52002 +1140.199951 +1139.449951 +1129.900024 +1144.800049 +1170.349976 +1167.099976 +1158.310059 +1139.930054 +1136.76001 +1137.069946 +1119.380005 +1123.089966 +1134.359985 +1142.920044 +1149.560059 +1139.930054 +1144.890015 +1144.650024 +1149.369995 +1157.130005 +1161.02002 +1148.079956 +1154.670044 +1165.27002 +1172.51001 +1164.890015 +1160.709961 +1155.140015 +1156.550049 +1145.599976 +1138.410034 +1146.189941 +1127.569946 +1138.880005 +1127.579956 +1119.310059 +1128.180054 +1132.150024 +1133.280029 +1133.060059 +1100.640015 +1113.569946 +1130.199951 +1122.199951 +1094.439941 +1090.02002 +1083.51001 +1080.170044 +1096.219971 +1111.939941 +1107.5 +1118.51001 +1116.47998 +1104.180054 +1083.339966 +1097.97998 +1080.949951 +1089.839966 +1109.430054 +1109.380005 +1109.890015 +1106.72998 +1131.780029 +1153.839966 +1146.140015 +1162.77002 +1157.540039 +1164.310059 +1168.26001 +1165.579956 +1154.089966 +1153.040039 +1166.160034 +1165.550049 +1170.290039 +1151.849976 +1153.589966 +1148.699951 +1131.869995 +1138.48999 +1144.579956 +1147.390015 +1146.540039 +1136.76001 +1125.400024 +1126.339966 +1122.72998 +1125.290039 +1117.800049 +1130.469971 +1103.689941 +1111.01001 +1102.550049 +1128.369995 +1126.069946 +1124.469971 +1125.170044 +1107.829956 +1100.959961 +1093.140015 +1091.47998 +1076.319946 +1065.449951 +1076.920044 +1086.459961 +1084.560059 +1073.430054 +1052.670044 +1049.48999 +1088.849976 +1073.01001 +1054.98999 +1074.560059 +1097.280029 +1091.069946 +1098.22998 +1106.589966 +1091.880005 +1079.880005 +1086.02002 +1097.079956 +1083.819946 +1074.550049 +1067.660034 +1064.660034 +1067.140015 +1040.680054 +1040.689941 +1049.900024 +1029.150024 +1027.530029 +1030.73999 +1013.599976 +1020.26001 +1009.559998 +1007.27002 +1036.170044 +1037.140015 +1019.98999 +1006.289978 +989.140015 +992.719971 +976.140015 +973.530029 +990.640015 +989.820007 +968.650024 +948.090027 +953.98999 +989.030029 +976.97998 +952.830017 +920.469971 +927.369995 +921.390015 +917.929993 +900.940002 +906.039978 +881.559998 +847.75 +819.849976 +797.700012 +843.429993 +838.679993 +852.840027 +898.960022 +902.780029 +911.619995 +884.659973 +864.23999 +834.599976 +859.570007 +876.77002 +905.460022 +908.640015 +903.799988 +884.210022 +919.619995 +930.25 +928.77002 +950.700012 +937.429993 +949.359985 +962.700012 +940.859985 +947.950012 +934.820007 +917.869995 +917.799988 +916.070007 +878.02002 +893.400024 +879.150024 +893.919983 +902.960022 +909.580017 +909.450012 +886.909973 +889.809998 +891.099976 +873.52002 +869.460022 +843.320007 +845.390015 +833.700012 +819.289978 +839.659973 +854.950012 +827.369995 +815.280029 +847.909973 +827.909973 +818.950012 +800.580017 +785.280029 +798.549988 +776.76001 +803.919983 +835.320007 +841.440002 +881.27002 +860.02002 +879.200012 +884.390015 +899.719971 +890.159973 +896.140015 +882.5 +897.650024 +890.22998 +882.150024 +890.710022 +885.76001 +900.960022 +908.349976 +915.390015 +923.76001 +902.650024 +894.73999 +876.190002 +882.950012 +882.530029 +904.27002 +909.830017 +900.359985 +896.73999 +914.150024 +933.76001 +930.549988 +932.869995 +913.309998 +938.869995 +936.309998 +934.530029 +920.75 +917.580017 +906.549988 +912.22998 +892 +904.450012 +904.960022 +901.580017 +889.47998 +910.400024 +902.98999 +891.119995 +884.25 +895.76001 +897.380005 +892.469971 +889.659973 +875.400024 +879.390015 +879.820007 +909.030029 +908.590027 +929.01001 +922.929993 +909.929993 +927.570007 +927.570007 +926.26001 +931.659973 +918.219971 +914.599976 +901.780029 +887.619995 +878.359985 +887.340027 +861.400024 +847.47998 +858.539978 +864.359985 +844.609985 +855.700012 +860.320007 +848.200012 +843.590027 +838.150024 +829.690002 +835.969971 +829.200012 +818.679993 +817.369995 +834.890015 +851.169983 +845.130005 +837.099976 +848.169983 +832.580017 +838.570007 +827.549988 +837.280029 +841.150024 +834.809998 +821.98999 +829.849976 +822.099976 +828.890015 +807.47998 +800.72998 +804.190002 +831.900024 +833.27002 +862.789978 +866.450012 +874.02002 +875.669983 +895.789978 +864.22998 +874.73999 +869.950012 +868.52002 +863.5 +848.179993 +858.47998 +880.900024 +876.450012 +878.849976 +879.929993 +878.289978 +865.98999 +871.580017 +868.299988 +885.22998 +890.809998 +879.909973 +893.580017 +892.01001 +911.369995 +919.02002 +911.429993 +898.809998 +914.840027 +917.840027 +916.919983 +916.299988 +930.080017 +926.549988 +934.390015 +929.619995 +920.27002 +933.409973 +945.109985 +942.299988 +939.280029 +946.669983 +944.299988 +920.77002 +919.72998 +923.419983 +931.869995 +933.219971 +951.47998 +953.219971 +949.640015 +963.590027 +967 +971.559998 +986.23999 +990.140015 +987.76001 +975.929993 +984.840027 +997.47998 +998.51001 +988.609985 +1010.73999 +1011.659973 +1010.090027 +994.700012 +995.690002 +981.640015 +983.450012 +975.320007 +985.820007 +976.219971 +974.5 +982.320007 +993.75 +985.700012 +1004.419983 +1007.840027 +1002.210022 +988.700012 +998.140015 +1003.859985 +1000.419983 +994.090027 +981.72998 +993.320007 +978.799988 +988.109985 +988.609985 +981.599976 +998.679993 +996.52002 +989.280029 +987.48999 +990.309998 +980.150024 +982.820007 +965.460022 +967.080017 +974.119995 +977.590027 +980.590027 +990.349976 +984.030029 +990.51001 +990.669983 +999.73999 +1002.349976 +1000.299988 +1003.27002 +993.059998 +993.710022 +996.72998 +996.789978 +1002.840027 +1008.01001 +1021.98999 +1026.27002 +1027.969971 +1021.390015 +1031.640015 +1023.169983 +1010.919983 +1016.419983 +1018.630005 +1014.809998 +1029.319946 +1025.969971 +1039.579956 +1036.300049 +1022.820007 +1029.030029 +1009.380005 +1003.27002 +996.849976 +1006.580017 +995.969971 +1018.219971 +1020.23999 +1029.849976 +1034.349976 +1039.25 +1033.780029 +1038.72998 +1038.060059 +1045.349976 +1049.47998 +1046.76001 +1050.069946 +1039.319946 +1044.680054 +1046.030029 +1030.359985 +1033.77002 +1028.910034 +1031.130005 +1046.790039 +1048.109985 +1046.939941 +1050.709961 +1059.02002 +1053.25 +1051.810059 +1058.050049 +1053.209961 +1047.109985 +1046.569946 +1058.530029 +1058.410034 +1050.349976 +1043.630005 +1034.150024 +1042.439941 +1033.650024 +1035.280029 +1052.079956 +1053.890015 +1058.449951 +1058.199951 +1070.119995 +1066.619995 +1064.72998 +1069.719971 +1061.5 +1069.300049 +1060.180054 +1059.050049 +1071.209961 +1074.140015 +1068.040039 +1075.130005 +1076.47998 +1089.180054 +1088.660034 +1092.939941 +1096.02002 +1094.040039 +1095.890015 +1109.47998 +1109.640015 +1111.920044 +1108.47998 +1122.219971 +1123.670044 +1126.329956 +1131.920044 +1121.859985 +1127.22998 +1121.219971 +1130.52002 +1132.050049 +1139.829956 +1138.77002 +1147.619995 +1143.939941 +1141.550049 +1155.369995 +1144.050049 +1128.47998 +1134.109985 +1131.130005 +1135.26001 +1136.030029 +1126.52002 +1128.589966 +1142.76001 +1139.810059 +1145.540039 +1157.76001 +1152.109985 +1145.810059 +1156.98999 +1151.819946 +1147.060059 +1144.109985 +1140.98999 +1139.089966 +1143.670044 +1144.910034 +1144.939941 +1155.969971 +1149.099976 +1151.030029 +1154.869995 +1156.859985 +1147.199951 +1140.579956 +1123.890015 +1106.780029 +1120.569946 +1104.48999 +1110.699951 +1123.75 +1122.319946 +1109.780029 +1095.400024 +1093.949951 +1091.329956 +1109.189941 +1108.060059 +1122.469971 +1127 +1126.209961 +1132.170044 +1141.810059 +1150.569946 +1148.160034 +1140.530029 +1139.319946 +1145.199951 +1129.439941 +1128.170044 +1128.839966 +1134.609985 +1135.819946 +1118.150024 +1124.089966 +1139.930054 +1140.599976 +1135.530029 +1138.109985 +1122.410034 +1113.890015 +1107.300049 +1117.48999 +1119.550049 +1121.530029 +1113.98999 +1098.699951 +1087.119995 +1095.449951 +1097.280029 +1096.439941 +1095.699951 +1084.099976 +1091.48999 +1088.680054 +1089.189941 +1093.560059 +1095.410034 +1113.050049 +1114.939941 +1121.280029 +1120.680054 +1121.199951 +1124.98999 +1116.640015 +1122.5 +1140.420044 +1142.180054 +1131.329956 +1136.469971 +1125.290039 +1132.01001 +1133.560059 +1132.050049 +1135.02002 +1130.300049 +1134.410034 +1144.060059 +1140.650024 +1134.430054 +1133.349976 +1136.199951 +1140.839966 +1128.939941 +1125.380005 +1116.209961 +1118.329956 +1109.109985 +1112.810059 +1114.349976 +1115.140015 +1111.469971 +1106.689941 +1101.390015 +1100.900024 +1108.670044 +1093.880005 +1096.839966 +1086.199951 +1084.069946 +1094.829956 +1095.420044 +1100.430054 +1101.719971 +1106.619995 +1099.689941 +1098.630005 +1080.699951 +1063.969971 +1065.219971 +1079.040039 +1075.790039 +1063.22998 +1064.800049 +1079.339966 +1081.709961 +1095.170044 +1091.22998 +1098.349976 +1095.680054 +1096.189941 +1104.959961 +1105.089966 +1107.77002 +1099.150024 +1104.23999 +1105.910034 +1118.310059 +1113.630005 +1121.300049 +1116.27002 +1118.380005 +1123.920044 +1125.819946 +1128.329956 +1120.369995 +1123.5 +1128.550049 +1122.199951 +1129.300049 +1113.560059 +1108.359985 +1110.109985 +1103.52002 +1110.060059 +1114.800049 +1114.579956 +1131.5 +1135.170044 +1134.47998 +1142.050049 +1130.650024 +1122.140015 +1124.390015 +1121.839966 +1113.650024 +1103.290039 +1108.199951 +1114.02002 +1103.22998 +1103.660034 +1106.48999 +1095.73999 +1094.800049 +1111.089966 +1125.400024 +1127.439941 +1130.199951 +1130.51001 +1130.560059 +1143.199951 +1161.670044 +1166.170044 +1164.890015 +1164.079956 +1162.910034 +1173.47998 +1184.170044 +1183.810059 +1175.430054 +1181.939941 +1183.550049 +1170.339966 +1177.23999 +1176.939941 +1181.76001 +1182.650024 +1178.569946 +1173.819946 +1191.369995 +1190.329956 +1191.170044 +1190.25 +1177.069946 +1182.810059 +1189.23999 +1188 +1198.680054 +1203.380005 +1205.719971 +1203.209961 +1194.199951 +1194.650024 +1205.449951 +1209.569946 +1210.130005 +1204.920044 +1213.540039 +1213.449951 +1213.550049 +1211.920044 +1202.079956 +1188.050049 +1183.73999 +1187.890015 +1186.189941 +1190.25 +1182.98999 +1187.699951 +1177.449951 +1184.52002 +1195.97998 +1184.630005 +1175.410034 +1167.869995 +1163.75 +1168.410034 +1174.069946 +1174.550049 +1171.359985 +1181.27002 +1189.410034 +1193.189941 +1189.890015 +1203.030029 +1201.719971 +1202.300049 +1191.98999 +1197.01001 +1205.300049 +1206.140015 +1210.119995 +1210.339966 +1200.75 +1201.589966 +1184.160034 +1190.800049 +1200.199951 +1211.369995 +1203.599976 +1210.410034 +1210.079956 +1210.469971 +1222.119995 +1225.310059 +1219.430054 +1207.01001 +1209.25 +1200.079956 +1206.829956 +1197.75 +1188.069946 +1190.209961 +1189.650024 +1183.780029 +1171.709961 +1172.530029 +1171.420044 +1174.280029 +1165.359985 +1181.410034 +1180.589966 +1172.920044 +1176.119995 +1181.390015 +1184.069946 +1191.140015 +1181.199951 +1181.209961 +1187.76001 +1173.790039 +1162.050049 +1142.619995 +1145.97998 +1152.780029 +1137.5 +1159.949951 +1152.119995 +1162.099976 +1151.829956 +1156.380005 +1143.219971 +1156.849976 +1162.160034 +1161.170044 +1175.650024 +1172.630005 +1171.349976 +1178.839966 +1166.219971 +1171.109985 +1159.359985 +1154.050049 +1165.689941 +1173.800049 +1185.560059 +1191.079956 +1189.280029 +1193.859985 +1194.069946 +1190.01001 +1197.619995 +1198.780029 +1191.5 +1202.219971 +1204.290039 +1196.02002 +1197.51001 +1197.26001 +1194.670044 +1200.930054 +1198.109985 +1200.819946 +1203.910034 +1206.579956 +1210.959961 +1216.959961 +1216.099976 +1213.609985 +1213.880005 +1200.72998 +1191.569946 +1190.689941 +1201.569946 +1199.849976 +1191.329956 +1194.439941 +1204.98999 +1194.939941 +1197.869995 +1211.859985 +1219.439941 +1222.209961 +1223.290039 +1226.5 +1227.920044 +1221.130005 +1229.349976 +1235.199951 +1227.040039 +1233.680054 +1229.030029 +1231.160034 +1236.790039 +1243.719971 +1234.180054 +1235.349976 +1244.119995 +1245.040039 +1235.859985 +1226.420044 +1223.130005 +1231.380005 +1229.130005 +1237.810059 +1230.390015 +1233.869995 +1219.339966 +1220.23999 +1219.02002 +1219.709961 +1221.72998 +1217.589966 +1209.589966 +1212.369995 +1205.099976 +1212.280029 +1208.410034 +1220.329956 +1221.589966 +1218.02002 +1233.390015 +1236.359985 +1231.670044 +1241.47998 +1240.560059 +1231.199951 +1227.160034 +1227.72998 +1237.910034 +1231.02002 +1221.339966 +1210.199951 +1214.619995 +1215.290039 +1215.630005 +1215.660034 +1216.890015 +1227.680054 +1228.810059 +1226.699951 +1214.469971 +1196.390015 +1191.48999 +1195.900024 +1187.329956 +1184.869995 +1177.680054 +1176.839966 +1186.569946 +1190.099976 +1178.140015 +1195.76001 +1177.800049 +1179.589966 +1199.380005 +1196.540039 +1191.380005 +1178.900024 +1198.410034 +1207.01001 +1202.76001 +1214.76001 +1219.939941 +1220.140015 +1222.810059 +1218.589966 +1220.650024 +1230.959961 +1234.719971 +1233.76001 +1229.01001 +1231.209961 +1242.800049 +1248.27002 +1254.849976 +1261.22998 +1265.609985 +1268.25 +1257.459961 +1257.47998 +1249.47998 +1264.670044 +1265.079956 +1262.089966 +1263.699951 +1257.369995 +1255.839966 +1259.369995 +1260.430054 +1267.430054 +1272.73999 +1270.939941 +1267.319946 +1259.920044 +1259.619995 +1262.790039 +1268.119995 +1268.660034 +1256.540039 +1258.170044 +1254.420044 +1248.290039 +1268.800049 +1273.459961 +1273.47998 +1285.449951 +1290.150024 +1289.689941 +1294.180054 +1286.060059 +1287.609985 +1282.930054 +1277.930054 +1285.040039 +1261.48999 +1263.819946 +1266.859985 +1264.680054 +1273.829956 +1283.719971 +1285.189941 +1280.079956 +1282.459961 +1270.839966 +1264.030029 +1265.02002 +1254.780029 +1265.650024 +1263.780029 +1266.98999 +1262.859985 +1275.530029 +1280 +1289.380005 +1287.23999 +1283.030029 +1292.670044 +1287.790039 +1289.430054 +1294.119995 +1280.660034 +1291.23999 +1289.140015 +1287.22998 +1278.26001 +1275.880005 +1278.469971 +1272.22998 +1281.420044 +1284.130005 +1297.47998 +1303.02002 +1305.329956 +1307.25 +1305.079956 +1297.22998 +1305.040039 +1301.670044 +1302.949951 +1301.609985 +1293.22998 +1302.890015 +1300.25 +1294.869995 +1297.810059 +1305.930054 +1311.560059 +1309.040039 +1295.5 +1296.619995 +1286.569946 +1288.119995 +1289.119995 +1285.329956 +1307.280029 +1309.930054 +1311.459961 +1311.280029 +1308.109985 +1301.73999 +1305.410034 +1309.719971 +1310.609985 +1305.189941 +1313.209961 +1308.119995 +1312.25 +1325.76001 +1324.660034 +1325.140015 +1322.849976 +1305.920044 +1291.23999 +1294.5 +1292.079956 +1270.319946 +1261.810059 +1267.030029 +1262.069946 +1256.579956 +1258.569946 +1272.880005 +1280.160034 +1259.869995 +1270.089966 +1285.709961 +1288.219971 +1265.290039 +1263.849976 +1256.150024 +1257.930054 +1252.300049 +1237.439941 +1223.689941 +1230.040039 +1256.160034 +1251.540039 +1240.130005 +1240.119995 +1252.199951 +1245.599976 +1244.5 +1250.560059 +1239.199951 +1246 +1272.869995 +1270.199951 +1280.189941 +1270.910034 +1274.079956 +1265.47998 +1267.339966 +1272.430054 +1258.599976 +1242.280029 +1236.199951 +1234.48999 +1236.859985 +1259.810059 +1249.130005 +1240.290039 +1260.910034 +1268.880005 +1268.400024 +1263.199951 +1278.550049 +1276.660034 +1270.920044 +1277.410034 +1280.27002 +1279.359985 +1275.77002 +1271.47998 +1265.949951 +1271.810059 +1266.73999 +1268.209961 +1285.579956 +1295.430054 +1297.47998 +1302.300049 +1297.52002 +1298.819946 +1292.98999 +1296.060059 +1295.089966 +1301.780029 +1304.280029 +1305.369995 +1303.819946 +1311.01001 +1313.25 +1300.26001 +1294.02002 +1298.920044 +1299.540039 +1313 +1318.069946 +1316.280029 +1319.660034 +1321.180054 +1317.640015 +1325.180054 +1318.030029 +1314.780029 +1326.369995 +1336.349976 +1336.589966 +1338.880005 +1335.849976 +1331.319946 +1334.109985 +1350.199951 +1353.219971 +1349.589966 +1350.660034 +1353.420044 +1349.949951 +1362.829956 +1365.619995 +1369.060059 +1364.050049 +1365.800049 +1366.959961 +1368.599976 +1377.02002 +1377.380005 +1382.219971 +1389.079956 +1377.339966 +1377.930054 +1377.939941 +1367.810059 +1367.339966 +1364.300049 +1379.780029 +1382.839966 +1385.719971 +1378.329956 +1380.900024 +1384.420044 +1393.219971 +1396.569946 +1399.76001 +1401.199951 +1400.5 +1402.810059 +1406.089966 +1400.949951 +1381.959961 +1386.719971 +1399.47998 +1400.630005 +1396.709961 +1409.119995 +1414.76001 +1412.900024 +1407.290039 +1409.839966 +1413.040039 +1411.560059 +1413.209961 +1425.48999 +1427.089966 +1422.47998 +1425.550049 +1423.530029 +1418.300049 +1410.76001 +1416.900024 +1426.839966 +1424.72998 +1418.300049 +1416.599976 +1418.339966 +1409.709961 +1412.839966 +1412.109985 +1414.849976 +1423.819946 +1430.72998 +1431.900024 +1430.619995 +1426.369995 +1430.5 +1422.949951 +1427.98999 +1440.130005 +1423.900024 +1422.180054 +1420.619995 +1428.819946 +1438.23999 +1445.939941 +1448.390015 +1446.98999 +1448 +1450.02002 +1448.310059 +1438.060059 +1433.369995 +1444.26001 +1455.300049 +1456.810059 +1455.540039 +1459.680054 +1457.630005 +1456.380005 +1451.189941 +1449.369995 +1399.040039 +1406.819946 +1403.170044 +1387.170044 +1374.119995 +1395.410034 +1391.969971 +1401.890015 +1402.839966 +1406.599976 +1377.949951 +1387.170044 +1392.280029 +1386.949951 +1402.060059 +1410.939941 +1435.040039 +1434.540039 +1436.109985 +1437.5 +1428.609985 +1417.22998 +1422.530029 +1420.859985 +1424.550049 +1437.77002 +1439.369995 +1443.76001 +1444.609985 +1448.390015 +1438.869995 +1447.800049 +1452.849976 +1468.329956 +1471.47998 +1472.5 +1470.72998 +1484.349976 +1480.930054 +1480.410034 +1495.420044 +1494.25 +1494.069946 +1482.369995 +1486.300049 +1495.920044 +1502.390015 +1505.619995 +1509.47998 +1507.719971 +1512.579956 +1491.469971 +1505.849976 +1503.150024 +1501.189941 +1514.140015 +1512.75 +1522.75 +1525.099976 +1524.119995 +1522.280029 +1507.51001 +1515.72998 +1518.109985 +1530.22998 +1530.619995 +1536.339966 +1539.180054 +1530.949951 +1517.380005 +1490.719971 +1507.670044 +1509.119995 +1493 +1515.670044 +1522.969971 +1532.910034 +1531.050049 +1533.699951 +1512.839966 +1522.189941 +1502.560059 +1497.73999 +1492.890015 +1506.339966 +1505.709961 +1503.349976 +1519.430054 +1524.869995 +1525.400024 +1530.439941 +1531.849976 +1510.119995 +1518.76001 +1547.699951 +1552.5 +1549.52002 +1549.369995 +1546.170044 +1553.079956 +1534.099976 +1541.569946 +1511.040039 +1518.089966 +1482.660034 +1458.949951 +1473.910034 +1455.27002 +1465.810059 +1472.199951 +1433.060059 +1467.670044 +1476.709961 +1497.48999 +1453.089966 +1453.640015 +1452.920044 +1426.540039 +1406.699951 +1411.27002 +1445.939941 +1445.550049 +1447.119995 +1464.069946 +1462.5 +1479.369995 +1466.790039 +1432.359985 +1463.76001 +1457.640015 +1473.98999 +1489.420044 +1472.290039 +1478.550049 +1453.550049 +1451.699951 +1471.48999 +1471.560059 +1483.949951 +1484.25 +1476.650024 +1519.780029 +1529.030029 +1518.75 +1525.75 +1517.72998 +1517.209961 +1525.420044 +1531.380005 +1526.75 +1547.040039 +1546.630005 +1539.589966 +1542.839966 +1557.589966 +1552.579956 +1565.150024 +1562.469971 +1554.410034 +1561.800049 +1548.709961 +1538.530029 +1541.23999 +1540.079956 +1500.630005 +1506.329956 +1519.589966 +1515.880005 +1514.400024 +1535.280029 +1540.97998 +1531.02002 +1549.380005 +1508.439941 +1509.650024 +1502.170044 +1520.27002 +1475.619995 +1474.77002 +1453.699951 +1439.180054 +1481.050049 +1470.579956 +1451.150024 +1458.73999 +1433.27002 +1439.699951 +1416.77002 +1440.699951 +1407.219971 +1428.22998 +1469.02002 +1469.719971 +1481.140015 +1472.420044 +1462.790039 +1485.01001 +1507.339966 +1504.660034 +1515.959961 +1477.650024 +1486.589966 +1488.410034 +1467.949951 +1445.900024 +1454.97998 +1453 +1460.119995 +1484.459961 +1496.449951 +1497.660034 +1476.27002 +1478.48999 +1468.359985 +1447.160034 +1447.160034 +1411.630005 +1416.180054 +1390.189941 +1409.130005 +1420.329956 +1401.02002 +1416.25 +1380.949951 +1373.199951 +1333.25 +1325.189941 +1310.5 +1338.599976 +1352.069946 +1330.609985 +1353.959961 +1362.300049 +1355.810059 +1378.550049 +1395.420044 +1380.819946 +1336.640015 +1326.449951 +1336.910034 +1331.290039 +1339.130005 +1348.859985 +1367.209961 +1348.859985 +1349.98999 +1348.780029 +1360.030029 +1342.530029 +1353.109985 +1371.800049 +1381.290039 +1380.02002 +1367.680054 +1330.630005 +1331.339966 +1326.75 +1333.699951 +1304.339966 +1293.369995 +1273.369995 +1320.650024 +1308.77002 +1315.47998 +1288.140015 +1276.599976 +1330.73999 +1298.420044 +1329.51001 +1349.880005 +1352.98999 +1341.130005 +1325.76001 +1315.219971 +1322.699951 +1370.180054 +1367.530029 +1369.310059 +1370.400024 +1372.540039 +1365.540039 +1354.48999 +1360.550049 +1332.829956 +1328.319946 +1334.430054 +1364.709961 +1365.560059 +1390.329956 +1388.170044 +1375.939941 +1379.930054 +1388.819946 +1397.839966 +1396.369995 +1390.939941 +1385.589966 +1409.339966 +1413.900024 +1407.48999 +1418.26001 +1392.569946 +1397.680054 +1388.280029 +1403.579956 +1403.040039 +1408.660034 +1423.569946 +1425.349976 +1426.630005 +1413.400024 +1390.709961 +1394.349976 +1375.930054 +1385.349976 +1390.839966 +1398.26001 +1400.380005 +1385.670044 +1377.650024 +1377.199951 +1404.050049 +1360.680054 +1361.76001 +1358.439941 +1335.48999 +1339.869995 +1360.030029 +1360.140015 +1350.930054 +1337.810059 +1342.829956 +1317.930054 +1318 +1314.290039 +1321.969971 +1283.150024 +1278.380005 +1280 +1284.910034 +1261.52002 +1262.900024 +1252.310059 +1273.699951 +1244.689941 +1253.390015 +1239.48999 +1228.300049 +1214.910034 +1245.359985 +1260.319946 +1260.680054 +1260 +1277 +1282.189941 +1252.540039 +1257.76001 +1234.369995 +1263.199951 +1284.26001 +1267.380005 +1260.310059 +1249.01001 +1284.880005 +1289.189941 +1266.069946 +1296.319946 +1305.319946 +1289.589966 +1285.829956 +1292.930054 +1298.199951 +1278.599976 +1266.689941 +1274.540039 +1277.719971 +1292.199951 +1266.839966 +1271.51001 +1281.660034 +1300.680054 +1282.829956 +1277.579956 +1274.97998 +1236.829956 +1242.310059 +1267.790039 +1224.51001 +1232.040039 +1249.050049 +1251.699951 +1192.699951 +1213.599976 +1156.390015 +1206.51001 +1255.079956 +1207.089966 +1188.219971 +1185.869995 +1209.180054 +1213.27002 +1106.420044 +1166.359985 +1161.060059 +1114.280029 +1099.22998 +1056.890015 +996.22998 +984.940002 +909.919983 +899.219971 +1003.349976 +998.01001 +907.840027 +946.429993 +940.549988 +985.400024 +955.049988 +896.780029 +908.109985 +876.77002 +848.919983 +940.51001 +930.090027 +954.090027 +968.75 +966.299988 +1005.75 +952.77002 +904.880005 +930.98999 +919.210022 +898.950012 +852.299988 +911.289978 +873.289978 +850.75 +859.119995 +806.580017 +752.440002 +800.030029 +851.809998 +857.390015 +887.679993 +896.23999 +816.210022 +848.809998 +870.73999 +845.219971 +876.070007 +909.700012 +888.669983 +899.23999 +873.590027 +879.72998 +868.570007 +913.179993 +904.419983 +885.280029 +887.880005 +871.630005 +863.159973 +868.150024 +872.799988 +869.419983 +890.640015 +903.25 +931.799988 +927.450012 +934.700012 +906.650024 +909.72998 +890.349976 +870.26001 +871.789978 +842.619995 +843.73999 +850.119995 +805.219971 +840.23999 +827.5 +831.950012 +836.570007 +845.710022 +874.090027 +845.140015 +825.880005 +825.440002 +838.51001 +832.22998 +845.849976 +868.599976 +869.890015 +827.159973 +833.73999 +835.190002 +826.840027 +789.169983 +788.419983 +778.940002 +770.049988 +743.330017 +773.140015 +764.900024 +752.830017 +735.090027 +700.820007 +696.330017 +712.869995 +682.549988 +683.380005 +676.530029 +719.599976 +721.359985 +750.73999 +756.549988 +753.890015 +778.119995 +794.349976 +784.039978 +768.539978 +822.919983 +806.119995 +813.880005 +832.859985 +815.940002 +787.530029 +797.869995 +811.080017 +834.380005 +842.5 +835.47998 +815.549988 +825.159973 +856.559998 +858.72998 +841.5 +852.059998 +865.299988 +869.599976 +832.390015 +850.080017 +843.549988 +851.919983 +866.22998 +857.51001 +855.159973 +873.640015 +872.809998 +877.52002 +907.23999 +903.799988 +919.530029 +907.390015 +929.22998 +909.23999 +908.349976 +883.919983 +893.070007 +882.880005 +909.710022 +908.130005 +903.469971 +888.330017 +887 +910.330017 +893.059998 +906.830017 +919.140015 +942.869995 +944.73999 +931.76001 +942.460022 +940.090027 +939.140015 +942.429993 +939.150024 +944.890015 +946.210022 +923.719971 +911.969971 +910.710022 +918.369995 +921.22998 +893.039978 +895.099976 +900.940002 +920.26001 +918.900024 +927.22998 +919.320007 +923.330017 +896.419983 +898.719971 +881.030029 +879.559998 +882.679993 +879.130005 +901.049988 +905.840027 +932.679993 +940.73999 +940.380005 +951.130005 +954.580017 +954.070007 +976.289978 +979.26001 +982.179993 +979.619995 +975.150024 +986.75 +987.47998 +1002.630005 +1005.650024 +1002.719971 +997.080017 +1010.47998 +1007.099976 +994.349976 +1005.809998 +1012.72998 +1004.090027 +979.72998 +989.669983 +996.460022 +1007.369995 +1026.130005 +1025.569946 +1028 +1028.119995 +1030.97998 +1028.930054 +1020.619995 +998.039978 +994.75 +1003.23999 +1016.400024 +1025.390015 +1033.369995 +1044.140015 +1042.72998 +1049.339966 +1052.630005 +1068.76001 +1065.48999 +1068.300049 +1064.660034 +1071.660034 +1060.869995 +1050.780029 +1044.380005 +1062.97998 +1060.609985 +1057.079956 +1029.849976 +1025.209961 +1040.459961 +1054.719971 +1057.579956 +1065.47998 +1071.48999 +1076.189941 +1073.189941 +1092.02002 +1096.560059 +1087.680054 +1097.910034 +1091.060059 +1081.400024 +1092.910034 +1079.599976 +1066.949951 +1063.410034 +1042.630005 +1066.109985 +1036.189941 +1042.880005 +1045.410034 +1046.5 +1066.630005 +1069.300049 +1093.079956 +1093.01001 +1098.51001 +1087.23999 +1093.47998 +1109.300049 +1110.319946 +1109.800049 +1094.900024 +1091.380005 +1106.23999 +1105.650024 +1110.630005 +1091.48999 +1095.630005 +1108.859985 +1109.23999 +1099.920044 +1105.97998 +1103.25 +1091.939941 +1095.949951 +1102.349976 +1106.410034 +1114.109985 +1107.930054 +1109.180054 +1096.079956 +1102.469971 +1114.050049 +1118.02002 +1120.589966 +1126.47998 +1127.780029 +1126.199951 +1126.420044 +1115.099976 +1132.98999 +1136.52002 +1137.140015 +1141.689941 +1144.97998 +1146.97998 +1136.219971 +1145.680054 +1148.459961 +1136.030029 +1150.22998 +1138.040039 +1116.47998 +1091.76001 +1096.780029 +1092.170044 +1097.5 +1084.530029 +1073.869995 +1089.189941 +1103.319946 +1097.280029 +1063.109985 +1066.189941 +1056.73999 +1070.52002 +1068.130005 +1078.469971 +1075.51001 +1094.869995 +1099.51001 +1106.75 +1109.170044 +1108.01001 +1094.599976 +1105.23999 +1102.939941 +1104.48999 +1115.709961 +1118.310059 +1118.790039 +1122.969971 +1138.699951 +1138.5 +1140.449951 +1145.609985 +1150.23999 +1149.98999 +1150.51001 +1159.459961 +1166.209961 +1165.829956 +1159.900024 +1165.810059 +1174.170044 +1167.719971 +1165.72998 +1166.589966 +1173.219971 +1173.27002 +1169.430054 +1178.099976 +1187.439941 +1189.439941 +1182.449951 +1186.439941 +1194.369995 +1196.47998 +1197.300049 +1210.650024 +1211.670044 +1192.130005 +1197.52002 +1207.170044 +1205.939941 +1208.670044 +1217.280029 +1212.050049 +1183.709961 +1191.359985 +1206.780029 +1186.689941 +1202.26001 +1173.599976 +1165.869995 +1128.150024 +1110.880005 +1159.72998 +1155.790039 +1171.670044 +1157.439941 +1135.680054 +1136.939941 +1120.800049 +1115.050049 +1071.589966 +1087.689941 +1073.650024 +1074.030029 +1067.949951 +1103.060059 +1089.410034 +1070.709961 +1098.380005 +1102.829956 +1064.880005 +1050.469971 +1062 +1055.689941 +1086.839966 +1091.599976 +1089.630005 +1115.22998 +1114.609985 +1116.040039 +1117.51001 +1113.199951 +1095.310059 +1092.040039 +1073.689941 +1076.76001 +1074.569946 +1041.23999 +1030.709961 +1027.369995 +1022.580017 +1028.060059 +1060.27002 +1070.25 +1077.959961 +1078.75 +1095.339966 +1095.170044 +1096.47998 +1064.880005 +1071.25 +1083.47998 +1069.589966 +1093.670044 +1102.660034 +1115.01001 +1113.839966 +1106.130005 +1101.530029 +1101.599976 +1125.859985 +1120.459961 +1127.23999 +1125.810059 +1121.640015 +1127.790039 +1121.060059 +1089.469971 +1083.609985 +1079.25 +1079.380005 +1092.540039 +1094.160034 +1075.630005 +1071.689941 +1067.359985 +1051.869995 +1055.329956 +1047.219971 +1064.589966 +1048.920044 +1049.329956 +1080.290039 +1090.099976 +1104.51001 +1091.839966 +1098.869995 +1104.180054 +1109.550049 +1121.900024 +1121.099976 +1125.069946 +1124.660034 +1125.589966 +1142.709961 +1139.780029 +1134.280029 +1124.829956 +1148.670044 +1142.160034 +1147.699951 +1144.72998 +1141.199951 +1146.23999 +1137.030029 +1160.75 +1159.969971 +1158.060059 +1165.150024 +1165.319946 +1169.77002 +1178.099976 +1173.810059 +1176.189941 +1184.709961 +1165.900024 +1178.170044 +1180.26001 +1183.079956 +1185.619995 +1185.640015 +1182.449951 +1183.780029 +1183.26001 +1184.380005 +1193.569946 +1197.959961 +1221.060059 +1225.849976 +1223.25 +1213.400024 +1218.709961 +1213.540039 +1199.209961 +1197.75 +1178.339966 +1178.589966 +1196.689941 +1199.72998 +1197.839966 +1180.72998 +1198.349976 +1189.400024 +1187.76001 +1180.550049 +1206.069946 +1221.530029 +1224.709961 +1223.119995 +1223.75 +1228.280029 +1233 +1240.400024 +1240.459961 +1241.589966 +1235.22998 +1242.869995 +1243.910034 +1247.079956 +1254.599976 +1258.839966 +1256.77002 +1257.540039 +1258.51001 +1259.780029 +1257.880005 +1257.640015 +1271.869995 +1270.199951 +1276.560059 +1273.849976 +1271.5 +1269.75 +1274.47998 +1285.959961 +1283.76001 +1293.23999 +1295.02002 +1281.920044 +1280.26001 +1283.349976 +1290.839966 +1291.180054 +1296.630005 +1299.540039 +1276.339966 +1286.119995 +1307.589966 +1304.030029 +1307.099976 +1310.869995 +1319.050049 +1324.569946 +1320.880005 +1321.869995 +1329.150024 +1332.319946 +1328.01001 +1336.319946 +1340.430054 +1343.01001 +1315.439941 +1307.400024 +1306.099976 +1319.880005 +1327.219971 +1306.329956 +1308.439941 +1330.969971 +1321.150024 +1310.130005 +1321.819946 +1320.02002 +1295.109985 +1304.280029 +1296.390015 +1281.869995 +1256.880005 +1273.719971 +1279.209961 +1298.380005 +1293.77002 +1297.540039 +1309.660034 +1313.800049 +1310.189941 +1319.439941 +1328.26001 +1325.829956 +1332.410034 +1332.869995 +1332.630005 +1335.540039 +1333.51001 +1328.170044 +1324.459961 +1314.160034 +1314.410034 +1314.52002 +1319.680054 +1305.140015 +1312.619995 +1330.359985 +1337.380005 +1335.25 +1347.23999 +1355.660034 +1360.47998 +1363.609985 +1361.219971 +1356.619995 +1347.319946 +1335.099976 +1340.199951 +1346.290039 +1357.160034 +1342.079956 +1348.650024 +1337.77002 +1329.469971 +1328.97998 +1340.680054 +1343.599976 +1333.27002 +1317.369995 +1316.280029 +1320.469971 +1325.689941 +1331.099976 +1345.199951 +1314.550049 +1312.939941 +1300.160034 +1286.170044 +1284.939941 +1279.560059 +1289 +1270.97998 +1271.829956 +1287.869995 +1265.420044 +1267.640015 +1271.5 +1278.359985 +1295.52002 +1287.140015 +1283.5 +1268.449951 +1280.099976 +1296.670044 +1307.410034 +1320.640015 +1339.670044 +1337.880005 +1339.219971 +1353.219971 +1343.800049 +1319.48999 +1313.640015 +1317.719971 +1308.869995 +1316.140015 +1305.439941 +1326.72998 +1325.839966 +1343.800049 +1345.02002 +1337.430054 +1331.939941 +1304.890015 +1300.670044 +1292.280029 +1286.939941 +1254.050049 +1260.339966 +1200.069946 +1199.380005 +1119.459961 +1172.530029 +1120.76001 +1172.640015 +1178.810059 +1204.48999 +1192.76001 +1193.890015 +1140.650024 +1123.530029 +1123.819946 +1162.349976 +1177.599976 +1159.27002 +1176.800049 +1210.079956 +1212.920044 +1218.890015 +1204.420044 +1173.969971 +1165.23999 +1198.619995 +1185.900024 +1154.22998 +1162.27002 +1172.869995 +1188.680054 +1209.109985 +1216.01001 +1204.089966 +1202.089966 +1166.76001 +1129.560059 +1136.430054 +1162.949951 +1175.380005 +1151.060059 +1160.400024 +1131.420044 +1099.22998 +1123.949951 +1144.030029 +1164.969971 +1155.459961 +1194.890015 +1195.540039 +1207.25 +1203.660034 +1224.579956 +1200.859985 +1225.380005 +1209.880005 +1215.390015 +1238.25 +1254.189941 +1229.050049 +1242 +1284.589966 +1285.089966 +1253.300049 +1218.280029 +1237.900024 +1261.150024 +1253.22998 +1261.119995 +1275.920044 +1229.099976 +1239.699951 +1263.849976 +1251.780029 +1257.810059 +1236.910034 +1216.130005 +1215.650024 +1192.97998 +1188.040039 +1161.790039 +1158.670044 +1192.550049 +1195.189941 +1246.959961 +1244.579956 +1244.280029 +1257.079956 +1258.469971 +1261.01001 +1234.349976 +1255.189941 +1236.469971 +1225.72998 +1211.819946 +1215.75 +1219.660034 +1205.349976 +1241.300049 +1243.719971 +1254 +1265.329956 +1265.430054 +1249.640015 +1263.02002 +1257.599976 +1277.060059 +1277.300049 +1281.060059 +1277.810059 +1280.699951 +1292.079956 +1292.47998 +1295.5 +1289.089966 +1293.670044 +1308.040039 +1314.5 +1315.380005 +1316 +1314.650024 +1326.060059 +1318.430054 +1316.329956 +1313.01001 +1312.410034 +1324.089966 +1325.540039 +1344.900024 +1344.329956 +1347.050049 +1349.959961 +1351.949951 +1342.640015 +1351.77002 +1350.5 +1343.22998 +1358.040039 +1361.22998 +1362.209961 +1357.660034 +1363.459961 +1365.73999 +1367.589966 +1372.180054 +1365.680054 +1374.089966 +1369.630005 +1364.329956 +1343.359985 +1352.630005 +1365.910034 +1370.869995 +1371.089966 +1395.949951 +1394.280029 +1402.599976 +1404.170044 +1409.75 +1405.52002 +1402.890015 +1392.780029 +1397.109985 +1416.51001 +1412.52002 +1405.540039 +1403.280029 +1408.469971 +1419.040039 +1413.380005 +1398.959961 +1398.079956 +1382.199951 +1358.589966 +1368.709961 +1387.569946 +1370.26001 +1369.569946 +1390.780029 +1385.140015 +1376.920044 +1378.530029 +1366.939941 +1371.969971 +1390.689941 +1399.97998 +1403.359985 +1397.910034 +1405.819946 +1402.310059 +1391.569946 +1369.099976 +1369.579956 +1363.719971 +1354.579956 +1357.98999 +1353.390015 +1338.349976 +1330.660034 +1324.800049 +1304.859985 +1295.219971 +1315.98999 +1316.630005 +1318.859985 +1320.680054 +1317.819946 +1332.420044 +1313.319946 +1310.329956 +1278.040039 +1278.180054 +1285.5 +1315.130005 +1314.98999 +1325.660034 +1308.930054 +1324.180054 +1314.880005 +1329.099976 +1342.839966 +1344.780029 +1357.97998 +1355.689941 +1325.51001 +1335.02002 +1313.719971 +1319.98999 +1331.849976 +1329.040039 +1362.160034 +1365.51001 +1374.02002 +1367.579956 +1354.680054 +1352.459961 +1341.469971 +1341.449951 +1334.76001 +1356.780029 +1353.640015 +1363.670044 +1372.780029 +1376.51001 +1362.660034 +1350.52002 +1338.310059 +1337.890015 +1360.02002 +1385.969971 +1385.300049 +1379.319946 +1375.319946 +1365 +1390.98999 +1394.22998 +1401.349976 +1402.219971 +1402.800049 +1405.869995 +1404.109985 +1403.930054 +1405.530029 +1415.51001 +1418.160034 +1418.130005 +1413.170044 +1413.48999 +1402.079956 +1411.130005 +1410.439941 +1409.300049 +1410.48999 +1399.47998 +1406.579956 +1404.939941 +1403.439941 +1432.119995 +1437.920044 +1429.079956 +1433.560059 +1436.560059 +1459.98999 +1465.77002 +1461.189941 +1459.319946 +1461.050049 +1460.26001 +1460.150024 +1456.890015 +1441.589966 +1433.319946 +1447.150024 +1440.670044 +1444.48999 +1445.75 +1450.98999 +1461.400024 +1460.930054 +1455.880005 +1441.47998 +1432.560059 +1432.839966 +1428.589966 +1440.130005 +1454.920044 +1460.910034 +1457.339966 +1433.189941 +1433.819946 +1413.109985 +1408.75 +1412.969971 +1411.939941 +1412.160034 +1427.589966 +1414.199951 +1417.26001 +1428.390015 +1394.530029 +1377.51001 +1379.849976 +1380.030029 +1374.530029 +1355.48999 +1353.329956 +1359.880005 +1386.890015 +1387.810059 +1391.030029 +1409.150024 +1406.290039 +1398.939941 +1409.930054 +1415.949951 +1416.180054 +1409.459961 +1407.050049 +1409.280029 +1413.939941 +1418.069946 +1418.550049 +1427.839966 +1428.47998 +1419.449951 +1413.579956 +1430.359985 +1446.790039 +1435.810059 +1443.689941 +1430.150024 +1426.660034 +1419.829956 +1418.099976 +1402.430054 +1426.189941 +1462.420044 +1459.369995 +1466.469971 +1461.890015 +1457.150024 +1461.02002 +1472.119995 +1472.050049 +1470.680054 +1472.339966 +1472.630005 +1480.939941 +1485.97998 +1492.560059 +1494.810059 +1494.819946 +1502.959961 +1500.180054 +1507.839966 +1501.959961 +1498.109985 +1513.170044 +1495.709961 +1511.290039 +1512.119995 +1509.390015 +1517.930054 +1517.01001 +1519.430054 +1520.329956 +1521.380005 +1519.790039 +1530.939941 +1511.949951 +1502.420044 +1515.599976 +1487.849976 +1496.939941 +1515.98999 +1514.680054 +1518.199951 +1525.199951 +1539.790039 +1541.459961 +1544.26001 +1551.180054 +1556.219971 +1552.47998 +1554.52002 +1563.22998 +1560.699951 +1552.099976 +1548.339966 +1558.709961 +1545.800049 +1556.890015 +1551.689941 +1563.77002 +1562.849976 +1569.189941 +1562.170044 +1570.25 +1553.689941 +1559.97998 +1553.280029 +1563.069946 +1568.609985 +1587.72998 +1593.369995 +1588.849976 +1552.359985 +1574.569946 +1552.01001 +1541.609985 +1555.25 +1562.5 +1578.780029 +1578.790039 +1585.160034 +1582.23999 +1593.609985 +1597.569946 +1582.699951 +1597.589966 +1614.420044 +1617.5 +1625.959961 +1632.689941 +1626.670044 +1633.699951 +1633.77002 +1650.339966 +1658.780029 +1650.469971 +1667.469971 +1666.290039 +1669.160034 +1655.349976 +1650.51001 +1649.599976 +1660.060059 +1648.359985 +1654.410034 +1630.73999 +1640.420044 +1631.380005 +1608.900024 +1622.560059 +1643.380005 +1642.810059 +1626.130005 +1612.52002 +1636.359985 +1626.72998 +1639.040039 +1651.810059 +1628.930054 +1588.189941 +1592.430054 +1573.089966 +1588.030029 +1603.26001 +1613.199951 +1606.280029 +1614.959961 +1614.079956 +1615.410034 +1631.890015 +1640.459961 +1652.319946 +1652.619995 +1675.02002 +1680.189941 +1682.5 +1676.26001 +1680.910034 +1689.369995 +1692.089966 +1695.530029 +1692.390015 +1685.939941 +1690.25 +1691.650024 +1685.329956 +1685.959961 +1685.72998 +1706.869995 +1709.670044 +1707.140015 +1697.369995 +1690.910034 +1697.47998 +1691.420044 +1689.469971 +1694.160034 +1685.390015 +1661.319946 +1655.829956 +1646.060059 +1652.349976 +1642.800049 +1656.959961 +1663.5 +1656.780029 +1630.47998 +1634.959961 +1638.170044 +1632.969971 +1639.77002 +1653.079956 +1655.079956 +1655.170044 +1671.709961 +1683.98999 +1689.130005 +1683.420044 +1687.98999 +1697.599976 +1704.76001 +1725.52002 +1722.339966 +1709.910034 +1701.839966 +1697.420044 +1692.77002 +1698.670044 +1691.75 +1681.550049 +1695 +1693.869995 +1678.660034 +1690.5 +1676.119995 +1655.449951 +1656.400024 +1692.560059 +1703.199951 +1710.140015 +1698.060059 +1721.540039 +1733.150024 +1744.5 +1744.660034 +1754.670044 +1746.380005 +1752.069946 +1759.77002 +1762.109985 +1771.949951 +1763.310059 +1756.540039 +1761.640015 +1767.930054 +1762.969971 +1770.48999 +1747.150024 +1770.609985 +1771.890015 +1767.689941 +1782 +1790.619995 +1798.180054 +1791.530029 +1787.869995 +1781.369995 +1795.849976 +1804.76001 +1802.47998 +1802.75 +1807.22998 +1805.810059 +1800.900024 +1795.150024 +1792.810059 +1785.030029 +1805.089966 +1808.369995 +1802.619995 +1782.219971 +1775.5 +1775.319946 +1786.540039 +1781 +1810.650024 +1809.599976 +1818.319946 +1827.98999 +1833.319946 +1842.02002 +1841.400024 +1841.069946 +1848.359985 +1831.97998 +1831.369995 +1826.77002 +1837.880005 +1837.48999 +1838.130005 +1842.369995 +1819.199951 +1838.880005 +1848.380005 +1845.890015 +1838.699951 +1843.800049 +1844.859985 +1828.459961 +1790.290039 +1781.560059 +1792.5 +1774.199951 +1794.189941 +1782.589966 +1741.890015 +1755.199951 +1751.640015 +1773.430054 +1797.02002 +1799.839966 +1819.75 +1819.26001 +1829.829956 +1838.630005 +1840.76001 +1828.75 +1839.780029 +1836.25 +1847.609985 +1845.119995 +1845.160034 +1854.290039 +1859.449951 +1845.72998 +1873.910034 +1873.810059 +1877.030029 +1878.040039 +1877.170044 +1867.630005 +1868.199951 +1846.339966 +1841.130005 +1858.829956 +1872.25 +1860.77002 +1872.01001 +1866.52002 +1857.439941 +1865.619995 +1852.560059 +1849.040039 +1857.619995 +1872.339966 +1885.52002 +1890.900024 +1888.77002 +1865.089966 +1845.040039 +1851.959961 +1872.180054 +1833.079956 +1815.689941 +1830.609985 +1842.97998 +1862.310059 +1864.849976 +1871.890015 +1879.550049 +1875.390015 +1878.609985 +1863.400024 +1869.430054 +1878.329956 +1883.949951 +1883.680054 +1881.140015 +1884.660034 +1867.719971 +1878.209961 +1875.630005 +1878.47998 +1896.650024 +1897.449951 +1888.530029 +1870.849976 +1877.859985 +1885.079956 +1872.829956 +1888.030029 +1892.48999 +1900.530029 +1911.910034 +1909.780029 +1920.030029 +1923.569946 +1924.969971 +1924.23999 +1927.880005 +1940.459961 +1949.439941 +1951.27002 +1950.790039 +1943.890015 +1930.109985 +1936.160034 +1937.780029 +1941.98999 +1956.97998 +1959.47998 +1962.869995 +1962.609985 +1949.97998 +1959.530029 +1957.219971 +1960.959961 +1960.22998 +1973.319946 +1974.619995 +1985.439941 +1977.650024 +1963.709961 +1972.829956 +1964.680054 +1967.569946 +1977.099976 +1973.280029 +1981.569946 +1958.119995 +1978.219971 +1973.630005 +1983.530029 +1987.01001 +1987.97998 +1978.339966 +1978.910034 +1969.949951 +1970.069946 +1930.670044 +1925.150024 +1938.98999 +1920.209961 +1920.23999 +1909.569946 +1931.589966 +1936.920044 +1933.75 +1946.719971 +1955.180054 +1955.060059 +1971.73999 +1981.599976 +1986.51001 +1992.369995 +1988.400024 +1997.920044 +2000.02002 +2000.119995 +1996.73999 +2003.369995 +2002.280029 +2000.719971 +1997.650024 +2007.709961 +2001.540039 +1988.439941 +1995.689941 +1997.449951 +1985.540039 +1984.130005 +1998.97998 +2001.569946 +2011.359985 +2010.400024 +1994.290039 +1982.77002 +1998.300049 +1965.98999 +1982.849976 +1977.800049 +1972.290039 +1946.160034 +1946.170044 +1967.900024 +1964.819946 +1935.099976 +1968.890015 +1928.209961 +1906.130005 +1874.73999 +1877.699951 +1862.48999 +1862.76001 +1886.76001 +1904.01001 +1941.280029 +1927.109985 +1950.819946 +1964.579956 +1961.630005 +1985.050049 +1982.300049 +1994.650024 +2018.050049 +2017.810059 +2012.099976 +2023.569946 +2031.209961 +2031.920044 +2038.26001 +2039.680054 +2038.25 +2039.329956 +2039.819946 +2041.319946 +2051.800049 +2048.719971 +2052.75 +2063.5 +2069.409912 +2067.030029 +2072.830078 +2067.560059 +2053.439941 +2066.550049 +2074.330078 +2071.919922 +2075.370117 +2060.310059 +2059.820068 +2026.140015 +2035.329956 +2002.329956 +1989.630005 +1972.73999 +2012.890015 +2061.22998 +2070.649902 +2078.540039 +2082.169922 +2081.879883 +2088.77002 +2090.570068 +2080.350098 +2058.899902 +2058.199951 +2020.579956 +2002.609985 +2025.900024 +2062.139893 +2044.810059 +2028.26001 +2023.030029 +2011.27002 +1992.670044 +2019.420044 +2022.550049 +2032.119995 +2063.149902 +2051.820068 +2057.090088 +2029.550049 +2002.160034 +2021.25 +1994.98999 +2020.849976 +2050.030029 +2041.51001 +2062.52002 +2055.469971 +2046.73999 +2068.590088 +2068.530029 +2088.47998 +2096.98999 +2100.340088 +2099.679932 +2097.449951 +2110.300049 +2109.659912 +2115.47998 +2113.860107 +2110.73999 +2104.5 +2117.389893 +2107.780029 +2098.530029 +2101.040039 +2071.26001 +2079.429932 +2044.160034 +2040.23999 +2065.949951 +2053.399902 +2081.189941 +2074.280029 +2099.5 +2089.27002 +2108.100098 +2104.419922 +2091.5 +2061.050049 +2056.149902 +2061.02002 +2086.23999 +2067.889893 +2059.689941 +2066.959961 +2080.620117 +2076.330078 +2081.899902 +2091.179932 +2102.060059 +2092.429932 +2095.840088 +2106.629883 +2104.98999 +2081.179932 +2100.399902 +2097.290039 +2107.959961 +2112.929932 +2117.689941 +2108.919922 +2114.76001 +2106.850098 +2085.51001 +2108.290039 +2114.48999 +2089.459961 +2080.149902 +2088 +2116.100098 +2105.330078 +2099.120117 +2098.47998 +2121.100098 +2122.72998 +2129.199951 +2127.830078 +2125.850098 +2130.820068 +2126.060059 +2104.199951 +2123.47998 +2120.790039 +2107.389893 +2111.72998 +2109.600098 +2114.070068 +2095.840088 +2092.830078 +2079.280029 +2080.149902 +2105.199951 +2108.860107 +2094.110107 +2084.429932 +2096.290039 +2100.439941 +2121.23999 +2109.98999 +2122.850098 +2124.199951 +2108.580078 +2102.310059 +2101.48999 +2057.639893 +2063.110107 +2077.419922 +2076.780029 +2068.76001 +2081.340088 +2046.680054 +2051.310059 +2076.620117 +2099.600098 +2108.949951 +2107.399902 +2124.290039 +2126.639893 +2128.280029 +2119.209961 +2114.149902 +2102.149902 +2079.649902 +2067.639893 +2093.25 +2108.570068 +2108.629883 +2103.840088 +2098.040039 +2093.320068 +2099.840088 +2083.560059 +2077.570068 +2104.179932 +2084.070068 +2086.050049 +2083.389893 +2091.540039 +2102.439941 +2096.919922 +2079.610107 +2035.72998 +1970.890015 +1893.209961 +1867.609985 +1940.51001 +1987.660034 +1988.869995 +1972.180054 +1913.849976 +1948.859985 +1951.130005 +1921.219971 +1969.410034 +1942.040039 +1952.290039 +1961.050049 +1953.030029 +1978.089966 +1995.310059 +1990.199951 +1958.030029 +1966.969971 +1942.73999 +1938.76001 +1932.23999 +1931.339966 +1881.77002 +1884.089966 +1920.030029 +1923.819946 +1951.359985 +1987.050049 +1979.920044 +1995.829956 +2013.430054 +2014.890015 +2017.459961 +2003.689941 +1994.23999 +2023.859985 +2033.109985 +2033.660034 +2030.77002 +2018.939941 +2052.51001 +2075.149902 +2071.179932 +2065.889893 +2090.350098 +2089.409912 +2079.360107 +2104.050049 +2109.790039 +2102.310059 +2099.929932 +2099.199951 +2078.580078 +2081.719971 +2075 +2045.969971 +2023.040039 +2053.189941 +2050.439941 +2083.580078 +2081.23999 +2089.169922 +2086.590088 +2089.139893 +2088.870117 +2090.110107 +2080.409912 +2102.629883 +2079.51001 +2049.620117 +2091.689941 +2077.070068 +2063.590088 +2047.619995 +2052.22998 +2012.369995 +2021.939941 +2043.410034 +2073.070068 +2041.890015 +2005.550049 +2021.150024 +2038.969971 +2064.290039 +2060.98999 +2056.5 +2078.360107 +2063.360107 +2043.939941 +2012.660034 +2016.709961 +1990.26001 +1943.089966 +1922.030029 +1923.670044 +1938.680054 +1890.280029 +1921.839966 +1880.329956 +1881.329956 +1859.329956 +1868.98999 +1906.900024 +1877.079956 +1903.630005 +1882.949951 +1893.359985 +1940.23999 +1939.380005 +1903.030029 +1912.530029 +1915.449951 +1880.050049 +1853.439941 +1852.209961 +1851.859985 +1829.079956 +1864.780029 +1895.579956 +1926.819946 +1917.829956 +1917.780029 +1945.5 +1921.27002 +1929.800049 +1951.699951 +1948.050049 +1932.22998 +1978.349976 +1986.449951 +1993.400024 +1999.98999 +2001.76001 +1979.26001 +1989.26001 +1989.569946 +2022.189941 +2019.640015 +2015.930054 +2027.219971 +2040.589966 +2049.580078 +2051.600098 +2049.800049 +2036.709961 +2035.939941 +2037.050049 +2055.01001 +2063.949951 +2059.73999 +2072.780029 +2066.129883 +2045.170044 +2066.659912 +2041.910034 +2047.599976 +2041.98999 +2061.719971 +2082.419922 +2082.780029 +2080.72998 +2094.340088 +2100.800049 +2102.399902 +2091.47998 +2091.580078 +2087.790039 +2091.699951 +2095.149902 +2075.810059 +2065.300049 +2081.429932 +2063.370117 +2051.120117 +2050.629883 +2057.139893 +2058.689941 +2084.389893 +2064.459961 +2064.110107 +2046.609985 +2066.659912 +2047.209961 +2047.630005 +2040.040039 +2052.320068 +2048.040039 +2076.060059 +2090.540039 +2090.100098 +2099.060059 +2096.949951 +2099.330078 +2105.26001 +2099.129883 +2109.409912 +2112.129883 +2119.120117 +2115.47998 +2096.070068 +2079.060059 +2075.320068 +2071.5 +2077.98999 +2071.219971 +2083.25 +2088.899902 +2085.449951 +2113.320068 +2037.410034 +2000.540039 +2036.089966 +2070.77002 +2098.860107 +2102.949951 +2088.550049 +2099.72998 +2097.899902 +2129.899902 +2137.159912 +2152.139893 +2152.429932 +2163.75 +2161.73999 +2166.889893 +2163.780029 +2173.02002 +2165.169922 +2175.030029 +2168.47998 +2169.179932 +2166.580078 +2170.060059 +2173.600098 +2170.840088 From 4dbbbafe6c8fd1555d82f35156a0ef721276a8c8 Mon Sep 17 00:00:00 2001 From: Tim Fox Date: Fri, 2 Feb 2018 16:55:01 -0500 Subject: [PATCH 16/19] Incorporated Yiheng's feedback on feedforward credit card fraud --- .../feedforward-credit-card-fraud.ipynb | 96 +++++++------------ 1 file changed, 34 insertions(+), 62 deletions(-) diff --git a/elephantscale/notebooks/feedforward-credit-card-fraud.ipynb b/elephantscale/notebooks/feedforward-credit-card-fraud.ipynb index 712d290..6de5d24 100644 --- a/elephantscale/notebooks/feedforward-credit-card-fraud.ipynb +++ b/elephantscale/notebooks/feedforward-credit-card-fraud.ipynb @@ -8,6 +8,8 @@ "\n", "Let us look at a BigDL example with Credit Card Fraud. We will train a simple, feedforward neural network with the credit card dataset.\n", "\n", + "This is using RDDs. The feedforward-credit-card-fraud-pipeline uses the Spark.ML pipeline interface using dataframes.\n", + "\n", "Let's look briefly at the credit card dataset\n", "\n", "| Time\" | \"V1\" | \"V2\" | \"V3\" | \"V4\" | \"V5\" | \"V6\" | \"V7\" | \"V8\" | \"V9\" | \"V10\" | \"V11\" | \"V12\" | \"V13\" | \"V14\" | \"V15\" | \"V16\" | \"V17\" | \"V18\" | \"V19\" | \"V20\" | \"V21\" | \"V22\" | \"V23\" | \"V24\" | \"V25\" | \"V26\" | \"V27\" | \"V28\" | \"Amount\" | \"Class\" | \n", @@ -25,7 +27,9 @@ { "cell_type": "code", "execution_count": 1, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "%matplotlib inline\n", @@ -58,7 +62,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "## Parameteters\n", @@ -69,10 +75,12 @@ { "cell_type": "code", "execution_count": 2, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ - "learning_rate = 0.1\n", + "learning_rate = 0.01\n", "training_epochs = 100\n", "batch_size = 10000\n", "display_step = 1\n", @@ -137,7 +145,9 @@ { "cell_type": "code", "execution_count": 2, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "cc_training = spark.read.csv(\"../data/creditcardfraud/creditcard.csv\", header=True, inferSchema=\"true\", mode=\"DROPMALFORMED\")" @@ -174,7 +184,9 @@ { "cell_type": "code", "execution_count": 3, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "cc_training = cc_training.select([col(c).cast(\"double\") for c in cc_training.columns])\n", @@ -183,46 +195,13 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "scrolled": true }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----+------------------+-------------------+------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------------------+-------------------+------------------+------------------+-------------------+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------+-----+\n", - "|Time| V1| V2| V3| V4| V5| V6| V7| V8| V9| V10| V11| V12| V13| V14| V15| V16| V17| V18| V19| V20| V21| V22| V23| V24| V25| V26| V27| V28|Amount|Class|\n", - "+----+------------------+-------------------+------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------------------+-------------------+------------------+------------------+-------------------+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------+-----+\n", - "| 0.0| -1.3598071336738|-0.0727811733098497| 2.53634673796914| 1.37815522427443| -0.338320769942518| 0.462387777762292| 0.239598554061257| 0.0986979012610507| 0.363786969611213| 0.0907941719789316|-0.551599533260813|-0.617800855762348| -0.991389847235408| -0.311169353699879| 1.46817697209427| -0.470400525259478| 0.207971241929242| 0.0257905801985591| 0.403992960255733| 0.251412098239705| -0.018306777944153| 0.277837575558899| -0.110473910188767| 0.0669280749146731| 0.128539358273528| -0.189114843888824| 0.133558376740387|-0.0210530534538215|149.62| 1.0|\n", - "| 0.0| 1.19185711131486| 0.26615071205963| 0.16648011335321| 0.448154078460911| 0.0600176492822243|-0.0823608088155687| -0.0788029833323113| 0.0851016549148104|-0.255425128109186| -0.166974414004614| 1.61272666105479| 1.06523531137287| 0.48909501589608| -0.143772296441519| 0.635558093258208| 0.463917041022171| -0.114804663102346| -0.183361270123994| -0.145783041325259|-0.0690831352230203| -0.225775248033138| -0.638671952771851| 0.101288021253234| -0.339846475529127| 0.167170404418143| 0.125894532368176|-0.00898309914322813| 0.0147241691924927| 2.69| 1.0|\n", - "| 1.0| -1.35835406159823| -1.34016307473609| 1.77320934263119| 0.379779593034328| -0.503198133318193| 1.80049938079263| 0.791460956450422| 0.247675786588991| -1.51465432260583| 0.207642865216696| 0.624501459424895| 0.066083685268831| 0.717292731410831| -0.165945922763554| 2.34586494901581| -2.89008319444231| 1.10996937869599| -0.121359313195888| -2.26185709530414| 0.524979725224404| 0.247998153469754| 0.771679401917229| 0.909412262347719| -0.689280956490685| -0.327641833735251| -0.139096571514147| -0.0553527940384261|-0.0597518405929204|378.66| 1.0|\n", - "| 1.0|-0.966271711572087| -0.185226008082898| 1.79299333957872| -0.863291275036453|-0.0103088796030823| 1.24720316752486| 0.23760893977178| 0.377435874652262| -1.38702406270197|-0.0549519224713749|-0.226487263835401| 0.178228225877303| 0.507756869957169| -0.28792374549456| -0.631418117709045| -1.0596472454325| -0.684092786345479| 1.96577500349538| -1.2326219700892| -0.208037781160366| -0.108300452035545|0.00527359678253453| -0.190320518742841| -1.17557533186321| 0.647376034602038| -0.221928844458407| 0.0627228487293033| 0.0614576285006353| 123.5| 1.0|\n", - "| 2.0| -1.15823309349523| 0.877736754848451| 1.548717846511| 0.403033933955121| -0.407193377311653| 0.0959214624684256| 0.592940745385545| -0.270532677192282| 0.817739308235294| 0.753074431976354|-0.822842877946363| 0.53819555014995| 1.3458515932154| -1.11966983471731| 0.175121130008994| -0.451449182813529| -0.237033239362776|-0.0381947870352842| 0.803486924960175| 0.408542360392758|-0.00943069713232919| 0.79827849458971| -0.137458079619063| 0.141266983824769| -0.206009587619756| 0.502292224181569| 0.219422229513348| 0.215153147499206| 69.99| 1.0|\n", - "| 2.0|-0.425965884412454| 0.960523044882985| 1.14110934232219| -0.168252079760302| 0.42098688077219|-0.0297275516639742| 0.476200948720027| 0.260314333074874| -0.56867137571251| -0.371407196834471| 1.34126198001957| 0.359893837038039| -0.358090652573631| -0.137133700217612| 0.517616806555742| 0.401725895589603| -0.0581328233640131| 0.0686531494425432|-0.0331937877876282| 0.0849676720682049| -0.208253514656728| -0.559824796253248|-0.0263976679795373| -0.371426583174346| -0.232793816737034| 0.105914779097957| 0.253844224739337| 0.0810802569229443| 3.67| 1.0|\n", - "| 4.0| 1.22965763450793| 0.141003507049326|0.0453707735899449| 1.20261273673594| 0.191880988597645| 0.272708122899098|-0.00515900288250983| 0.0812129398830894| 0.464959994783886|-0.0992543211289237| -1.41690724314928|-0.153825826253651| -0.75106271556262| 0.16737196252175| 0.0501435942254188| -0.443586797916727| 0.00282051247234708| -0.61198733994012|-0.0455750446637976| -0.21963255278686| -0.167716265815783| -0.270709726172363| -0.154103786809305| -0.780055415004671| 0.75013693580659| -0.257236845917139| 0.0345074297438413|0.00516776890624916| 4.99| 1.0|\n", - "| 7.0|-0.644269442348146| 1.41796354547385| 1.0743803763556| -0.492199018495015| 0.948934094764157| 0.428118462833089| 1.12063135838353| -3.80786423873589| 0.615374730667027| 1.24937617815176|-0.619467796121913| 0.291474353088705| 1.75796421396042| -1.32386521970526| 0.686132504394383|-0.0761269994382006| -1.2221273453247| -0.358221569869078| 0.324504731321494| -0.156741852488285| 1.94346533978412| -1.01545470979971| 0.057503529867291| -0.649709005559993| -0.415266566234811|-0.0516342969262494| -1.20692108094258| -1.08533918832377| 40.8| 1.0|\n", - "| 7.0| -0.89428608220282| 0.286157196276544|-0.113192212729871| -0.271526130088604| 2.6695986595986| 3.72181806112751| 0.370145127676916| 0.851084443200905|-0.392047586798604| -0.410430432848439|-0.705116586646536|-0.110452261733098| -0.286253632470583| 0.0743553603016731| -0.328783050303565| -0.210077268148783| -0.499767968800267| 0.118764861004217| 0.57032816746536| 0.0527356691149697| -0.0734251001059225| -0.268091632235551| -0.204232669947878| 1.0115918018785| 0.373204680146282| -0.384157307702294| 0.0117473564581996| 0.14240432992147| 93.2| 1.0|\n", - "| 9.0| -0.33826175242575| 1.11959337641566| 1.04436655157316| -0.222187276738296| 0.49936080649727| -0.24676110061991| 0.651583206489972| 0.0695385865186387|-0.736727316364109| -0.366845639206541| 1.01761446783262| 0.836389570307029| 1.00684351373408| -0.443522816876142| 0.150219101422635| 0.739452777052119| -0.540979921943059| 0.47667726004282| 0.451772964394125| 0.203711454727929| -0.246913936910008| -0.633752642406113| -0.12079408408185| -0.385049925313426|-0.0697330460416923| 0.0941988339514961| 0.246219304619926| 0.0830756493473326| 3.68| 1.0|\n", - "|10.0| 1.44904378114715| -1.17633882535966| 0.913859832832795| -1.37566665499943| -1.97138316545323| -0.62915213889734| -1.4232356010359| 0.0484558879088564| -1.72040839292037| 1.62665905834133| 1.1996439495421|-0.671439778462005| -0.513947152539479|-0.0950450453999549| 0.230930409124119| 0.0319674667862076| 0.253414715863197| 0.854343814324194| -0.221365413645481| -0.387226474431156|-0.00930189652490052| 0.313894410791098| 0.0277401580170247| 0.500512287104917| 0.25136735874921| -0.129477953726618| 0.0428498709381461| 0.0162532619375515| 7.8| 1.0|\n", - "|10.0| 0.38497821518095| 0.616109459176472|-0.874299702595052|-0.0940186259679115| 2.92458437838817| 3.31702716826156| 0.470454671805879| 0.53824722837695|-0.558894612428441| 0.30975539423728|-0.259115563735702|-0.326143233995877|-0.0900467227020648| 0.362832368569793| 0.928903660629178| -0.129486811402759| -0.809978925963589| 0.359985390219981| 0.70766382644648| 0.12599157561542| 0.049923685888971| 0.238421512225103|0.00912986861262866| 0.996710209581086| -0.767314827174801| -0.492208295340017| 0.042472441919027|-0.0543373883732122| 9.99| 1.0|\n", - "|10.0| 1.249998742053| -1.22163680921816| 0.383930151282291| -1.23489868766892| -1.48541947377961| -0.753230164566149| -0.689404975426345| -0.227487227519552| -2.09401057344842| 1.32372927445937| 0.227666231237246|-0.242681998944186| 1.20541680770748| -0.317630527025074| 0.725674990179153| -0.815612186027305| 0.873936447614439| -0.847788598847099| -0.683192626267037| -0.102755941505071| -0.231809239223849| -0.483285330117712| 0.0846676908596583| 0.392830885335013| 0.161134553588505| -0.354990039673962| 0.0264155490776107| 0.0424220887282304| 121.5| 1.0|\n", - "|11.0| 1.0693735878819| 0.287722129331455| 0.828612726634281| 2.71252042961718| -0.178398016248009| 0.337543730282968| -0.0967168617395962| 0.115981735546597|-0.221082566236194| 0.460230444301678|-0.773656930526689| 0.32338724546722|-0.0110758870883779| -0.178485175177916| -0.65556427824926| -0.19992517131173| 0.1240054151819| -0.980496201537345| -0.982916082135047| -0.153197231044512| -0.0368755317335273| 0.0744124028162195|-0.0714074332998586| 0.104743752596029| 0.548264725394119| 0.104094153162781| 0.0214910583643189| 0.021293311477486| 27.5| 1.0|\n", - "|12.0| -2.7918547659339| -0.327770756658658| 1.64175016056605| 1.76747274389883| -0.136588446465306| 0.80759646826532| -0.422911389711497| -1.90710747624096| 0.755712908314791| 1.1510869876677| 0.844555470974377| 0.7929439518176| 0.370448092803246| -0.734975105820311| 0.406795710431001| -0.303057623825763| -0.155868714793874| 0.778265457041536| 2.22186801373788| -1.58212204356551| 1.15166304848789| 0.222181966098225| 1.02058620426601| 0.0283166513238872| -0.232746324289105| -0.23555721754117| -0.16477751177654|-0.0301536365592253| 58.8| 1.0|\n", - "|12.0|-0.752417042956605| 0.345485415344747| 2.05732291276727| -1.46864329840046| -1.1583936804082|-0.0778498291166733| -0.608581418236123|0.00360348436201849|-0.436166983515744| 0.747730827192802|-0.793980602837221|-0.770406728847129| 1.04762699748088| -1.06660368148653| 1.10695345662141| 1.66011355713381| -0.279265373246772| -0.419994141181313| 0.432535348618175| 0.263450864446125| 0.499624954671111| 1.35365048557231| -0.256573280448308|-0.0650837078816517|-0.0391243535426488|-0.0870864732146962| -0.180997500092721| 0.129394059390202| 15.99| 1.0|\n", - "|12.0| 1.10321543528383|-0.0402962145973447| 1.2673320885949| 1.28909146962552| -0.735997163604068| 0.288069162976262| -0.586056786337461| 0.189379713679593| 0.782332891785191| -0.267975066537173|-0.450311279515466| 0.936707714991982| 0.708380406186981| -0.468647287707221| 0.354574063407955| -0.246634655717582|-0.00921237772707382| -0.595912405700819| -0.57568162226261| -0.113910176982092| -0.0246120063374677| 0.196001952806192| 0.0138016541409422| 0.103758331023198| 0.364297540595235| -0.382260574113217| 0.092809187460487| 0.0370505169810008| 12.99| 1.0|\n", - "|13.0|-0.436905071360625| 0.918966212909322| 0.92459077438817| -0.727219053596792| 0.915678718106307| -0.127867352079254| 0.707641607333935| 0.0879623554672504| -0.66527135413364| -0.737979823596458| 0.32409781346169| 0.277192107214981| 0.252624256310781| -0.291896460370468| -0.184520169327133| 1.14317370716197| -0.92870926272403| 0.680469592634687| 0.0254364616880793|-0.0470212823165035| -0.194795823794671| -0.672637997017793| -0.156857514491897| -0.888386320943716| -0.342413218776576| -0.049026728633951| 0.0796923991551505| 0.131023789452311| 0.89| 1.0|\n", - "|14.0| -5.40125766315825| -5.45014783420644| 1.18630463143652| 1.73623880012095| 3.04910587764025| -1.76340557365201| -1.55973769907953| 0.160841747266769| 1.23308974041888| 0.345172827050629| 0.917229867699146| 0.970116716069048| -0.266567764915222| -0.479129929276704| -0.526608502569153| 0.47200411177674| -0.725480944982201| 0.075081351540202| -0.406866573198217| -2.19684802485647| -0.503600328973703| 0.984459785590244| 2.45858857639219| 0.0421188969891572| -0.481630823956716| -0.621272013713977| 0.392053289557744| 0.949594245504846| 46.8| 1.0|\n", - "|15.0| 1.4929359769862| -1.02934573189487| 0.45479473374366| -1.43802587991702| -1.55543410136344| -0.720961147043557| -1.08066413038614|-0.0531271179483221| -1.9786815953872| 1.63807603690446| 1.07754241162743| -0.63204651464934| -0.41695716661602| 0.0520105153724404|-0.0429789228232019| -0.166432496451972| 0.304241418614353| 0.554432499062278| 0.0542295152184719| -0.387910172646258| -0.177649846438814| -0.175073809074822| 0.0400022190621329| 0.295813862676508| 0.33293059939425| -0.220384850672322| 0.0222984359135846|0.00760225559997897| 5.0| 1.0|\n", - "+----+------------------+-------------------+------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------------------+-------------------+------------------+------------------+-------------------+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------+-----+\n", - "only showing top 20 rows\n", - "\n" - ] - } - ], + "outputs": [], "source": [ - "cc_training.show()" + "cc_training.show(1)" ] }, { @@ -274,7 +253,9 @@ { "cell_type": "code", "execution_count": 6, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "# get the time to split the data.\n", @@ -478,22 +459,11 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": { "scrolled": false }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAugAAAK7CAYAAACgQHrJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3XucXXV97//XZ2ZygwTCZYCE+zUK\niBEQUaskXgEVvNWCrYK9UM/R369aa9XiT63Wo22t2ovVQ48c8VSRVKtNFRW8jOgRq6AJ90uIQWBC\nAoTcrzPz+f2x1yR7JjPJJOyZvdbK6/l47Mfs/V1rr/3dn+yZvOc73/VdkZlIkiRJKoeOdndAkiRJ\n0g4GdEmSJKlEDOiSJElSiRjQJUmSpBIxoEuSJEklYkCXJEmSSsSALkn7iIhYFhEvaXc/JEm7ZkCX\nJEmSSsSALkmSJJWIAV2S9jERMSUiPh0RvcXt0xExpdh2aER8MyJWR8SqiPhxRHQU294TEY9ExLqI\nuDciXtzedyJJ9dTV7g5IkibclcC5wFwggf8A3g/8f8C7gIeB7mLfc4GMiDnA24FnZ2ZvRBwHdE5s\ntyVp3+AIuiTte34X+HBmrszMx4C/BN5UbNsGzAKOzcxtmfnjzEygH5gCnBoRkzJzWWY+0JbeS1LN\nGdAlad8zG3iw6fGDRRvA3wJLgBsiYmlEvBcgM5cA7wA+BKyMiK9ExGwkSS1nQJekfU8vcGzT42OK\nNjJzXWa+KzNPAF4F/OngXPPM/HJm/lbx3AT+emK7LUn7BgO6JO17rgXeHxHdEXEo8AHgXwEi4pUR\ncVJEBLCWxtSW/oiYExEvKk4m3QxsKrZJklrMgC5J+56/Am4BbgNuB35ZtAGcDHwPWA/cDPxzZvbQ\nmH/+ceBx4FHgMOAvJrTXkrSPiMa5P5IkSZLKwBF0SZIkqUQM6JIkSVKJGNAlSZKkEjGgS5IkSSXS\n1e4OjKeZM2fmSSed1O5u1MaGDRvYf//9292NWrCWrWMtW8t6to61bB1r2TrWsrWa63nrrbc+npnd\nrThurQP64Ycfzi233NLubtRGT08P8+bNa3c3asFato61bC3r2TrWsnWsZetYy9ZqrmdEPLjrvcfO\nKS6SJElSiRjQJUmSpBIxoEuSJEklYkCXJEmSSsSALkmSJJWIAV2SJEkqEQO6JEmSVCIGdEmSJKlE\nDOiSJElSiRjQJUmSpBIxoEuSJEklYkBX5Vz789/wuR890O5uSJIkjYuudndA2lPv+/fbAXjreSe2\nuSeSJEmt5wi6JEmSVCIGdEmSJKlEDOiSJElSiRjQJUmSpBIxoEuSJEklYkCXJEmSSsSALkmSJJWI\nAV2SJEkqEQO6JEmSVCIGdEmSJKlEDOiSJElSiRjQJUmSpBJpW0CPiKsjYmVE3NHUdl1ELCpuyyJi\nUdF+XERsatr2uXb1W5IkSRpPXW187S8A/wR8cbAhM39n8H5E/B2wpmn/BzJz7oT1TpIkSWqDtgX0\nzLwpIo4baVtEBPAG4EUT2SdJkiSp3SIz2/fijYD+zcw8fVj7C4FPZubZTfvdCdwHrAXen5k/HuWY\nVwBXAHR3d5+1YMGCcer9vmf9+vVMnz693d3g8u9sAOAL5+/f5p7svbLUsg6sZWtZz9axlq1jLVvH\nWrZWcz3nz59/62B2faraOcVlVy4Frm16vBw4JjOfiIizgG9ExGmZuXb4EzPzKuAqgDlz5uS8efMm\nor/7hJ6eHkpRz+98C6AcfdlLpallDVjL1rKerWMtW8dato61bK3xqmfpVnGJiC7gtcB1g22ZuSUz\nnyju3wo8AJzSnh5KkiRJ46d0AR14CXBPZj482BAR3RHRWdw/ATgZWNqm/kmSJEnjpp3LLF4L3AzM\niYiHI+IPik2XMHR6C8ALgdsiYjHwVeCtmblq4norSZIkTYx2ruJy6Sjtl4/Q9jXga+PdJ0mSJKnd\nyjjFRZIkSdpnGdAlSZKkEjGgS5IkSSViQJckSZJKxIAuSZIklYgBXZIkSSoRA7okSZJUIgZ0SZIk\nqUQM6JIkSVKJGNAlSZKkEjGgS5IkSSViQJckSZJKxIAuSZIklYgBXZIkSSoRA7okSZJUIgZ0SZIk\nqUQM6JIkSVKJGNAlSZKkEjGgS5IkSSViQJckSZJKxIAuSZIklYgBXZIkSSoRA7okSZJUIgZ0SZIk\nqUQM6JIkSVKJGNAlSZKkEjGgS5IkSSXS1oAeEVdHxMqIuKOp7UMR8UhELCpuFzZte19ELImIeyPi\n5e3ptSRJkjR+2j2C/gXg/BHaP5WZc4vb9QARcSpwCXBa8Zx/jojOCeupJEmSNAHaGtAz8yZg1Rh3\nvxj4SmZuycxfA0uAc8atc5IkSVIbtHsEfTRvj4jbiikwBxVtRwIPNe3zcNEmSZIk1UZkZns7EHEc\n8M3MPL14fDjwOJDAR4BZmfn7EfEZ4ObM/Ndiv88D12fm14Yd7wrgCoDu7u6zFixYMFFvpfbWr1/P\n9OnT290NLv/OBgC+cP7+be7J3itLLevAWraW9Wwda9k61rJ1rGVrNddz/vz5t2bm2a04blcrDtJK\nmbli8H5E/AvwzeLhw8DRTbseBfSO8PyrgKsA5syZk/PmzRu3vu5renp6KEU9v/MtgHL0ZS+VppY1\nYC1by3q2jrVsHWvZOtaytcarnqWb4hIRs5oevgYYXOFlIXBJREyJiOOBk4GfT3T/JEmSpPHU1hH0\niLgWmAccGhEPAx8E5kXEXBpTXJYBfwyQmXdGxALgLqAPeFtm9rej35IkSdJ4aWtAz8xLR2j+/C72\n/yjw0fHrkaokM4mIdndDkiSppUo3xUUaqzaf3yxJkjQuDOiqLPO5JEmqIwO6KqvdS4RKkiSNBwO6\nJEmSVCIGdFWW4+eSJKmODOiqLGe4SJKkOjKgq7LSMXRJklRDBnRVliPokiSpjgzokiRJUokY0FVZ\njqBLkqQ6MqCrspyDLkmS6siArspyBF2SJNWRAV2VZT6XJEl1ZEBXZaVD6JIkqYYM6Kos47kkSaoj\nA7oqywF0SZJURwZ0SZIkqUQM6KouR9AlSVINGdBVWa6DLkmS6siArspyDrokSaojA7oqy3wuSZLq\nyICuynIddEmSVEcGdFWW8VySJNWRAV2V5QC6JEmqIwO6KstVXCRJUh0Z0FVd5nNJklRDBnRVlvlc\nkiTVUdsCekRcHRErI+KOpra/jYh7IuK2iPh6RMws2o+LiE0Rsai4fa5d/VZ5OAddkiTVUTtH0L8A\nnD+s7Ubg9Mw8A7gPeF/Ttgcyc25xe+sE9VGSJEmaUG0L6Jl5E7BqWNsNmdlXPPwZcNSEd0yV4Umi\nkiSpjso8B/33gW83PT4+In4VET+KiBe0q1MqD6e4SJKkOop2Xo0xIo4DvpmZpw9rvxI4G3htZmZE\nTAGmZ+YTEXEW8A3gtMxcO8IxrwCuAOju7j5rwYIF4/wu9h3r169n+vTp7e4Gl39nAwB/d940DplW\n5t8xR1eWWtaBtWwt69k61rJ1rGXrWMvWaq7n/Pnzb83Ms1tx3K5WHKSVIuIy4JXAi7P47SEztwBb\nivu3RsQDwCnALcOfn5lXAVcBzJkzJ+fNmzdBPa+/np4eSlHP73wLgOecey5HHbRfmzuzd0pTyxqw\nlq1lPVvHWraOtWwda9la41XPUg0/RsT5wHuAizJzY1N7d0R0FvdPAE4GlranlyoLp7hIkqQ6atsI\nekRcC8wDDo2Ih4EP0li1ZQpwY0QA/KxYseWFwIcjog/oB96amatGPLAkSZJUYW0L6Jl56QjNnx9l\n368BXxvfHkmSJEntV6opLtKecIqLJEmqIwO6JEmSVCIGdFWWFyqSJEl1ZECXJEmSSsSArspyDrok\nSaojA7okSZJUIgZ0VZYD6JIkqY4M6JIkSVKJGNBVWekkdEmSVEMGdEmSJKlEDOiqLMfPJUlSHRnQ\nJUmSpBIxoEuSJEklYkBXZXmOqCRJqiMDuiRJklQiBnRVmEPokiSpfgzokiRJUokY0FVZzkGXJEl1\nZECXJEmSSsSArspyAF2SJNWRAV2SJEkqEQO6Kss56JIkqY4M6JIkSVKJGNBVWeksdEmSVEMGdEmS\nJKlEDOiqLOegS5KkOjKgS5IkSSViQFdlOYIuSZLqqK0BPSKujoiVEXFHU9vBEXFjRNxffD2oaI+I\n+IeIWBIRt0XEme3ruSRJkjQ+2j2C/gXg/GFt7wW+n5knA98vHgNcAJxc3K4APjtBfVRJuYqLJEmq\no7YG9My8CVg1rPli4Jri/jXAq5vav5gNPwNmRsSsiempJEmSNDEi2zyRNyKOA76ZmacXj1dn5sym\n7U9m5kER8U3g45n5k6L9+8B7MvOWYce7gsYIO93d3WctWLBgYt7IPmD9+vVMnz693d3g8u9sAOAv\nnzeVYw/obHNv9k5ZalkH1rK1rGfrWMvWsZatYy1bq7me8+fPvzUzz27FcbtacZAJEiO07fTbRWZe\nBVwFMGfOnJw3b944d2vf0dPTQynq+Z1vAXDWWWdz+pEHtrkze6c0tawBa9la1rN1rGXrWMvWsZat\nNV71bPcc9JGsGJy6UnxdWbQ/DBzdtN9RQO8E902SJEkaV2UM6AuBy4r7lwH/0dT+5mI1l3OBNZm5\nvB0dlCRJksZLW6e4RMS1wDzg0Ih4GPgg8HFgQUT8AfAb4LeL3a8HLgSWABuBt0x4h1UqroMuSZLq\nqK0BPTMvHWXTi0fYN4G3jW+PJEmSpPYq4xQXaUxcB12SJNWRAV2SJEkqEQO6Kss56JIkqY4M6JIk\nSVKJGNAlSZKkEjGgq7Kc4SJJkurIgC5JkiSViAFdlZWeJSpJkmrIgC5JkiSViAFdleX4uSRJqiMD\nuiRJklQiBnRVllPQJUlSHRnQJUmSpBIxoKvCHEKXJEn1Y0CXJEmSSsSArspyDrokSaojA7okSZJU\nIgZ0VZYD6JIkqY4M6JIkSVKJGNBVWc5BlyRJdWRAV6WkqVySJNWcAV2VZViXJEl1ZEBXpZjJJUlS\n3RnQVVlmdUmSVEcGdFWKoVySJNWdAV2V5XQXSZJURwZ0VYonhkqSpLrrancHRhIRc4DrmppOAD4A\nzAT+CHisaP+LzLx+grunkkgnvEiSpBoqZUDPzHuBuQAR0Qk8AnwdeAvwqcz8RBu7pzYykkuSpLqr\nwhSXFwMPZOaD7e6ISsa0LkmSaijKPqc3Iq4GfpmZ/xQRHwIuB9YCtwDvyswnh+1/BXAFQHd391kL\nFiyY2A7X2Pr165k+fXpb+9A3kPzhDRsB+PNnT+XUQzrb2p+9VYZa1oW1bC3r2TrWsnWsZetYy9Zq\nruf8+fNvzcyzW3HcUgf0iJgM9AKnZeaKiDgceJzG2OlHgFmZ+fujPX/OnDl57733Tkxn9wE9PT3M\nmzevrX3Y2jfAKe//NgBf+sPn8PyTDm1rf/ZWGWpZF9aytaxn61jL1rGWrWMtW6u5nhHRsoBe9iku\nF9AYPV8BkJkrMrM/MweAfwHOaWvvNOE8MVSSJNVd2QP6pcC1gw8iYlbTttcAd0x4jyRJkqRxVMpV\nXAAiYj/gpcAfNzX/TUTMpTHFZdmwbdoHNM/IKvHsLEmSpL1W2oCemRuBQ4a1valN3ZEkSZImRNmn\nuEijcj66JEmqIwO6JEmSVCIGdFWKc9AlSVLdGdAlSZKkEjGgq1Ka5507gC5JkurIgK5K6RswlkuS\npHozoKtS+vqbRtCLSei3LFvFus3b2tUlSZKkljKgq1K29Q8Mebxm4zZe/7mb+X+u/VWbeiRJktRa\nBnRVSnNAT2BtMXJ+/4r1beqRJElSaxnQVSnb+ofOQd+0rR+AaZM729EdSZKkljOgq1L6mqe4JGza\n2gjo+xnQJUlSTRjQVSmjjaBP7TKgS5KkejCgq1KGzkHPHQHdEXRJklQTBnRVSt/A0FVctgzOQZ/k\nR1mSJNWDqUaVsrWveR102Lh1MKA7gi5JkurBgK5KGT6CPjjlpavTj7IkSaoHU40qZcgc9GzcADoj\n2tQjSZKk1jKgq1KGr+LSXyT0Dj/JkiSpJow1qpThVxIdGGgE9HAEXZIk1YQBXZXSN3wEvQjoHeZz\nSZJUEwZ0VcrQOejJYF7vcARdkiTVhAFdlZLDHg9sH0E3oEuSpHowoKtacujdwZNEzeeSJKkuDOiq\nlGS0OegmdEmSVA8GdFVK5tD7A54kKkmSasaArkoZPgd9xxQXE7okSaoHA7oqJYcl9IHhiV2SJKni\nutrdgdFExDJgHdAP9GXm2RFxMHAdcBywDHhDZj7Zrj5q4g2dg57bp7jk8OQuSZJUUWUfQZ+fmXMz\n8+zi8XuB72fmycD3i8fahwzP4YNTXMznkiSpLsoe0Ie7GLimuH8N8Oo29kVt0DxS3nySqPlckiTV\nRZR1akBE/Bp4kkb2+p+ZeVVErM7MmU37PJmZBw173hXAFQDd3d1nLViwYCK7XWvr169n+vTpbe3D\n9x7cxr/evRWAt82dwpLV/Xx3WR8vO7aLNz59Slv7tifKUMu6sJatZT1bx1q2jrVsHWvZWs31nD9/\n/q1Nsz6ektLOQQeen5m9EXEYcGNE3DOWJ2XmVcBVAHPmzMl58+aNYxf3LT09PbS7ng/+dBncfScA\np512GhuXrYJlyzjyqKOYN++0tvZtT5ShlnVhLVvLeraOtWwda9k61rK1xquepZ3ikpm9xdeVwNeB\nc4AVETELoPi6sn09VDsM/4vPjpNE29EbSZKk1itlQI+I/SNixuB94GXAHcBC4LJit8uA/2hPD9Uu\nQ9ZwyR0niUqSJNVFWae4HA58vbj4TBfw5cz8TkT8AlgQEX8A/Ab47Tb2UW2w0youA4PtBnVJklQP\npQzombkUeOYI7U8AL574Hqkshq6Cnq7iIkmSaqeUU1yk0QwfKXcddEmSVDcGdFXW0HXQTeiSJKke\nDOiqlOEj5QOOoEuSpJoxoKtSmkfKE+gvHg4Y0CVJUk0Y0FUpO42gb0/mJnRJklQPBnRVytB10JN+\nL1QkSZJqxoCuStlpHXTnoEuSpJoxoKtShq/W4ioukiSpbgzoqhRH0CVJUt0Z0FVZmbBl20Djfpv7\nIkmS1CoGdFXK8CuJbunrL9rb0RtJkqTWM6CrUpqDeJJs3j6CbkKXJEn1YEBXpQyP4YMj6OZzSZJU\nFwZ0VcqQEfSELX3OQZckSfViQFelDJ/Ksn2Ki5PQJUlSTRjQVSk7j6AXJ4m2qT+SJEmtZkBXpey8\nisvgCHo7eiNJktR6BnRVSnMOH8hkaxHQB0zokiSpJgzoqpTmHD44eg5OcZEkSfVhQFelNJ8k2j8w\nZFF0SZKkWjCgq1KaR9Cbp7V4oSJJklQXBnRVytA56E3t5nNJklQTBnRVytBlFnPEdkmSpCozoKtS\nmqeyOMVFkiTVkQFd1dKUw/sHmprN55IkqSYM6KqU4eugj9QuSZJUZQZ0VcrQeefOQZckSfVTuoAe\nEUdHxA8j4u6IuDMi/qRo/1BEPBIRi4rbhe3uqybe0GUWh2yZ6K5IkiSNi652d2AEfcC7MvOXETED\nuDUibiy2fSozP9HGvqnNmmN484WKHEGXJEl1UbqAnpnLgeXF/XURcTdwZHt7pbIYdZnFNvRFkiRp\nPESWeOgxIo4DbgJOB/4UuBxYC9xCY5T9yRGecwVwBUB3d/dZCxYsmKDe1t/69euZPn16W/vwpbu3\ncOODfQC88oRJfHPpNgDOOLSTPz17aju7tkfKUMu6sJatZT1bx1q2jrVsHWvZWs31nD9//q2ZeXYr\njlu6EfRBETEd+BrwjsxcGxGfBT5CY7D0I8DfAb8//HmZeRVwFcCcOXNy3rx5E9bnuuvp6aHd9exZ\neyc8uAyAo485BpY+AMBBBx/MvHnntLFne6YMtawLa9la1rN1rGXrWMvWsZatNV71LN1JogARMYlG\nOP9SZv47QGauyMz+zBwA/gWoThrTuHCZRUmSVEelC+gREcDngbsz85NN7bOadnsNcMdE903tN9rS\nimWeqiVJkrQnyjjF5fnAm4DbI2JR0fYXwKURMZfGYOky4I/b0z2102iruEiSJNVF6QJ6Zv4EiBE2\nXT/RfVH5DF0H3WUWJUlS/ZRuiou0K8koU1ychS5JkmrCgK5KcQRdkiTVnQFdldI87dwriUqSpDoy\noKtikijOUBhwioskSaohA7oqJRM6ioQ+2pKLkiRJVWZAV6Vk7ljixwsVSZKkOjKgq1KS3D6C3j8w\nZIMkSVItGNBVKZlsH0JvnuIy4BwXSZJUEwZ0VUoCHUVA73eKiyRJqiEDuiqlMQe9kdCHrOLiCLok\nSaoJA7oqpTEHvXF/cFpLhCPokiSpPgzoqpaEGLbMYmeEyyxKkqTaMKCrUprOEd1+JdGOjnAEXZIk\n1YYBXZWSufOVRDsjvFKRJEmqDQO6KiVpnuLSaOvsiCEnjEqSJFWZAV2VkslOJ4lO7uqgz4QuSZJq\nwoCuSmkeQd8e0Ds72DbksqKSJEnVZUBXpWTuWGZx8CTRyV0d9BnQJUlSTRjQVSnFyueN+8Wslsld\nHWzrd4qLJEmqBwO6KqOvfwBGmoPuFBdJklQjXe3ugDQW2/oHOPnKbwNw+AFTAE8SlSRJ9eQIuiph\nw5a+7fc7Bk8SLQbNp3Q5gi5JkurDgK5K2LStf/v9jVsb95tH0A3okiSpLgzoqoTN23YE8DWbtgE7\nAvqUrg76PElUkiTVhAFdlbBpa/9ObQNNq7j0DSSZhnRJklR9BnRVwua+kQL6jlVcAJdalCRJtWBA\nVyVsHnEEfcccdIC+AeehS5Kk6qtcQI+I8yPi3ohYEhHvbXd/NDGaTxIddMcja4EdAd0RdEmSVAeV\nCugR0Ql8BrgAOBW4NCJObW+vNBGaTxIdbnJnJ4AruUiSpFqo2oWKzgGWZOZSgIj4CnAxcNdIO6/a\nnHz0W3ex/5Qupk/pIiIIIILia2y/z5BtjfaO4j7N+w/u07Qfw7cRjec2Djzk9QYN3ovt+xWvVWwM\nGpeyT7LpOTueP+QgIz8c8nojbYcd00Qmde74XW34uZaDfbjj8X46739shO1Dbdraz5a+fg7ab/KQ\nOsVInRhh0Ls/k//141/zo/se47XPOpJnHXsQ9z26boTeNwyOoH/79uVMm9zF1r4BZkzt2qmuOx4z\n5PFgS1dH87/n0H+Poc+L7Y9He421m7Zx34r1POOoA7av297stsf64N6Vo76n0V6nrJq/FybaXU/0\nM2nJ49sfb+sfYMOWfg7ab1J7OjSSkv/7Nbv7iX6mPPDEqNtzpG/anXcakz35m9fwn7lVcM+qfqYu\nbdTSc9j3XPO/dXMttbM9+Xzt7ntce+aJTeMzOBhVWvkiIl4PnJ+Zf1g8fhPwnMx8+0j77z/75Ox+\n86cnsosaR087Ygaf+O1nsmlbP9OndPHJG+9jycr1vOMlJ/MnX1nU7u5JkqR9zOtPmcQnfv9lAETE\nrZl5diuOW7UR9JHGTob8hhERVwBXAHR3d3P1y/ejbwAGZ0gM/j6Sg0/Mwfu5/f727bnja/MLZY5y\nf8jxIDOH7DdSx7P5wbC2ISOnuxm13unwu9l/UBTbBobtMHyUKoCNmzax37Rpox5nUGdH4/HWkWqe\nIx97uMP3C6Z2Bau3JJ0BG7YlB0/r5/H7fwXABuB3jwGOgXzyPv7mhdPoK16vI6B/yD/KsH+jJoO/\noDbXIHf1vGHtQ/YZdvBJHTDa79WbNm5i2n4j13K012m3ZPRvwHb+nr9p0yamDftcbhto1L8MdlWa\nkb4f2m3jxp3rOdxY+tzqt9Xuz9neGOmzqb1jLXcY7WfxWFnL1to/N9HT09Py41YtoD8MHN30+Cig\nt3mHzLwKuApgzpw5+aL58yeudzXX09PDvHnz2t2NWrCWrWMtW8t6to61bB1r2TrWsrXGq54lGWMa\ns18AJ0fE8RExGbgEWNjmPkmSJEktU6kR9Mzsi4i3A98FOoGrM/PONndLkiRJaplKBXSAzLweuL7d\n/ZAkSZLGQ9WmuEiSJEm1ZkCXJEmSSsSALkmSJJWIAV2SJEkqEQO6JEmSVCIGdEmSJKlEDOiSJElS\niRjQJUmSpBKJzGx3H8ZNRKwD7m13P2rkUODxdneiJqxl61jL1rKerWMtW8dato61bK3meh6bmd2t\nOGjlriS6h+7NzLPb3Ym6iIhbrGdrWMvWsZatZT1bx1q2jrVsHWvZWuNVT6e4SJIkSSViQJckSZJK\npO4B/ap2d6BmrGfrWMvWsZatZT1bx1q2jrVsHWvZWuNSz1qfJCpJkiRVTd1H0CVJkqRKMaBLkiRJ\nJVLbgB4R50fEvRGxJCLe2+7+lF1EHB0RP4yIuyPizoj4k6L9QxHxSEQsKm4XNj3nfUV9742Il7ev\n9+UTEcsi4vaiZrcUbQdHxI0RcX/x9aCiPSLiH4pa3hYRZ7a39+USEXOaPn+LImJtRLzDz+bYRMTV\nEbEyIu5oatvjz2JEXFbsf39EXNaO99Juo9TybyPinqJeX4+ImUX7cRGxqenz+bmm55xV/HxYUtQ7\n2vF+2m2Ueu7x97X/349ay+ua6rgsIhYV7X42d2EXeWhif25mZu1uQCfwAHACMBlYDJza7n6V+QbM\nAs4s7s8A7gNOBT4E/NkI+59a1HUKcHxR7852v4+y3IBlwKHD2v4GeG9x/73AXxf3LwS+DQRwLvBf\n7e5/WW/F9/ajwLF+NsdcsxcCZwJ3NLXt0WcROBhYWnw9qLh/ULvfW0lq+TKgq7j/1021PK55v2HH\n+Tnw3KLO3wYuaPd7K1E99+j72v/vR6/lsO1/B3yguO9nc9e1HC0PTejPzbqOoJ8DLMnMpZm5FfgK\ncHGb+1Rqmbk8M39Z3F8H3A0cuYunXAx8JTO3ZOavgSU06q7RXQxcU9y/Bnh1U/sXs+FnwMyImNWO\nDlbAi4EHMvPBXezjZ7NJZt4ErBrWvKefxZcDN2bmqsx8ErgROH/8e18uI9UyM2/IzL7i4c+Ao3Z1\njKKeB2Tmzdn4X/yL7Kj/PmWUz+ZoRvu+9v97dl3LYhT8DcC1uzqGn82GXeShCf25WdeAfiTwUNPj\nh9l12FSTiDgOeBbwX0XT24s/21w9+CcdrPHuJHBDRNwaEVcUbYdn5nJo/AAADivareXYXcLQ/2T8\nbO6dPf0sWtOx+X0aI2mDjo/yM1LxAAAgAElEQVSIX0XEjyLiBUXbkTTqN8ha7mxPvq/9bO7eC4AV\nmXl/U5ufzTEYlocm9OdmXQP6SHOmXE9yDCJiOvA14B2ZuRb4LHAiMBdYTuPPZGCNd+f5mXkmcAHw\ntoh44S72tZZjEBGTgYuAfyua/Gy23mi1s6a7ERFXAn3Al4qm5cAxmfks4E+BL0fEAVjL3dnT72vr\nuXuXMnRgw8/mGIyQh0bddYS2p/zZrGtAfxg4uunxUUBvm/pSGRExicaH8UuZ+e8AmbkiM/szcwD4\nF3ZMFbDGu5CZvcXXlcDXadRtxeDUleLrymJ3azk2FwC/zMwV4GfzKdrTz6I13YXi5K9XAr9bTA2g\nmIrxRHH/VhrzpE+hUcvmaTDWsslefF/72dyFiOgCXgtcN9jmZ3P3RspDTPDPzboG9F8AJ0fE8cWo\n2yXAwjb3qdSKOWqfB+7OzE82tTfPhX4NMHiG+ELgkoiYEhHHAyfTOLlknxcR+0fEjMH7NE4iu4NG\nzQbP4r4M+I/i/kLgzcWZ4OcCawb/jKYhhowC+dl8Svb0s/hd4GURcVAx5eBlRds+LyLOB94DXJSZ\nG5vauyOis7h/Ao3P4dKinusi4tzi5+6b2VH/fd5efF/7//2uvQS4JzO3T13xs7lro+UhJvrn5nie\nCdvOG42zau+j8Zvhle3uT9lvwG/R+NPLbcCi4nYh8H+A24v2hcCspudcWdT3XvbBM713UcsTaKwk\nsBi4c/DzBxwCfB+4v/h6cNEewGeKWt4OnN3u91C2G7Af8ARwYFObn82x1e5aGn/S3kZjROcP9uaz\nSGN+9ZLi9pZ2v68S1XIJjXmmgz83P1fs+7ri+38x8EvgVU3HOZtG8HwA+CeKq3rva7dR6rnH39f+\nfz9yLYv2LwBvHbavn81d13K0PDShPzejOIAkSZKkEqjrFBdJkiSpkgzokiRJUokY0CVJkqQSMaBL\nkiRJJWJAlyRJkkrEgC5JkiSViAFdkiRJKhEDuiRJklQiBnRJkiSpRAzokiRJUokY0CVJkqQSMaBL\nkiRJJWJAlyRJkkrEgC5JkiSViAFdkiRJKhEDuiRJklQiBnRJkiSpRAzokiRJUokY0CVJkqQSMaBL\nkiRJJWJAlyRJkkrEgC5JkiSViAFdkiRJKhEDuiRJklQiBnRJkiSpRAzokiRJUokY0CVJkqQSMaBL\nkiRJJWJAl6QJFBHLIuIlLTrW5Ij4anHMjIh5rTiuJKm9DOiSVG0/AX4PeLTdHdmdiOhsdx8kqQoM\n6JI0QSLi/wDHAP8ZEesj4s+L9osi4s6IWB0RPRHx9KbnLIuI90XEXRHxZET874iYCpCZWzPz05n5\nE6B/DK//loi4OyLWRcTSiPjjYdsvjohFEbE2Ih6IiPOL9oOL1+0t+vCNov3yiPjJsGNkRJxU3P9C\nRHw2Iq6PiA3A/Ih4RUT8qniNhyLiQ8Oe/1sR8dOiFg8Vr/HsiFgREV1N+70uIhbtQfklqTIM6JI0\nQTLzTcBvgFdl5vTM/JuIOAW4FngH0A1cTyPAT2566u8CLwdOBE4B3r+XXVgJvBI4AHgL8KmIOBMg\nIs4Bvgi8G5gJvBBYVjzv/wD7AacBhwGf2oPXfCPwUWAGjdH+DcCbi9d4BfDfIuLVRR+OAb4N/CON\nWswFFmXmL4AngJc2Hff3in5JUu0Y0CWpvX4H+FZm3piZ24BPANOA5zXt80+Z+VBmrqIRdi/dmxfK\nzG9l5gPZ8CPgBuAFxeY/AK4u+jGQmY9k5j0RMQu4AHhrZj6ZmduK547Vf2Tm/y2OuTkzezLz9uLx\nbTR+OTmv2Pd3ge9l5rXF6zyRmYOj5NfQCOVExME0fmH58t7UQZLKzoAuSe01G3hw8EFmDgAPAUc2\n7fNQ0/0Hi+fssYi4ICJ+FhGrImI1cCFwaLH5aOCBEZ52NLAqM5/cm9dkaN+JiOdExA8j4rGIWAO8\ndQx9APhX4FURMR14A/DjzFy+l32SpFIzoEvSxMphj3uBYwcfRETQCKqPNO1zdNP9Y4rn7JGImAJ8\njcYI/eGZOZPGdJoodnmIxhSa4R4CDo6ImSNs20Bj6svgaxwxwj7D3++XgYXA0Zl5IPC5MfSBzHwE\nuBl4DfAmnN4iqcYM6JI0sVYAJzQ9XgC8IiJeHBGTgHcBW4CfNu3ztog4qpja8RfAdYMbImLK4Emj\nwOSImFqE/OEmA1OAx4C+iLgAeFnT9s8Dbyn60RERR0bE04pR6m8D/xwRB0XEpIh4YfGcxcBpETG3\n6MOHxvD+Z9AYkd9czHt/Y9O2LwEviYg3RERXRBwSEXObtn8R+HPgGcDXx/BaklRJBnRJmlgfA95f\nrFLyZ5l5L4251f8IPA68isZJpFubnvNlGvPFlxa3v2radi+wicaUmO8W949lmMxcB/y/NH4heJJG\nMF7YtP3nFCeOAmuAHzUd503ANuAeGieavqN4zn3Ah4HvAffTOAl0d/478OGIWAd8oOjPYB9+Q2Pa\nzbuAVcAi4JlNz/160aevZ+aGMbyWJFVSZA7/66MkqSwiYhnwh5n5vXb3pQwi4gHgj62HpDpzBF2S\nVAkR8Toac9p/0O6+SNJ46tr9LpIktVdE9ACnAm8qVrqRpNpyioskSZJUIk5xkSRJkkqk1lNcZs6c\nmSeddFK7u1EbGzZsYP/99293N2rDeraW9Wwda9la1rO1rGfrWMvWuvXWWx/PzO5WHKvWAf3www/n\nlltuaXc3aqOnp4d58+a1uxu1YT1by3q2jrVsLevZWtazdaxla0XEg7vfa2yc4iJJkiSViAFdkiRJ\nKhEDuiRJklQiBnRJkiSpRAzokiRJUokY0CVJkqQSMaBLkiRJJWJAlyRJkkrEgC5JkiSViAFdkiRJ\nKhEDuiRJklQiBvSa2NY/wCv/8cfceNeKdndFkiRJT4EBvSYeXbOZOx5Zy8+WPtHurkiSJOkpMKDX\nxCOrNwHQW3yVJElSNRnQa2L5miKgr9nc5p5IkiTpqTCg10Tv6s3FV0fQJUmSqsyAXhODU1weW7eF\nLX39be6NJEmS9pYBvSaWN42cr1izpY09kSRJ0lNhQK+J3tWbmTGlC9gxmi5JkqTqMaDXRO/qTTzr\n2IO235ckSVI1jSmgR8T5EXFvRCyJiPeOsP3YiPh+RNwWET0RcVTTtr+OiDuK2+80tb8oIn5ZtF8T\nEV1Fe0TEPxSvdVtEnNn0nP6IWFTcFj61t14fazdvY92WPs46phHQB1d0kSRJUvXsNqBHRCfwGeAC\n4FTg0og4ddhunwC+mJlnAB8GPlY89xXAmcBc4DnAuyPigIjoAK4BLsnM04EHgcuKY10AnFzcrgA+\n2/Q6mzJzbnG7aG/ecB0tL1ZwOaF7fw7ZfzKPrHapRUmSpKoaywj6OcCSzFyamVuBrwAXD9vnVOD7\nxf0fNm0/FfhRZvZl5gZgMXA+cAiwJTPvK/a7EXhdcf9iGmE/M/NnwMyImLUX722fMTilZfbMacya\nOdUpLpIkSRXWNYZ9jgQeanr8MI3R8GaLaQTsvwdeA8yIiEOK9g9GxCeB/YD5wF3A48CkiDg7M28B\nXg8cvYvXOxJYDkyNiFuAPuDjmfmN4Z2NiCtojLzT3d1NT0/PGN5itfX8ZhsAD979KyZv28qS3nXj\n8r7Xr1+/T9RzoljP1rKerWMtW8t6tpb1bB1rWV5jCegxQlsOe/xnwD9FxOXATcAjQF9m3hARzwZ+\nCjwG3Fy0Z0RcAnwqIqYAN9AI3bt7vWMyszciTgB+EBG3Z+YDQ3bMvAq4CmDOnDk5b968MbzFavvF\nd++h856lXPyy+dy29S7+7ZaHOO+884gYqZR7r6enh32hnhPFeraW9Wwda9la1rO1rGfrWMvyGssU\nl4fZMboNcBTQ27xDZvZm5msz81nAlUXbmuLrR4s54y+lEb7vL9pvzswXZOY5NEL9/bt7vcwc/LoU\n6AGeNfa3Wl+9qzdzxAFT6ewIZs+cyoat/azd3Lf7J0qSJKl0xhLQfwGcHBHHR8Rk4BJgyAoqEXFo\nceInwPuAq4v2zmKqCxFxBnAGjdFyIuKw4usU4D3A54rnLwTeXKzmci6wJjOXR8RBxb5ExKHA82lM\nl9nn9a7exOyZU4HGPHRwJRdJkqSq2u0Ul8zsi4i3A98FOoGrM/POiPgwcEtmLgTmAR+LiKQxGv62\n4umTgB8XUy3WAr+XmYNDu++OiFfS+CXhs5n5g6L9euBCYAmwEXhL0f504H9GxEDxnI9npgEd6F2z\niTOLJRYHA3rv6k087YgD2tktSZIk7YWxzEEnM6+nEZyb2z7QdP+rwFdHeN5mGiu5jHTMdwPvHqE9\n2RHwm9t/CjxjLP3dl/QPJI+u2bw9mM8+sPHVpRYlSZKqySuJVtzj67ewrT+ZfWBjikv3jCl0dQTL\nXWpRkiSpkgzoFde8BjpAZ0dwxIGuhS5JklRVBvSK6y2msgwGdGhMc+l1ioskSVIlGdArbvsI+oFN\nAX3mVHpdxUWSJKmSDOgV17tmE/tP7uSAaTvO9509cxqPrtlM/8Dw60lJkiSp7AzoFddYA33akKuG\nzpo5jb6B5LF1W9rYM0mSJO0NA3rF9a7ezKym+ecARxYXLXKaiyRJUvUY0Ctu+ZpN2wP5oOaLFUmS\nJKlaDOgVtnlbP4+v3zrkBFGAWQca0CVJkqrKgF5hy9fsvMQiwAFTu5g+pculFiVJkirIgF5hg1cL\nnTVsiktENJZaHMcR9M3b+sft2JIkSfsyA3qFPVIE8COHjaBDY5rLeJ0kesOdj/KMD32XRQ+tHpfj\nS5Ik7csM6BU2OIXliAOn7rRt9sxpLB+HKS4r123mvf9+O9v6k6/e+lDLjy9JkrSvM6BX2PI1mzh0\n+hSmdHXutO3ImVN5YsPWlk5FyUze89Xb2LCljzOPmcm3blvOtv6Blh1fkiRJBvRKe2T1zkssDhqP\nlVx6Hurjh/c+xvsueBpvPe9Enty4jZ8sebxlx5ckSZIBvdIGryI6ksH2wZVenqqlj63n2nu38oKT\nD+XNzz2O8+Z0c8DULv5zUW9Lji9JkqQGA3pFZSbL12zePlI+3OCJo4+0YAS9r3+Ady5YzKQO+NvX\nP5OOjmBKVycXnD6L7975qCu6SJIktZABvaLWbNrGxq39zB5lisvhB04BWjPF5TM/fIDFD63mslOn\nDDkh9aK5s9mwtZ8f3LPyKb+GJEmSGgzoFbWrJRYBpnR10j1jylNeyWXRQ6v5hx/cz6vnzuacWV1D\ntp17wiF0z5jCQqe5SJIktYwBvaIGg/esUQI6NOahP5W10Ddu7eOd1y3i8BlT+MuLT99pe2dH8Ipn\nzOIH965k7eZte/06kiRJ2qFr97uojAaD92hTXABmHziVe1es2+Vx7l6+lhvvWjHitsUPrebXj2/g\ny3/0HA6cNmnEfS6aO5sv/HQZ373jUX777KPH2HtJkqT6+M4dj7b0eAb0inpk9SYmd3Zw6P5TRt1n\n9sxp9Nz7GJlJRIy4z5Vfv51f/mbkK4JGwJ+8+GSed+Kho77Gs46eydEHT2Ph4l4DuiRJ2uc8umYz\n7/nabS09pgG9onpXb+aIA6fS0TFy8IZGQN+0rZ/VG7dx0P6Td9r+0KqN/PI3q3n3y+fw1vNOHPEY\nnbs4PkBEcNEzZ/O5Hy3l8fVbOHT66L8wSJIk1cnAQPLury5ma19rL9zoHPSKWr560y6nt0BjiguM\nvtTif97WOLnzomfOprMjRryNxUXPPJL+geT625fvwTuQJEmqti/evIwf3/8473/l01t6XAN6Re3q\nIkWDdnexooWLennWMTM5+uD9nlJf5hwxgzmHz3A1F0mStM9YsnIdH/v2PbzoaYfxxnOOaemxDegV\n1Nc/wKNrNzN7lIsUDRoM6COthX7/inXc8+g6Lnrm7Jb06aK5s7nlwSd5+MmNLTmeJElSWW3tG+Ad\n1y1i/yldfPx1zxj1XL+9ZUCvoJXrtjCQ7HYE/ZD9JzO5s2PEgL5wcS8dAa84Y1ZL+vSqMxpB/z8X\nO81FkiTV2z98/37ueGQt/+M1z+CwGbuecrw3DOgVNBi4dzcHvaMjmDVzKr3DprhkJgsX9/LcEw9p\n2YfqmEP2Y+7RM1m42GkukiSpvm59cBX/3LOE3z7rKM4//YhxeQ0DegXt7iqizWYfOG2nEfTbHl7D\ng09sbNn0lkEXPXM2dy9fy5KVu157XZIkqYrWb+njndct5siDpvHBi04bt9cxoFfQ4Emfu7qK6KBZ\nM6eyfFhAX7i4l0mdwfmntWZ6y6BXnjGLjsCTRSVJUi391Tfv4qEnN/LJN8xl+pTxW63cddBL6Af3\nrODqnyzjHy59FgePsH557+pNHDC1a0wfjCNnTuPRtZvp6x+gq7OD/oHkm7f1ct4ph3HgfiNfHXRv\nHXbAVJ574iF8fdEjvO1FJzGlq7Olx5ckSbt2410r+B/X3z2mdbk3b97M1J/9YAJ6VR+PrN7Ef5t3\nIs8+7uBxfR0Desk8umYz77xuMWs2beP937idz7zxzJ3ODB7LEouDZs+cxkDCinVbOHLmNH7+61Ws\nWLuFK1/R2uktg6544YlcdvXP+eQN9/G+C1u7JqgkSRrd8jWbeNeCRXTPmMK5Jxyy2/0fffRRjjhi\n9/tph1kHTuX/ffHJ4/46YwroEXE+8PdAJ/C/MvPjw7YfC1wNdAOrgN/LzIeLbX8NvKLY9SOZeV3R\n/iLgE8Bk4FbgDzKzLxpp9O+BC4GNwOWZ+cviOZcB7y+O9VeZec1eveuSykz+/Gu3saWvn999zjF8\n6b9+wzcWPcJrnnXUkP16V28ec0CfVVysaPnqTRw5cxoLF/cybVInL3n6YS3vP8B5p3Tzxuccw1U/\nXsqLnnYYzxnDDwhJkvTUDAwkf/Zvi+kbSD5/2bM57tD9d/ucnp4nmTfvmRPQO+2p3c5Bj4hO4DPA\nBcCpwKURceqw3T4BfDEzzwA+DHyseO4rgDOBucBzgHdHxAER0QFcA1ySmacDDwKXFce6ADi5uF0B\nfLY41sHAB4vjnAN8MCIO2sv3XUr/+rMHuem+x7jywqfz4YtP5+xjD+ID37hzpyuB9q7Z/VVEBw2e\nSPrI6k1s7Rvg23cs56WnHs5+k8fvjydXXvh0jjl4P/50wWLWbd42bq8jSZIarrl5Gf93yRO8/xWn\njimcq9zGcpLoOcCSzFyamVuBrwAXD9vnVOD7xf0fNm0/FfhRZvZl5gZgMXA+cAiwJTPvK/a7EXhd\ncf9iGmE/M/NnwMyImAW8HLgxM1dl5pPFc87fw/dbWg88tp6PXn83Lzylm98791g6O4JPvmEuA5n8\n2YLFDAwkABu39rF647axj6Bvv1jRZn6y5DFWb9zW8tVbhtt/SheffMNclq/ZxIf/865xfS1JkvZ1\nS1au4+PFFS0vPefodndHLTCWYdQjgYeaHj9MYxS72WIaAfvvgdcAMyLikKL9gxHxSWA/YD5wF/A4\nMCkizs7MW4DXA4OfqJFe78hdtA8REVfQGHmnu7ubnp6eMbzF9uobSD76X5vpZIDXzF7Pj370o+3b\n3nBKJ//7jie48ovf4+XHTaJ3feOkj9W9y+jpeXhMx9+vC265awk3LU72nwQ8ehc9K+/e436uX79+\nj+r5iuMn8W+3PswRA49x1uGe7jDcntZTu2Y9W8datpb1bC3rOVTfQPJXP9vMpBjgolnrhmSI3bGW\n5TWW1DTStUtz2OM/A/4pIi4HbgIeAfoy84aIeDbwU+Ax4OaiPSPiEuBTETEFuAHo283rjaUfZOZV\nwFUAc+bMyXnz5u363ZXAp793H79ecz+feeOZO13Z87xMHvriLXzt/sd5ywXn0rFmM/zk57zkuWdy\nzvFjO4P4mEU3sXnyZG57dDWvmns0L3nRGXvVz56eHvakns/7rQF+/dn/y5fu28ybLjx3XK60VWV7\nWk/tmvVsHWvZWtaztaznUJ+84V6WrV3C537vTM4/fc+WT7aW5TWWKS4Ps2N0G+AoYMhC15nZm5mv\nzcxnAVcWbWuKrx/NzLmZ+VIaIfv+ov3mzHxBZp5DI9Tfv5vX220/qmjxQ6v5xx8s4dVzZ+8UzgEi\ngo+99gxmTOnindct4sFVG4HdX0W02ZEzp/HTB55gw9b+cZ/e0mxyVwefesNc1m/p431fu53MnX6f\nkiRJe+mXv3mSz/Q8wOvOPGqPw7nKbSwB/RfAyRFxfERMBi4BFjbvEBGHFid+AryPxoouRERnMdWF\niDgDOIPGaDkRcVjxdQrwHuBzxfMXAm+OhnOBNZm5HPgu8LKIOKg4OfRlRVtlbdrazzuvW8RhM6bw\nlxefPup+3TOm8LHXPoM7e9fy99+7jwg4/ICxB/RZM6eSCYfNmDLhq6qcfPgM3nv+0/j+PSv5yi8e\n2v0TJEnSbm3c2sefXreIIw6YygcvGr52h6put1NciqUP304jDHcCV2fmnRHxYeCWzFwIzAM+FhFJ\nYzT8bcXTJwE/LtbxXktj+cXBqSzvjohX0vgl4bOZObhS/vU0llhcQmOZxbcU/VgVER+h8QsDwIcz\nc9Xev/X2+/i372bp4xv40h8+hwOn7fqiQS877QjecPZRLLjlYY44YCqTOsd+EdjBE0pfccYsOjtG\nmik0vi5/3nF8/54VfOSbd/G8Ew/h2EM8u1ySpN1Zs3EbSx9fP+K2L//Xb3hw1Uau/aNzOWBqay88\nqPYb05l7mXk9jeDc3PaBpvtfBb46wvM201jJZaRjvht49wjtyY6AP3zb1RSj81W3ZOV6rrn5QS5/\n3nE8/6RDx/ScD7zqNG5e+sT2pRPH6oRDpwPw6rk7nVM7ITo6gr99/TN5+adv4p3XLWLBHz+Xrj34\nBUOSpH3Nqg1bOf/TN7Fy3ZZR9/mjFxw/pgsSqXpcWqNNFi56hI6A/z7/xDE/Z/qULv79vz1/j+dy\nv+zUw7nxnS/k5MNn7Gk3W2b2zGl85OLTecd1i/ifNy3lbfNPaltfJEkqs8zkff9+G6s3buPvL5nL\nASP8lX3apE7OGefLzat9DOhtkJksXNzLc088ZI9XNumeMWWPX6+jI9oazgddPHc237t7BZ+68T5e\neHI3zzjqwHZ3SZKk0vnqrQ/z3TtX8L4LnsbFbfrrt9rLeQZtcNvDa1j2xMYJXVGlDCKCv3r16Rw6\nfQrvuO5XbN7W3+4uSZJUKg+t2shf/uddnHP8wfzhC05od3fUJgb0Nli4uJdJncH5p+17SyLN3G8y\nf/vbZ/DAYxv4+LfvaXd3JEkqjf6B5F0LFgPwyTc8sy0LO6gcDOgTrH8g+eZtvZx3ymEcuN++edb1\nC07u5vLnHccXfrqMm+57rN3dkSSpFP7lx0v5+bJV/OVFp3HUQfu1uztqIwP6BPv5r1exYu0WLp67\nb01vGe69FzyNkw6bzru/upjVG7e2uzuSJLXVnb1r+Lsb7uXCZxzBa8903vm+zoA+wRYu7mW/yZ28\n5OmHt7srbTV1Uief/p25PLF+K1d+4w6vMipJ2mdt3ta4cOFB+03mo69+BsX1Y7QPM6BPoK19A3z7\njuW89NTDmTa5s93dabvTjzyQd770FL5123L+Y1Fvu7sjSVJbfOK793LfivX8zevP4KD9J7e7OyoB\nl1mcQD++/zH+//buPDyu+srz//uoSi553yQ27wRjMMEYbCCEkIjQCUsWGkgHmKydEKanQz/dJPD7\nhcnODCFJEzKZSTrd9C90N/mFAIGQdroJkBAEYTOWjQUGY7wgGVvGthYvkqylVGf+qCtTFmWrJN3S\nreXzep56XLpbnTrPler41Pd+756uvrKbveVI/ut7j+ePr+7i6/++jnPeMZOjpwxv2kkREZGwtXb0\n8KPHNtKbTOX9tXr7U/x6zXY+fc48ahcdlffXk+KgAn0MrWhoZur4Ss5bWBN1KAUjHqvgGx9ezKU/\neZrntrRqvlcREYnc3Su3ctezTRw9Zfj3HhmJ8xZWc9PFJ4/Ja0lxUIE+Rg709vP7V3Zy6dLjGBfX\nyKJMMyelv87rGYNOhYiIyJEM3EzwrPkzuO+vzok6HClTqhTHyB/W76Srt5+PaHjL2yTi6fH4Pbpx\nkYiIROzVN/ezcVcHHynz2dYkWirQx8iKhmaOnpLg7AUzow6l4FRVpk9DddBFRCRqKxqaiVUYl7zz\nmKhDkTKmAn0M7O3q44kNu/nwkuN0V7AsBjro3eqgi4hIhNyd3zY0854Tqpk5aWzGn4tkowJ9DDz8\n8g56+1OaveUwKmNGhamDLiIi0VqzdQ/b2g/o81oipwJ9DKxoaGbezAksmT016lAKkpmRiMfUQRcR\nkUj9tqGZRLyCD55S3jcTlOipQM+zXfu7eXZzKx897TjdGewIqior1EEXEZHIJPtT/MeLO3j/SUcx\nuaoy6nCkzKlAz7P/fHEHKUdflw1BHXQREYnSc1vaaOno0ee1FAQV6Hn2u5fe5KRjJrPw6MlRh1LQ\nEuqgi4hIhFY0bGdSIs75J+lunhI9Feh59uqb+1g+f3rUYRS8KnXQRUQkIj3Jfn637k0+uPhoqipj\nUYcjogI9n/Z09bKvO8n8mROjDqXgqYMuIiJReWLDbvZ3J3VzIikYKtDzqLG1C4B5KtCHpA66iIhE\nZUVDM9MnVPKeE6qjDkUEUIGeV02tnQDMmzkh4kgKnzroIiIShc6eJH9Yv5NLTj2WypjKIikMOhPz\nqCnooM+doQJ9KOlZXFSgi4jI2PrD+p109+lmglJYVKDnUWNrJ8dMqdIFJzlId9A1xEVERMbWbxua\nOXZqFWfOnxF1KCIHqUDPo62tXRrekqOqeIweddBFRGQM7enq5YnXdvPhJcdSUaGbCUrhUIGeR42t\nXZrBJUfqoIuIyFh7eN2b9PU7Hz1tVtShiBxCBXqedPQkaenoYa466Dmp0hh0EREZYysamllQPZF3\nzpoSdSgih1CBnidbgwtE1UHPjTroIiIylvYe6OO5La18eMmxmGl4ixQWFeh5oikWh6cqHqOv3+lP\nedShiIhIGViztZ2Uw216fA0AACAASURBVDnHz4w6FJG3UYGeJ01twRSLKtBzkqhMn4rqoouIyFio\nb2wjVmEsnTst6lBE3kYFep40tXYyc+I4plRVRh1KUaiKBwW6xqGLiMgYqG9s55TjpjBhXDzqUETe\nJqcC3cwuMrMNZrbJzL6SZf08M3vMzF40szozm52x7ntmti54XJmx/AIzW2Nma83sKTM7IYdj9Qfb\nrzWzFaN76/nV2NKl7vkwJIK54rvVQRcRkTzrTaZo2LaHZfOmRx2KSFZDFuhmFgN+AlwMLAauNrPF\ngza7DbjL3ZcANwO3Bvt+CDgDWAqcDdxoZgOXSv8U+IS7LwXuBr52pGMFDrj70uDx0WG/2zG0tU1T\nLA5HVaU66CIiMjZebt5Ld19KNyeSgpVLB/0sYJO7b3H3XuAe4NJB2ywGHgueP56xfjHwhLsn3b0T\naAAuCtY5MFCsTwWahzhW0eju66d57wFdIDoMibg66CIiMjZWN7UDsFwddClQuQy8mgW8kfHzNtLd\n8EwNwBXAj4DLgMlmNjNY/k0zux2YAJwPvBLscw3wkJkdAPYB7zrSsdy9Fagys3ogCXzX3X8zOFgz\nuxa4FqCmpoa6uroc3mK4mjtSuEPXrq3U1TUPvUOR6OjoyFs+X9uVBOCZlat4c2osL69RaPKZz3Kk\nfIZHuQyX8hmuMPL5uxe6qRlvvLLmuYNFSTnSuVm4cinQs00OOnguvBuAH5vZZ4Enge1A0t0fNbMz\ngWeA3cCzpItrgOuBS9x9pZndCNxOumjPeqxgn7nu3mxmxwN/NLOX3H3zIYG53wHcAbBo0SKvra3N\n4S2G67H1O+Gpei56zzLOmFs6/zuvq6sjX/ms3NQCa1ZyyqlLObtMprzKZz7LkfIZHuUyXMpnuEab\nT3fnhqf+wHtPOoba2qXhBVaEdG4WrlwK9G3AnIyfZ/PWcBQA3L0ZuBzAzCYBV7j73mDdLcAtwbq7\ngY1mVgOc5u4rg0PcCzycw7Gag3+3mFkdcDpwSIFeCBqDmxTNm6EhLrk6OAY9qTHoIiKSP02tXbR0\n9LJsfuk00KT05DIGfRWw0MwWmNk44CrgkBlUzKzazAaOdRNwZ7A8Fgx1wcyWAEuAR4F2YKqZnRjs\n8wFg/RDHmm5miYFtgHOhML+Z2trayeREnBkTx0UdStE4OAa9T2PQRUQkf1Y1tgGwfJ4uEJXCNWQH\n3d2TZnYd8AgQA+5095fN7Gag3t1XALXArWbmpIelfDHYvRL4U3AL3X3AJ909CWBmXwAeMLMU6YL9\nc8E+hzvWycA/BdtXkB6DXpAFemNrF/OqJ+jWwcOgDrqIiIyF1U3tTKmKs/CoSVGHInJYOc3O7+4P\nAQ8NWvaNjOf3A/dn2a+b9Kws2Y75IPBgluWHO9YzwKm5xBu1ptZOTjluatRhFBV10EVEZCzUN7Wz\nbN50KirURJPCpTuJhizZn2Jbu6ZYHK5EXB10ERHJr/bOXjbt6mC55j+XAqcCPWTNe7pJplwF+jAd\nvJOoOugiIpInmv9cioUK9JA1tXUCME93ER0WddBFRCTf6pvaqYwZp82ZFnUoIkekAj1kA1MszleB\nPiwHC3R10EVEJE/qG9s45bipVFWWxw3xpHipQA9ZU0sniXgFR01ORB1KUTEzEvEKddBFRCQvepL9\nvLh9L2dq/nMpAirQQ9bU1sW8mRN0dfgIVFXGNAZdRETyYt32vfQmUyzT/OdSBFSgh6yptZO5MzS8\nZSTUQRcRkXxZ1Zi+QHSZLhCVIqACPUSplLO1rYv5msFlRNRBFxGRfKlvbGdB9URqNARVioAK9BDt\n2t9Dd1+KedXqoI+EOugiIpIP7s7qpjZ1z6VoqEAPUWNrMMXiDHXQR0IddBERyYfNuztp7+rTBaJS\nNFSgh2irplgcFXXQRUQkH1Y3tQHoAlEpGirQQ9TY2km8wjhuWlXUoRQlddBFRCQfVjW2M31CJe+o\nUQNNioMK9BA1tXUxe/p44jGldSTUQRcRkXxY3dTOsnkzMNMUyFIcVEmGqKm1k3ka3jJiVZUxFegi\nIhKqlo4eXm/pZLnGn0sRUYEeEnenqSV9kyIZmUS8QkNcREQkVPXB/OfLNYOLFJF41AGUivauPvb3\nJNVBH4WEOugiIjICew/08eCabfT2v/0z5E8bWxgXr+DU2VMjiExkZFSgh0RTLI6eOugiIjJc7s7f\n3vMCdRt2H3abPzv5KBLx2BhGJTI6KtBDcnCKxWoV6COlMegiIjJcv1i5lboNu/n6hxdz1Zlzsm4z\nvlLFuRQXFeghaWztxAxmT1eBPlKJeAW9yRSplFNRoSvtRUTkyLbs7uCW/1zPeQur+ct3z9dnh5QM\nXSQakqbWLo6dUkWV/pc+YgO5yzaGUEREJFNff4rr713LuHgFt/3FaSrOpaSoQA+JplgcvUQ8fTpq\nHLqIiAzlJ49vomHbXr5z2akcPUU3CJTSogI9JE2tmmJxtBKV6dNR49BFRORIXtjazv/54yYuO30W\nH1pybNThiIROBXoI9nf30drZqw76KFUFV9irgy4iIofTk3S+dF8Dx0yp4tuXnhJ1OCJ5oYtEQ9A0\nMIOLOuijog66iIgM5Z4NvTS2Jrn7mncxpaoy6nBE8kId9BAMFOhzVaCPijroIiJyJI+/uovH30hy\nzXsWcM47ZkYdjkjeqIMegqa24CZFGuIyKuqgi4iUN3fnlv9cz+MbdmVdv2NvN7MnGTdcuGiMIxMZ\nWyrQQ7BzbzdTquJMSiidozEwzaI66CIi5en+1dv4/556nXe/YybTJ4572/rT5kzjrIltuiuolDxV\nlCFo6eylelIi6jCK3sA0iz196qCLiJSbN9q6+PZvX+HsBTP4+efPJnaYec3r6urGNjCRCGgMegha\nO3qYOent/9OX4TnYQU+qgy4iUk76U86X72vAgB98/LTDFuci5UIFeghaO3qZOVEd9NFSB11EpDzd\n8eQWnm9s49uXnsLs6ZpwQUQFeghaOnqonqwO+mipgy4iUn5ebt7L7b/fwCWnHsNlp8+KOhyRgqAC\nfZSS/Snau/rUQQ+BOugiIuWlu6+f6+9dy/QJ47jlz0/FTENbRCDHAt3MLjKzDWa2ycy+kmX9PDN7\nzMxeNLM6M5udse57ZrYueFyZsfwCM1tjZmvN7CkzOyGHY33GzDYGj8+M7q2Ho62rF4BqjUEfNXXQ\nRUTKy98/soHXdnbw/Y8tyTpri0i5GrJAN7MY8BPgYmAxcLWZLR602W3AXe6+BLgZuDXY90PAGcBS\n4GzgRjObEuzzU+AT7r4UuBv42hDHmgF8MzjOWcA3zWz6SN50mFo70gX6TM3iMmrjYuqgi4iUi6c3\ntfCzp17n0+fMo3bRUVGHI1JQcplm8Sxgk7tvATCze4BLgVcytlkMXB88fxz4TcbyJ9w9CSTNrAG4\nCLgPcGCgWJ8KNA9xrAuB37t7WxDH74Nj/TKnd5onLR09AJpmMQQVFca4eIU66CIiJWLz7g7aO3vf\ntjyZcm74VQPHV0/kpotPjiAykcKWS4E+C3gj4+dtpLvYmRqAK4AfAZcBk81sZrD8m2Z2OzABOJ+3\nCvtrgIfM7ACwD3jXEMfKFkfkV5O81UHXV3NhSMQr1EEXESkB//FiM9fd/cJh18crjAf+27sZP043\nHRIZLJcCPdsVGz7o5xuAH5vZZ4Enge1A0t0fNbMzgWeA3cCzQDLY53rgEndfaWY3AreTLtqzHivH\nODCza4FrAWpqavJ+Q4OVjX0AvLp2FW9UlvbFLR0dHXnPZ4X307h1G3V1u/P6OoVgLPJZTpTP8CiX\n4SrHfLZ3p/ja0wc4fmoFly/M3sCqGW+0b15L3ebhHbsc85kvymXhyqVA3wbMyfh5Nm8NRwHA3ZuB\nywHMbBJwhbvvDdbdAtwSrLsb2GhmNcBp7r4yOMS9wMNHOpaZbQNqB8VRNzhYd78DuANg0aJFXltb\nO3iTUK18+FUqN27hkj+rLfmrz+vq6sh3Piev/CMzamZQW7s0r69TCMYin+VE+QyPchmucsunu/Pp\nO58nRS8/+8J5LKieGOrxyy2f+aRcFq5cZnFZBSw0swVmNg64CliRuYGZVZvZwLFuAu4MlseC4SmY\n2RJgCfAo0A5MNbMTg30+AKw/0rGAR4APmtn04OLQDwbLItWyv4eZExMlX5yPlarKGD1JDXERESlW\nP3+uiT9tbOGrHzo59OJcpFwM2UF396SZXUe6GI4Bd7r7y2Z2M1Dv7itId7ZvNTMnPSzli8HulcCf\nguJ1H/DJ4IJRzOwLwANmliJdsH8u2Cfrsdy9zcz+B+n/MADcPHDBaJRaO3s1/jxEiXgF3X26SFRE\npBht3t3Bdx5aT+2iGj5x9tyowxEpWrkMccHdHwIeGrTsGxnP7wfuz7JfN+lZWbId80HgwSzLsx4r\nWHcnb3XUC0JrR4+mWAyROugiIsWprz/Fl+5dS1VljO9fsUTfLIuMgu4kOkotHb1U6+YKoVEHXUSk\nOP34j5to2LaX71x2KkdNqYo6HJGipgJ9FNydlo4eqiergx6WRLxCHXQRkSKz9o09/PjxTVx++iwu\nOfXYqMMRKXoq0Eehs7efnmSKmeqgh6aqMqYOuohIETnQ28+X7l3L0ZMTfOvSU6IOR6Qk5DQGXbJr\nDe4iqjHo4VEHXURk9Oob2/iPF3eMyWtt3LWfLS2d3H3N2UypqhyT1xQpdSrQR6EluItotWZxCY06\n6CIio7OtvYu//JdV9PanqKrM/106zeDLHziRd59QnffXEikXKtBHoSXooFergx4addBFREYulXK+\nfF8DKXf+8KX3MWfGhKhDEpER0Bj0UWgNOuiaBz086qCLiIzcz556nZWvt/HNj56i4lykiKlAH4WB\nMegzdJFoaAY66O4edSgiIkVl/Y59/P0jG7jwlKP5i2Wzow5HREZBBfootHb2MqUqTiKe/zF+5SIR\njJfUMBcRkdz1JPu5/t61TBlfyXcuO1U3CRIpcirQR2F3R4/Gn4csEU+fkirQRURy94NHX+PVN/fz\n/Y+dqpnFREqACvRRaO3o0fjzkA3MONCjcegiIjl5dnMr//ynLfyXs+fy/pOOjjocEQmBCvRRaO3o\nZeZEdSrCpA66iEju9nX3ccOvGpg3YwJf+9DJUYcjIiFRgT4KrZ29VE9WBz1MAx10zeQiIjK0b/37\ny7y5r5sfXrmUCeM0c7JIqVCBPkLJ/hTtXeqgh00ddBGR3Dy9qYVfv7CdL55/AqfPnR51OCISIhXo\nI9TW1Yu77iIaNnXQRURy8/iruxgXr+Cva98RdSgiEjIV6CP01k2K1EEPkzroIiK5qW9q57TZUw82\nNkSkdKhAH6GBAl3TLIZLHXQRkaEd6O1n3fa9LJ8/I+pQRCQPVKCPUEtwF1FNsxiuRKU66CIiQ2nY\ntodkylk+T2PPRUqRCvQRGijQq3WRaKiq4gN3ElUHXUTkcFY3tQOwTAW6SElSgT5CrZ29VMaMKeM1\nrVWYBjro3X3qoIuIHM6qxjYWHjWJaRP0La5IKVKBPkKtHT3MnJjAzKIOpaQc7KBrDLqISFaplLOm\nqZ3l89U9FylVKtBHqLWjV+PP8+BgB11j0EVEstq4q4N93UmWzdMFoiKlSgX6CLV09GiKxTxIHOyg\nq0AXEclmVWMbAGeqgy5SslSgj1BLRy/VE9VBD1uswqiMGd26SFREJKvVTe1UT0owd8aEqEMRkTxR\ngT4C7k5rZw/Vk9VBz4dEPKYOuojIYaxqbGP5vOm6BkqkhKlAH4Gu3n66+1LMVAc9L6oqK9RBFxHJ\nYue+bra1H9AFoiIlTgX6CLx1kyJ10PNBHXQRkezqG9Pzn+sOoiKlTQX6CLR09AK6i2i+JNRBFxHJ\nalVjG1WVFZxy3JSoQxGRPFKBPgKtQQe9Rh30vFAHXUQku9VN7SydM43KmD6+RUqZfsNHoLVTHfR8\nqqqsoEcddBGRQ3T2JHllxz6Wa/5zkZKnAn0EWvanO+gzdJFoXiTiFeqgi4gMsvaNPfSnXBeIipQB\nFegj0NrZy+Sq+MGb6ki4qipjGoMuIjJIfWM7ZnDGPBXoIqUupwLdzC4ysw1mtsnMvpJl/Twze8zM\nXjSzOjObnbHue2a2LnhcmbH8AjNbY2ZrzewpMzshWD7XzB43sxeC410SLJ9vZgeC7dea2T+O/u2P\nTEtHj8af55E66CIib1ff1Maioyczpaoy6lBEJM+GLNDNLAb8BLgYWAxcbWaLB212G3CXuy8BbgZu\nDfb9EHAGsBQ4G7jRzAYuPf8p8Al3XwrcDXwtWP414D53Px24CviHjNfZ7O5Lg8dfDfvdhqS1o1fj\nz/NIHXQRkUP1p5wXtu7R8BaRMpFLB/0sYJO7b3H3XuAe4NJB2ywGHgueP56xfjHwhLsn3b0TaAAu\nCtY5MFCsTwWah1heMFo6epg5UR30fFEHXUTkUK++uY+OnqQuEBUpE7kU6LOANzJ+3hYsy9QAXBE8\nvwyYbGYzg+UXm9kEM6sGzgfmBNtdAzxkZtuATwHfDZZ/C/hksPwh4G8yXmdBMPTlCTM7L5c3mA+t\nneqg55M66CIih1rdlL5B0TKNPxcpC/EctrEsy3zQzzcAPzazzwJPAtuBpLs/amZnAs8Au4FngWSw\nz/XAJe6+0sxuBG4nXbRfDfyru//AzM4Bfm5m7wR2AHPdvdXMlgG/MbNT3H3fIcGaXQtcC1BTU0Nd\nXV0ObzF3/SmnvbOXjpYd1NW1hnrsQtfR0RF6PrPZtaOHrp7kmLxWlMYqn+VC+QyPchmuMPL5n2u7\nmZ4wNjWsZLNl+1guHzo/w6NcFq5cCvRtvNX1BpjNoGEn7t4MXA5gZpOAK9x9b7DuFuCWYN3dwEYz\nqwFOc/eVwSHuBR4Onn+eYBiMuz9rZlVAtbvvAnqC5avNbDNwIlA/KJY7gDsAFi1a5LW1tTm8xdzt\n3t+DP/oHlr/zRGrPmR/qsQtdXV0dYeczm9W9G3ikaRPve9/7sBL+IBqrfJYL5TM8ymW4wsjnf3/2\nMd69aDrnn39GOEEVMZ2f4VEuC1cuQ1xWAQvNbIGZjSN94eaKzA3MrNrMBo51E3BnsDwWDHXBzJYA\nS4BHgXZgqpmdGOzzAWB98HwrcEGwz8lAFbDbzGqCC1Yxs+OBhcCW4b/l0WkJ7iI6U7O45E0iXoE7\n9PUP/qJGRKT8bN9zgOa93SzX8BaRsjFkB93dk2Z2HfAIEAPudPeXzexmoN7dVwC1wK1m5qSHuHwx\n2L0S+FPQBd0HfNLdkwBm9gXgATNLkS7YPxfs82Xgn83setJDaT7r7m5m7wVuNrMk0A/8lbu3jT4F\nw9PaEdxFVDcpypuqyvT88t3JfsbFNVW/iJS3+sb0R92Z83WBqEi5yGWIC+7+EOkLNjOXfSPj+f3A\n/Vn26yY9k0u2Yz4IPJhl+SvAuVmWPwA8kEu8+dTame6gV09WBz1fEkFR3tOXSn9/IiJSxlY3tTNh\nXIyTjpkcdSgiMkZyKtDlLS1BB71a0yzmTSLooPdoJhcRKRP9KWffgb6s655/vY3T504jHtM3iiLl\nQgX6MLV09BCvMKaMV+ryZaCD3q250EWkTPz1L1bzyMs7D7v+by9YOIbRiEjUVGUOU2tHDzMnjSvp\n2UWiVqUOuoiUkb7+FE++1sJ5C6u54KSj3rY+FqvgI0uOjSAyEYmKCvRhau3opVozuOSVOugiUk7W\n79jHgb5+Pr58Dh857biowxGRAqABbcPU0tmrKRbzLBFXB11EyseqxvRdQpfP1zSKIpKmAn2YWvb3\nUK0pFvOqqjJjFhcRkRK3uqmNWdPGc+zU8VGHIiIFQgX6MLg7rZ3pMeiSP+qgi0i5cHfqG9vVPReR\nQ6hAH4au3n66+1Iag55nAx10jUEXkVL3RtsBdu3v0V1CReQQKtCH4eBdRFWg55XmQReRclHflL5L\n6HLdJVREMqhAH4bdHem7iGqIS35VaRYXESkT9U3tTE7EOfFo3SVURN6iAn0YWoMCXXcRzS910EWk\nXKxubOf0edOJVejeGiLyFhXow9DamR7iUj1ZHfR8UgddRMrB3q4+Nuzcz5kafy4ig6hAH4aBDvoM\nTbOYV/FYBbEKUwddREramq3p+c+XaQYXERlEBfowtHT0MrkqfnAaQMmfqniFOugiUtLqm9qIVxhL\n50yLOhQRKTAq0IehpaNHUyyOkURlTB10ESlpqxrbOeW4KUwYF486FBEpMCrQh6G1o5dqzeAyJtRB\nF5FS1ptM0fDGHpbN0/SKIvJ2KtCHobWzh5mawWVMpDvoKtBFpDS93LyXnmSKMzX+XESyUIE+DC0d\nvZoDfYwk4hV092mIi4iUpvpGXSAqIoenAj1Hyf4U7V29uovoGFEHXURKWX1TG3NnTOCoyVVRhyIi\nBUgFeo7au/pwhxp10MdElTroIlKi3J3VTe0sV/dcRA5DBXqOWoI50NVBHxvqoItIqWps7aKlo5fl\nukBURA5DBXqO1u/YB8CC6okRR1IequIV9KiDLiIlqL6xDUAddBE5LBXoOapvamdyIs6JR0+OOpSy\noA66iJSq1U3tTB1fyQk1k6IORUQKlAr0HK1ubOeMedOJVVjUoZQFddBFpFStamxj2bzpVOjzREQO\nQwV6DvZ29bFh536Wz9PXkWMlUVlBtzroIlJi2jt72by7U8NbROSIVKDnYM1WzVc71hLxmDroIlJy\nVjelP090gaiIHIkK9BysamwjXmEsnTMt6lDKRpU66CJSglY1tVEZM5bMnhp1KCJSwFSg56C+qZ1T\njpvChHHxqEMpG4l4jP6Uk+xXkS4ipWN1YzunzppKVWUs6lBEpICpQB9CbzJFwxt7WKavI8dUVWX6\n1FQXXURKRXdfPy9u28vy+fo8EZEjU4E+hHXNe+lJpjhT48/HVCKe7i5pHLqIlIp12/fS259imSYc\nEJEhqEAfwupGXSAaBXXQRaTUPPjCdszQjGAiMiQV6EOob2pj7owJHDW5KupQyoo66CJSSp54bTe/\nWLmVz527gJmTElGHIyIFTgX6Ebg79Y3tmq82Agc76H3qoItIcevodW78VQMLj5rEjRcuijocESkC\nORXoZnaRmW0ws01m9pUs6+eZ2WNm9qKZ1ZnZ7Ix13zOzdcHjyozlF5jZGjNba2ZPmdkJwfK5Zva4\nmb0QHO+SjH1uCmLYYGYXju6tD62xtYvWzl7NVxuBgx30pDroIlK83J27XumhrbOXH165VLO3iEhO\nhizQzSwG/AS4GFgMXG1miwdtdhtwl7svAW4Gbg32/RBwBrAUOBu40cymBPv8FPiEuy8F7ga+Fiz/\nGnCfu58OXAX8Q3CsxcHPpwAXAf8QxJY39Y1tAOqgRyChDrqIlIAVDc08/2Y/13/gRN45S3Ofi0hu\ncumgnwVscvct7t4L3ANcOmibxcBjwfPHM9YvBp5w96S7dwINpItrAAcGivWpQPMQyy8F7nH3Hnd/\nHdgUxJY39Y3tTB1fyQk1k/L5MpKFOugiUuya9xzg679ZxwnTKviv7z0+6nBEpIjkcuedWcAbGT9v\nI90Nz9QAXAH8CLgMmGxmM4Pl3zSz24EJwPnAK8E+1wAPmdkBYB/wrmD5t4BHzexvgInAn2XE8dyg\nOGYNDtbMrgWuBaipqaGuri6Ht5jdk+u7mD+pgieffGLExyglHR0do8rncDTtSxfmq9e+CDtK8wZR\nY5nPcqB8hke5HL2UO7fVd9PTl+ITpzhP/enJqEMqGTo/w6NcFq5cKh/LsswH/XwD8GMz+yzwJLAd\nSLr7o2Z2JvAMsBt4FkgG+1wPXOLuK83sRuB20kX71cC/uvsPzOwc4Odm9s4c48Dd7wDuAFi0aJHX\n1tbm8Bbfrq2zlx0P/55PnXcCtbUnjOgYpaauro6R5nO4Nu3qgGee4IRFJ1O79G3/DysJY5nPcqB8\nhke5HL1/efp1Xml9he9cdirHHdiifIZI52d4lMvClcsQl23AnIyfZ/PWsBMA3L3Z3S8Pxo1/NVi2\nN/j3Fndf6u4fIF1kbzSzGuA0d18ZHOJe4N3B888D9wX7PgtUAdW5xBGm1U3p+c91gWg0BmZx6dEY\ndBEpMpt27ee7v3uV9590FFefNWfoHUREBsmlQF8FLDSzBWY2jvSFmisyNzCzajMbONZNwJ3B8lgw\n1AUzWwIsAR4F2oGpZnZisM8HgPXB863ABcE+J5Mu0HcHr3mVmSXMbAGwEHh++G85N/VNbVTGjCWz\ndVFPFDQGXUSKUbI/xfX3NjBhXIzvXnEqZtm+/BURObIhh7i4e9LMrgMeAWLAne7+spndDNS7+wqg\nFrjVzJz0EJcvBrtXAn8K/kDtAz7p7kkAM/sC8ICZpUgX7J8L9vky8M9mdj3pISyfdXcHXjaz+0iP\nYU8CX3T3vFVv9Y3tnDprqqbEiojmQReRYrRm6x5e2r6X2/7iNN3gTkRGLKer79z9IeChQcu+kfH8\nfuD+LPt1k57JJdsxHwQezLL8FeDcw+xzC3BLLjGPRndfPy9t28tnz52f75eSw1AHXUSKUX1Tenre\n9590VMSRiEgx051Es1i3fS+9/SmWzdP851GpjBkVpg66iBSX1Y3tHF8zkRkTx0UdiogUMRXoWaxq\nHLhAVAV6VMyMRDymDrqIFI1UyqlvaudMTS4gIqOkAj2L1U1tHF89kZmTElGHUtaqKivUQReRorF5\ndwd7D/SxTHefFpFRUoE+SCrlrG5q1/CWAqAOuogUk/omffsqIuFQgT7IlpYO2rv6OHO+vqKMWlVl\nBT1JddBFpDisamxj5sRxLKieGHUoIlLkVKAPUh+MP9dXlNFLxGN096mDLiLFYeDbV819LiKjpQJ9\nkPqmdmZMHMfx6oBELqEOuogUiV37u2lq7WK5mjsiEoKc5kEvNTf/9hUefGFb1nX7u5PULjpKHZAC\nUKUOuogUidUDs39peKSIhKAsC/TT5kwlmcremTXgY8vmjG1AklWisoKOnmTUYYiIDKm+qZ1EvIJ3\nHjc16lBEpASUwCaIQQAADjpJREFUZYF+6dJZXLp0VtRhyBAS8RgtHb1RhyEiMqT6xjZOmz2NcXGN\nHBWR0dNfEilY6THoGuIiIoXtQG8/Lzfv0/hzEQmNCnQpWFXxGD26UZGIFLi1b+whmXIV6CISGhXo\nUrDUQReRYlDf2AbAsrm6QFREwqECXQpWehYXddBFpLDVN7Vz4tGTmDqhMupQRKREqECXgqUOuogU\nuv6Us2ZrO8vmqXsuIuFRgS4Fqyoeo6/f6U951KGIiGT12s797O9OcqbGn4tIiFSgS8FKVKZPT3XR\nRaRQ1TcFNyhSB11EQqQCXQpWVTCfsMahi0ihWt3YRs3kBHNmjI86FBEpISrQpWAlKmOAOugiUrhW\nNbZz5vzpmFnUoYhICVGBLgWrqlIddBEpXDv2HmD7ngO6QFREQqcCXQpWIq4OuogUrvrGgfHnukBU\nRMKlAl0KljroIlLIVje1M74yxuLjpkQdioiUGBXoUrAOdtD71EEXkcJT39TG0jnTqIzpo1REwqW/\nKlKwDnbQk+qgi0hh6ehJ8krzPpZr/nMRyQMV6FKw1EEXkUK1duseUg7L5+sCUREJnwp0KVjqoItI\noapvasMMTp87LepQRKQEqUCXgqUOuogUqudfb+OkY6Ywpaoy6lBEpASpQJeClQjuJNqjDrqIFJA/\nvrqTZza3cuEpR0cdioiUKBXoUrAG7iTarQ66iBSI1o4e/p/7X+KkYybz32rfEXU4IlKi4lEHIHI4\n6qCLSCFxd2769UvsO9DH/3/NWQeH4YmIhE0ddClYBwt0ddBFpAD8avU2Hn1lJzdeuIiTjtHNiUQk\nf1SgS8EyMxLxCnXQRSRyb7R18e0VL3PO8TP5/HsWRB2OiJS4nAp0M7vIzDaY2SYz+0qW9fPM7DEz\ne9HM6sxsdsa675nZuuBxZcbyC8xsjZmtNbOnzOyEYPkPg2Vrzew1M9uTsU9/xroVo3vrUgyqKmMa\ngy4ikepPOdffu5YKM277+GlUVFjUIYlIiRtyDLqZxYCfAB8AtgGrzGyFu7+SsdltwF3u/m9m9n7g\nVuBTZvYh4AxgKZAAnjCz37n7PuCnwKXuvt7M/hr4GvBZd78+47X/Bjg943UOuPvS0bxhKS7qoItI\n1P7pyc3UN7XzwytPY9a08VGHIyJlIJcO+lnAJnff4u69wD3ApYO2WQw8Fjx/PGP9YuAJd0+6eyfQ\nAFwUrHNgYBDfVKA5y2tfDfwylzcipUkddBGJ0rrte/nh71/jQ0uO5c+Xzoo6HBEpE+buR97A7GPA\nRe5+TfDzp4Cz3f26jG3uBla6+4/M7HLgAaAaWAZ8k3T3fQLwPPATd/+BmZ0H/AY4AOwD3hV01geO\nOQ94Dpjt7v3BsiSwFkgC33X332SJ91rgWoCamppl99133/CzIll1dHQwadKkMX3N//5UF8dNrOC6\n06vG9HXHQhT5LGXKZ3iKOZc9/Uf+TBuOZApuWXmArj74n+eOZ9K4kQ1tKeZ8FiLlMzzKZbjOP//8\n1e6+PIxj5TLNYra/SIP/At4A/NjMPgs8CWwHku7+qJmdCTwD7AaeJV1cA1wPXOLuK83sRuB24JqM\nY14F3D9QnAfmunuzmR0P/NHMXnL3zYcE5n4HcAfAokWLvLa2Noe3KLmoq6tjrPM586WnmDxpHLW1\nZ43p646FKPJZypTP8BRrLm/69Yv88vk3Qj/uv33uLN53Ys2I9y/WfBYq5TM8ymXhyqVA3wbMyfh5\nNoOGo7h7M3A5gJlNAq5w973BuluAW4J1dwMbzawGOM3dVwaHuBd4eNDrXgV8Mcvr4O5bzKyO9Pj0\nzUjJ0hh0EcnFv6/dzi+ff4PLTp/FomMmh3bcRUdPHlVxLiIyErkU6KuAhWa2gHRn/Crgv2RuYGbV\nQJu7p4CbgDuD5TFgmru3mtkSYAnwaLDbVDM70d1fIz0EZn3G8RYB00l33AeWTQe63L0neL1zge+P\n4D1LEamqjNHVmxx6QxEpWzv2HuDrv1nHGXOn8fcfW0I8phmERaS4DVmgu3vSzK4DHgFiwJ3u/rKZ\n3QzUu/sKoBa41cyc9BCXgc53JfAnM4P0OPNPunsSwMy+ADxgZimgHfhcxsteDdzjhw6QPxn4p2D7\nCtJj0DNnkpESlIhX0N6lDrqIZJdKOTf8qoFkyrn940tVnItIScilg467PwQ8NGjZNzKe3w/cn2W/\nbtIzuWQ75oPAg4dZ960sy54BTs0lXikdmsVFRI7kX59p5OlNrdx6+anMr54YdTgiIqFQq0EKmsag\ni8jhbNy5n+8+/CoXnHQUV505Z+gdRESKhAp0KWiJyhjdfSrQReRQvckUf3fvWiYn4nz3iiUEQylF\nREpCTkNcRKKS7qBriIuIHOp//eE1Xm7exx2fWkbN5ETU4YiIhEoddCloVZUxetRBF5EMqxrb+Mcn\nNnPl8jl88JRjog5HRCR06qBLQUvEK+jtT/EvT7+e9Y5ZxWxjUx+NT78edRglQ/kMT6Hn8mdPv87s\n6RP4+keyzkEgIlL0VKBLQZtfPQGAb/+2RGfUXF+i7ysqymd4CjiXE8bFuOtzZzEpoY8wESlN+usm\nBe2y02fz/pOOJpXyoTcuMk8//TTnnntu1GGUDOUzPIWey6rKGOPHxaIOQ0Qkb1SgS8GbOr4y6hDy\nYtI4Y/rEcVGHUTKUz/AolyIi0dJFoiIiIiIiBUQFuoiIiIhIAVGBLiIiIiJSQFSgi4iIiIgUEBXo\nIiIiIiIFRAW6iIiIiEgBUYEuIiIiIlJAVKCLiIiIiBQQFegiIiIiIgVEBbqIiIiISAFRgS4iIiIi\nUkDM3aOOIW/MbD+wIeo4Skg10BJ1ECVE+QyX8hke5TJcyme4lM/wKJfhWuTuk8M4UDyMgxSwDe6+\nPOogSoWZ1Suf4VE+w6V8hke5DJfyGS7lMzzKZbjMrD6sY2mIi4iIiIhIAVGBLiIiIiJSQEq9QL8j\n6gBKjPIZLuUzXMpneJTLcCmf4VI+w6Nchiu0fJb0RaIiIiIiIsWm1DvoIiIiIiJFRQW6iIiIiEgB\nKdkC3cwuMrMNZrbJzL4SdTyFzszmmNnjZrbezF42s78Nln/LzLab2drgcUnGPjcF+d1gZhdGF31h\nMrNGM3spyFt9sGyGmf3ezDYG/04PlpuZ/e8gny+a2RnRRl9YzGxRxjm41sz2mdnf6fzMnZndaWa7\nzGxdxrJhn49m9plg+41m9pko3kvUDpPLvzezV4N8PWhm04Ll883sQMY5+o8Z+ywL/kZsCvJtUbyf\nqB0mn8P+3dbnftph8nlvRi4bzWxtsFzn5xEcoTbK/99Ody+5BxADNgPHA+OABmBx1HEV8gM4Fjgj\neD4ZeA1YDHwLuCHL9ouDvCaABUG+Y1G/j0J6AI1A9aBl3we+Ejz/CvC94PklwO8AA94FrIw6/kJ9\nBL/fbwLzdH4OK2/vBc4A1mUsG9b5CMwAtgT/Tg+eT4/6vRVILj8IxIPn38vI5fzM7QYd53ngnCDP\nvwMujvq9FVA+h/W7rc/9I+dz0PofAN8Inuv8PHIuD1cb5f1vZ6l20M8CNrn7FnfvBe4BLo04poLm\n7jvcfU3wfD+wHph1hF0uBe5x9x53fx3YRDrvcmSXAv8WPP834M8zlt/lac8B08zs2CgCLAIXAJvd\nvekI2+j8HMTdnwTaBi0e7vl4IfB7d29z93bg98BF+Y++sGTLpbs/6u7J4MfngNlHOkaQzynu/qyn\nP8Hv4q38l5XDnJuHc7jfbX3uB46Uz6AL/nHgl0c6hs7PtCPURnn/21mqBfos4I2Mn7dx5GJTMpjZ\nfOB0YGWw6Lrgq5o7B77GQTnOhQOPmtlqM7s2WHa0u++A9C8+cFSwXPnM3VUc+uGi83Pkhns+Kq+5\n+RzpLtqABWb2gpk9YWbnBctmkc7fAOXy7Ybzu61zMzfnATvdfWPGMp2fORhUG+X9b2epFujZxklp\nPskcmNkk4AHg79x9H/BT4B3AUmAH6a/GQDnOxbnufgZwMfBFM3vvEbZVPnNgZuOAjwK/Chbp/MyP\nw+VPeR2CmX0VSAK/CBbtAOa6++nAl4C7zWwKyuVQhvu7rXzm5moObXDo/MxBltrosJtmWTai87NU\nC/RtwJyMn2cDzRHFUjTMrJL0CfgLd/81gLvvdPd+d08B/8xbwwSU4yG4e3Pw7y7gQdK52zkwdCX4\nd1ewufKZm4uBNe6+E3R+hmC456PyegTBhV8fBj4RDAsgGIrRGjxfTXqc9Imkc5k5DEa5zDCC322d\nm0MwszhwOXDvwDKdn0PLVhsxBn87S7VAXwUsNLMFQcftKmBFxDEVtGBc2s+A9e5+e8byzHHQlwED\nV4WvAK4ys4SZLQAWkr6gRAAzm2hmkweek76AbB3pvA1cvf0Z4N+D5yuATwdXgL8L2Dvw9Zkc4pDu\nj87PURvu+fgI8EEzmx4MOfhgsKzsmdlFwP8LfNTduzKW15hZLHh+POlzcUuQz/1m9q7g7++neSv/\nZW8Ev9v63B/anwGvuvvBoSs6P4/scLURY/G3c6yuhB3rB+kraV8j/b/Br0YdT6E/gPeQ/rrlRWBt\n8LgE+DnwUrB8BXBsxj5fDfK7gTK8unuIfB5PehaBBuDlgXMQmAk8BmwM/p0RLDfgJ0E+XwKWR/0e\nCu0BTABagakZy3R+5p6/X5L+OruPdDfn8yM5H0mPr94UPP4y6vdVQLncRHqM6cDfz38Mtr0i+BvQ\nAKwBPpJxnOWkC8/NwI8J7u5dbo/D5HPYv9v63D98PoPl/wr81aBtdX4eOZeHq43y/rfTgp1ERERE\nRKQAlOoQFxERERGRoqQCXURERESkgKhAFxEREREpICrQRUREREQKiAp0EREREZECogJdRERERKSA\nqEAXERERESkg/xeCBzSYfQK7xQAAAABJRU5ErkJggg==\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "\n", "loss = np.array(train_summary.read_scalar(\"Loss\"))\n", @@ -544,7 +514,9 @@ { "cell_type": "code", "execution_count": 17, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "predictions = trained_model.predict(cc_rdd_test).collect()\n", @@ -596,21 +568,21 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 2", + "display_name": "Python 3", "language": "python", - "name": "python2" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.14" + "pygments_lexer": "ipython3", + "version": "3.5.4" } }, "nbformat": 4, From dc48602f3f6ee6ffe23126028dba82ed1447db03 Mon Sep 17 00:00:00 2001 From: Tim Fox Date: Fri, 2 Feb 2018 17:05:06 -0500 Subject: [PATCH 17/19] Added some more explanation on feedforward notebooks --- ...edforward-credit-card-fraud-pipeline.ipynb | 126 +++++------------- .../feedforward-credit-card-fraud.ipynb | 2 + .../notebooks/feedforward-iris-pipeline.ipynb | 45 ++++--- .../notebooks/feedforward-iris.ipynb | 46 +++++-- 4 files changed, 98 insertions(+), 121 deletions(-) diff --git a/elephantscale/notebooks/feedforward-credit-card-fraud-pipeline.ipynb b/elephantscale/notebooks/feedforward-credit-card-fraud-pipeline.ipynb index ccf223e..dba172b 100644 --- a/elephantscale/notebooks/feedforward-credit-card-fraud-pipeline.ipynb +++ b/elephantscale/notebooks/feedforward-credit-card-fraud-pipeline.ipynb @@ -4,27 +4,17 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Feedforward Network with Credit Card Fraud\n", + "# Feedforward Network with Credit Card Fraud (Pipeline API version)\n", "\n", - "Credit Card Transactions Fraud Detection Example:\n", - "\n", - "The notebook demonstrates how to develop a fraud detection application with the BigDL deep learning library on Apache Spark. We'll try to introduce some techniques that can be used for training a fraud detection model, but some advanced skills is not applicable since the dataset is highly simplified.\n", - "\n", - "Dataset: Credit Card Fraud Detection https://www.kaggle.com/dalpozz/creditcardfraud\n", - "\n", - "This dataset presents transactions that occurred in two days, where we got 492 frauds out of 284,807 transactions. The dataset is highly unbalanced, the positive class (frauds) account for 0.172% of all transactions.\n", - "\n", - "It contains only numerical input variables which are the result of a PCA transformation. \n", - "\n", - "Unfortunately, due to confidentiality issues, we cannot find the original features and more background information about the data. Features V1, V2, ... V28 are the principal components obtained with PCA, the only features which have not been transformed with PCA are 'Time' and 'Amount'. Feature 'Time' contains the seconds elapsed between each transaction and the first transaction in the dataset. The feature 'Amount' is the transaction Amount, this feature can be used for example-dependant cost-senstive learning. Feature 'Class' is the response variable and it takes value 1 in case of fraud and 0 otherwise.\n", - "\n", - "Contact: yuhao.yang@intel.com\n" + "Please view the credit-card-fraud example, which will show how to do this in with an RDD-based approach. Here, we will follow the same approach with a dataframe-based pipeline API based on Spark MLLib.\n" ] }, { "cell_type": "code", "execution_count": 36, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "%matplotlib inline\n", @@ -124,7 +114,9 @@ { "cell_type": "code", "execution_count": 38, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "cc_training = spark.read.csv(\"../data/creditcardfraud/creditcard.csv\", header=True, inferSchema=\"true\", mode=\"DROPMALFORMED\")" @@ -161,7 +153,9 @@ { "cell_type": "code", "execution_count": 40, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "cc_training = cc_training.select([col(c).cast(\"double\") for c in cc_training.columns])\n", @@ -206,7 +200,9 @@ { "cell_type": "code", "execution_count": 42, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "# get the time to split the data.\n", @@ -273,7 +269,9 @@ { "cell_type": "code", "execution_count": 43, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "assembler = VectorAssembler(inputCols=cols, outputCol=\"assembled\")\n", @@ -289,23 +287,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### step 2. split the dataset into training and validation dataset.\n", - "\n", - "Unlike some other training dataset, where the data does not have a time of occurance. For this case, we can know the sequence of the transactions from the Time column. Thus randomly splitting the data into training and validation does not make much sense, since in real world applications, we can only use the history transactions for training and use the latest transactions for validation. Thus we'll split the dataset according the time of occurance." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### step 3. Build the model with BigDL\n", + "## step 2. Build the model with BigDL\n", "From the research community and industry feedback, a simple neural network turns out be the perfect candidate for the fraud detection training. We will quickly build a multiple layer Perceptron with linear layers.\n", "```\n", " val bigDLModel = Sequential()\n", @@ -380,12 +362,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "# Evaluate our Model\n", "\n", - "Now we have finished the training of our first model (which is certainly not the best, keep reading!).\n", - "\n", - "We'll need to think about how do evaluate the trained model:\n", - "\n", - "Given the class imbalance ratio, we recommend measuring the accuracy using the Area Under the Precision-Recall Curve (AUPRC). Confusion matrix accuracy is not meaningful for unbalanced classification. Since even if the model predicts all the records as normal transactions, it will still get an accuracy above 99%." + "Now we are goin to do the Precision, Recall, and AUC. To evaluate our model." ] }, { @@ -433,7 +412,9 @@ { "cell_type": "code", "execution_count": 47, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "y_pred = np.array(predictionDF.select('prediction').collect())\n", @@ -443,7 +424,9 @@ { "cell_type": "code", "execution_count": 48, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [ { "name": "stdout", @@ -474,74 +457,33 @@ "sn.heatmap(df_cm, annot=True,fmt='d');" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To this point, we have finished the training and evaluation with a simple BigDL model. We can see that even though the recall and precision are high, the area under precision-recall curve is not optimistic. That's because we haven't really apply any technique to handle the imbalanced training data.\n", - "\n", - "Next we'll try to optimize the training process.\n", - "\n", - "### step 4. handle the data imbalance\n", - "There are several ways to approach this classification problem taking into consideration this unbalance.\n", - "\n", - "Collect more data? Nice strategy but not applicable in this case.\n", - "\n", - "Resampling the dataset Essentially this is a method that will process the data to have an approximate 50-50 ratio. One way to achieve this is by OVER-sampling, which is adding copies of the under-represented class (better when there're little data) Another is UNDER-sampling, which deletes instances from the over-represented class (better when there are lots of data)\n", - "Apart from under and over sampling, there is a very popular approach called SMOTE (Synthetic Minority Over-Sampling Technique), which is a combination of oversampling and undersampling, but the oversampling approach is not by replicating minority class but constructing new minority class data instance via an algorithm.\n", - "\n", - "We'll start with Resampling.\n", - "\n", - "Since there're 492 frauds out of 284,807 transactions, to build a reasonable training dataset, we'll use UNDER-sampling for normal transactions and use OVER-sampling for fraud transactions. By using the sampling rate as fraud -> 10, normal -> 0.05, we can get a training dataset of (5K fraud + 14K normal) transactions. We can use the training data to fit a model.\n", - "\n", - "Yet we'll soon find that since there're only 5% of all the normal transactions are included in the training data, the model can only cover 5% of all the normal transactions, which is obviousely not optimistic. So how can we get a better converage for the normal transactions without breaking the ideal ratio in the training dataset?\n", - "\n", - "An immediate improvement would be to train multiple models. For each model, we will run the resampling from the original dataset and get a new training data set. After training, we can select best voting strategy for all the models to make the prediction.\n", - "\n", - "We'll use Ensembling of neural networks. That's where a Bagging classifier becomes handy. Bagging is an Estimator we developed for ensembling of multiple other Estimator.\n", - "\n", - "```\n", - "package org.apache.spark.ml.ensemble\n", - "\n", - "class Bagging[M <: Model[M]](override val uid: String)\n", - " extends Estimator[BaggingModel[M]]\n", - " with BaggingParams[M] {\n", - "For usage, user need to set the specific Estimator to use and the number of models to be trained:\n", - " val estimator = new Bagging()\n", - " .setPredictor(dlClassifier)\n", - " .setLabelCol(\"Class\")\n", - " .setIsClassifier(true)\n", - " .setNumModels(10)\n", - "```\n", - "\n", - "Internally, Bagging will train $(numModels) models. Each model is trained with the resampled data from the original dataset." - ] - }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 2", + "display_name": "Python 3", "language": "python", - "name": "python2" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.14" + "pygments_lexer": "ipython3", + "version": "3.5.4" } }, "nbformat": 4, diff --git a/elephantscale/notebooks/feedforward-credit-card-fraud.ipynb b/elephantscale/notebooks/feedforward-credit-card-fraud.ipynb index 6de5d24..43a100b 100644 --- a/elephantscale/notebooks/feedforward-credit-card-fraud.ipynb +++ b/elephantscale/notebooks/feedforward-credit-card-fraud.ipynb @@ -197,6 +197,7 @@ "cell_type": "code", "execution_count": null, "metadata": { + "collapsed": true, "scrolled": true }, "outputs": [], @@ -461,6 +462,7 @@ "cell_type": "code", "execution_count": null, "metadata": { + "collapsed": true, "scrolled": false }, "outputs": [], diff --git a/elephantscale/notebooks/feedforward-iris-pipeline.ipynb b/elephantscale/notebooks/feedforward-iris-pipeline.ipynb index dd6f003..878eff7 100644 --- a/elephantscale/notebooks/feedforward-iris-pipeline.ipynb +++ b/elephantscale/notebooks/feedforward-iris-pipeline.ipynb @@ -4,10 +4,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Feedforward Neural Network on the Iris Dataset\n", - "# Using Pipeline API\n", + "# Feedforward Neural Network on the Iris Dataset Using Pipeline API\n", "\n", - "Fisher's Iris dataset \n", + "This notebook shows how to do the iris classification on the Spark mllib dataframe-based pipeline API (as opposed to the RDD API). For introduction, first see the feedforward-iris.ipynb.\n", + "\n", + "### Fisher's Iris dataset \n", "\n", "We are going to try the same Iris dataset as on feedforward-iris.ipynb, but this time using\n", "Spark MLLib's Pipeline-based API.\n", @@ -41,7 +42,9 @@ { "cell_type": "code", "execution_count": 1, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "%matplotlib inline\n", @@ -103,7 +106,9 @@ { "cell_type": "code", "execution_count": 2, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "learning_rate = 0.01\n", @@ -194,7 +199,9 @@ { "cell_type": "code", "execution_count": 4, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "iris_training = spark.read.csv(\"../data/iris/iris_training.csv\", header=True, inferSchema=\"true\", mode=\"DROPMALFORMED\")\n", @@ -204,7 +211,9 @@ { "cell_type": "code", "execution_count": 5, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "iris_training = iris_training.select([col(c).cast(\"double\") for c in iris_training.columns])\n", @@ -227,7 +236,9 @@ { "cell_type": "code", "execution_count": 6, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "assembler = VectorAssembler(inputCols=['c1','c2','c3','c4'], outputCol=\"assembled\")\n", @@ -450,7 +461,9 @@ { "cell_type": "code", "execution_count": 33, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "y_pred = np.array(predictionDF.select('prediction').collect())\n", @@ -496,28 +509,30 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 2", + "display_name": "Python 3", "language": "python", - "name": "python2" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.14" + "pygments_lexer": "ipython3", + "version": "3.5.4" } }, "nbformat": 4, diff --git a/elephantscale/notebooks/feedforward-iris.ipynb b/elephantscale/notebooks/feedforward-iris.ipynb index 7872678..dc1c6ce 100644 --- a/elephantscale/notebooks/feedforward-iris.ipynb +++ b/elephantscale/notebooks/feedforward-iris.ipynb @@ -6,7 +6,9 @@ "source": [ "# Feedforward Neural Network on the Iris Dataset\n", "\n", - "Fisher's Iris dataset \n", + "This notebook shows how to do a simple feedforward neural Network on BigDL using the RDD based API. If you are interested in seeing the same example using the dataframe-based Spark MLLib pipeline, please see feedforward-iris-pipeline.ipynb.\n", + "\n", + "### Fisher's Iris dataset \n", "\n", "This dataset contains 150 samples, with 4 dimensions, as follows:\n", "\n", @@ -36,7 +38,9 @@ { "cell_type": "code", "execution_count": 1, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "%matplotlib inline\n", @@ -96,7 +100,9 @@ { "cell_type": "code", "execution_count": 2, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "learning_rate = 0.1\n", @@ -186,7 +192,9 @@ { "cell_type": "code", "execution_count": 4, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "iris_training = spark.read.csv(\"../data/iris/iris_training.csv\", header=True, inferSchema=\"true\", mode=\"DROPMALFORMED\")\n", @@ -201,7 +209,9 @@ { "cell_type": "code", "execution_count": 5, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "iris_k_train = iris_training.rdd.map(list)\n", @@ -591,7 +601,9 @@ { "cell_type": "code", "execution_count": 14, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "predictions = trained_model.predict(iris_rdd_test).collect()\n", @@ -645,7 +657,9 @@ { "cell_type": "code", "execution_count": 16, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "def map_predict_label(outputs, threshold):\n", @@ -659,7 +673,9 @@ { "cell_type": "code", "execution_count": 17, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "true_positive_rate = list()\n", @@ -673,28 +689,30 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 2", + "display_name": "Python 3", "language": "python", - "name": "python2" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.14" + "pygments_lexer": "ipython3", + "version": "3.5.4" } }, "nbformat": 4, From 77b536df5860ca58b2ad4da2d62b87feb51d9adf Mon Sep 17 00:00:00 2001 From: Tim Fox Date: Fri, 2 Feb 2018 17:39:23 -0500 Subject: [PATCH 18/19] Added warning about --executor memory 16gb --- elephantscale/notebooks/lstm-20news.ipynb | 41 ++++++++++++++++------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/elephantscale/notebooks/lstm-20news.ipynb b/elephantscale/notebooks/lstm-20news.ipynb index 04cb4be..7df4484 100644 --- a/elephantscale/notebooks/lstm-20news.ipynb +++ b/elephantscale/notebooks/lstm-20news.ipynb @@ -6,13 +6,20 @@ "source": [ "# LSTM Example: 20 Newsgroup Classification\n", "\n", - "This shows how to do an LSTM classification on the 20 newsgroup collection" + "This shows how to do an LSTM classification on the 20 newsgroup collection\n", + "\n", + "## Requirements\n", + "\n", + "Running this notebook requires a minimum of 16GB executor memory. Starting it up in an ordinary jupyter notebook \n", + "won't work as you'll get an out of memory exception. Quit jupyter and make sure you start this with at least --executor-memory 16gb" ] }, { "cell_type": "code", "execution_count": 1, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "import itertools\n", @@ -32,7 +39,9 @@ { "cell_type": "code", "execution_count": 2, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "\n", @@ -90,7 +99,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "\n", @@ -112,7 +123,9 @@ { "cell_type": "code", "execution_count": 3, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "\n", @@ -132,7 +145,9 @@ { "cell_type": "code", "execution_count": 4, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "\n", @@ -283,28 +298,30 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 2", + "display_name": "Python 3", "language": "python", - "name": "python2" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.14" + "pygments_lexer": "ipython3", + "version": "3.5.4" } }, "nbformat": 4, From 56a0dacc57333d4d82753a567d026c7c0a6fe12b Mon Sep 17 00:00:00 2001 From: Tim Fox Date: Fri, 2 Feb 2018 18:32:53 -0500 Subject: [PATCH 19/19] Added README.md --- elephantscale/scripts/README.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 elephantscale/scripts/README.md diff --git a/elephantscale/scripts/README.md b/elephantscale/scripts/README.md new file mode 100644 index 0000000..4af92eb --- /dev/null +++ b/elephantscale/scripts/README.md @@ -0,0 +1,4 @@ +# Scripts Directory + +This contains scripts. For video 3, run the run-transfer.sh script. You will need to start with plenty +of memory (16gb minimum).